Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// How much verification output is kept and fed back to the fixer.
64const OUTPUT_TAIL: usize = 8_000;
65
66/// Bytes of a failing command's output kept in an event, so the reason a run
67/// stopped is readable from the report without opening `run.json`.
68const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
71/// command's pid before releasing the build cache's lease.
72const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
75/// command's pid to actually exit before giving up and releasing anyway.
76///
77/// A timeout means the process was asked to die (`kill_on_drop`,
78/// `start_kill`), not that it already has — on Windows in particular that can
79/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
80/// the instant the command returns would let the very next acquirer (this
81/// run's own next round, another run's verification, the janitor's prune)
82/// start touching the same directory while it might still be writing to it,
83/// so this polls the actual pid — real confirmation, not a fixed guess —
84/// until it is gone or this ceiling is reached. It is still not full
85/// process-tree reaping: a grandchild the timed-out process spawned and that
86/// outlives it independently is invisible to a pid check, and continuing to
87/// observe and collect *that* stays a different piece of work with its own
88/// owner. Set generously because the common case returns early the moment
89/// the pid is confirmed gone, not because every timeout pays this in full.
90const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92/// Consecutive review rounds with no tree progress (see
93/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
94/// instead of spending the rest of the round budget.
95///
96/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
97/// legitimately finds nothing left to change (its previous round's fix already
98/// covered it, and this round's reviewers re-raised only nits) looks the same
99/// as one that is spinning, for exactly one round. Two in a row is where the
100/// two stop being distinguishable, and a review round on this workload has
101/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
102/// third attempt at a tree that has not moved twice running is pure cost.
103/// This does not touch `review_rounds` itself, which stays the operator's
104/// call.
105pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
108/// a base that moved before giving up and leaving the run `Blocked` for a
109/// person.
110///
111/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
112/// that keeps moving faster than a run can catch it is not something more
113/// rebasing fixes, it is a person's call. Not the same *number as*
114/// `land_rounds` - this budget is spent before a pull request exists, land's
115/// after - but bounded for the identical reason, so it uses the same
116/// default. Counted across both call sites in [`Runner::finish_after_tally`]
117/// (once before review, once before the gate), because either one finding
118/// the base still moving is the same signal.
119pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
120
121/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
122/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
123/// reply held no [`FixReport`].
124///
125/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
126/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
127/// "I'll pause here until the `cargo make check` background run reports
128/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
129/// No `FixReport` was ever collected from that seat, and the run moved on to
130/// the next review round regardless.
131///
132/// Bounded independently of `review_rounds` and `graph.retries`: this
133/// recovers one seat's missing report mid-round, not a new round of review or
134/// an ordinary parse retry, and must not itself become the unbounded wait the
135/// rest of this module exists to avoid.
136const MAX_FIX_CONTINUATIONS: usize = 2;
137
138/// One queued agent invocation.
139///
140/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
141/// CLI hung up on its own stream is asked again from the same job rather than
142/// rebuilt from scratch. See [`Runner::resume_undelivered`].
143#[derive(Clone)]
144struct SeatJob {
145    spec: AgentSpec,
146    seat: SeatState,
147    cwd: PathBuf,
148    prompt: String,
149    timeout: Duration,
150    allow_write: bool,
151    sessions: bool,
152    artifacts: PathBuf,
153    stem: String,
154    /// The prompt for a seat that has been handed to another roster agent
155    /// (a fresh session): everything the original seat would have
156    /// remembered. `None` when `prompt` already carries it, as the first
157    /// ranking and the implement prompt do. Never a resume-style prompt.
158    handover: Option<String>,
159}
160
161/// How the graph reads one agent invocation.
162///
163/// Quota is split out from an ordinary failure on purpose: a rate-limited call
164/// is known to fail again if retried now, so the retry loop must not spend an
165/// attempt on it. `Dropped` is split out for the opposite reason: unlike
166/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
167/// error JSON, never the agent's answer — a caller that matched only
168/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
169/// left to read that JSON as if it were usable output. `resume_undelivered`
170/// is the only caller that acts on it; everywhere else it is reported like an
171/// ordinary failure.
172enum AgentOutcome {
173    /// A usable output.
174    Ok(AgentOutput),
175    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
176    Quota(AgentOutput),
177    /// The CLI hung up on its own stream after billed work. See
178    /// [`agent::AgentOutput::work_undelivered`].
179    Dropped(AgentOutput),
180    /// Any other failure: a timeout, a bad exit code, an empty reply.
181    Failed(String),
182    /// A resource prevented admission or interrupted execution. Not an agent
183    /// failure: never advance a fallback chain or spend an attempt on it.
184    /// Post-spawn interruptions carry a typed error into wave, which records
185    /// implementation preservation before returning this outcome. This outcome
186    /// alone is not proof that no source changes were made.
187    /// [`RunState::resource_block`] says why.
188    ResourceBlocked(String),
189}
190
191/// A request to park the run at its next node boundary.
192///
193/// Cloning is how the request travels: the loop keeps one handle and hands a
194/// clone to each [`Runner`], and every clone points at the same flag. There
195/// is no channel because there is nothing to send - the only message is
196/// "park", it is idempotent, and a flag cannot be missed by a receiver that
197/// was not listening yet.
198///
199/// The boundary is what makes this cheap. Every node writes the run's state
200/// before the next one starts, and every node skips what is already recorded:
201/// `prep` returns early once candidates exist, `implement` asks only the seats
202/// with nothing on disk, `judge` returns early once judgements exist. So a
203/// parked run resumes into exactly the node it stopped before, and no agent
204/// work is thrown away. Killing the process mid-node, by contrast, loses
205/// whatever the seats in flight had not yet written - which for an implement
206/// wave is an hour of paid work.
207///
208/// A [`Runner`] watches two independent handles of this type - see
209/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
210/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
211/// clone covering the whole daemon's lifetime and is never asked to un-park,
212/// which is correct exactly because nothing is dispatched after it fires.
213/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
214/// that parks for an interrupted task must go on to run other tasks
215/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
216/// reusing the daemon-wide one.
217#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221    /// A pause nobody has asked for yet.
222    #[must_use]
223    pub fn new() -> Self {
224        Self::default()
225    }
226
227    /// Ask the run to park at its next node boundary. Idempotent.
228    pub fn park(&self) {
229        self.0.store(true, Ordering::SeqCst);
230    }
231
232    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
233    /// fold into the run's own `park` event - so an operator reading the run
234    /// later knows this was a deliberate interrupt rather than a shutdown or
235    /// a binary swap. The first reason recorded wins; a park already in
236    /// flight is not relabelled by a second, unrelated request.
237    pub fn park_because(&self, reason: impl Into<String>) {
238        self.park_for(ParkCause::Interrupt {
239            reason: reason.into(),
240        });
241    }
242
243    /// [`Pause::park`], recording the structured cause. The first cause
244    /// recorded wins, like [`Pause::park_because`].
245    pub fn park_for(&self, cause: ParkCause) {
246        let mut reason_guard = self
247            .1
248            .lock()
249            .unwrap_or_else(std::sync::PoisonError::into_inner);
250        if reason_guard.is_none() {
251            *reason_guard = Some(cause);
252        }
253        drop(reason_guard);
254        self.park();
255    }
256
257    /// Has a park been asked for?
258    #[must_use]
259    pub fn parked(&self) -> bool {
260        self.0.load(Ordering::SeqCst)
261    }
262
263    /// Why the park was asked for, when the caller used [`Pause::park_because`].
264    #[must_use]
265    pub fn reason(&self) -> Option<String> {
266        self.cause().map(|c| c.label())
267    }
268
269    /// The structured cause, when one was recorded.
270    #[must_use]
271    pub fn cause(&self) -> Option<ParkCause> {
272        self.1
273            .lock()
274            .unwrap_or_else(std::sync::PoisonError::into_inner)
275            .clone()
276    }
277}
278
279/// Drives one run.
280pub struct Runner {
281    /// Run state; public so the CLI can report on it.
282    pub state: RunState,
283    roles: ResolvedRoles,
284    sem: Arc<Semaphore>,
285    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
286    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
287    /// this is never the same handle as `interrupt`.
288    pause: Pause,
289    /// Set when `magi serve`'s interrupt scheduler wants this specific run
290    /// parked at its next node boundary, to let a task marked
291    /// [`crate::queue::Task::interrupt`] run alone before this one carries
292    /// on. Unlike `pause`, a fresh, unshared handle per run - see
293    /// [`Runner::watch_interrupt`].
294    interrupt: Pause,
295}
296
297/// Where the branch's own commits start: its merge base with the base branch
298/// as the remote has it now, else with the recorded `base_commit`. A branch
299/// rebased onto a base that moved past `base_commit` would otherwise count
300/// the base's commits as its own under `base_commit..branch`.
301async fn review_base(
302    repo: &Path,
303    remote: &str,
304    base_branch: &str,
305    base_commit: &str,
306    branch: &str,
307) -> String {
308    review_base_checked(repo, remote, base_branch, base_commit, branch)
309        .await
310        .0
311}
312
313/// [`review_base`] plus whether the base was read from a freshly fetched
314/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
315/// never older than `base_commit`'s view of the base), but the answer is then
316/// not trusted to rewrite a pull request's title.
317async fn review_base_checked(
318    repo: &Path,
319    remote: &str,
320    base_branch: &str,
321    base_commit: &str,
322    branch: &str,
323) -> (String, bool) {
324    let tracking = format!("{remote}/{base_branch}");
325    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326    if git::rev_exists(repo, &tracking).await
327        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328        && !mb.is_empty()
329    {
330        return (mb, fresh);
331    }
332    let mb = git::merge_base(repo, base_commit, branch)
333        .await
334        .ok()
335        .filter(|mb| !mb.is_empty())
336        .unwrap_or_else(|| base_commit.to_owned());
337    (mb, false)
338}
339
340/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
341/// when git cannot say or finds nothing: a stale list is better than a wrong
342/// or empty one.
343pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344    if !is_review_run(state) {
345        return;
346    }
347    let base = review_base(
348        &state.repo,
349        &state.config.merge.remote,
350        &state.base_branch,
351        &state.base_commit,
352        branch,
353    )
354    .await;
355    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356        && !subjects.is_empty()
357        && state.reviewed_commits.as_ref() != Some(&subjects)
358    {
359        state.reviewed_commits = Some(subjects);
360        state.save().ok();
361    }
362}
363
364/// Subjects of the base's commits between the recorded start and the branch's
365/// merge base: what a stale `base_commit..branch` would have mistaken for the
366/// branch's own work.
367async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368    let (base, trusted) = review_base_checked(
369        &state.repo,
370        &state.config.merge.remote,
371        &state.base_branch,
372        &state.base_commit,
373        branch,
374    )
375    .await;
376    if !trusted {
377        return None;
378    }
379    git::subjects(&state.repo, &state.base_commit, &base)
380        .await
381        .ok()
382}
383
384/// May an adopted pull request's title be replaced with `computed`? Only when
385/// it is empty, magi's own shape, or a base commit's subject that leaked in;
386/// a title a person wrote stays. Never when `computed` is itself a leak.
387pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389    if is_leak(computed) {
390        return false;
391    }
392    let cur = current.trim();
393    cur.is_empty()
394        || cur.starts_with(REVIEW_PROMPT_OPENING)
395        || cur.starts_with("chore: land candidate")
396        || cur.starts_with("magi: candidate")
397        || is_leak(cur)
398}
399
400/// The commit a run branches from: the base branch as the remote has it.
401///
402/// Two failures this replaces. A run used to branch off `HEAD` and so refused
403/// to start on a dirty tree, which made `magi serve` decline every task for as
404/// long as the operator had work in progress - most of the time. Branching off
405/// the *local* base branch fixed that and introduced a worse one: `land` merges
406/// the winner on GitHub, nothing updates the local ref, and the next run
407/// branches off a base missing everything the previous runs landed. Two tasks
408/// in a row from a phone would have had the second silently re-implementing
409/// against stale code and opening a pull request that reverted the first.
410///
411/// Only refs move here - no checkout, no local branch, no merge - so it is safe
412/// with uncommitted work in the tree. A machine with no network still starts:
413/// the fetch may fail and the local tip is used with a warning, because
414/// refusing to run offline is a worse failure than running against a base the
415/// operator can see for themselves.
416///
417/// One function, called by both entry points. Two answers to "where does a run
418/// branch from" is the kind of drift nobody notices until a diff is wrong.
419/// Bring the local `branch` in line with `<remote>/<branch>` before a review
420/// checks it out.
421///
422/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
423/// anything other than plain `git push` from this checkout (a jj colocated
424/// workspace, another clone) moves only the remote-tracking ref - so the local
425/// one can be a stale placeholder. It moves only when local is behind the remote or is an
426/// empty placeholder that diverged from it; unpushed local work is kept, and a real
427/// divergence is refused rather than guessed at.
428async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429    let tracking = format!("{remote}/{branch}");
430    let fetched = git::fetch(repo, remote, branch).await;
431    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432    let local_exists = git::branch_exists(repo, branch).await?;
433    if !fresh {
434        if !local_exists {
435            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436        }
437        tracing::warn!(
438            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439        );
440        return Ok(());
441    }
442    let remote_sha = git::rev_parse(repo, &tracking).await?;
443    if !local_exists {
444        git::git(repo, &["branch", branch, &tracking]).await?;
445        return Ok(());
446    }
447    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449        return Ok(());
450    }
451    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452        // Diverged. `reconcile` settles it only when it can prove nothing is
453        // lost: a local tip that is the remote's change rebased is pushed over
454        // it (lease pinned to the tip read here), a tip whose every extra
455        // commit is empty is a placeholder the remote's work replaced, and
456        // anything else is two different changes - a question for a person.
457        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458            .await?
459        {
460            crate::reconcile::Reconciliation::Pushed => {
461                tracing::warn!(
462                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463                    short(&local_sha),
464                    short(&remote_sha)
465                );
466                return Ok(());
467            }
468            crate::reconcile::Reconciliation::Placeholder => {}
469            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470        }
471    }
472    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473    if !out.ok() {
474        bail!(
475            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476            short(&local_sha),
477            short(&remote_sha),
478            out.stderr
479        );
480    }
481    tracing::warn!(
482        "local `{branch}` was stale: fast-forwarded {} -> {}",
483        short(&local_sha),
484        short(&remote_sha)
485    );
486    Ok(())
487}
488
489async fn resolve_base(
490    repo: &Path,
491    base_branch: &str,
492    remote: &str,
493    coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495    let _lease = crate::repo_lock::SeatLease::operation_with(
496        repo,
497        coordination,
498        &crate::repo_lock::WaitPolicy::current(),
499    )
500    .await?;
501    let tracking = format!("{remote}/{base_branch}");
502    let fetched = git::fetch(repo, remote, base_branch).await;
503    if let Ok(out) = &fetched
504        && out.ok()
505        && git::rev_exists(repo, &tracking).await
506    {
507        return Ok(crate::run::FetchedBase {
508            remote: remote.to_owned(),
509            branch: base_branch.to_owned(),
510            commit: git::rev_parse(repo, &tracking).await?,
511            fetched_at: jiff::Timestamp::now(),
512        });
513    }
514    let why = match &fetched {
515        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516        Ok(_) => format!("{remote} has no {base_branch}"),
517        Err(e) => e.to_string(),
518    };
519    // No fallback to the local branch: it may be behind, and branching off an
520    // old commit is the stale-checkout bug this check exists to prevent.
521    bail!(
522        "cannot read {tracking} ({why}); refusing to branch off the local \
523         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524         base / remote in magi.toml. This is launcher preparation: check the \
525         launcher's network, Git access and repository coordination before \
526         retrying"
527    )
528}
529
530/// Outcome of the shared fixer chain, before caller-owned stop policy.
531enum FixStep {
532    Resource,
533    Recorded { report_lost: bool },
534}
535
536/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
537/// on an early return or a panic.
538///
539/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
540/// manual `magi fix` invocations against the same run are otherwise
541/// invisible to each other and would race to remove and recreate the same
542/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
543/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
544/// which is only ever reclaimed later, out of band, by
545/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
546/// `magi fix` invocation is not necessarily running under either of those, so
547/// nothing would ever sweep a lock a killed or crashed process left behind.
548/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
549/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
550/// lease use: an unreadable or unparsable pid, or a liveness query the
551/// platform cannot answer, reads as alive and the lock is left in place.
552struct FixClaim {
553    path: PathBuf,
554}
555
556impl FixClaim {
557    fn acquire(dir: &Path) -> Result<Self> {
558        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559        let path = dir.join("fix.lock");
560        let retry = crate::lock_file::Retry::new(None);
561        match Self::create(&path, retry) {
562            Ok(claim) => Ok(claim),
563            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564                if Self::reclaim_if_dead(&path) {
565                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566                } else {
567                    bail!(
568                        "another repair or `magi fix` is already running for this run ({} exists)",
569                        path.display()
570                    )
571                }
572            }
573            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574        }
575    }
576
577    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578        let mut f = retry.create(path)?;
579        use std::io::Write as _;
580        // Read back by `reclaim_if_dead` on a later, stuck invocation.
581        writeln!(f, "{}", std::process::id())?;
582        Ok(Self {
583            path: path.to_owned(),
584        })
585    }
586
587    /// True if the lock named a process confirmed dead, in which case it was
588    /// also removed. Never true on an unreadable file, an unparsable pid, or
589    /// a liveness query the platform cannot answer — see this type's own doc.
590    fn reclaim_if_dead(path: &Path) -> bool {
591        let dead = std::fs::read_to_string(path)
592            .ok()
593            .and_then(|body| body.trim().parse::<u32>().ok())
594            .is_some_and(|pid| !crate::proc::pid_alive(pid));
595        dead && std::fs::remove_file(path).is_ok()
596    }
597}
598
599impl Drop for FixClaim {
600    fn drop(&mut self) {
601        let _ = std::fs::remove_file(&self.path);
602    }
603}
604
605/// Holds the mutable runner as well as its durable lease: cancellation must
606/// clear both copies of ownership, including state a caller may save later.
607struct DriverExecution<'a> {
608    runner: &'a mut Runner,
609    driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613    fn drop(&mut self) {
614        self.driver.release_local(&mut self.runner.state);
615    }
616}
617
618impl Runner {
619    /// Start a fresh run against `repo`.
620    pub async fn start(
621        repo: &Path,
622        instruction: String,
623        config: Config,
624        origin: Origin,
625    ) -> Result<Self> {
626        Self::start_naming(repo, instruction, "", config, origin).await
627    }
628
629    /// [`Runner::start`] for a queued task. Titles and instructions never
630    /// select implementation sources. The title argument is retained for callers.
631    pub async fn start_naming(
632        repo: &Path,
633        instruction: String,
634        _also_scan: &str,
635        config: Config,
636        origin: Origin,
637    ) -> Result<Self> {
638        let repo = git::toplevel(repo).await?;
639        let missing = agent::missing_programs(&config.agents);
640        if !missing.is_empty() {
641            bail!(
642                "these agent programs are not on PATH: {}. Fix the roster in \
643                 magi.toml or install them.",
644                missing.join(", ")
645            );
646        }
647        let base_branch =
648            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649                .await?;
650        let fetched_base = resolve_base(
651            &repo,
652            &base_branch,
653            &config.merge.remote,
654            config.coordination.as_ref(),
655        )
656        .await?;
657        let base_commit = fetched_base.commit.clone();
658        let roles = config.resolve_roles()?;
659        let max_parallel = config.graph.max_parallel.max(1);
660        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661        // Recorded before the first save, so a crash right after minting
662        // cannot leave a run with no origin. Legibility only: nothing reads it
663        // to decide anything.
664        state.fetched_base = Some(fetched_base);
665        state.origin = Some(origin);
666        state.reserve()?;
667        state.event("start", format!("run {} created", state.id));
668        state.save()?;
669        Ok(Self {
670            state,
671            roles,
672            sem: Arc::new(Semaphore::new(max_parallel)),
673            pause: Pause::new(),
674            interrupt: Pause::new(),
675        })
676    }
677
678    /// Open a review-only run against work that already exists on `branch`.
679    ///
680    /// The expensive half of the graph is the implement wave — measured at
681    /// 111 and 134 internal tool-loop turns on this repository, against a
682    /// handful for a judge or a reviewer. The cheap half is worth running on
683    /// hand-written work too, and there was no way to reach it.
684    ///
685    /// No new state and no schema change are needed: a run with **one** viable
686    /// candidate and a tally already decided degrades `execute` to exactly
687    /// review → gate → merge, because `judge` skips a single-candidate field,
688    /// `deliberate` has fewer than two first choices to reconcile, `vote`
689    /// returns early, `tally` is already present and `fold_losers` has no
690    /// losers. Resuming such a run therefore does the right thing as well.
691    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692        Self::review_taking_over(repo, branch, config, None, origin).await
693    }
694
695    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
696    /// at the same task still has `branch` checked out, its worktree is
697    /// released first if that is safe (see [`crate::handover`]), and the
698    /// review refuses with the reason if it is not. `None` is a hand-run
699    /// review: it has no earlier attempts, so only a worktree of a dead run
700    /// magi recorded itself can be released.
701    pub async fn review_taking_over(
702        repo: &Path,
703        branch: &str,
704        config: Config,
705        takeover: Option<crate::handover::Takeover>,
706        origin: Origin,
707    ) -> Result<Self> {
708        let repo = git::toplevel(repo).await?;
709        let missing = agent::missing_programs(&config.agents);
710        if !missing.is_empty() {
711            bail!(
712                "these agent programs are not on PATH: {}. Fix the roster in \
713                 magi.toml or install them.",
714                missing.join(", ")
715            );
716        }
717        let base_branch =
718            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719                .await?;
720        if base_branch == branch {
721            bail!("`{branch}` is the base branch; there is nothing to review against");
722        }
723        let fetched_base = resolve_base(
724            &repo,
725            &base_branch,
726            &config.merge.remote,
727            config.coordination.as_ref(),
728        )
729        .await?;
730        let base_commit = fetched_base.commit.clone();
731
732        let roles = config.resolve_roles()?;
733        let max_parallel = config.graph.max_parallel.max(1);
734        let mut state = RunState::new(
735            repo.clone(),
736            base_branch,
737            base_commit.clone(),
738            String::new(),
739            config,
740        );
741        state.fetched_base = Some(fetched_base);
742        state.origin = Some(origin);
743        state.reserve()?;
744
745        // Released before anything else touches the branch: a stale local
746        // branch is moved with `git branch -f`, which git refuses while an
747        // earlier attempt's worktree still has it checked out. Everything
748        // after this point that can fail puts the old run back.
749        // A hand-run review has no task, hence no earlier attempts, but a
750        // worktree of a dead run magi made may still be released.
751        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752            earlier: Vec::new(),
753            home: crate::run::home(),
754            choice: None,
755            task_instruction: None,
756        });
757        let wait = repository_wait(&state, None);
758        let coordination_config = state.config.coordination.clone();
759        let coordination =
760            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761                .await?;
762        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763        if let Some(released) = &released {
764            state.event(
765                "release",
766                format!(
767                    "took `{branch}` over from run {}: its worktree was released: {}",
768                    crate::run::short_of(&released.old_id),
769                    released.audit
770                ),
771            );
772        }
773        // The owner's answer to an earlier divergence question is applied
774        // here: after the release (git will not move a checked-out branch)
775        // and before the sync that would otherwise ask again.
776        if let Some(choice) = takeover.choice.as_ref()
777            && let Err(e) =
778                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779                    .await
780        {
781            if let Some(released) = &released {
782                released.restore(&repo, branch).await;
783            }
784            return Err(e.context("applying the owner's answer about the diverged branch"));
785        }
786        // An earlier attempt of this task that published the task head, and
787        // whose work is the branch under review, hands that head over: the
788        // review run updates the same pull request. Without one (a legacy
789        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
790        // own head and its own open pull request is adopted as before.
791        for id in takeover.earlier.iter().rev() {
792            if let Ok(old) = RunState::load_under(id, &takeover.home)
793                && let Some(head) = old.pr_head.clone()
794                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795            {
796                state.pr_head = Some(head);
797                break;
798            }
799        }
800        drop(coordination);
801        let opened = Self::open_review(
802            &repo,
803            branch,
804            state,
805            roles,
806            max_parallel,
807            base_commit,
808            &takeover,
809        )
810        .await;
811        if opened.is_err()
812            && let Some(released) = &released
813        {
814            let _coordination = crate::repo_lock::SeatLease::operation_with(
815                &repo,
816                coordination_config.as_ref(),
817                &wait,
818            )
819            .await?;
820            released.restore(&repo, branch).await;
821        }
822        opened
823    }
824
825    /// Recover exactly the operator-selected public implementation, without touching old sources.
826    pub async fn recover_pr(
827        repo: &Path,
828        task: &str,
829        selection: crate::pr_recovery::Selection,
830        mut config: Config,
831    ) -> Result<Self> {
832        selection.configure(&mut config)?;
833        selection.check_task(task, &selection.instruction)?;
834        let base_branch =
835            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836                .await?;
837        let fetched_base = resolve_base(
838            repo,
839            &base_branch,
840            &config.merge.remote,
841            config.coordination.as_ref(),
842        )
843        .await?;
844        let base_commit = fetched_base.commit.clone();
845        selection
846            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847            .await?;
848        Box::pin(selection.verify_source(repo)).await?;
849        if let Some(proof) = &selection.source_proof {
850            anyhow::ensure!(
851                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852                "source rebase base is outside the fetched landing base"
853            );
854        }
855        let roles = config.resolve_roles()?;
856        let max_parallel = config.graph.max_parallel.max(1);
857        let mut state = RunState::new(
858            repo.to_path_buf(),
859            base_branch,
860            base_commit.clone(),
861            selection.instruction.clone(),
862            config,
863        );
864        state.fetched_base = Some(fetched_base);
865        state.origin = Some(crate::run::Origin::queue(task));
866        state.pr_head = Some(selection.branch.clone());
867        state.pr_recovery = Some(selection.clone());
868        state.reserve()?;
869        if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870            // Never leave a reserved record behind as a phantom owner of the head.
871            state.discard_reserved();
872            return Err(e);
873        }
874        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875    }
876
877    async fn open_selected(
878        repo: &Path,
879        state: RunState,
880        roles: ResolvedRoles,
881        max_parallel: usize,
882        base_commit: String,
883        selection: crate::pr_recovery::Selection,
884    ) -> Result<Self> {
885        let mut state = state;
886        let branch = state.branch_for('A');
887        let wait = repository_wait(&state, None);
888        let prepared = async {
889            let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890            git::git(repo, &["branch", &branch, selection.source()]).await?;
891            drop(coordination);
892            anyhow::Ok(())
893        }
894        .await;
895        if let Err(e) = prepared {
896            // Nothing was exposed yet: do not leave a reserved record as a
897            // phantom owner of the public head.
898            state.discard_reserved();
899            return Err(e);
900        }
901        state.event(
902            "recovery",
903            format!(
904                "selected implementation {} for {} (public lease {})",
905                selection.source(),
906                selection.url,
907                selection.head
908            ),
909        );
910        let takeover = crate::handover::Takeover {
911            earlier: Vec::new(),
912            home: crate::run::home(),
913            choice: None,
914            task_instruction: Some(selection.instruction.clone()),
915        };
916        if let Err(e) = state.save() {
917            state.discard_reserved();
918            return Err(e);
919        }
920        let id = state.id.clone();
921        let opened = Self::open_review(
922            repo,
923            &branch,
924            state,
925            roles,
926            max_parallel,
927            base_commit,
928            &takeover,
929        )
930        .await;
931        match opened {
932            Ok(runner) => Ok(runner),
933            Err(e) => {
934                if let Ok(mut partial) = RunState::load(&id) {
935                    partial.status = RunStatus::Blocked;
936                    partial.driver_exited = true;
937                    partial.event(
938                        "recovery",
939                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
940                    );
941                    partial.save()?;
942                }
943                Err(crate::pr_recovery::StartupFailed {
944                    run: id,
945                    cause: format!("{e:#}"),
946                }
947                .into())
948            }
949        }
950    }
951
952    /// What the newest earlier attempt that reviewed `branch` leaves for this
953    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
954    /// no earlier attempts. Reads the earlier record only: ownership of its
955    /// worktree and branch is [`crate::handover`]'s business.
956    async fn inherit(
957        repo: &Path,
958        branch: &str,
959        base_commit: &str,
960        takeover: &crate::handover::Takeover,
961    ) -> Option<crate::recovery::RecoveryRecord> {
962        use crate::recovery::{Relation, assess};
963        let newest = takeover.earlier.last()?;
964        let mut chosen = None;
965        for id in takeover.earlier.iter().rev() {
966            if let Ok(old) = RunState::load_under(id, &takeover.home)
967                && !old.reviews.is_empty()
968                && (old.winner().is_some_and(|w| w.branch == branch)
969                    || old.pr_head.as_deref() == Some(branch))
970            {
971                chosen = Some(old);
972                break;
973            }
974        }
975        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976            .await
977            .unwrap_or_default();
978        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979            Some(last) if last.head == tip => Relation::Same,
980            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981            _ => Relation::Diverged,
982        };
983        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984        Some(assess(from, chosen.as_ref(), base_commit, relation))
985    }
986
987    /// The half of [`Runner::review_taking_over`] that can fail after an
988    /// earlier attempt's worktree was released.
989    async fn open_review(
990        repo: &Path,
991        branch: &str,
992        mut state: RunState,
993        roles: ResolvedRoles,
994        max_parallel: usize,
995        base_commit: String,
996        takeover: &crate::handover::Takeover,
997    ) -> Result<Self> {
998        let wait = repository_wait(&state, None);
999        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000        if state.pr_recovery.is_none() {
1001            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002        }
1003        // The commit subjects are the closest thing to a task statement that
1004        // existing work carries, and the reviewers are told as much.
1005        let start = review_base(
1006            repo,
1007            &state.config.merge.remote,
1008            &state.base_branch,
1009            &base_commit,
1010            branch,
1011        )
1012        .await;
1013        let log = git::log_oneline(repo, &start, branch)
1014            .await
1015            .unwrap_or_default();
1016        let log_text = if log.trim().is_empty() {
1017            "(no commit messages)"
1018        } else {
1019            log.trim()
1020        };
1021        let mut instruction = format!(
1022            "Review the work already on branch `{branch}`. There is no task \
1023             statement: what the change claims to do is whatever its commits \
1024             say.\n\n{log_text}"
1025        );
1026        // A retry of a task whose earlier attempt left reviews behind carries
1027        // that attempt's evidence, and the task's own statement with it: the
1028        // completion criteria are the task's, not whatever the commits claim.
1029        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030            if let Some(task) = takeover
1031                .task_instruction
1032                .as_deref()
1033                .filter(|t| !t.trim().is_empty())
1034                && rec.reviewed_head.is_some()
1035            {
1036                instruction = format!(
1037                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038                    task.trim()
1039                );
1040                rec.criteria_inherited = true;
1041            }
1042            state.event("recovery", rec.summary());
1043            state.recovery = Some(rec);
1044        }
1045        if let Some(selected) = &state.pr_recovery {
1046            instruction = selected.instruction.clone();
1047        }
1048        state.task_instruction = takeover.task_instruction.clone();
1049        state.instruction = instruction;
1050        state.reviewed_commits = Some(
1051            git::subjects(repo, &start, branch)
1052                .await
1053                .unwrap_or_default(),
1054        );
1055
1056        // A takeover must not forget a hold that a previous review observed,
1057        // even if repairs have since removed it from the branch.
1058        for id in &takeover.earlier {
1059            if let Ok(old) = RunState::load_under(id, &takeover.home)
1060                && old.repo == state.repo
1061                && (old.pr_head.as_deref() == Some(branch)
1062                    || old.winner().is_some_and(|w| w.branch == branch))
1063            {
1064                for hold in old.branch_owner_holds {
1065                    if !state.branch_owner_holds.contains(&hold) {
1066                        state.event(
1067                            "approval-policy",
1068                            format!(
1069                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070                                hold.control.key(),
1071                                hold.file,
1072                                hold.head
1073                            ),
1074                        );
1075                        state.branch_owner_holds.push(hold);
1076                    }
1077                }
1078            }
1079        }
1080        let reviewed_head = git::rev_parse(repo, branch).await?;
1081        state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083        // An attached worktree, so the fixer's commits land on the branch under
1084        // review rather than on a detached head nobody will look at again.
1085        let worktree = state.worktree_root().join("under-review");
1086        if let Some(parent) = worktree.parent() {
1087            tokio::fs::create_dir_all(parent).await.ok();
1088        }
1089        let path = worktree.to_string_lossy().to_string();
1090        git::git(repo, &["worktree", "add", &path, branch])
1091            .await
1092            .with_context(|| {
1093                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094            })?;
1095
1096        let setup = match ensure_setup_config(&mut state, repo).await {
1097            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098            Err(e) => Err(e),
1099        };
1100        if let Err(e) = setup {
1101            git::worktree_remove_clean(repo, &worktree).await.ok();
1102            return Err(e);
1103        }
1104        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105            .await
1106            .unwrap_or(0);
1107        if commits == 0 {
1108            git::worktree_remove_clean(repo, &worktree).await.ok();
1109            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110        }
1111        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112            .await
1113            .map(|f| f.len())
1114            .unwrap_or(0);
1115        if files == 0
1116            && let (Ok(head_tree), Ok(base_tree)) = (
1117                git::tree_of(&worktree, "HEAD").await,
1118                git::tree_of(&worktree, &base_commit).await,
1119            )
1120            && head_tree == base_tree
1121        {
1122            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123            git::worktree_remove_clean(repo, &worktree).await.ok();
1124            bail!(
1125                "`{branch}` at {} has a tree identical to base {}; this usually means \
1126                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127                 against `{}/{branch}`) rather than an empty change",
1128                short(&head),
1129                short(&base_commit),
1130                state.config.merge.remote
1131            );
1132        }
1133        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134            .await
1135            .unwrap_or_default();
1136
1137        state.candidates.push(Candidate {
1138            index: 0,
1139            label: 'A',
1140            // Not an agent id on purpose: nothing in the roster wrote this, and
1141            // the stats tables must not credit anyone with a win for it.
1142            agent: EXISTING_BRANCH.to_owned(),
1143            branch: branch.to_owned(),
1144            worktree,
1145            summary: String::new(),
1146            stat,
1147            files,
1148            commits,
1149            empty: false,
1150            failed: None,
1151            verified_noop: None,
1152            duration_ms: 0,
1153            folded: false,
1154        });
1155        state.tally = Some(Tally {
1156            first_choice: BTreeMap::from([('A', 0)]),
1157            borda: BTreeMap::new(),
1158            winner: 'A',
1159            rankings: 0,
1160            unanimous_initial: false,
1161            deliberated: false,
1162            changed_votes: 0,
1163            unanimous_final: false,
1164            tie_break: None,
1165            // No panel sat, so no quorum applies. Zero judges is the correct
1166            // number for work that never competed, and must not be reported as
1167            // a collapsed panel.
1168            judges: 0,
1169            present: 0,
1170            quorum: 0,
1171            met_quorum: true,
1172            uncontested: Some("review-only run: nothing competed".to_owned()),
1173        });
1174        state.status = RunStatus::Reviewing;
1175        state.prep_complete = Some(true);
1176        state.event(
1177            "start",
1178            format!(
1179                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180                state.id
1181            ),
1182        );
1183        state.save()?;
1184        Ok(Self {
1185            state,
1186            roles,
1187            sem: Arc::new(Semaphore::new(max_parallel)),
1188            pause: Pause::new(),
1189            interrupt: Pause::new(),
1190        })
1191    }
1192
1193    /// Authorize exactly one additional rebase, under the caller's queue task
1194    /// claim and the same owner/execution locks used by run maintenance.
1195    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1196        Self::authorize_base_recovery_request(home, id, task, false).await
1197    }
1198
1199    /// Queue release uses its durable approval generation exactly once.
1200    pub(crate) async fn authorize_base_recovery_from_resumption(
1201        home: &Path,
1202        id: &str,
1203        task: &str,
1204    ) -> Result<()> {
1205        Self::authorize_base_recovery_request(home, id, task, true).await
1206    }
1207
1208    async fn authorize_base_recovery_request(
1209        home: &Path,
1210        id: &str,
1211        task: &str,
1212        resumption: bool,
1213    ) -> Result<()> {
1214        let _owner = crate::pr_head::owner_lock(home, id)?;
1215        let mut state = RunState::load_under(id, home)?;
1216        crate::pr_head::ensure_owner(&state)?;
1217        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1218        anyhow::ensure!(
1219            queued.runs.last().is_some_and(|run| run == id)
1220                && queued.status != crate::queue::TaskStatus::Done
1221                && !queued.fresh_start
1222                && queued.review_branch.is_none(),
1223            "run task ownership changed"
1224        );
1225        anyhow::ensure!(
1226            state
1227                .origin
1228                .as_ref()
1229                .and_then(|o| o.task.as_deref())
1230                .is_none_or(|owner| owner == task),
1231            "run belongs to another task"
1232        );
1233        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1234        anyhow::ensure!(
1235            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1236            "run is live or its ownership is unknown"
1237        );
1238        // Judge the previous driver's lease before acquiring our maintenance
1239        // lease. The owner lock excludes new driver claims across both steps.
1240        let _execution = crate::driver::maintenance(home, id)?;
1241        anyhow::ensure!(
1242            state.status.resumable()
1243                && state.released_to.is_none()
1244                && state.retired_to.is_none()
1245                && state.pr_taken_over_by.is_none(),
1246            "run ownership is final"
1247        );
1248        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1249        anyhow::ensure!(
1250            !state.pr_publication_pending
1251                && !crate::handover::landing_checkpoint(&state)
1252                && state.merge_decision.is_none(),
1253            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1254        );
1255        let winner = state.winner().context("run has no retained winner")?;
1256        anyhow::ensure!(
1257            git::is_clean(&winner.worktree).await?,
1258            "retained worktree has uncommitted changes"
1259        );
1260        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1261        anyhow::ensure!(
1262            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1263            "retained worktree does not match the winner branch"
1264        );
1265        state.record_base_sync_stop(&head);
1266        state.save()?;
1267        if resumption {
1268            anyhow::ensure!(
1269                crate::daemon::compatible_run_instruction(&state, &queued),
1270                "resumption instruction does not match the retained run"
1271            );
1272            let approval = queued
1273                .resumption
1274                .as_ref()
1275                .context("no operator resumption")?;
1276            anyhow::ensure!(
1277                approval.instruction == queued.instruction,
1278                "resumption approval does not match the current task instruction"
1279            );
1280            state.authorize_base_sync_resumption(task, approval)?;
1281        } else {
1282            state.authorize_base_sync(task)?;
1283        }
1284        state.event(
1285            "base-recovery",
1286            "owner authorized one additional base rebase; fresh verification required",
1287        );
1288        state.save()?;
1289        Ok(())
1290    }
1291
1292    /// Reopen an existing run.
1293    pub fn resume(id: &str) -> Result<Self> {
1294        let state = RunState::load(id)?;
1295        crate::pr_head::ensure_owner(&state)?;
1296        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1297            bail!(
1298                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1299            );
1300        }
1301        if let Some(to) = &state.released_to {
1302            bail!(
1303                "run {} cannot be resumed: its worktree was released to run {}",
1304                state.short(),
1305                crate::run::short_of(to)
1306            );
1307        }
1308        let roles = state.config.resolve_roles()?;
1309        if let Some(id) = &state.config.graph.selected_implementer
1310            && state.candidates.iter().any(|c| c.agent != *id)
1311        {
1312            bail!(
1313                "replacement implementer pin does not match the saved candidates; task must stay held"
1314            );
1315        }
1316        let max_parallel = state.config.graph.max_parallel.max(1);
1317        Ok(Self {
1318            state,
1319            roles,
1320            sem: Arc::new(Semaphore::new(max_parallel)),
1321            pause: Pause::new(),
1322            interrupt: Pause::new(),
1323        })
1324    }
1325
1326    /// Walk the graph to a terminal state, skipping nodes already recorded.
1327    ///
1328    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1329    /// the phone - ends here, so this is the one place a run that ended
1330    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1331    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1332    pub async fn execute(&mut self) -> Result<()> {
1333        if let Some(selection) = &self.state.pr_recovery
1334            && (selection.source_proof.is_some() || selection.source_run.is_some())
1335        {
1336            Box::pin(selection.verify_source(&self.state.repo)).await?;
1337        }
1338        // Queue claims cover daemon work; this also serializes manual drivers
1339        // consuming a repair answer, including the new review and decision.
1340        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1341            || self.state.land_approval.is_some()
1342            || self.state.merge_decision.is_some()
1343            || !self.state.merge_repairs.is_empty()
1344            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1345        {
1346            let claim = FixClaim::acquire(&self.state.dir())?;
1347            let disk = RunState::load(&self.state.id)?;
1348            let checkpoint = |s: &RunState| {
1349                (
1350                    s.merge_repairs.len(),
1351                    s.merge_repairs
1352                        .last()
1353                        .map(|r| (r.question.clone(), r.phase)),
1354                    s.land_approval
1355                        .as_ref()
1356                        .map(|a| (a.question.clone(), a.head.clone())),
1357                )
1358            };
1359            if checkpoint(&disk) != checkpoint(&self.state) {
1360                bail!(
1361                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1362                );
1363            }
1364            Some(claim)
1365        } else {
1366            None
1367        };
1368        let mut execution = self.claim_driver().await?;
1369        let runner = &mut *execution.runner;
1370        let wait = runner.repository_wait();
1371        // Keep the graph's large future off the callers' stack, including
1372        // standalone task attempts running on a default-sized test thread.
1373        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1374        let result = match result {
1375            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1376                record_repository_block(&mut runner.state, &e);
1377                runner.park_here()?;
1378                Ok(())
1379            }
1380            other => other,
1381        };
1382        execution.driver.finish(&mut runner.state).await?;
1383        let ended = if result.is_err() {
1384            Some({
1385                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1386                if let Err(error) = &result {
1387                    notice.message = format!(
1388                        "Run {} stopped with an error: {}",
1389                        runner.state.id,
1390                        error.to_string().lines().next().unwrap_or("unknown error")
1391                    );
1392                }
1393                notice
1394            })
1395        } else {
1396            crate::notices::run_ended(&runner.state)
1397        };
1398        if let Some(notice) = ended {
1399            crate::notices::raise(notice);
1400        }
1401        result
1402    }
1403
1404    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1405        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1406    }
1407
1408    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1409        let wait = self.repository_wait();
1410        coordinate_operation(&mut self.state, cwd, &wait).await
1411    }
1412
1413    /// Validate ownership and persist the driver claim in one critical section.
1414    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1415        let owner = match crate::pr_head::owner_lock_wait(
1416            &crate::run::home(),
1417            &self.state.id,
1418            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1419        )
1420        .await
1421        {
1422            Ok(owner) => owner,
1423            Err(error) => {
1424                let detail = format!("worktree maintenance or ownership update: {error:#}");
1425                // Admission did not own run.json: retain this resource outcome
1426                // in memory for queue settlement without overwriting a concurrent
1427                // retirement or publication transaction.
1428                self.state.resource_block = Some(crate::run::ResourceBlock {
1429                    cause: crate::run::ResourceBlockCause::Other,
1430                    node: "start".into(),
1431                    kind: "worktree maintenance".into(),
1432                    detail: detail.clone(),
1433                    at: Timestamp::now(),
1434                });
1435                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1436            }
1437        };
1438        let disk = RunState::load(&self.state.id)?;
1439        crate::pr_head::ensure_owner(&disk)?;
1440        let claimed =
1441            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1442        // Only proven-live ownership refuses here. Unknown records (legacy runs
1443        // with no claim marker or process identity) fall through to the
1444        // exclusive execution lease taken just below, which is what actually
1445        // keeps a second driver out.
1446        anyhow::ensure!(
1447            disk.liveness(claimed) != crate::run::Liveness::Live,
1448            "run already has a live driver; wait for its execution to end"
1449        );
1450        if let Some(to) = &disk.released_to {
1451            bail!(
1452                "run {} cannot continue: its worktree was released to run {}",
1453                self.state.short(),
1454                crate::run::short_of(to)
1455            );
1456        }
1457        anyhow::ensure!(
1458            serde_json::to_value(&disk.base_sync_recovery)?
1459                == serde_json::to_value(&self.state.base_sync_recovery)?,
1460            "base recovery changed since this run was loaded; reload the saved run"
1461        );
1462        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1463        let mut execution = DriverExecution {
1464            runner: self,
1465            driver,
1466        };
1467        // Declare after the execution guard: on any claim error this short
1468        // lock must drop before that guard reloads and releases ownership.
1469        let _owner = owner;
1470        let state = &mut execution.runner.state;
1471        // Moving again, so it is no longer parked. Set before the walk rather
1472        // than in `resume`, so every way of re-entering the graph clears it
1473        // and a card cannot claim a run is waiting to be resumed while the
1474        // agents are already working.
1475        state.parked = false;
1476        state.resource_wait = None;
1477        state.resource_block = None;
1478        // Any seat this state still lists as answering belongs to whatever
1479        // process last drove this run — this one included, if it crashed
1480        // mid-wave. Cleared and flushed immediately, before anything else
1481        // runs, so a resume can never show a seat as live when nothing is
1482        // asking it anything yet; the node that actually dispatches the next
1483        // wave repopulates it.
1484        state.clear_active();
1485        // Recorded in the same spot, and flushed together with the clear
1486        // above: this is the pid a reader checks (`RunState::liveness`) when
1487        // no daemon claim exists to answer "is a process still driving this
1488        // run" — a plain `magi run` / `magi review` typed into a terminal
1489        // claims nothing there. Always overwritten, never only-if-absent, so
1490        // a resumed run's stale pid from a previous, possibly-dead process
1491        // can never survive into this one's own report. Unlike
1492        // `clear_active`, this changes on every single `execute()` call, so
1493        // the save below is now unconditional rather than only-if-cleared.
1494        //
1495        // `driver_started_at` is recorded in the same breath, from this same
1496        // pid, so `liveness` can tell a live pid that is genuinely still us
1497        // apart from one the OS has since handed to an unrelated process —
1498        // see that field's own doc for why the pid alone is not enough.
1499        let pid = std::process::id();
1500        state.driver_pid = Some(pid);
1501        state.driver_started_at = crate::proc::process_started_at(pid);
1502        state.driver_exited = false;
1503        state.driver_claimed = Some(true);
1504        state.begin_pass();
1505        execution.driver.bind(state);
1506        state.save()?;
1507        execution.driver.record(state)?;
1508        Ok(execution)
1509    }
1510
1511    async fn execute_graph(&mut self) -> Result<()> {
1512        self.state.configure_recovery_resume()?;
1513        // Preservation may invalidate an old empty classification. Resolve it
1514        // before any resume shortcut can reuse a verdict or publish a branch.
1515        self.persist_candidates().await?;
1516        // A saved base stop outranks stage markers, including legacy Voting.
1517        // Recovery never reenters implementation, deliberation or voting.
1518        if self.state.base_sync_blocked() {
1519            self.sync_to_base().await?;
1520            if !self.state.base_sync_blocked()
1521                && self.state.status != RunStatus::AlreadyInBase
1522                && !self.resource_halt()?
1523            {
1524                self.finish_after_tally().await?;
1525            }
1526            return Ok(());
1527        }
1528        if self
1529            .state
1530            .merge_decision
1531            .as_ref()
1532            .is_some_and(|decision| !decision.closed)
1533            && self.state.status != RunStatus::Landing
1534        {
1535            if crate::merge_decision::resumable_hold(&self.state) {
1536                self.state.merge = Some(MergeOutcome {
1537                    mode: MergeMode::Pr,
1538                    ok: false,
1539                    detail: self
1540                        .state
1541                        .merge_decision
1542                        .as_ref()
1543                        .unwrap()
1544                        .target
1545                        .pr
1546                        .clone(),
1547                    empty: false,
1548                });
1549                self.run_land().await?;
1550                self.settle_questions();
1551                return Ok(());
1552            }
1553            bail!(
1554                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1555            );
1556        }
1557        if self.state.config.graph.selected_implementer.is_some()
1558            && self.state.status == RunStatus::Blocked
1559            && self.state.tally.is_none()
1560            && self.state.candidate_preservation.is_empty()
1561            && self.state.interrupted_implementations.is_empty()
1562        {
1563            bail!(
1564                "selected implementation is held; explicitly release the owning task to retry it"
1565            );
1566        }
1567
1568        if self.state.merge_repairs.last().is_some_and(|r| {
1569            matches!(
1570                r.phase,
1571                crate::run::MergeRepairPhase::Requested
1572                    | crate::run::MergeRepairPhase::Fixing
1573                    | crate::run::MergeRepairPhase::Reviewing
1574            )
1575        }) {
1576            if self.repair_merge_review().await? {
1577                self.run_land().await?;
1578            }
1579            self.settle_questions();
1580            return Ok(());
1581        }
1582        // Publication already passed review and gate. Resume only that node:
1583        // syncing to a newer base here could change the verified commit.
1584        if self.state.pr_publication_pending {
1585            self.merge().await?;
1586            if self.resource_halt()? {
1587                return Ok(());
1588            }
1589            self.state.save()?;
1590            self.settle_questions();
1591            return Ok(());
1592        }
1593        // A run that already lost its quorum never resumes into the verdict
1594        // machinery: `deliberate` and `vote` would otherwise clobber the
1595        // stalled marker back to Voting and the run would keep going past a
1596        // verdict that is no longer trustworthy. Everything already recorded is
1597        // kept, so the run stays resumable (or foldable) for a human to pick up.
1598        //
1599        // On --resume the run gets one chance to repair itself: the seats a
1600        // rate limit took out are re-asked. If their quota has since reset and
1601        // the quorum is restored, the run picks up and finishes; otherwise it
1602        // stays stale and still-resumable for a later retry. If it does not
1603        // recover, the returned status stays `Stalled` and nothing was
1604        // clobbered (the recovery only mutates entries for the lost seats).
1605        if self.state.status == RunStatus::Stalled {
1606            if self.recover_stall().await? {
1607                self.finish_after_tally().await?;
1608            } else {
1609                // Still below quorum: persist the marker and stay resumable.
1610                self.state.save()?;
1611            }
1612            return Ok(());
1613        }
1614        // A run parked inside `land` - watching CI, mid fix-round, or
1615        // waiting on the owner's merge approval - resumes directly into it,
1616        // never back through `prep`. Everything before `merge` already
1617        // concluded; that is the only way `status` reaches `Landing` in the
1618        // first place. Re-walking `review_loop` first would also be actively
1619        // wrong: its own status recomputation (see its doc) treats any
1620        // clean round as reason to set `status` to `Gating`, which would
1621        // clobber this marker before `merge` ever ran, and this run would
1622        // never find its way back into `land` at all.
1623        if self.state.status == RunStatus::Landing
1624            || crate::handover::landing_checkpoint(&self.state)
1625        {
1626            self.state.land_sync_pending = true;
1627            self.state.save()?;
1628            self.run_land().await?;
1629            // `run_land` may have settled the run right here - CI came back
1630            // green and the PR merged, say - without ever passing back
1631            // through `merge`'s own trailing call. Whatever it left `status`
1632            // as is what this has to read.
1633            self.settle_questions();
1634            return Ok(());
1635        }
1636        // A run parked at `merge` on the owner's word about a withheld PR text
1637        // resumes straight into `merge`. Re-walking the tail would sync to the
1638        // base again (possibly rebasing, possibly spending the fixer) after the
1639        // gate had passed, and then push a tree nobody reviewed or gated.
1640        if self.state.github_text.is_some()
1641            && self.state.merge.is_none()
1642            && self.state.status == RunStatus::Gating
1643        {
1644            self.merge().await?;
1645            if self.resource_halt()? {
1646                return Ok(());
1647            }
1648            self.state.save()?;
1649            return Ok(());
1650        }
1651        if let Err(e) = self.prep().await {
1652            if e.is::<crate::repo_lock::ResourceBlocked>() {
1653                return Err(e);
1654            }
1655            self.state.status = RunStatus::Blocked;
1656            self.state
1657                .prep_failure
1658                .get_or_insert_with(|| format!("{e:#}"));
1659            self.state
1660                .event("prep", format!("preparation stopped: {e:#}"));
1661            self.state.save()?;
1662            return Err(e);
1663        }
1664        if self.park_here()? {
1665            return Ok(());
1666        }
1667        self.advise().await?;
1668        if self.park_here()? {
1669            return Ok(());
1670        }
1671        self.implement().await?;
1672        if self.park_here()? {
1673            return Ok(());
1674        }
1675        // `after_implement` already saved the state and settled any open
1676        // questions when it set this; nothing later in the graph has
1677        // anything to judge.
1678        if self.state.status == RunStatus::VerifiedNoop {
1679            return Ok(());
1680        }
1681        self.judge().await?;
1682        if self.park_here()? {
1683            return Ok(());
1684        }
1685        self.deliberate().await?;
1686        if self.park_here()? {
1687            return Ok(());
1688        }
1689        self.vote().await?;
1690        if self.park_here()? {
1691            return Ok(());
1692        }
1693        self.tally()?;
1694        // A verdict that lost its quorum is not trustworthy: do not review,
1695        // gate, or merge on it. Everything already done is kept, so the run
1696        // stays resumable (or foldable); the human can replace the agent that
1697        // ran out of quota and pick it up.
1698        if self.state.status == RunStatus::Stalled {
1699            // Persist the stalled marker now — the normal end-of-execute save
1700            // below is below this early return, and without it a resumed run
1701            // would reload a pre-tally status and keep going.
1702            self.state.save()?;
1703            return Ok(());
1704        }
1705        self.finish_after_tally().await?;
1706        Ok(())
1707    }
1708
1709    /// A wave could not start its agents because a shared resource stayed busy
1710    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1711    /// resumable. Unstarted seats have no result; completed peers retain
1712    /// their saved candidate state.
1713    fn stop_for_resource(&mut self) -> Result<()> {
1714        // Resource acquisition observes cancellation inside a node. Complete
1715        // the same durable park as every other node boundary before returning.
1716        if self.pause.parked() || self.interrupt.parked() {
1717            self.park_here()?;
1718            return Ok(());
1719        }
1720        if self.state.resource_block.is_some() {
1721            self.state.status = RunStatus::Blocked;
1722        }
1723        self.state.save()
1724    }
1725
1726    /// Stop walking the graph if a node ended on a resource block.
1727    fn resource_halt(&mut self) -> Result<bool> {
1728        if self.state.resource_block.is_none() {
1729            return Ok(false);
1730        }
1731        self.stop_for_resource()?;
1732        Ok(true)
1733    }
1734
1735    /// Park here if asked to, recording it in the run's own timeline.
1736    ///
1737    /// Returns whether the caller should stop walking the graph. The state is
1738    /// saved either way by the node that just finished; this adds the event so
1739    /// the operator's card says why a run that is neither finished nor moving
1740    /// is sitting where it is.
1741    fn park_here(&mut self) -> Result<bool> {
1742        let parking = self.pause.parked() || self.interrupt.parked();
1743        if !parking && self.state.resource_block.is_some() {
1744            self.stop_for_resource()?;
1745            return Ok(true);
1746        }
1747        // Either handle asking is enough - see `Pause`'s own doc for why
1748        // they are never the same one. `interrupt` is checked second so a
1749        // reason it carries is preferred in the message below over a plain
1750        // shutdown park racing it at the same boundary.
1751        if !self.pause.parked() && !self.interrupt.parked() {
1752            return Ok(false);
1753        }
1754        if self.state.parked {
1755            return Ok(true);
1756        }
1757        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1758        if let Some(block) = &mut self.state.resource_block
1759            && matches!(
1760                block.cause,
1761                crate::run::ResourceBlockCause::Cancelled { .. }
1762            )
1763        {
1764            block.cause = crate::run::ResourceBlockCause::Cancelled {
1765                park: cause.clone(),
1766            };
1767        }
1768        let why = match cause.as_ref().map(ParkCause::label) {
1769            Some(reason) => format!(
1770                "parked after `{}` ({reason}) — resume to carry on from here",
1771                self.state.status.as_str()
1772            ),
1773            None => format!(
1774                "parked after `{}` — resume to carry on from here",
1775                self.state.status.as_str()
1776            ),
1777        };
1778        self.state.event("park", why);
1779        self.state
1780            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1781        self.state.parked = true;
1782        self.state.save()?;
1783        Ok(true)
1784    }
1785
1786    /// Only a new explicit operator approval may retry failed preparation.
1787    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1788        if self.state.prep_failure.is_none() {
1789            return Ok(());
1790        }
1791        let failed_at = self
1792            .state
1793            .events
1794            .iter()
1795            .rev()
1796            .find(|event| event.node == "prep")
1797            .map_or(self.state.updated_at, |event| event.at);
1798        anyhow::ensure!(
1799            approval_at > failed_at,
1800            "preparation retry requires a new explicit release after the failure"
1801        );
1802        self.state.prep_failure = None;
1803        self.state.prep_complete = Some(false);
1804        self.state
1805            .event("prep-retry", "operator approved retry of preparation");
1806        Ok(())
1807    }
1808
1809    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1810    pub fn on_pause(&mut self, pause: Pause) {
1811        self.pause = pause;
1812    }
1813
1814    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1815    /// scheduler asking this one run - and no other - to park so a task
1816    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1817    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1818    /// handle.
1819    pub fn watch_interrupt(&mut self, pause: Pause) {
1820        self.interrupt = pause;
1821    }
1822
1823    /// Abandon this run's own open questions, once `status` has actually
1824    /// settled rather than merely paused.
1825    ///
1826    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1827    /// either back up with the candidates, the review round and the seat
1828    /// sessions already on disk, so a question an implementer asked mid-round
1829    /// may still get a real answer read by a real resume. Only the statuses
1830    /// `resumable` excludes are actually final: the run merged, it reached
1831    /// `Ready` with nothing left to do, it failed outright with no
1832    /// established point to continue from, or every candidate agreed, with
1833    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1834    /// every one of those the seat that asked is gone for good, exactly like
1835    /// the run being deleted under `magi run rm` - so the same cleanup
1836    /// applies, worded for what actually happened instead of "the run was
1837    /// deleted".
1838    ///
1839    /// Best-effort and silent on success: called from every place `status`
1840    /// can land on one of those three, including ones a resumed run revisits,
1841    /// so it must cost nothing when there was nothing open to begin with.
1842    fn settle_questions(&mut self) {
1843        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1844            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1845        }
1846    }
1847
1848    /// The tail of the graph after a trustworthy tally: fold losers, review,
1849    /// gate, merge, and persist.
1850    async fn finish_after_tally(&mut self) -> Result<()> {
1851        // Before the losers are folded: their branches are what the graft
1852        // reads from.
1853        self.graft().await?;
1854        if self.resource_halt()? {
1855            return Ok(());
1856        }
1857        self.fold_losers().await?;
1858        // Before review starts, and again right before the gate: a run's
1859        // review rounds can themselves take long enough for the base to move
1860        // a second time, and the gate is the one node whose "green" gets
1861        // acted on.
1862        self.sync_to_base().await?;
1863        if self.hold_base_sync().await? || self.resource_halt()? {
1864            return Ok(());
1865        }
1866        if self.state.status == RunStatus::AlreadyInBase {
1867            return Ok(());
1868        }
1869        self.review_loop().await?;
1870        if self.resource_halt()? {
1871            return Ok(());
1872        }
1873        self.sync_to_base().await?;
1874        if self.hold_base_sync().await? || self.resource_halt()? {
1875            return Ok(());
1876        }
1877        if self.state.status == RunStatus::AlreadyInBase {
1878            return Ok(());
1879        }
1880        self.gate().await?;
1881        self.merge().await?;
1882        if self.resource_halt()? {
1883            return Ok(());
1884        }
1885        self.state.save()?;
1886        Ok(())
1887    }
1888
1889    // ---------------------------------------------------------------- prep
1890
1891    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1892    /// run with the step and its output in the event; it is never swallowed.
1893    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1894    /// resume starts over instead of finding candidates and missing seats.
1895    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1896        let result = match ensure_setup_config(&mut self.state, repo).await {
1897            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1898            Err(e) => Err(e),
1899        };
1900        if let Err(e) = result {
1901            git::worktree_remove_clean(repo, wt).await.ok();
1902            if in_prep {
1903                self.rollback_prep(repo).await;
1904            }
1905            self.state.status = RunStatus::Blocked;
1906            self.state.event(
1907                "setup",
1908                format!("worktree setup failed in {}: {e:#}", wt.display()),
1909            );
1910            self.state.save()?;
1911            return Err(e);
1912        }
1913        Ok(())
1914    }
1915
1916    /// Undo everything `prep` created, so a resume re-runs it whole.
1917    async fn rollback_prep(&mut self, repo: &Path) {
1918        let root = self.state.worktree_root();
1919        for c in std::mem::take(&mut self.state.candidates) {
1920            if matches!(
1921                git::worktree_remove_clean(repo, &c.worktree).await,
1922                Ok(true)
1923            ) {
1924                git::branch_delete(repo, &c.branch).await.ok();
1925            } else {
1926                self.state.candidates.push(c);
1927            }
1928        }
1929        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1930        for j in 1..=seats {
1931            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1932                .await
1933                .ok();
1934        }
1935        for k in 1..=self.state.config.graph.advisors {
1936            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1937                .await
1938                .ok();
1939        }
1940    }
1941
1942    async fn prep(&mut self) -> Result<()> {
1943        if let Some(reason) = &self.state.prep_failure {
1944            bail!(
1945                "candidate preparation previously failed: {reason}; correct the source before retrying"
1946            );
1947        }
1948        if self
1949            .state
1950            .prep_complete
1951            .unwrap_or(!self.state.candidates.is_empty())
1952        {
1953            return Ok(());
1954        }
1955        let repo = self.state.repo.clone();
1956        // Includes rollback, config, refs, checkout creation and setup. No
1957        // repository mutation may precede the real shared lock acquisition.
1958        let _coordination = self.coordinate(&repo).await?;
1959        let base = self.state.base_commit.clone();
1960        let plan = self.state.sources.plan(&repo, &base).await?;
1961        self.rollback_prep(&repo).await;
1962        if !self.state.candidates.is_empty() {
1963            bail!(
1964                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1965            );
1966        }
1967        self.state.prep_complete = Some(false);
1968        self.state.status = RunStatus::Prep;
1969        self.state.save()?;
1970        let repo = self.state.repo.clone();
1971        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1972        let root = self.state.worktree_root();
1973        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1974
1975        // The hook is the write-time half of the blindness contract; the
1976        // presentation filter in `blind` is the half that cannot be bypassed.
1977        let hooks_dir = self.state.dir().join("hooks");
1978        if self.state.config.blind.commit_msg_hook {
1979            std::fs::create_dir_all(&hooks_dir)
1980                .with_context(|| format!("create {}", hooks_dir.display()))?;
1981            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1982            let path = hooks_dir.join("commit-msg");
1983            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1984            make_executable(&path)?;
1985            // Ref-counted rather than a plain idempotent set: with more than
1986            // one run able to be in flight in the same repository at once
1987            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1988            // true?" check cannot tell "another run of mine still needs
1989            // this" from "nobody does", and the run that happens to finish
1990            // first would disable the hook out from under a sibling still
1991            // relying on it.
1992            git::acquire_worktree_config(&repo).await?;
1993            self.state.enabled_worktree_config = true;
1994        }
1995
1996        // `[worktree] setup` hides its products per worktree, which needs the
1997        // same per-worktree config the hook does. Held until fold, like it.
1998        ensure_setup_config(&mut self.state, &repo).await?;
1999
2000        for (index, (spec, label)) in self
2001            .roles
2002            .implementers
2003            .clone()
2004            .into_iter()
2005            .zip(labels)
2006            .enumerate()
2007        {
2008            let branch = self.state.branch_for(label);
2009            let worktree = root.join(format!("cand-{label}"));
2010            // Persist intent before Git can leave a partially created tree.
2011            // Candidate registration follows setup, so it cannot inventory
2012            // crashes or refusals during creation itself.
2013            self.state.event(
2014                "prep-intent",
2015                serde_json::json!({
2016                    "branch": branch,
2017                    "worktree": worktree,
2018                    "start": start,
2019                    "picks": plan.picks,
2020                })
2021                .to_string(),
2022            );
2023            self.state.save()?;
2024            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2025            if self.state.config.blind.commit_msg_hook {
2026                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2027            }
2028            git::local_exclude(&worktree, "/.magi/").await?;
2029            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2030                // Nothing of this prep may survive: a resume must start over,
2031                // not find half the candidates and skip the rest.
2032                git::worktree_remove_clean(&repo, &worktree).await.ok();
2033                git::branch_delete(&repo, &branch).await.ok();
2034                self.rollback_prep(&repo).await;
2035                self.state.status = RunStatus::Blocked;
2036                self.state
2037                    .event("prep", format!("worktree setup failed: {e:#}"));
2038                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2039                self.state.save()?;
2040                return Err(e);
2041            }
2042            self.state.candidates.push(Candidate {
2043                index,
2044                label,
2045                agent: spec.id.clone(),
2046                branch,
2047                worktree,
2048                summary: String::new(),
2049                stat: String::new(),
2050                files: 0,
2051                commits: 0,
2052                empty: false,
2053                failed: None,
2054                verified_noop: None,
2055                duration_ms: 0,
2056                folded: false,
2057            });
2058            self.state.save()?;
2059            for pick in &plan.picks {
2060                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2061                {
2062                    self.state.status = RunStatus::Blocked;
2063                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2064                    self.state
2065                        .event("prep", format!("cannot apply referenced commit: {e}"));
2066                    self.state.save()?;
2067                    return Err(e);
2068                }
2069            }
2070        }
2071
2072        for j in 1..=self.roles.judges.len() {
2073            let wt = root.join(format!("judge-{j}"));
2074            if !wt.exists() {
2075                git::worktree_add_detached(&repo, &wt, &base).await?;
2076                self.setup_seat_worktree(&repo, &wt, true).await?;
2077            }
2078        }
2079
2080        // Disposable, detached checkouts for the design-deliberation stage's
2081        // advisor seats — the same shape as the judges' above, at the same
2082        // base commit, since advisors also only ever read. Sized off the
2083        // configured count directly rather than a resolved roster: unlike
2084        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2085        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2086        // `prep` has no `ResolvedRoles` field to read a count from here.
2087        if self.state.config.graph.advise {
2088            for k in 1..=self.state.config.graph.advisors {
2089                let wt = root.join(format!("advisor-{k}"));
2090                if !wt.exists() {
2091                    git::worktree_add_detached(&repo, &wt, &base).await?;
2092                    self.setup_seat_worktree(&repo, &wt, true).await?;
2093                }
2094            }
2095        }
2096
2097        // A judge cannot tell it is looking at its own patch — the seats keep
2098        // separate conversations — but a panel that shares agents with the
2099        // field is less independent than it looks, and that is worth saying out
2100        // loud once per run rather than leaving it in the config.
2101        let authors: Vec<&str> = self
2102            .roles
2103            .implementers
2104            .iter()
2105            .map(|a| a.id.as_str())
2106            .collect();
2107        let overlap: Vec<String> = self
2108            .roles
2109            .judges
2110            .iter()
2111            .enumerate()
2112            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2113            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2114            .collect();
2115        if !overlap.is_empty() {
2116            let note = format!(
2117                "{} also authored a candidate; blind, but the panel is less \
2118                 independent than {} distinct agents would be",
2119                overlap.join(", "),
2120                self.roles.judges.len()
2121            );
2122            self.state.event("prep", note);
2123        }
2124
2125        self.state.event(
2126            "prep",
2127            format!(
2128                "{} candidates, {} judges, base {} ({})",
2129                self.state.candidates.len(),
2130                self.roles.judges.len(),
2131                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2132                self.state.base_branch
2133            ),
2134        );
2135        self.state.prep_complete = Some(true);
2136        self.state.status = RunStatus::Implementing;
2137        self.state.save()?;
2138        Ok(())
2139    }
2140
2141    // -------------------------------------------------------------- advise
2142
2143    /// The design-deliberation stage: independent, read-only advisor seats
2144    /// each sketch a design before any implementer touches the repository,
2145    /// and (when at least one produced a usable proposal) a synthesis seat
2146    /// blends them into a brief `implement` carries in every candidate's
2147    /// prompt.
2148    ///
2149    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2150    /// advisors` is the proposal count. Everything here is best-effort and
2151    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2152    /// that cannot reach quota, or a synthesis seat that produced nothing
2153    /// usable all leave `implement` exactly as it was before this stage
2154    /// existed — the task instruction alone — rather than failing the whole
2155    /// competition over an enrichment stage. Every outcome is still recorded
2156    /// as an event, so a run that got nothing from this stage says why.
2157    ///
2158    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2159    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2160    /// resumed run whose stage failed would re-run it, and re-spend the
2161    /// agent calls, on every reentry before `implement`.
2162    ///
2163    /// Also skipped once any candidate shows implementation progress — the
2164    /// exact predicate `implement` itself uses to decide a candidate is no
2165    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2166    /// is not enough: a run created by an older binary that predates this
2167    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2168    /// an already-`Implementing`-or-later run under this build would
2169    /// otherwise walk straight back through `prep` (a no-op once candidates
2170    /// exist) into this node and spawn every advisor seat against worktrees
2171    /// `prep` never recreated — after implementation has already started,
2172    /// which is exactly the invariant this stage exists to guarantee.
2173    async fn advise(&mut self) -> Result<()> {
2174        let implement_untouched = self
2175            .state
2176            .candidates
2177            .iter()
2178            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2179        if !self.state.config.graph.advise || self.state.advise_attempted {
2180            return Ok(());
2181        }
2182        if !implement_untouched {
2183            self.state.event(
2184                "advise",
2185                "skipping the design-deliberation stage: at least one \
2186                 candidate already shows implementation progress, so this \
2187                 run is past the point the stage exists to run before"
2188                    .to_owned(),
2189            );
2190            self.state.advise_attempted = true;
2191            self.state.save()?;
2192            return Ok(());
2193        }
2194        let run_id = self.state.id.clone();
2195        let prompts = self.state.config.prompts.clone();
2196        let instruction = self.state.instruction.clone();
2197        let language = self.state.config.graph.language.clone();
2198        let root = self.state.worktree_root();
2199        let n = self.state.config.graph.advisors;
2200        let where_recorded = self.state.dir().join("run.json");
2201
2202        let seats = match self.state.config.advisors() {
2203            Ok(seats) if !seats.is_empty() => seats,
2204            Ok(_) => {
2205                self.state.event(
2206                    "advise",
2207                    format!(
2208                        "[graph] advisors is 0; skipping the design-deliberation \
2209                         stage and continuing without a synthesis brief (see {})",
2210                        where_recorded.display()
2211                    ),
2212                );
2213                self.state.advise_attempted = true;
2214                self.state.save()?;
2215                return Ok(());
2216            }
2217            Err(e) => {
2218                self.state.event(
2219                    "advise",
2220                    format!(
2221                        "could not resolve advisor seats ({e:#}); continuing \
2222                         without a design-deliberation brief (see {})",
2223                        where_recorded.display()
2224                    ),
2225                );
2226                self.state.advise_attempted = true;
2227                self.state.save()?;
2228                return Ok(());
2229            }
2230        };
2231
2232        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2233        let artifacts = agent::artifacts_dir(&self.state.dir());
2234        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2235
2236        let mut jobs = Vec::new();
2237        for (i, spec) in seats.iter().cloned().enumerate() {
2238            let seat_key = format!("advisor-{}", i + 1);
2239            let seat = self.seat(&seat_key, &spec.id);
2240            jobs.push(SeatJob {
2241                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2242                spec,
2243                seat,
2244                cwd: worktrees[i % worktrees.len()].clone(),
2245                timeout,
2246                allow_write: false,
2247                sessions: false,
2248                artifacts: artifacts.clone(),
2249                stem: seat_key,
2250                handover: None,
2251            });
2252        }
2253
2254        self.state.event(
2255            "advise",
2256            format!(
2257                "{} advisor seat(s) sketching a design in parallel",
2258                jobs.len()
2259            ),
2260        );
2261        let mut quota_losses = Vec::new();
2262        let cache = self.state.config.cache_dir();
2263        let ctx = WaveCtx {
2264            carry_seats: false,
2265            pauses: Some((&self.pause, &self.interrupt)),
2266            run: &run_id,
2267            node: "advise",
2268            prompts: &prompts,
2269            cache: cache.as_deref(),
2270            round: None,
2271        };
2272        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2273        let results = ask_json_wave::<Proposal>(
2274            jobs,
2275            Arc::clone(&self.sem),
2276            self.state.config.graph.retries,
2277            &advisor_roster,
2278            &ctx,
2279            &mut quota_losses,
2280            &mut self.state,
2281            &|p: &Proposal| p.validate(),
2282        )
2283        .await;
2284        if self.state.resource_block.is_some() {
2285            return self.stop_for_resource();
2286        }
2287        self.state.quota.extend(quota_losses);
2288
2289        let mut records = Vec::with_capacity(results.len());
2290        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2291            let agent_id = seat.agent.clone();
2292            self.state.seats.insert(seat.key.clone(), seat);
2293            match res {
2294                Ok((proposal, out)) => {
2295                    self.state
2296                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2297                    records.push(advise::AdvisorRecord::proposed(
2298                        i + 1,
2299                        agent_id,
2300                        proposal,
2301                        out.duration_ms,
2302                    ));
2303                }
2304                Err(e) => {
2305                    self.state.event(
2306                        "advise",
2307                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2308                    );
2309                    records.push(advise::AdvisorRecord::failed(
2310                        i + 1,
2311                        agent_id,
2312                        e.to_string(),
2313                    ));
2314                }
2315            }
2316        }
2317
2318        let mut advice = advise::Advice {
2319            records,
2320            synthesis: None,
2321        };
2322        if advice.proposals().is_empty() {
2323            self.state.event(
2324                "advise",
2325                "no advisor produced a usable proposal; continuing without a \
2326                 synthesis brief"
2327                    .to_owned(),
2328            );
2329        } else {
2330            match self
2331                .synthesize_brief(
2332                    &advice,
2333                    &instruction,
2334                    &language,
2335                    &worktrees[0],
2336                    &artifacts,
2337                    &run_id,
2338                    &prompts,
2339                    cache.as_deref(),
2340                )
2341                .await
2342            {
2343                Ok(Some(text)) => {
2344                    self.state.event(
2345                        "advise",
2346                        "synthesized a design brief for the implementer".to_owned(),
2347                    );
2348                    advice.synthesis = Some(text);
2349                }
2350                Ok(None) => {
2351                    self.state.event(
2352                        "advise",
2353                        "the synthesis seat produced nothing usable; continuing \
2354                         without a design brief"
2355                            .to_owned(),
2356                    );
2357                }
2358                Err(e) => {
2359                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2360                        return Err(e);
2361                    }
2362                    self.state.event(
2363                        "advise",
2364                        format!("could not synthesize a design brief: {e:#}"),
2365                    );
2366                }
2367            }
2368        }
2369        advise::apply_reflection(&mut advice);
2370
2371        self.state.advice = Some(advice);
2372        self.state.advise_attempted = true;
2373        self.state.save()?;
2374        Ok(())
2375    }
2376
2377    /// The synthesis seat: reads every advisor's proposal and blends them
2378    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2379    /// out of [`Runner::advise`] only for readability — it is not called
2380    /// anywhere else.
2381    ///
2382    /// Picked the same way [`crate::talk`]'s standing conversation and
2383    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2384    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2385    /// order (a claude seat, else the first runnable agent in roster order)
2386    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2387    /// in [`crate::config`] for why a dedicated field exists here at all.
2388    #[allow(clippy::too_many_arguments)]
2389    async fn synthesize_brief(
2390        &mut self,
2391        advice: &advise::Advice,
2392        instruction: &str,
2393        language: &str,
2394        cwd: &Path,
2395        artifacts: &Path,
2396        run_id: &str,
2397        prompts: &Prompts,
2398        cache: Option<&Path>,
2399    ) -> Result<Option<String>> {
2400        let chain = agent::pick_chain(
2401            &self.state.config.agents,
2402            self.state.config.roles.synthesizer.as_ref(),
2403            &agent::installed,
2404            "synthesizer",
2405        )?;
2406        let proposals = advice.proposals();
2407        let mut prompt = prompt::with_overlay(
2408            prompt::synthesize_brief(instruction, &proposals, language),
2409            prompts.overlay("advise"),
2410        );
2411        if cache.is_some() {
2412            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2413            // below — see `prompt::build_cache_note`'s doc for why telling a
2414            // read-only seat to build through the shared cache is exactly how
2415            // a sandbox's write refusal gets misread as a defect.
2416            prompt.push('\n');
2417            prompt.push_str(&prompt::build_cache_note("advise", false));
2418        }
2419        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2420        // Each id is tried once, in order; a quota hit, error or unusable
2421        // answer moves to the next. The seat is single-turn (`sessions:
2422        // false`) and the prompt is the whole context, so a fallback agent
2423        // needs nothing carried over.
2424        let mut last = None;
2425        for (n, spec) in chain.iter().enumerate() {
2426            if n > 0 {
2427                self.state
2428                    .event("advise", format!("synthesis falling back to {}", spec.id));
2429            }
2430            let mut seat = self.seat("advise-synthesis", &spec.id);
2431            let wait = self.repository_wait();
2432            let outcome = wait
2433                .scope(agent::invoke(
2434                    spec,
2435                    &mut seat,
2436                    &Invocation {
2437                        coordination: self.state.config.coordination.as_ref(),
2438                        fetched_base: None,
2439                        cwd,
2440                        prompt: &prompt,
2441                        timeout,
2442                        allow_write: false,
2443                        sessions: false,
2444                        artifacts,
2445                        stem: &if n == 0 {
2446                            "advise-synthesis".to_owned()
2447                        } else {
2448                            format!("advise-synthesis-{}", spec.id)
2449                        },
2450                        run: run_id,
2451                        node: "advise",
2452                        cache_dir: None,
2453                        attachments: &[],
2454                        writable: &[],
2455                    },
2456                ))
2457                .await;
2458            if outcome.is_ok() {
2459                self.state.seats.insert(seat.key.clone(), seat);
2460            }
2461            let advance = agent::chain_advances(&outcome);
2462            last = Some(outcome);
2463            if !advance {
2464                break;
2465            }
2466        }
2467        // Exhausted: the last attempt's result is what a single failed seat
2468        // would have produced.
2469        let out = last.expect("a chain holds at least one agent")?;
2470        if !out.usable() {
2471            return Ok(None);
2472        }
2473        let text =
2474            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2475        Ok((!text.trim().is_empty()).then_some(text))
2476    }
2477
2478    // ----------------------------------------------------------- implement
2479
2480    async fn implement(&mut self) -> Result<()> {
2481        self.persist_candidates().await?;
2482        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2483        // agent files with `magi task add` name the run that paid for it. The
2484        // prompt overlay is cloned alongside it because the waves borrow it
2485        // while `self` is mutably borrowed by the node's own bookkeeping.
2486        let run_id = self.state.id.clone();
2487        let prompts = self.state.config.prompts.clone();
2488        let todo: Vec<usize> = self
2489            .state
2490            .candidates
2491            .iter()
2492            .enumerate()
2493            .filter(|(_, c)| {
2494                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2495                    && c.failed.is_none()
2496                    && !c.empty
2497            })
2498            .map(|(i, _)| i)
2499            .collect();
2500        if todo.is_empty() {
2501            return self.after_implement();
2502        }
2503        self.state.status = RunStatus::Implementing;
2504
2505        let language = self.state.config.graph.language.clone();
2506        let github_language = self.state.config.graph.github_language.clone();
2507        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2508        let sessions = self.state.config.graph.sessions;
2509        let artifacts = agent::artifacts_dir(&self.state.dir());
2510        // The design-deliberation stage's blended brief, when `advise` found
2511        // one — carried into every implementer's prompt the same way
2512        // regardless of which candidate it is.
2513        let brief = self
2514            .state
2515            .advice
2516            .as_ref()
2517            .and_then(|a| a.synthesis.as_deref())
2518            .map(str::to_owned);
2519        let attachments = self.state.attachments.clone();
2520
2521        let mut jobs = Vec::new();
2522        for &i in &todo {
2523            let (index, label, worktree) = {
2524                let c = &self.state.candidates[i];
2525                (c.index, c.label, c.worktree.clone())
2526            };
2527            let spec = self.roles.implementers[index].clone();
2528            let seat_key = format!("impl-{label}");
2529            let seat = self.seat(&seat_key, &spec.id);
2530            let instruction = seeded_instruction(&self.state);
2531            jobs.push(SeatJob {
2532                spec,
2533                seat,
2534                prompt: prompt::implement_in(
2535                    &instruction,
2536                    &worktree.to_string_lossy(),
2537                    &language,
2538                    &github_language,
2539                    brief.as_deref(),
2540                    &attachments,
2541                ),
2542                cwd: worktree,
2543                timeout,
2544                allow_write: true,
2545                sessions,
2546                artifacts: artifacts.clone(),
2547                stem: format!("impl-{label}"),
2548                handover: None,
2549            });
2550        }
2551
2552        self.state.event(
2553            "implement",
2554            format!("{} candidates in parallel", jobs.len()),
2555        );
2556        // Kept so a seat whose CLI hung up can be asked again from the same
2557        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2558        // can update a seat's own entry once a fallback agent takes it over —
2559        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2560        // whichever agent actually answered, not the one that quota'd out.
2561        let mut sent = jobs.clone();
2562        let cache = self.state.config.cache_dir();
2563        let ctx = WaveCtx {
2564            carry_seats: false,
2565            pauses: Some((&self.pause, &self.interrupt)),
2566            run: &run_id,
2567            node: "implement",
2568            prompts: &prompts,
2569            cache: cache.as_deref(),
2570            round: None,
2571        };
2572        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2573        if !wave_blocked(&results) {
2574            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2575                .await;
2576            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2577                .await?;
2578            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2579                .await;
2580        }
2581        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2582
2583        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2584            && results
2585                .iter()
2586                .any(|(_, _, out)| FailClass::of(out).is_some());
2587        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2588            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2589                // Executed turns interrupted by disk pressure already carry
2590                // a preservation barrier from wave. Admission-only waits do
2591                // not. Keep the interrupted seat's session for recovery too.
2592                if self
2593                    .state
2594                    .candidate_preservation
2595                    .contains_key(&self.state.candidates[i].label)
2596                {
2597                    self.state.seats.insert(seat.key.clone(), seat);
2598                }
2599                continue;
2600            }
2601            let seat_key = seat.key.clone();
2602            // A quota fallback (`resume_seat_handovers`) may have handed this
2603            // seat to a different agent than the one `prep` recorded on the
2604            // candidate; the stats tables and any later fixer-defaults-to-
2605            // winner's-author lookup must credit whoever actually answered —
2606            // unless every fallback also quota'd out, in which case nobody
2607            // actually answered and crediting the last agent tried would
2608            // erase every earlier agent's own quota loss from the stats
2609            // tables instead of just this one seat's.
2610            let agent = seat.agent.clone();
2611            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2612            self.state.seats.insert(seat.key.clone(), seat);
2613            let label = self.state.candidates[i].label;
2614            self.state.interrupted_implementations.remove(&label);
2615
2616            let (summary, duration, failed, verified_claim) = match out {
2617                AgentOutcome::Ok(o) => {
2618                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2619                    let failed = (!o.usable()).then(|| {
2620                        if o.timed_out {
2621                            "agent timed out".to_owned()
2622                        } else {
2623                            format!(
2624                                "agent exited with {}",
2625                                crate::notices::exit_status(o.exit_code)
2626                            )
2627                        }
2628                    });
2629                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2630                    (text, o.duration_ms, failed, verified_claim)
2631                }
2632                // Left un-resumed by `resume_undelivered` (a dirty tree
2633                // already rescues the work, or there was no session left to
2634                // resume into) — reported like the ordinary failure it is,
2635                // never as if `o.text` (the CLI's raw error JSON) were an
2636                // answer.
2637                AgentOutcome::Dropped(o) => {
2638                    let why = o
2639                        .dropped
2640                        .as_ref()
2641                        .map(|d| d.why.as_str())
2642                        .unwrap_or("the CLI ended the stream without delivering its answer");
2643                    (
2644                        String::new(),
2645                        o.duration_ms,
2646                        Some(format!("the CLI dropped the stream ({why})")),
2647                        None,
2648                    )
2649                }
2650                AgentOutcome::Quota(o) => {
2651                    self.state.quota.push(QuotaLoss {
2652                        seat: seat_key,
2653                        node: "implement".to_owned(),
2654                        at: Timestamp::now(),
2655                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2656                    });
2657                    (
2658                        String::new(),
2659                        o.duration_ms,
2660                        Some("rate limited (quota); produced no change".to_owned()),
2661                        None,
2662                    )
2663                }
2664                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2665                    (String::new(), 0, Some(e), None)
2666                }
2667            };
2668
2669            // Save every completed turn before any fallible Git operation. A failed
2670            // rescue must not erase the reply or prevent collection of other seats.
2671            let c = &mut self.state.candidates[i];
2672            if !exhausted_the_fallback_chain {
2673                c.agent = agent;
2674            }
2675            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2676            c.duration_ms = duration;
2677            c.failed = failed;
2678            c.verified_noop = verified_claim;
2679            self.state
2680                .candidate_preservation
2681                .insert(label, "awaiting persistence".to_owned());
2682        }
2683        self.state.save()?;
2684        self.persist_candidates().await?;
2685
2686        if resource_blocked {
2687            return self.stop_for_resource();
2688        }
2689        if selected_failed {
2690            self.state.status = RunStatus::Blocked;
2691            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2692            self.state.save()?;
2693            bail!(
2694                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2695            );
2696        }
2697        self.after_implement()
2698    }
2699
2700    /// Retry only persistence after a completed implementation turn.
2701    async fn persist_candidates(&mut self) -> Result<()> {
2702        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2703        for label in labels {
2704            let i = self
2705                .state
2706                .candidates
2707                .iter()
2708                .position(|c| c.label == label)
2709                .context("pending candidate is missing")?;
2710            let c = self.state.candidates[i].clone();
2711            let interrupted = self.state.interrupted_implementations.contains(&label);
2712            let result = async {
2713                let wait = crate::repo_lock::WaitPolicy::new(
2714                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2715                    || false,
2716                );
2717                let _coordination =
2718                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2719                // Best effort evidence requires no index writes. A successful snapshot
2720                // never authorizes removal of a tree whose rescue failed.
2721                if let Err(e) = git::snapshot_uncommitted(
2722                    &c.worktree,
2723                    &self
2724                        .state
2725                        .dir()
2726                        .join("artifacts")
2727                        .join(format!("candidate-{label}-recovery")),
2728                )
2729                .await
2730                {
2731                    self.state.event(
2732                        "preserve",
2733                        format!("candidate {label} snapshot incomplete: {e:#}"),
2734                    );
2735                }
2736                let rescued = git::rescue_commit(
2737                    &c.worktree,
2738                    &format!("magi: candidate {label} (uncommitted work)"),
2739                )
2740                .await?;
2741                self.state.note_withheld("implement", &rescued.withheld);
2742                let base = &self.state.base_commit;
2743                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2744                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2745                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2746                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2747                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2748                if c.empty && commits > 0 && !patch.trim().is_empty()
2749                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2750                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2751                        || self.state.tally.is_some())
2752                {
2753                    // Keep the old classification and pending marker so every
2754                    // retry stops too. Retain decisions as recovery evidence;
2755                    // only a supported recovery operation may replace them.
2756                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2757                }
2758                let c = &mut self.state.candidates[i];
2759                c.commits = commits;
2760                c.stat = stat;
2761                c.files = files;
2762                let no_changes = commits == 0 || patch.trim().is_empty();
2763                c.empty = no_changes && !interrupted;
2764                if interrupted && no_changes {
2765                    c.failed = None;
2766                    c.verified_noop = None;
2767                }
2768                if !c.empty {
2769                    c.failed = None;
2770                    c.verified_noop = None;
2771                }
2772                self.state.event(
2773                    "preserve",
2774                    format!("candidate {label}: {files} files, {commits} commits"),
2775                );
2776                Ok::<_, anyhow::Error>(())
2777            }
2778            .await;
2779            match result {
2780                Ok(()) => {
2781                    self.state.candidate_preservation.remove(&label);
2782                    if !interrupted || self.state.candidates[i].files > 0 {
2783                        self.state.interrupted_implementations.remove(&label);
2784                    }
2785                }
2786                Err(e) => {
2787                    let reason = format!("candidate {label} preservation failed: {e:#}");
2788                    self.state.event("preserve", &reason);
2789                    self.state.candidate_preservation.insert(label, reason);
2790                }
2791            }
2792            if self.state.candidate_preservation.is_empty()
2793                && self.state.status == RunStatus::Blocked
2794                && self.state.candidates.iter().all(|c| c.failed.is_none())
2795            {
2796                // Persist the recovered phase with the cleared pending marker;
2797                // a crash between saves must not turn this into a selected-seat
2798                // failure hold on the next resume.
2799                self.state.status = RunStatus::Implementing;
2800                self.state.event(
2801                    "preserve",
2802                    "candidate persistence recovered; implementation can continue",
2803                );
2804            }
2805            self.state.save()?;
2806        }
2807        if !self.state.candidate_preservation.is_empty() {
2808            self.state.status = RunStatus::Blocked;
2809            self.state.save()?;
2810            bail!(
2811                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2812            );
2813        }
2814        Ok(())
2815    }
2816
2817    /// Ask again, once, for work a CLI did and then failed to hand over.
2818    ///
2819    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2820    /// status with an empty response and a usage report showing output tokens,
2821    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2822    /// seven minutes and 14,267 output tokens that arrived as an empty
2823    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2824    ///
2825    /// Two conditions, and both matter:
2826    ///
2827    /// - **Only when the tree is untouched.** Often the agent has already
2828    ///   written its files and only the closing message was lost; the rescue
2829    ///   commit below picks that up and there is nothing to ask for. Re-asking
2830    ///   then would pay for a second implementation of work already on disk.
2831    /// - **Once.** A CLI that drops one stream can drop the next, and this
2832    ///   node is the most expensive in the graph.
2833    ///
2834    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2835    /// dropped reply still carried its `conversation_id`, so the seat is asked
2836    /// to finish what it was doing rather than sent the whole task again. It
2837    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2838    /// node's - for the same reason a re-ranked judge does: restating finished
2839    /// work is not the work.
2840    ///
2841    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2842    /// same way until it resets, while an abandoned conversation is still
2843    /// there to be picked up.
2844    async fn resume_undelivered(
2845        &mut self,
2846        results: &mut [(usize, SeatState, AgentOutcome)],
2847        sent: &[SeatJob],
2848        prompts: &Prompts,
2849        run_id: &str,
2850    ) {
2851        for (wi, seat, out) in results.iter_mut() {
2852            let Some(dropped) = (match &*out {
2853                AgentOutcome::Dropped(o) => o.dropped.clone(),
2854                _ => None,
2855            }) else {
2856                continue;
2857            };
2858            let Some(job) = sent.get(*wi) else { continue };
2859            // Already on disk? Then only the closing message was lost.
2860            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2861                self.state.event(
2862                    "implement",
2863                    format!(
2864                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2865                         work is in the tree",
2866                        seat.key, dropped.output_tokens, dropped.why
2867                    ),
2868                );
2869                continue;
2870            }
2871            // The re-ask only makes sense as a resume: `resume_after_drop`
2872            // says nothing about the task, trusting the seat to still hold it.
2873            // Without a session to resume — sessions disabled, or this CLI's
2874            // drop shape happened not to carry a session id — that prompt
2875            // would open a brand-new conversation with no context at all,
2876            // which is worse than leaving this as the ordinary failure it
2877            // already is.
2878            if !has_context(&job.spec, seat, job.sessions) {
2879                self.state.event(
2880                    "implement",
2881                    format!(
2882                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2883                         is no session left to resume",
2884                        seat.key, dropped.output_tokens, dropped.why
2885                    ),
2886                );
2887                continue;
2888            }
2889            self.state.event(
2890                "implement",
2891                format!(
2892                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2893                     conversation",
2894                    seat.key, dropped.output_tokens, dropped.why
2895                ),
2896            );
2897            let mut retry = job.clone();
2898            retry.seat = seat.clone();
2899            retry.prompt = prompt::resume_after_drop(&dropped.why);
2900            retry.timeout = retry_budget(job.timeout, true);
2901            retry.stem = format!("{}-resume", job.stem);
2902            let cache = self.state.config.cache_dir();
2903            let ctx = WaveCtx {
2904                carry_seats: false,
2905                pauses: Some((&self.pause, &self.interrupt)),
2906                run: run_id,
2907                node: "implement",
2908                prompts,
2909                cache: cache.as_deref(),
2910                round: None,
2911            };
2912            let (resumed_seat, resumed) =
2913                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2914            *seat = resumed_seat;
2915            *out = resumed;
2916        }
2917    }
2918
2919    /// Fall an implement seat through to the next untried agent in the
2920    /// implementer roster when it lost to quota — or, since the handover was
2921    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2922    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2923    /// quota path itself being unchanged — instead of leaving the
2924    /// seat's loss final the moment one agent's account runs dry.
2925    ///
2926    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2927    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2928    /// truncates to the single slot rotation picked, so a solo task whose one
2929    /// implementer hits quota mid-run used to have nothing else to try. This
2930    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2931    /// unrotated roster — which is the only place the *other* candidates in
2932    /// the machine's roster still exist once `implementers` has been cut down
2933    /// to size.
2934    ///
2935    /// Walks forward from just past the seat's own original position in the
2936    /// roster, never wrapping back to the front: a later candidate slot (say
2937    /// `beta`, the roster's second entry) must fall through to the *next*
2938    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2939    /// almost certainly a different candidate's own agent already — and once
2940    /// the roster's tail is exhausted there is nothing left to fall through
2941    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2942    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2943    /// let this retry that id forever. The loop keeps falling through until
2944    /// an attempt lands something other than `Quota` or the roster's tail
2945    /// runs out of untried ids, at which point the seat is left exactly as
2946    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2947    /// `QuotaLoss` recorded, the candidate failed/empty.
2948    ///
2949    /// `sent` is taken mutably and updated with the fallback agent's spec:
2950    /// `resume_unconfirmed_commands`, which runs after this and also reads
2951    /// `sent`, must see whichever agent actually ended up answering the seat
2952    /// — reading the stale, original spec there would check session
2953    /// eligibility against the wrong CLI and could hand a fallback agent's
2954    /// session id to the agent that just lost the seat to quota.
2955    ///
2956    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2957    /// — `self.seat` only reuses state when the agent id is unchanged, so
2958    /// handing it a different id already gets this for free. Reusing the old
2959    /// seat would resume a different CLI's session as if it were a
2960    /// continuation of this one.
2961    ///
2962    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2963    /// a quota loss cuts an agent off mid-turn, so anything already in the
2964    /// tree is unfinished work, not a completed candidate a re-ask would pay
2965    /// for twice. A dirty tree is rescued into a commit first (the same
2966    /// neutral-identity rescue `implement`'s own outcome loop gives every
2967    /// candidate) so the next agent starts clean.
2968    ///
2969    /// The new agent gets the implementer's full prompt and full
2970    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2971    /// it has no session and no context, and is implementing the task from
2972    /// nothing, unlike a resumed drop which is only restating work already
2973    /// done.
2974    ///
2975    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2976    /// `implement` event, never into `self.state.quota` — that is what
2977    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2978    /// go unspent, and a seat that ultimately recovered on its second or
2979    /// third agent is not the stalled panel that check exists to catch. Only
2980    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2981    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2982    /// outcome loop already has — this helper never pushes to it itself.
2983    async fn resume_seat_handovers(
2984        &mut self,
2985        results: &mut [(usize, SeatState, AgentOutcome)],
2986        sent: &mut [SeatJob],
2987        prompts: &Prompts,
2988        run_id: &str,
2989    ) -> Result<()> {
2990        let instruction = seeded_instruction(&self.state);
2991        let language = self.state.config.graph.language.clone();
2992        let github_language = self.state.config.graph.github_language.clone();
2993        let brief = self
2994            .state
2995            .advice
2996            .as_ref()
2997            .and_then(|a| a.synthesis.as_deref())
2998            .map(str::to_owned);
2999        let attachments = self.state.attachments.clone();
3000        for (wi, seat, out) in results.iter_mut() {
3001            // Who holds the other candidate seats of this wave right now
3002            // (earlier handovers already written back to `sent`).
3003            let others: BTreeSet<String> = sent
3004                .iter()
3005                .enumerate()
3006                .filter(|(j, _)| j != wi)
3007                .map(|(_, j)| j.spec.id.clone())
3008                .collect();
3009            let Some(job) = sent.get_mut(*wi) else {
3010                continue;
3011            };
3012            // Where the seat's own original agent sits in the roster — the
3013            // fallback walk starts just past here, never at the front, so a
3014            // later candidate slot's quota loss does not fall back onto an
3015            // earlier slot's own agent.
3016            let start = self
3017                .roles
3018                .implementer_roster
3019                .iter()
3020                .position(|s| s.id == job.spec.id)
3021                .unwrap_or(0);
3022            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3023            let mut fallback_attempt = 0usize;
3024            let mut prev: Option<FailClass> = None;
3025            while let Some(cur) = FailClass::of(&*out) {
3026                if !should_hand_over(prev.as_ref(), &cur) {
3027                    break;
3028                }
3029                let Some(next) =
3030                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3031                        .cloned()
3032                else {
3033                    break;
3034                };
3035                tried.insert(next.id.clone());
3036                fallback_attempt += 1;
3037
3038                let coordination = self.coordinate(&job.cwd).await?;
3039                if let Ok(r) = git::rescue_commit(
3040                    &job.cwd,
3041                    &format!(
3042                        "magi: candidate {} (uncommitted work before {} fallback)",
3043                        seat.key,
3044                        if cur == FailClass::Quota {
3045                            "quota"
3046                        } else {
3047                            "handover"
3048                        }
3049                    ),
3050                )
3051                .await
3052                {
3053                    self.state.note_withheld("implement", &r.withheld);
3054                }
3055                drop(coordination);
3056
3057                record_handover(
3058                    &mut self.state,
3059                    "implement",
3060                    &seat.key,
3061                    &seat.agent,
3062                    &next.id,
3063                    &cur,
3064                    &fail_reason(&*out),
3065                );
3066                prev = Some(cur.clone());
3067
3068                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3069                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3070                // Kept in sync on `sent` itself, not just the local retry: a
3071                // later helper (`resume_unconfirmed_commands`) reads `sent`
3072                // after this one returns and must see whichever agent is now
3073                // occupying the seat, not the one that just quota'd out —
3074                // otherwise it would judge session/continuation eligibility
3075                // by the wrong CLI and could resend a fallback's session id
3076                // to the agent that lost it the seat in the first place.
3077                job.spec = next.clone();
3078                let mut retry = job.clone();
3079                retry.seat = new_seat;
3080                retry.prompt = prompt::implement_in(
3081                    &instruction,
3082                    &job.cwd.to_string_lossy(),
3083                    &language,
3084                    &github_language,
3085                    brief.as_deref(),
3086                    &attachments,
3087                );
3088                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3089                let cache = self.state.config.cache_dir();
3090                let ctx = WaveCtx {
3091                    carry_seats: false,
3092                    pauses: Some((&self.pause, &self.interrupt)),
3093                    run: run_id,
3094                    node: "implement",
3095                    prompts,
3096                    cache: cache.as_deref(),
3097                    round: None,
3098                };
3099                let (fallback_seat, fallback_out) = run_one(
3100                    retry,
3101                    Arc::clone(&self.sem),
3102                    &ctx,
3103                    &mut self.state,
3104                    fallback_attempt,
3105                )
3106                .await;
3107                *seat = fallback_seat;
3108                *out = fallback_out;
3109            }
3110        }
3111        Ok(())
3112    }
3113
3114    /// Ask an implement seat's own CLI to confirm what it started, once, when
3115    /// its reply reported a command whose completion status it never
3116    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3117    /// that does and does not mean.
3118    ///
3119    /// The completion contract this task asks for, extended to `implement`
3120    /// with the same signal `continue_fix_report` reads for the fixer,
3121    /// rather than a keyword search over the reply or a hard requirement on
3122    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3123    /// a candidate's CLI turn ended cleanly while a test run it had started
3124    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3125    /// named in it at all is untouched by this: `commands` is empty, so
3126    /// there is nothing to be unconfirmed.
3127    ///
3128    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3129    /// this is not about recovering edits that might already be on disk, it
3130    /// is about a result the seat itself never vouched for, which resuming
3131    /// asks for regardless of what the tree already holds. Bounded to one
3132    /// attempt for the same reason `resume_undelivered` is — this is the
3133    /// most expensive node in the graph — and a seat that still cannot
3134    /// confirm on that attempt is left as whatever its (possibly still
3135    /// unconfirmed) reply says; this does not invent a new "failed" reason
3136    /// for a candidate that otherwise produced a real, committed change.
3137    async fn resume_unconfirmed_commands(
3138        &mut self,
3139        results: &mut [(usize, SeatState, AgentOutcome)],
3140        sent: &[SeatJob],
3141        prompts: &Prompts,
3142        run_id: &str,
3143    ) {
3144        for (wi, seat, out) in results.iter_mut() {
3145            let AgentOutcome::Ok(o) = &*out else {
3146                continue;
3147            };
3148            if !has_unconfirmed_command(&o.commands) {
3149                continue;
3150            }
3151            let Some(job) = sent.get(*wi) else { continue };
3152            if !has_context(&job.spec, seat, job.sessions) {
3153                self.state.event(
3154                    "implement",
3155                    format!(
3156                        "{}: the reply named a command whose own CLI never confirmed the exit \
3157                         status of, but there is no session left to resume",
3158                        seat.key
3159                    ),
3160                );
3161                continue;
3162            }
3163            self.state.event(
3164                "implement",
3165                format!(
3166                    "{}: the reply named a command whose own CLI never confirmed the exit \
3167                     status of; resuming the conversation",
3168                    seat.key
3169                ),
3170            );
3171            let mut retry = job.clone();
3172            retry.seat = seat.clone();
3173            retry.prompt = prompt::resume_incomplete(
3174                "a command in your last reply had no confirmed exit status",
3175            );
3176            retry.timeout = retry_budget(job.timeout, true);
3177            retry.stem = format!("{}-confirm", job.stem);
3178            let cache = self.state.config.cache_dir();
3179            let ctx = WaveCtx {
3180                carry_seats: false,
3181                pauses: Some((&self.pause, &self.interrupt)),
3182                run: run_id,
3183                node: "implement",
3184                prompts,
3185                cache: cache.as_deref(),
3186                round: None,
3187            };
3188            let (resumed_seat, resumed) =
3189                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3190            *seat = resumed_seat;
3191            *out = resumed;
3192        }
3193    }
3194
3195    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3196    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3197    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3198    /// that motivated this.
3199    ///
3200    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3201    /// own nudge loop already covers for judge/review/vote seats, and not a
3202    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3203    /// implement seats: here the CLI turn genuinely finished while the node's
3204    /// own work — the fixer's account of what it did — had not. Gated purely
3205    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3206    /// reply, never on any wording in it, so a fixer whose valid, first-try
3207    /// `FixReport` happens to mention having waited on a background test is
3208    /// never resumed — the `Ok(report)` branch at the call site returns
3209    /// before this is ever invoked.
3210    ///
3211    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3212    /// attempt ([`retry_budget`]), nothing attempted once the session is
3213    /// gone, and a quota hit ends the loop immediately rather than retrying a
3214    /// rate limit that fails the same way again.
3215    async fn continue_fix_report(
3216        &mut self,
3217        mut seat: SeatState,
3218        parse_err: String,
3219        job: &SeatJob,
3220        prompts: &Prompts,
3221        run_id: &str,
3222        round: usize,
3223    ) -> (
3224        SeatState,
3225        Option<FixReport>,
3226        Option<String>,
3227        ContinuationRecord,
3228    ) {
3229        let mut last_err = parse_err;
3230        let mut cumulative_wait_ms = 0u64;
3231        let mut attempts = 0usize;
3232        loop {
3233            if !has_context(&job.spec, &seat, job.sessions) {
3234                self.state.event(
3235                    "fix",
3236                    format!(
3237                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3238                         session left to resume into"
3239                    ),
3240                );
3241                let outcome = if attempts == 0 {
3242                    ContinuationOutcome::NoSession
3243                } else {
3244                    ContinuationOutcome::Exhausted
3245                };
3246                return (
3247                    seat,
3248                    None,
3249                    Some(format!("unparsable fix report: {last_err}")),
3250                    ContinuationRecord {
3251                        attempts,
3252                        cumulative_wait_ms,
3253                        outcome,
3254                    },
3255                );
3256            }
3257            if attempts >= MAX_FIX_CONTINUATIONS {
3258                self.state.event(
3259                    "fix",
3260                    format!(
3261                        "round {round}: fixer's reply still had no adoption report after \
3262                         {attempts} continuation(s) ({last_err}); giving up"
3263                    ),
3264                );
3265                return (
3266                    seat,
3267                    None,
3268                    Some(format!(
3269                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3270                    )),
3271                    ContinuationRecord {
3272                        attempts,
3273                        cumulative_wait_ms,
3274                        outcome: ContinuationOutcome::Exhausted,
3275                    },
3276                );
3277            }
3278            attempts += 1;
3279            self.state.event(
3280                "fix",
3281                format!(
3282                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3283                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3284                ),
3285            );
3286            let mut retry = job.clone();
3287            retry.seat = seat.clone();
3288            retry.prompt = prompt::resume_incomplete(&last_err);
3289            retry.timeout = retry_budget(job.timeout, true);
3290            retry.stem = format!("{}-continue{attempts}", job.stem);
3291            let cache = self.state.config.cache_dir();
3292            let ctx = WaveCtx {
3293                carry_seats: false,
3294                pauses: Some((&self.pause, &self.interrupt)),
3295                run: run_id,
3296                node: "fix",
3297                prompts,
3298                cache: cache.as_deref(),
3299                round: Some(round),
3300            };
3301            let (resumed_seat, resumed_out) = run_one(
3302                retry,
3303                Arc::clone(&self.sem),
3304                &ctx,
3305                &mut self.state,
3306                attempts,
3307            )
3308            .await;
3309            seat = resumed_seat;
3310            match resumed_out {
3311                AgentOutcome::Ok(o) => {
3312                    cumulative_wait_ms += o.duration_ms;
3313                    match verdict::extract_json::<FixReport>(&o.text) {
3314                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3315                            self.state.event(
3316                                "fix",
3317                                format!(
3318                                    "round {round}: fixer's adoption report recovered after \
3319                                     {attempts} continuation(s)"
3320                                ),
3321                            );
3322                            return (
3323                                seat,
3324                                Some(report),
3325                                None,
3326                                ContinuationRecord {
3327                                    attempts,
3328                                    cumulative_wait_ms,
3329                                    outcome: ContinuationOutcome::Resumed,
3330                                },
3331                            );
3332                        }
3333                        // The report parsed, but this same reply's own
3334                        // CommandEvidence — the identical record `state.jobs`
3335                        // renders — names a command whose CLI never
3336                        // confirmed an exit status. Read together, that is
3337                        // not a resolved answer: keep nudging rather than
3338                        // accept a report standing next to a command the
3339                        // seat's own CLI cannot vouch for.
3340                        Ok(_) => {
3341                            last_err = "the reply parsed, but it reported a command whose own CLI \
3342                                 never confirmed an exit status"
3343                                .to_owned();
3344                        }
3345                        Err(e) => last_err = e.to_string(),
3346                    }
3347                }
3348                AgentOutcome::Quota(o) => {
3349                    cumulative_wait_ms += o.duration_ms;
3350                    self.state.quota.push(QuotaLoss {
3351                        seat: seat.key.clone(),
3352                        node: "fix".to_owned(),
3353                        at: Timestamp::now(),
3354                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3355                    });
3356                    self.state.event(
3357                        "fix",
3358                        format!(
3359                            "round {round}: continuation rate limited (quota); not retrying now"
3360                        ),
3361                    );
3362                    return (
3363                        seat,
3364                        None,
3365                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3366                        ContinuationRecord {
3367                            attempts,
3368                            cumulative_wait_ms,
3369                            outcome: ContinuationOutcome::QuotaLost,
3370                        },
3371                    );
3372                }
3373                AgentOutcome::Dropped(o) => {
3374                    cumulative_wait_ms += o.duration_ms;
3375                    let why = o
3376                        .dropped
3377                        .as_ref()
3378                        .map(|d| d.why.as_str())
3379                        .unwrap_or("the CLI ended the stream without delivering its answer");
3380                    last_err = format!("the CLI dropped the stream ({why})");
3381                }
3382                AgentOutcome::Failed(e) => last_err = e,
3383                // The continuation was never started; the report stays lost
3384                // and the caller records that, without any handover.
3385                AgentOutcome::ResourceBlocked(e) => {
3386                    return (
3387                        seat,
3388                        None,
3389                        Some(format!("the continuation was not started: {e}")),
3390                        ContinuationRecord {
3391                            attempts: attempts.saturating_sub(1),
3392                            cumulative_wait_ms,
3393                            outcome: ContinuationOutcome::NoSession,
3394                        },
3395                    );
3396                }
3397            }
3398        }
3399    }
3400
3401    fn after_implement(&mut self) -> Result<()> {
3402        if !self.state.candidate_preservation.is_empty() {
3403            bail!("candidate preservation incomplete; judging is unsafe");
3404        }
3405        // Scan every candidate patch once the set is complete.
3406        if self.state.leaks.is_empty() {
3407            let cfg = self.state.config.blind.clone();
3408            let mut leaks = Vec::new();
3409            for c in &self.state.candidates {
3410                let Some(patch) =
3411                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3412                else {
3413                    continue;
3414                };
3415                leaks.extend(blind::scan(
3416                    &format!("candidate {} patch", c.label),
3417                    &patch,
3418                    &cfg.vendor_tokens,
3419                ));
3420            }
3421            if !leaks.is_empty() {
3422                let summary = leaks
3423                    .iter()
3424                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3425                    .collect::<Vec<_>>()
3426                    .join(", ");
3427                match cfg.on_leak {
3428                    LeakPolicy::Fail => {
3429                        self.state.status = RunStatus::Failed;
3430                        self.state
3431                            .event("blind", format!("vendor text in a patch: {summary}"));
3432                        self.state.leaks = leaks;
3433                        self.state.save()?;
3434                        self.settle_questions();
3435                        bail!(
3436                            "blind.on_leak = \"fail\" and vendor text reached a \
3437                             judged patch: {summary}"
3438                        );
3439                    }
3440                    LeakPolicy::Redact => self.state.event(
3441                        "blind",
3442                        format!("redacting vendor text for judging: {summary}"),
3443                    ),
3444                    LeakPolicy::Warn => self.state.event(
3445                        "blind",
3446                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3447                    ),
3448                }
3449                self.state.leaks = leaks;
3450            }
3451        }
3452
3453        if self.state.viable().is_empty() {
3454            if self.state.all_candidates_verified_noop() {
3455                // Every candidate agreed, with evidence the adoption guard
3456                // accepted, that nothing belongs in this worktree. That is
3457                // not the same fact as a candidate that simply failed to
3458                // write anything, and settling it as an ordinary `Failed`
3459                // (see `SCHEMA`'s doc for schema 10) is what let two of
3460                // task 391f's attempts burn a retry each re-discovering the
3461                // same already-landed fix. Terminal either way, so `judge`
3462                // must never run over an empty candidate set — unlike the
3463                // `Failed` branch below this returns `Ok`, not an error:
3464                // nothing here failed.
3465                self.state.status = RunStatus::VerifiedNoop;
3466                self.state.save()?;
3467                self.settle_questions();
3468                return Ok(());
3469            }
3470            self.state.status = RunStatus::Failed;
3471            self.state.save()?;
3472            self.settle_questions();
3473            bail!("no candidate produced a change; nothing to judge");
3474        }
3475        self.state.status = RunStatus::Judging;
3476        self.state.save()?;
3477        Ok(())
3478    }
3479
3480    // --------------------------------------------------------------- judge
3481
3482    async fn judge(&mut self) -> Result<()> {
3483        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3484        // agent files with `magi task add` name the run that paid for it. The
3485        // prompt overlay is cloned alongside it because the waves borrow it
3486        // while `self` is mutably borrowed by the node's own bookkeeping.
3487        let run_id = self.state.id.clone();
3488        let prompts = self.state.config.prompts.clone();
3489        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3490            return Ok(());
3491        }
3492        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3493        if viable.len() == 1 {
3494            // Recorded so this is a one-time event: `judgements` stays empty
3495            // either way, which without this flag is indistinguishable from
3496            // "not yet judged" on the next reentry — and status is left
3497            // untouched, so a later node's conclusion (e.g. `Blocked` after
3498            // the review budget ran out) survives a resume instead of being
3499            // clobbered back to `Judging` by this node running again.
3500            self.state.judge_skipped = true;
3501            self.state.event(
3502                "judge",
3503                format!(
3504                    "only candidate {} produced a change; judging skipped",
3505                    viable[0].label
3506                ),
3507            );
3508            self.state.save()?;
3509            return Ok(());
3510        }
3511        self.state.status = RunStatus::Judging;
3512
3513        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3514        let language = self.state.config.graph.language.clone();
3515        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3516        let sessions = self.state.config.graph.sessions;
3517        let artifacts = agent::artifacts_dir(&self.state.dir());
3518        let root = self.state.worktree_root();
3519        let base_short = short(&self.state.base_commit);
3520
3521        let mut jobs = Vec::new();
3522        let mut orders = Vec::new();
3523        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3524            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3525            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3526            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3527            let seat_key = format!("judge-{}", j + 1);
3528            let seat = self.seat(&seat_key, &spec.id);
3529            jobs.push(SeatJob {
3530                prompt: prompt::judge(
3531                    &self.state.instruction,
3532                    &views,
3533                    self.roles.judges.len(),
3534                    &base_short,
3535                    &language,
3536                ),
3537                spec,
3538                seat,
3539                cwd: root.join(format!("judge-{}", j + 1)),
3540                timeout,
3541                allow_write: false,
3542                sessions,
3543                artifacts: artifacts.clone(),
3544                stem: format!("judge-{}", j + 1),
3545                handover: None,
3546            });
3547        }
3548
3549        self.state.event(
3550            "judge",
3551            format!(
3552                "{} judges ranking {} candidates blind",
3553                jobs.len(),
3554                viable.len()
3555            ),
3556        );
3557        let labels_for_check = labels.clone();
3558        let mut quota_losses = Vec::new();
3559        let cache = self.state.config.cache_dir();
3560        let ctx = WaveCtx {
3561            carry_seats: false,
3562            pauses: Some((&self.pause, &self.interrupt)),
3563            run: &run_id,
3564            node: "judge",
3565            prompts: &prompts,
3566            cache: cache.as_deref(),
3567            round: None,
3568        };
3569        let results = ask_json_wave::<Ranking>(
3570            jobs,
3571            Arc::clone(&self.sem),
3572            self.state.config.graph.retries,
3573            &self.roles.judge_roster,
3574            &ctx,
3575            &mut quota_losses,
3576            &mut self.state,
3577            &move |r: &Ranking| r.validate(&labels_for_check),
3578        )
3579        .await;
3580        if self.state.resource_block.is_some() {
3581            return self.stop_for_resource();
3582        }
3583        self.state.quota.extend(quota_losses);
3584
3585        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3586            let agent_id = seat.agent.clone();
3587            self.state.seats.insert(seat.key.clone(), seat);
3588            let mut record = Judgement {
3589                judge: j + 1,
3590                seat: format!("judge-{}", j + 1),
3591                agent: agent_id,
3592                ranking: Vec::new(),
3593                reasons: BTreeMap::new(),
3594                confidence: None,
3595                strengths: BTreeMap::new(),
3596                order: orders[j].clone(),
3597                failed: None,
3598                duration_ms: 0,
3599            };
3600            match res {
3601                Ok((ranking, out)) => {
3602                    record.ranking = ranking.normalized();
3603                    record.reasons = ranking.reasons;
3604                    record.strengths = ranking.strengths;
3605                    record.confidence = ranking.confidence;
3606                    record.duration_ms = out.duration_ms;
3607                    self.state.event(
3608                        "judge",
3609                        format!(
3610                            "judge {} ranked {}",
3611                            j + 1,
3612                            record.ranking.iter().collect::<String>()
3613                        ),
3614                    );
3615                }
3616                Err(e) => {
3617                    record.failed = Some(e.to_string());
3618                    self.state
3619                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3620                }
3621            }
3622            self.state.judgements.push(record);
3623            self.state.save()?;
3624        }
3625        Ok(())
3626    }
3627
3628    // ---------------------------------------------------------- deliberate
3629
3630    async fn deliberate(&mut self) -> Result<()> {
3631        if self.hold_base_sync().await? {
3632            return Ok(());
3633        }
3634        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3635        // agent files with `magi task add` name the run that paid for it. The
3636        // prompt overlay is cloned alongside it because the waves borrow it
3637        // while `self` is mutably borrowed by the node's own bookkeeping.
3638        let run_id = self.state.id.clone();
3639        let prompts = self.state.config.prompts.clone();
3640        if !self.state.deliberation.is_empty() || self.state.tally.is_some() {
3641            return Ok(());
3642        }
3643        let tops: Vec<char> = self
3644            .state
3645            .judgements
3646            .iter()
3647            .filter_map(|j| j.ranking.first().copied())
3648            .collect();
3649        let rounds = self.state.config.graph.deliberate_rounds;
3650        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3651            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3652                self.state.event(
3653                    "deliberate",
3654                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3655                );
3656            }
3657            self.state.status = RunStatus::Voting;
3658            self.state.save()?;
3659            return Ok(());
3660        }
3661
3662        self.state.status = RunStatus::Deliberating;
3663        self.state.event(
3664            "deliberate",
3665            format!(
3666                "split: first choices were {} — opening {rounds} round(s)",
3667                tops.iter().collect::<String>()
3668            ),
3669        );
3670
3671        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3672        let language = self.state.config.graph.language.clone();
3673        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3674        let sessions = self.state.config.graph.sessions;
3675        let artifacts = agent::artifacts_dir(&self.state.dir());
3676        let root = self.state.worktree_root();
3677        let base_short = short(&self.state.base_commit);
3678
3679        // Judges argue in sequence so that a turn can answer the one before it;
3680        // that is the difference between deliberation and three parallel
3681        // monologues.
3682        for round in 1..=rounds {
3683            let mut turns: Vec<DeliberationTurn> = Vec::new();
3684            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3685                if self.state.judgements[j].failed.is_some() {
3686                    continue;
3687                }
3688                let seat_key = format!("judge-{}", j + 1);
3689                let spec = self.occupant(&seat_key, spec);
3690                let mut seat = self.seat(&seat_key, &spec.id);
3691                let transcript = self.transcript(&turns, j);
3692                let build = |context: Option<&str>| {
3693                    prompt::deliberate(
3694                        &self.state.instruction,
3695                        context,
3696                        &transcript,
3697                        round,
3698                        rounds,
3699                        &language,
3700                    )
3701                };
3702                let block = self.candidate_block(&viable, &base_short);
3703                let full = build(Some(&block));
3704                let text = if has_context(&spec, &seat, sessions) {
3705                    build(None)
3706                } else {
3707                    full.clone()
3708                };
3709                let job = SeatJob {
3710                    spec,
3711                    seat: seat.clone(),
3712                    prompt: text,
3713                    cwd: root.join(format!("judge-{}", j + 1)),
3714                    timeout,
3715                    allow_write: false,
3716                    sessions,
3717                    artifacts: artifacts.clone(),
3718                    stem: format!("delib-{round}-judge-{}", j + 1),
3719                    handover: Some(full),
3720                };
3721                let cache = self.state.config.cache_dir();
3722                let ctx = WaveCtx {
3723                    carry_seats: false,
3724                    pauses: Some((&self.pause, &self.interrupt)),
3725                    run: &run_id,
3726                    node: "deliberate",
3727                    prompts: &prompts,
3728                    cache: cache.as_deref(),
3729                    round: None,
3730                };
3731                // A turn is never nudged (`retries` 0): a failed seat is
3732                // handed to the next roster agent, which gets the full
3733                // context. An empty answer is a turn, not a failure.
3734                let mut losses = Vec::new();
3735                let mut results = ask_wave_with::<String>(
3736                    vec![job],
3737                    Arc::clone(&self.sem),
3738                    0,
3739                    &self.roles.judge_roster,
3740                    &ctx,
3741                    &mut losses,
3742                    &mut self.state,
3743                    &|text: &str| {
3744                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3745                    },
3746                )
3747                .await;
3748                self.state.quota.extend(losses);
3749                let (updated, res, _) = results.pop().expect("one job in, one result out");
3750                seat = updated;
3751                let agent_id = seat.agent.clone();
3752                self.state.seats.insert(seat.key.clone(), seat);
3753                let body = match res {
3754                    Ok((body, _)) => body,
3755                    // Skip the seat; a CLI's raw error JSON is never read as
3756                    // this judge's position.
3757                    Err(e) => {
3758                        self.state
3759                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3760                        continue;
3761                    }
3762                };
3763                let tentative = verdict::extract_json::<Position>(&body)
3764                    .ok()
3765                    .and_then(|p| p.tentative)
3766                    .and_then(|s| s.trim().chars().next())
3767                    .map(|c| c.to_ascii_uppercase());
3768                self.state.event(
3769                    "deliberate",
3770                    format!(
3771                        "round {round}: judge {} now favours {}",
3772                        j + 1,
3773                        tentative.map_or("—".to_owned(), |c| c.to_string())
3774                    ),
3775                );
3776                turns.push(DeliberationTurn {
3777                    judge: j + 1,
3778                    agent: agent_id,
3779                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3780                    tentative,
3781                });
3782            }
3783            self.state
3784                .deliberation
3785                .push(DeliberationRound { round, turns });
3786            self.state.save()?;
3787        }
3788
3789        self.state.status = RunStatus::Voting;
3790        self.state.save()?;
3791        Ok(())
3792    }
3793
3794    // ---------------------------------------------------------------- vote
3795
3796    async fn vote(&mut self) -> Result<()> {
3797        if self.hold_base_sync().await? {
3798            return Ok(());
3799        }
3800        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3801        // agent files with `magi task add` name the run that paid for it. The
3802        // prompt overlay is cloned alongside it because the waves borrow it
3803        // while `self` is mutably borrowed by the node's own bookkeeping.
3804        let run_id = self.state.id.clone();
3805        let prompts = self.state.config.prompts.clone();
3806        if !self.state.votes.is_empty() || self.state.tally.is_some() {
3807            return Ok(());
3808        }
3809        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3810        if viable.len() == 1 {
3811            return Ok(());
3812        }
3813        self.state.status = RunStatus::Voting;
3814
3815        let language = self.state.config.graph.language.clone();
3816        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3817        let sessions = self.state.config.graph.sessions;
3818        let artifacts = agent::artifacts_dir(&self.state.dir());
3819        let root = self.state.worktree_root();
3820        let base_short = short(&self.state.base_commit);
3821        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3822
3823        let mut jobs = Vec::new();
3824        let mut seats_at = Vec::new();
3825        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3826            if self
3827                .state
3828                .judgements
3829                .get(j)
3830                .is_some_and(|r| r.failed.is_some())
3831            {
3832                continue;
3833            }
3834            let seat_key = format!("judge-{}", j + 1);
3835            let spec = self.occupant(&seat_key, spec);
3836            let seat = self.seat(&seat_key, &spec.id);
3837            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3838            let text = if has_context(&spec, &seat, sessions) {
3839                prompt::final_vote(&viable, &language)
3840            } else {
3841                full.clone()
3842            };
3843            jobs.push(SeatJob {
3844                spec,
3845                seat,
3846                prompt: text,
3847                cwd: root.join(format!("judge-{}", j + 1)),
3848                timeout,
3849                allow_write: false,
3850                sessions,
3851                artifacts: artifacts.clone(),
3852                stem: format!("vote-judge-{}", j + 1),
3853                handover: Some(full),
3854            });
3855            seats_at.push(j);
3856        }
3857
3858        self.state.event(
3859            "vote",
3860            format!(
3861                "collecting {} final votes one by one, privately",
3862                jobs.len()
3863            ),
3864        );
3865        let allowed = viable.clone();
3866        let mut quota_losses = Vec::new();
3867        let cache = self.state.config.cache_dir();
3868        let ctx = WaveCtx {
3869            carry_seats: false,
3870            pauses: Some((&self.pause, &self.interrupt)),
3871            run: &run_id,
3872            node: "vote",
3873            prompts: &prompts,
3874            cache: cache.as_deref(),
3875            round: None,
3876        };
3877        let results = ask_json_wave::<FinalVote>(
3878            jobs,
3879            Arc::clone(&self.sem),
3880            self.state.config.graph.retries,
3881            &self.roles.judge_roster,
3882            &ctx,
3883            &mut quota_losses,
3884            &mut self.state,
3885            &move |v: &FinalVote| match v.label() {
3886                Some(c) if allowed.contains(&c) => Ok(()),
3887                other => bail!("vote {other:?} is not one of {allowed:?}"),
3888            },
3889        )
3890        .await;
3891        if self.state.resource_block.is_some() {
3892            return self.stop_for_resource();
3893        }
3894        self.state.quota.extend(quota_losses);
3895
3896        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3897            let agent_id = seat.agent.clone();
3898            self.state.seats.insert(seat.key.clone(), seat);
3899            let initial = self
3900                .state
3901                .judgements
3902                .get(j)
3903                .and_then(|r| r.ranking.first().copied());
3904            let mut record = VoteRecord {
3905                judge: j + 1,
3906                agent: agent_id,
3907                vote: None,
3908                reason: String::new(),
3909                changed: false,
3910            };
3911            match res {
3912                Ok((v, _)) => {
3913                    record.vote = v.label();
3914                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3915                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3916                    self.state.event(
3917                        "vote",
3918                        format!(
3919                            "judge {} voted {}{}",
3920                            j + 1,
3921                            record.vote.unwrap_or('?'),
3922                            if record.changed { " (changed)" } else { "" }
3923                        ),
3924                    );
3925                }
3926                Err(e) => {
3927                    self.state
3928                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3929                }
3930            }
3931            self.state.votes.push(record);
3932            self.state.save()?;
3933        }
3934        Ok(())
3935    }
3936
3937    // --------------------------------------------------------------- tally
3938
3939    fn tally(&mut self) -> Result<()> {
3940        if self.state.tally.is_some() {
3941            return Ok(());
3942        }
3943        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3944        let tops: Vec<char> = self
3945            .state
3946            .judgements
3947            .iter()
3948            .filter_map(|j| j.ranking.first().copied())
3949            .collect();
3950        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3951
3952        // A judge whose private vote failed still counted once, in the initial
3953        // ranking; using it beats discarding a whole seat.
3954        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3955        let mut cast: Vec<char> = Vec::new();
3956        for (i, j) in self.state.judgements.iter().enumerate() {
3957            let vote = self
3958                .state
3959                .votes
3960                .iter()
3961                .find(|v| v.judge == i + 1)
3962                .and_then(|v| v.vote)
3963                .or_else(|| j.ranking.first().copied());
3964            if let Some(v) = vote {
3965                *first_choice.entry(v).or_insert(0) += 1;
3966                cast.push(v);
3967            }
3968        }
3969
3970        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3971        for j in &self.state.judgements {
3972            let n = j.ranking.len();
3973            for (pos, label) in j.ranking.iter().enumerate() {
3974                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3975            }
3976        }
3977
3978        let best = first_choice.values().copied().max().unwrap_or(0);
3979        let mut leaders: Vec<char> = first_choice
3980            .iter()
3981            .filter(|(_, v)| **v == best)
3982            .map(|(k, _)| *k)
3983            .collect();
3984        let mut tie_break = None;
3985        if leaders.len() > 1 {
3986            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3987            let borda_leaders: Vec<char> = leaders
3988                .iter()
3989                .copied()
3990                .filter(|l| borda[l] == top_borda)
3991                .collect();
3992            tie_break = Some(if borda_leaders.len() == 1 {
3993                format!(
3994                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3995                    leaders.len()
3996                )
3997            } else {
3998                format!(
3999                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4000                    leaders.len()
4001                )
4002            });
4003            leaders = borda_leaders;
4004            leaders.sort_unstable();
4005        }
4006        let winner = *leaders
4007            .first()
4008            .or(viable.first())
4009            .context("no candidate to declare a winner from")?;
4010
4011        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4012        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4013        let deliberated = !self.state.deliberation.is_empty();
4014
4015        // Whose verdict is this? A rate-limited seat is absent even if it
4016        // ranked before the limit hit, so presence is measured against the
4017        // recorded losses, not just "did a ranking ever appear".
4018        let quota_seats: std::collections::BTreeSet<&str> =
4019            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4020        let mut present = 0usize;
4021        for (i, j) in self.state.judgements.iter().enumerate() {
4022            if quota_seats.contains(j.seat.as_str()) {
4023                continue;
4024            }
4025            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4026            let voted = self
4027                .state
4028                .votes
4029                .iter()
4030                .any(|v| v.judge == i + 1 && v.vote.is_some());
4031            if ranked || voted {
4032                present += 1;
4033            }
4034        }
4035        // Strict majority of the configured panel. A bare majority is real
4036        // signal we can act on, while a minority verdict must never stand in
4037        // for a healthy one. A one-candidate run needs no panel at all, and
4038        // `judges` stays `0` rather than the roster size a panel that never
4039        // sat would otherwise be credited with.
4040        let needs_quorum = viable.len() > 1;
4041        let judges_total = if needs_quorum {
4042            self.roles.judges.len()
4043        } else {
4044            0
4045        };
4046        let quorum = if needs_quorum {
4047            judges_total / 2 + 1
4048        } else {
4049            0
4050        };
4051        let met_quorum = !needs_quorum || present >= quorum;
4052        let uncontested = (!needs_quorum).then(|| {
4053            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4054        });
4055
4056        self.state.event(
4057            "tally",
4058            match &uncontested {
4059                Some(reason) => format!("winner {winner} — {reason}"),
4060                None => format!(
4061                    "winner {winner} — votes {} | initial {} | {} changed | \
4062                     {present}/{judges_total} judges{}",
4063                    first_choice
4064                        .iter()
4065                        .map(|(k, v)| format!("{k}:{v}"))
4066                        .collect::<Vec<_>>()
4067                        .join(" "),
4068                    if unanimous_initial {
4069                        "unanimous"
4070                    } else {
4071                        "split"
4072                    },
4073                    changed_votes,
4074                    if met_quorum {
4075                        String::new()
4076                    } else {
4077                        format!(" — below quorum ({quorum} required)")
4078                    },
4079                ),
4080            },
4081        );
4082        if !met_quorum {
4083            self.state.event(
4084                "stall",
4085                format!(
4086                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4087                     the run stops here, resumable"
4088                ),
4089            );
4090        }
4091        self.state.tally = Some(Tally {
4092            first_choice,
4093            borda,
4094            winner,
4095            rankings: tops.len(),
4096            unanimous_initial,
4097            deliberated,
4098            changed_votes,
4099            unanimous_final,
4100            tie_break,
4101            judges: judges_total,
4102            present,
4103            quorum,
4104            met_quorum,
4105            uncontested,
4106        });
4107        self.state.status = if met_quorum {
4108            RunStatus::Reviewing
4109        } else {
4110            RunStatus::Stalled
4111        };
4112        self.state.save()?;
4113        Ok(())
4114    }
4115
4116    // ------------------------------------------------------------- recover
4117
4118    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4119    /// resumed toward completion once the transient cause clears.
4120    ///
4121    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4122    /// it toward the quorum, which is exactly the set of seats whose absence
4123    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4124    /// not depend on which node happened to hit the limit), or an ordinary
4125    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4126    /// seat is never disturbed.
4127    ///
4128    /// A seat that now answers with a usable ranking is "recovered": its
4129    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4130    /// `tally` counts it present again), and its vote re-collected. A seat that
4131    /// still fails keeps its loss and stays absent.
4132    ///
4133    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4134    /// to review/gate/merge), `false` when it is still below quorum (the run
4135    /// stays `Stalled`, still resumable for a later retry).
4136    #[allow(clippy::too_many_lines)]
4137    async fn recover_stall(&mut self) -> Result<bool> {
4138        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4139        // agent files with `magi task add` name the run that paid for it. The
4140        // prompt overlay is cloned alongside it because the waves borrow it
4141        // while `self` is mutably borrowed by the node's own bookkeeping.
4142        let run_id = self.state.id.clone();
4143        let prompts = self.state.config.prompts.clone();
4144        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4145        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4146        // is what keeps a non-quota collapse — or a quota loss recorded at the
4147        // deliberate node — from being a permanent dead-end on `--resume`.
4148        let quota_seats: BTreeSet<&str> =
4149            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4150        let absent: Vec<String> = self
4151            .state
4152            .judgements
4153            .iter()
4154            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4155            .map(|j| j.seat.clone())
4156            .collect();
4157        if absent.is_empty() {
4158            return Ok(false);
4159        }
4160        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4161        if viable.len() <= 1 {
4162            return Ok(false);
4163        }
4164        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4165        let language = self.state.config.graph.language.clone();
4166        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4167        let sessions = self.state.config.graph.sessions;
4168        let artifacts = agent::artifacts_dir(&self.state.dir());
4169        let root = self.state.worktree_root();
4170        let base_short = short(&self.state.base_commit);
4171        let candidates: Vec<Candidate> = viable.clone();
4172
4173        // Map each absent seat key to its 0-based position in `roles.judges`.
4174        let mut positions: Vec<usize> = absent
4175            .iter()
4176            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4177            .collect();
4178        if positions.is_empty() {
4179            return Ok(false);
4180        }
4181        positions.sort_unstable();
4182        positions.dedup();
4183
4184        // Re-rank the lost seats, one blind prompt each.
4185        let mut judge_jobs = Vec::new();
4186        for &j in &positions {
4187            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4188            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4189            let seat_key = format!("judge-{}", j + 1);
4190            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4191            let seat = self.seat(&seat_key, &spec.id);
4192            judge_jobs.push(SeatJob {
4193                spec,
4194                seat,
4195                prompt: prompt::judge(
4196                    &self.state.instruction,
4197                    &views,
4198                    self.roles.judges.len(),
4199                    &base_short,
4200                    &language,
4201                ),
4202                cwd: root.join(seat_key),
4203                timeout,
4204                allow_write: false,
4205                sessions,
4206                artifacts: artifacts.clone(),
4207                stem: format!("judge-{}-recover", j + 1),
4208                handover: None,
4209            });
4210        }
4211
4212        let labels_for_check = labels.clone();
4213        let mut judge_losses = Vec::new();
4214        let retries = self.state.config.graph.retries;
4215        let cache = self.state.config.cache_dir();
4216        let ctx = WaveCtx {
4217            carry_seats: false,
4218            pauses: Some((&self.pause, &self.interrupt)),
4219            run: &run_id,
4220            node: "judge",
4221            prompts: &prompts,
4222            cache: cache.as_deref(),
4223            round: None,
4224        };
4225        let results = ask_json_wave::<Ranking>(
4226            judge_jobs,
4227            Arc::clone(&self.sem),
4228            retries,
4229            &self.roles.judge_roster,
4230            &ctx,
4231            &mut judge_losses,
4232            &mut self.state,
4233            &move |r: &Ranking| r.validate(&labels_for_check),
4234        )
4235        .await;
4236        if self.state.resource_block.is_some() {
4237            self.stop_for_resource()?;
4238            return Ok(false);
4239        }
4240
4241        // Refresh the judgement of every seat that ranked again.
4242        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4243        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4244            let agent_id = seat.agent.clone();
4245            self.state.seats.insert(seat.key.clone(), seat);
4246            let record = &mut self.state.judgements[j];
4247            match res {
4248                Ok((ranking, out)) => {
4249                    record.agent = agent_id;
4250                    record.ranking = ranking.normalized();
4251                    record.reasons = ranking.reasons;
4252                    record.strengths = ranking.strengths;
4253                    record.confidence = ranking.confidence;
4254                    record.failed = None;
4255                    record.duration_ms = out.duration_ms;
4256                    recovered.insert(j);
4257                    self.state.event(
4258                        "recover",
4259                        format!("judge {} ranked again after the limit", j + 1),
4260                    );
4261                }
4262                Err(e) => {
4263                    self.state
4264                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4265                }
4266            }
4267        }
4268
4269        // Re-ask the votes of the seats that recovered a ranking.
4270        let mut vote_jobs = Vec::new();
4271        let mut vote_pos: Vec<usize> = Vec::new();
4272        for &j in &recovered {
4273            let seat_key = format!("judge-{}", j + 1);
4274            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4275            let seat = self.seat(&seat_key, &spec.id);
4276            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4277            let text = if has_context(&spec, &seat, sessions) {
4278                prompt::final_vote(&labels, &language)
4279            } else {
4280                full.clone()
4281            };
4282            vote_jobs.push(SeatJob {
4283                spec,
4284                seat,
4285                prompt: text,
4286                cwd: root.join(seat_key),
4287                timeout,
4288                allow_write: false,
4289                sessions,
4290                artifacts: artifacts.clone(),
4291                stem: format!("vote-judge-{}-recover", j + 1),
4292                handover: Some(full),
4293            });
4294            vote_pos.push(j);
4295        }
4296        let allowed = labels.clone();
4297        let mut vote_losses = Vec::new();
4298        let vote_retries = self.state.config.graph.retries;
4299        let vote_cache = self.state.config.cache_dir();
4300        let ctx = WaveCtx {
4301            carry_seats: false,
4302            pauses: Some((&self.pause, &self.interrupt)),
4303            run: &run_id,
4304            node: "vote",
4305            prompts: &prompts,
4306            cache: vote_cache.as_deref(),
4307            round: None,
4308        };
4309        let votes = ask_json_wave::<FinalVote>(
4310            vote_jobs,
4311            Arc::clone(&self.sem),
4312            vote_retries,
4313            &self.roles.judge_roster,
4314            &ctx,
4315            &mut vote_losses,
4316            &mut self.state,
4317            &move |v: &FinalVote| match v.label() {
4318                Some(c) if allowed.contains(&c) => Ok(()),
4319                other => bail!("vote {other:?} is not one of {allowed:?}"),
4320            },
4321        )
4322        .await;
4323        if self.state.resource_block.is_some() {
4324            self.stop_for_resource()?;
4325            return Ok(false);
4326        }
4327        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4328            let agent_id = seat.agent.clone();
4329            self.state.seats.insert(seat.key.clone(), seat);
4330            match res {
4331                Ok((v, _)) => {
4332                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4333                        rec.vote = v.label();
4334                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4335                    } else {
4336                        self.state.votes.push(VoteRecord {
4337                            judge: j + 1,
4338                            agent: agent_id,
4339                            vote: v.label(),
4340                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4341                            changed: false,
4342                        });
4343                    }
4344                    self.state.event(
4345                        "recover",
4346                        format!("judge {} voted again after the limit", j + 1),
4347                    );
4348                }
4349                Err(e) => {
4350                    self.state
4351                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4352                }
4353            }
4354        }
4355
4356        // A seat that ranked again is present even if its re-vote failed —
4357        // `tally` falls back to the initial ranking's first choice — so clear
4358        // its quota loss. Seats that still fail keep theirs and stay absent.
4359        let recovered_keys: BTreeSet<String> = recovered
4360            .iter()
4361            .map(|&j| format!("judge-{}", j + 1))
4362            .collect();
4363        self.state
4364            .quota
4365            .retain(|q| !recovered_keys.contains(&q.seat));
4366        // A seat that hit the limit again is a fresh loss, not the old one:
4367        // replace the stale entry so the history stays one-per-seat and the
4368        // daemon can tell this attempt's loss from a previous session's.
4369        for loss in judge_losses.into_iter().chain(vote_losses) {
4370            if recovered_keys.contains(&loss.seat) {
4371                continue;
4372            }
4373            self.state.quota.retain(|q| q.seat != loss.seat);
4374            self.state.quota.push(loss);
4375        }
4376
4377        // Recompute the verdict from the refreshed panel.
4378        self.state.tally = None;
4379        self.tally()?;
4380        Ok(self
4381            .state
4382            .tally
4383            .as_ref()
4384            .map(|t| t.met_quorum)
4385            .unwrap_or(false))
4386    }
4387
4388    // ---------------------------------------------------------------- graft
4389
4390    /// Port the strengths the judges attributed to the losing candidates onto
4391    /// the winner, once, between the tally and the review loop.
4392    ///
4393    /// Best-effort by construction: it returns nothing and fails nothing. A
4394    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4395    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4396    /// the run carries on as if the phase were off. See [`graft`] for the
4397    /// decisions and `AGENTS.md` for the invariants.
4398    ///
4399    /// The one exception is a rollback that could not be confirmed: the
4400    /// rejected commit may still be the winner's tip, so the run is blocked
4401    /// (resumable; the record stays `Running`, which makes a resume retry the
4402    /// restore) instead of reviewing, gating and merging it.
4403    async fn graft(&mut self) -> Result<()> {
4404        let Err(e) = self.graft_inner().await else {
4405            return Ok(());
4406        };
4407        if e.is::<crate::repo_lock::ResourceBlocked>() {
4408            return Err(e);
4409        }
4410        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4411            self.state.status = RunStatus::Blocked;
4412            self.state.event("graft", format!("blocked: {e:#}"));
4413            self.state.save()?;
4414            return Err(e);
4415        }
4416        // An error after the `Running` record was saved may have left the
4417        // fixer's work in the tree: the record must not stay unsettled
4418        // (a later resume would reset past the review fixes), so confirm the
4419        // restore now, or block.
4420        if let Some(rec) = self.state.graft.clone()
4421            && !rec.outcome.settled()
4422        {
4423            if let Some(winner) = self.state.winner().cloned() {
4424                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4425                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4426                        return Err(re);
4427                    }
4428                    self.state.status = RunStatus::Blocked;
4429                    self.state.event("graft", format!("blocked: {re:#}"));
4430                    self.state.save()?;
4431                    return Err(re);
4432                }
4433            }
4434            if let Some(g) = self.state.graft.as_mut() {
4435                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4436            }
4437        }
4438        self.state.event("graft", format!("skipped: {e:#}"));
4439        if let Err(e) = self.state.save() {
4440            tracing::warn!("save after graft: {e:#}");
4441        }
4442        Ok(())
4443    }
4444
4445    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4446    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4447    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4448        let _coordination = self.coordinate(wt).await?;
4449        crate::recovery::restore_repair(wt, pre)
4450            .await
4451            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4452    }
4453
4454    /// A skip is a run event, recorded once however often the node is re-entered.
4455    fn graft_skipped(&mut self, why: &str) {
4456        let message = format!("skipped: {why}");
4457        if !self
4458            .state
4459            .events
4460            .iter()
4461            .any(|e| e.node == "graft" && e.message == message)
4462        {
4463            self.state.event("graft", message);
4464        }
4465    }
4466
4467    async fn graft_inner(&mut self) -> Result<()> {
4468        // A round that was started and never finished (crash, kill): the
4469        // fixer may have left anything in the tree, so put the winner back.
4470        if let Some(rec) = self.state.graft.clone()
4471            && !rec.outcome.settled()
4472        {
4473            // Reviews exist only if the graft ended without its record being
4474            // settled (a swallowed error): the tree has moved on since, and
4475            // a reset to `pre` would discard the review fixes.
4476            if self.state.reviews.is_empty()
4477                && let Some(winner) = self.state.winner().cloned()
4478            {
4479                self.graft_restore(&winner.worktree, &rec.pre).await?;
4480            }
4481            if rec.outcome == GraftOutcome::WaitingForRestore {
4482                self.state.graft = None;
4483                self.state.save()?;
4484            } else {
4485                if let Some(g) = self.state.graft.as_mut() {
4486                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4487                }
4488                self.state.event(
4489                    "graft",
4490                    "rolled back: the previous round was interrupted; the winner is untouched",
4491                );
4492                self.state.save()?;
4493                return Ok(());
4494            }
4495        }
4496        let plan = match graft::plan(&self.state) {
4497            Ok(plan) => plan,
4498            Err(skip) => {
4499                if skip != graft::Skip::AlreadyDone {
4500                    self.graft_skipped(skip.reason());
4501                    self.state.save()?;
4502                }
4503                return Ok(());
4504            }
4505        };
4506        let Some(winner) = self.state.winner().cloned() else {
4507            return Ok(());
4508        };
4509        let wt = winner.worktree.clone();
4510        if !git::is_clean(&wt).await? {
4511            self.graft_skipped("the winner's worktree has uncommitted changes");
4512            self.state.save()?;
4513            return Ok(());
4514        }
4515        let pre = git::rev_parse(&wt, "HEAD").await?;
4516        // Saved before the fixer is called: a stop from here on is repaired
4517        // by the branch above on the next entry.
4518        self.state.graft = Some(GraftRecord {
4519            pre: pre.clone(),
4520            post: None,
4521            lines: None,
4522            limit: None,
4523            agent: None,
4524            outcome: GraftOutcome::Running,
4525            requested: graft::requested(&plan),
4526            adoption: None,
4527            adoption_error: None,
4528        });
4529        self.state.event(
4530            "graft",
4531            format!(
4532                "porting strengths of candidate(s) {} onto {}",
4533                plan.strengths
4534                    .iter()
4535                    .map(|(l, _, _)| l.to_string())
4536                    .collect::<Vec<_>>()
4537                    .join(", "),
4538                plan.winner
4539            ),
4540        );
4541        self.state.save()?;
4542
4543        let fix_prompt = prompt::graft(
4544            &self.state.instruction,
4545            plan.winner,
4546            &plan.strengths,
4547            &self.state.config.graph.language,
4548        );
4549        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4550        let sessions = self.state.config.graph.sessions;
4551        let artifacts = agent::artifacts_dir(&self.state.dir());
4552        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4553        let (_, seat, out) = self
4554            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4555                SeatJob {
4556                    prompt: fix_prompt.clone(),
4557                    spec,
4558                    seat,
4559                    cwd: wt.clone(),
4560                    timeout,
4561                    allow_write: true,
4562                    sessions,
4563                    artifacts: artifacts.clone(),
4564                    stem: "graft".to_owned(),
4565                    handover: None,
4566                }
4567            })
4568            .await;
4569        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4570            // Preserve the need to restore earlier fallback edits without
4571            // confusing an unstarted seat with an interrupted invocation.
4572            self.state.graft.as_mut().expect("graft record").outcome =
4573                GraftOutcome::WaitingForRestore;
4574            return self.stop_for_resource();
4575        }
4576        let mut why: Option<String> = None;
4577        match &out {
4578            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4579                why = Some("the fixer produced nothing usable".to_owned());
4580            }
4581            AgentOutcome::Ok(_) => {}
4582            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4583            AgentOutcome::Quota(o) => {
4584                self.state.quota.push(QuotaLoss {
4585                    seat: seat.key.clone(),
4586                    node: "graft".to_owned(),
4587                    at: Timestamp::now(),
4588                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4589                });
4590                why = Some("rate limited (quota)".to_owned());
4591            }
4592            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4593                why = Some(format!(
4594                    "the fixer failed: {}",
4595                    e.lines()
4596                        .next()
4597                        .unwrap_or("")
4598                        .chars()
4599                        .take(160)
4600                        .collect::<String>()
4601                ));
4602            }
4603        }
4604        let agent_id = seat.agent.clone();
4605        self.state.seats.insert(seat.key.clone(), seat);
4606
4607        let mut post = None;
4608        let mut lines = None;
4609        let _coordination = self.coordinate(&wt).await?;
4610        if why.is_none() {
4611            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4612            if let Ok(r) = &rescued {
4613                self.state.note_withheld("graft", &r.withheld);
4614            }
4615            let head = git::rev_parse(&wt, "HEAD").await?;
4616            // What is measured must be what gets reviewed and gated: a failed
4617            // rescue, or anything still uncommitted (withheld lockfiles
4618            // aside, which are never part of the tree), rejects the graft.
4619            let dirty = match &rescued {
4620                Ok(r) => git::status_porcelain(&wt)
4621                    .await
4622                    .map(|st| {
4623                        st.lines().any(|l| {
4624                            !r.withheld
4625                                .iter()
4626                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4627                        })
4628                    })
4629                    .unwrap_or(true),
4630                Err(_) => true,
4631            };
4632            if dirty {
4633                why = Some("the fixer left changes that could not be committed".to_owned());
4634            } else if head == pre {
4635                why = Some("the fixer changed nothing".to_owned());
4636            } else {
4637                lines = git::changed_lines(&wt, &pre, &head).await;
4638                post = Some(head);
4639            }
4640        }
4641        let (adoption, adoption_error) = match &out {
4642            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4643            _ => (None, None),
4644        };
4645        let requested = graft::requested(&plan);
4646        let outcome = match why {
4647            None => {
4648                self.state.event(
4649                    "graft",
4650                    format!(
4651                        "applied: {}",
4652                        lines.map_or_else(
4653                            || "changed lines unavailable".to_owned(),
4654                            |n| format!("{n} changed lines")
4655                        )
4656                    ),
4657                );
4658                GraftOutcome::Applied
4659            }
4660            Some(why) => {
4661                // Not settled until the restore is confirmed: the record
4662                // stays `Running`, so a resume retries it.
4663                crate::recovery::restore_repair(&wt, &pre)
4664                    .await
4665                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4666                self.state.event(
4667                    "graft",
4668                    format!("rolled back: {why}; the winner is untouched"),
4669                );
4670                GraftOutcome::RolledBack(why)
4671            }
4672        };
4673        self.state.graft = Some(GraftRecord {
4674            pre,
4675            post,
4676            lines,
4677            limit: None,
4678            agent: Some(agent_id),
4679            outcome,
4680            requested,
4681            adoption,
4682            adoption_error,
4683        });
4684        self.state.save()?;
4685        Ok(())
4686    }
4687
4688    // ----------------------------------------------------------------- fold
4689
4690    async fn fold_losers(&mut self) -> Result<()> {
4691        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4692            return Ok(());
4693        };
4694        if !self.state.candidate_preservation.is_empty() {
4695            bail!("candidate preservation incomplete; refusing cleanup");
4696        }
4697        let repo = self.state.repo.clone();
4698        let _coordination = self.coordinate(&repo).await?;
4699        let mut folded = Vec::new();
4700        for i in 0..self.state.candidates.len() {
4701            let c = &self.state.candidates[i];
4702            if c.label == winner || c.folded {
4703                continue;
4704            }
4705            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4706            remove_preserved_worktree(&repo, &wt).await?;
4707            if git::branch_exists(&repo, &branch).await?
4708                && !git::branch_delete(&repo, &branch).await?
4709            {
4710                bail!("candidate branch removal refused");
4711            }
4712            self.state.candidates[i].folded = true;
4713            self.state.save()?;
4714            folded.push(label.to_string());
4715        }
4716        // The judges are finished; their checkouts are pure cost from here.
4717        let root = self.state.worktree_root();
4718        for j in 1..=self.roles.judges.len() {
4719            let wt = root.join(format!("judge-{j}"));
4720            if wt.exists() {
4721                remove_preserved_worktree(&repo, &wt).await?;
4722            }
4723        }
4724        // The design-deliberation stage is finished by the time a tally
4725        // exists — same reasoning as the judges above.
4726        if self.state.config.graph.advise {
4727            for k in 1..=self.state.config.graph.advisors {
4728                let wt = root.join(format!("advisor-{k}"));
4729                if wt.exists() {
4730                    remove_preserved_worktree(&repo, &wt).await?;
4731                }
4732            }
4733        }
4734        if !folded.is_empty() {
4735            self.state
4736                .event("fold", format!("folded candidates {}", folded.join(", ")));
4737            self.state.save()?;
4738        }
4739        Ok(())
4740    }
4741
4742    // ------------------------------------------------------------ base sync
4743
4744    async fn hold_base_sync(&mut self) -> Result<bool> {
4745        self.hold_base_sync_as(None).await
4746    }
4747
4748    async fn hold_base_sync_kind(
4749        &mut self,
4750        kind: crate::base_sync_recovery::StopKind,
4751    ) -> Result<()> {
4752        self.hold_base_sync_as(Some(kind)).await?;
4753        Ok(())
4754    }
4755
4756    async fn hold_base_sync_as(
4757        &mut self,
4758        kind: Option<crate::base_sync_recovery::StopKind>,
4759    ) -> Result<bool> {
4760        if !self.state.base_sync_blocked() {
4761            self.state.base_sync_recovery.stop = None;
4762            return Ok(false);
4763        }
4764        let head = match self.state.winner() {
4765            Some(w) => git::rev_parse(&w.worktree, "HEAD")
4766                .await
4767                .unwrap_or_default(),
4768            None => String::new(),
4769        };
4770        self.state.record_base_sync_stop(&head);
4771        if let Some(kind) = kind {
4772            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind;
4773        }
4774        self.state.save()?;
4775        Ok(true)
4776    }
4777
4778    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4779    /// anything verifies it.
4780    ///
4781    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4782    /// read whatever is checked out in the winner's worktree. Left alone that
4783    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4784    /// it when the run *branched* - and a run takes long enough that the base
4785    /// has usually moved by the time it gets here. A gate that ran there
4786    /// answers "green on the commit this run started from", not "green on
4787    /// what is about to land", and the difference showed up three times in
4788    /// one day as a green run whose merge would have reverted a file another
4789    /// pull request had already landed.
4790    ///
4791    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4792    /// `land::Step::Rebase` calls, rather than a second implementation of the
4793    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4794    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4795    /// doc warns about ("two answers to a question nobody notices until a
4796    /// diff is wrong").
4797    ///
4798    /// A conflict is not the end of the road: the standing rebase is handed
4799    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4800    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4801    /// land). Once it finishes, review and the gate run as usual on the
4802    /// rebased tree, which is where a breakage the new base caused is caught
4803    /// by the ordinary gate-fix round. magi resolves nothing itself.
4804    ///
4805    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4806    /// `state.base_sync.attempts`, is how many times the base is *rebased
4807    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4808    /// *conflict* was given to a fixer. When the fixer cannot finish the
4809    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4810    /// what was tried (rounds spent, paths still conflicted) and the branch
4811    /// and worktree stay exactly as they were - untouched, for a person to
4812    /// look at - which is also what makes re-entering this function
4813    /// afterwards a no-op instead of a second attempt at the same wall. A
4814    /// push failure ends the same way.
4815    async fn sync_to_base(&mut self) -> Result<()> {
4816        if self.state.status == RunStatus::AlreadyInBase {
4817            return Ok(());
4818        }
4819        self.hold_base_sync().await?;
4820        let conflicted = self
4821            .state
4822            .base_sync
4823            .as_ref()
4824            .is_some_and(|s| s.conflict.is_some());
4825        let Some(winner) = self.state.winner().cloned() else {
4826            return Ok(());
4827        };
4828
4829        let repo = self.state.repo.clone();
4830        let remote = self.state.config.merge.remote.clone();
4831        let base_branch = self.state.base_branch.clone();
4832        let tracking = format!("{remote}/{base_branch}");
4833
4834        let fetched_base =
4835            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
4836        // No network, or the remote never had this branch: the run already
4837        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4838        // otherwise), and one that got this far is not blocked by a fetch
4839        // that fails now. It just has no newer tip to compare against.
4840        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4841            return Ok(());
4842        };
4843
4844        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4845        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4846        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4847
4848        // Before any rebase, and before a recorded conflict is honoured: a
4849        // branch whose change reached the base under other commit ids has
4850        // nothing to rebase and nothing to conflict with, and a run that
4851        // already stopped on that phantom conflict recovers here on resume.
4852        // `behind == 0` with head == tip is a branch the base has since taken
4853        // in whole, whether or not a conflict was ever recorded: the ancestry
4854        // proof must still run (`classify` ignores a head still on the start
4855        // commit).
4856        if (behind > 0 || conflicted || head == tip)
4857            && self
4858                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4859                .await?
4860        {
4861            return Ok(());
4862        }
4863        if conflicted && self.state.base_recovery_pending() {
4864            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
4865            let owner =
4866                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
4867            if !fetched_base
4868                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
4869                || !git::is_clean(&winner.worktree).await?
4870                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
4871            {
4872                return Ok(());
4873            }
4874        }
4875        let recovery = conflicted && self.state.consume_base_recovery(&head);
4876        if conflicted {
4877            // Consumption (including a refused changed HEAD) is durable before Git.
4878            self.state.save()?;
4879            if !recovery {
4880                return Ok(());
4881            }
4882        }
4883
4884        if behind == 0 {
4885            // A fixer-finished rebase moves the branch ref before the
4886            // winner's worktree is told (`sync_to_head` below). A run that
4887            // died in between resumes here with `behind == 0` and a tree still
4888            // holding the pre-rebase files, which review and the gate would
4889            // then read. That state is exactly: HEAD moved off the tip the
4890            // rebase started from, yet the tree is still identical to that
4891            // tip. A tree with edits of its own differs from it, so nothing
4892            // is thrown away.
4893            if let Some(from) = self
4894                .state
4895                .rebase_fixes
4896                .iter()
4897                .rev()
4898                .find_map(|r| r.from.clone())
4899                && from != head
4900                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4901                    .await
4902                    .is_ok_and(|o| o.ok())
4903            {
4904                git::sync_to_head(&winner.worktree).await?;
4905            }
4906            // An earlier attempt may have rebased the branch locally and died
4907            // before pushing it (only the fresh-rebase arm below pushes).
4908            // Publish it now, so the plain push at PR time is not refused as
4909            // a non-fast-forward. A run already holding a recorded conflict
4910            // never reaches here; that case is out of scope.
4911            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4912            if let Some(why) = &conflict {
4913                self.state.status = RunStatus::Blocked;
4914                self.state.event("land", why.clone());
4915            }
4916            self.state.base_sync = Some(BaseSync {
4917                tip,
4918                behind: 0,
4919                attempts: attempts + usize::from(recovery),
4920                conflict,
4921                already_in: None,
4922            });
4923            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4924                .await?;
4925            self.state.save()?;
4926            return Ok(());
4927        }
4928
4929        if attempts >= BASE_SYNC_ROUNDS && !recovery {
4930            let why = format!(
4931                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4932                 rebase(s); rebasing again would only race it",
4933                winner.branch
4934            );
4935            self.state.status = RunStatus::Blocked;
4936            self.state.base_sync = Some(BaseSync {
4937                tip,
4938                behind,
4939                attempts,
4940                conflict: Some(why.clone()),
4941                already_in: None,
4942            });
4943            self.state.event("land", why);
4944            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
4945                .await?;
4946            self.state.save()?;
4947            return Ok(());
4948        }
4949
4950        self.state.event(
4951            "land",
4952            format!(
4953                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4954                winner.branch
4955            ),
4956        );
4957        self.state.save()?;
4958
4959        // The remote's copy of the branch, read now and only if the fetch
4960        // really succeeded (a stale tracking ref must never pin a lease). It is
4961        // pushed over after a rebase only when it is a commit this branch
4962        // already contains, by ancestry or by patch (an earlier rebase of ours
4963        // that never reached the remote): anything else is somebody else's work.
4964        let branch_tracking = format!("{remote}/{}", winner.branch);
4965        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4966        let remote_tip = if self.state.task_pr_branch().is_none()
4967            && matches!(&fetched_branch, Ok(o) if o.ok())
4968        {
4969            git::rev_parse(&repo, &branch_tracking).await.ok()
4970        } else {
4971            None
4972        };
4973        // A remote tip this branch does not contain is somebody else's work:
4974        // rebasing would leave a local tip that can never be pushed. Stop
4975        // before touching anything and say so.
4976        if let Some(theirs) = &remote_tip
4977            && !git::is_ancestor(&repo, theirs, &head).await
4978            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4979                .await
4980                .is_ok_and(|missing| missing.is_empty())
4981        {
4982            let why = format!(
4983                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4984                 them",
4985                short(theirs),
4986                winner.branch
4987            );
4988            self.state.status = RunStatus::Blocked;
4989            self.state.base_sync = Some(BaseSync {
4990                tip,
4991                behind,
4992                attempts: attempts + usize::from(recovery),
4993                conflict: Some(why.clone()),
4994                already_in: None,
4995            });
4996            self.state.event("land", why);
4997            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
4998                .await?;
4999            self.state.save()?;
5000            return Ok(());
5001        }
5002
5003        let scratch = self.state.dir().join("base-sync");
5004        let rebased =
5005            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5006                .await
5007            {
5008                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5009                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5010                Err(e) => Err(e),
5011            };
5012        let attempts = attempts + 1;
5013        let stop_kind = match &rebased {
5014            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5015            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5016            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5017        };
5018        match rebased {
5019            Ok(None) => {
5020                // The branch ref moved, but a worktree that already had it
5021                // checked out (the winner's) was not told; sync its index and
5022                // files before anything reads them.
5023                git::sync_to_head(&winner.worktree).await?;
5024                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5025                let mut conflict = None;
5026                if let Some(pinned) = &remote_tip {
5027                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5028                    match pushed {
5029                        Ok(o) if o.ok() => self.state.event(
5030                            "land",
5031                            format!("pushed rebased {} to {remote}", winner.branch),
5032                        ),
5033                        Ok(o) => {
5034                            conflict = Some(format!(
5035                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5036                                winner.branch,
5037                                short(pinned),
5038                                o.stderr.chars().take(600).collect::<String>()
5039                            ));
5040                        }
5041                        Err(e) => {
5042                            conflict = Some(format!(
5043                                "rebased {} locally but could not push it: {e:#}",
5044                                winner.branch
5045                            ));
5046                        }
5047                    }
5048                }
5049                if let Some(why) = &conflict {
5050                    self.state.status = RunStatus::Blocked;
5051                    self.state.event("land", why.clone());
5052                }
5053                self.state.base_sync = Some(BaseSync {
5054                    tip: tip.clone(),
5055                    behind: 0,
5056                    attempts,
5057                    conflict,
5058                    already_in: None,
5059                });
5060                self.state
5061                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5062            }
5063            Ok(Some(conflict)) => {
5064                let why = format!(
5065                    "{} conflicts with {tracking} and did not rebase: {}",
5066                    winner.branch,
5067                    conflict.chars().take(600).collect::<String>()
5068                );
5069                self.state.status = RunStatus::Blocked;
5070                self.state.base_sync = Some(BaseSync {
5071                    tip,
5072                    behind,
5073                    attempts,
5074                    conflict: Some(why.clone()),
5075                    already_in: None,
5076                });
5077                self.state.event("land", why);
5078            }
5079            // A refused Git write or a busy repository: nothing is wrong with
5080            // the branch. The scratch rebase stays standing and the run
5081            // resumes into it (`base_sync` must not claim a conflict).
5082            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5083                self.state.save()?;
5084                return Ok(());
5085            }
5086            Err(e) => {
5087                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5088                self.state.status = RunStatus::Blocked;
5089                self.state.base_sync = Some(BaseSync {
5090                    tip,
5091                    behind,
5092                    attempts,
5093                    conflict: Some(why.clone()),
5094                    already_in: None,
5095                });
5096                self.state.event("land", why);
5097            }
5098        }
5099        if !self.state.base_sync_blocked() {
5100            self.state.base_sync_recovery.stop = None;
5101        }
5102        self.hold_base_sync_kind(stop_kind).await?;
5103        self.state.save()?;
5104        Ok(())
5105    }
5106
5107    /// Push a branch an earlier attempt rebased locally but never published,
5108    /// pinned to the remote tip read right after a successful fetch. Returns
5109    /// the reason when the run must stop; `None` when there was nothing to do
5110    /// (no remote copy, the same tip, or a remote copy this branch already
5111    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5112    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5113        // Task competitions publish only after the PR text gate, to their
5114        // public head; never resurrect a candidate branch on the remote.
5115        if self.state.task_pr_branch().is_some() {
5116            return None;
5117        }
5118        let repo = self.state.repo.clone();
5119        let remote = self.state.config.merge.remote.clone();
5120        let fetched = git::fetch(&repo, &remote, branch).await;
5121        if !matches!(&fetched, Ok(o) if o.ok()) {
5122            return None;
5123        }
5124        let branch_tracking = format!("{remote}/{branch}");
5125        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5126        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5127            return None;
5128        }
5129        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5130            .await
5131            .is_ok_and(|missing| missing.is_empty())
5132        {
5133            return Some(format!(
5134                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5135                 them",
5136                short(&theirs)
5137            ));
5138        }
5139        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5140            Ok(o) if o.ok() => {
5141                self.state
5142                    .event("land", format!("pushed rebased {branch} to {remote}"));
5143                None
5144            }
5145            Ok(o) => Some(format!(
5146                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5147                 someone may have pushed): {}",
5148                short(&theirs),
5149                o.stderr.chars().take(600).collect::<String>()
5150            )),
5151            Err(e) => Some(format!(
5152                "{branch} is rebased locally but could not be pushed: {e:#}"
5153            )),
5154        }
5155    }
5156
5157    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5158    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5159    /// whether it did.
5160    ///
5161    /// Checked only when the base is ahead of the branch. A failing check is
5162    /// "not proven" - the ordinary rebase path then decides - never a reason to
5163    /// stop the run.
5164    ///
5165    /// The remote copy of the branch is held to the same standard as the local
5166    /// one: if it carries a tip this worktree does not, that tip must itself be
5167    /// proven in the base, or nothing is settled (a pull request would
5168    /// otherwise be closed over commits nobody checked). The pull request is
5169    /// closed *before* the terminal status is saved; if that fails for a
5170    /// reason other than a refusal (no network, a `gh` error) the run is left
5171    /// `Blocked` with the reason as its conflict, which a resume retries -
5172    /// the same recovery a phantom conflict gets.
5173    async fn settle_already_in(
5174        &mut self,
5175        branch: &str,
5176        tip: &str,
5177        head: &str,
5178        attempts: usize,
5179        behind: usize,
5180    ) -> Result<bool> {
5181        let repo = self.state.repo.clone();
5182        let remote = self.state.config.merge.remote.clone();
5183        let start = self.state.base_commit.clone();
5184        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5185            Ok(Some(e)) => e,
5186            Ok(None) => return Ok(false),
5187            Err(e) => {
5188                tracing::warn!("already-in-base check for {branch}: {e:#}");
5189                return Ok(false);
5190            }
5191        };
5192        let mut verified = vec![head.to_owned()];
5193        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5194        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5195        if matches!(&fetched, Ok(o) if o.ok())
5196            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5197            && theirs != head
5198        {
5199            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5200                Ok(Some(_)) => verified.push(theirs),
5201                _ => return Ok(false),
5202            }
5203        }
5204        let base_branch = self.state.base_branch.clone();
5205        let message = format!(
5206            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5207             land",
5208            evidence.names(),
5209            evidence.proof.as_str()
5210        );
5211        let closed =
5212            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5213                .await;
5214        match closed {
5215            Ok(Ok(url)) => self
5216                .state
5217                .event("land", format!("closed {url}: superseded on {base_branch}")),
5218            Ok(Err(why)) => self
5219                .state
5220                .event("land", format!("did not close a pull request: {why}")),
5221            Err(e) => {
5222                let why = format!(
5223                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
5224                     not be closed ({e:#}); resume to retry"
5225                );
5226                self.state.status = RunStatus::Blocked;
5227                self.state.base_sync = Some(BaseSync {
5228                    tip: tip.to_owned(),
5229                    behind,
5230                    attempts,
5231                    conflict: Some(why.clone()),
5232                    already_in: None,
5233                });
5234                self.state.event("land", why);
5235                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
5236                    .await?;
5237                self.state.save()?;
5238                return Ok(true);
5239            }
5240        }
5241        self.state.status = RunStatus::AlreadyInBase;
5242        self.state.base_sync_recovery.stop = None;
5243        self.state.base_sync = Some(BaseSync {
5244            tip: tip.to_owned(),
5245            behind,
5246            attempts,
5247            conflict: None,
5248            already_in: Some(evidence),
5249        });
5250        self.state.event("land", message);
5251        self.state.save()?;
5252        self.settle_questions();
5253        Ok(true)
5254    }
5255
5256    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
5257    /// last landed the winner on, once it has run, else the commit the run
5258    /// branched from.
5259    ///
5260    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
5261    /// and `vote` all happen before there is a winner to rebase, so they
5262    /// compare every candidate against the branch point on purpose, and a
5263    /// base that moves after they are already done cannot change an answer
5264    /// they already gave.
5265    fn landing_base(&self) -> String {
5266        self.state
5267            .base_sync
5268            .as_ref()
5269            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5270    }
5271
5272    // ------------------------------------------------------- operator fix
5273
5274    /// Route specific, already-recorded review findings to a fixer for a
5275    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
5276    /// entry point.
5277    ///
5278    /// Distinct from `review_loop`'s own fix step in three ways: it never
5279    /// runs a reviewer wave, it never spends review-round budget, and what
5280    /// happened is recorded as an [`OperatorFixRequest`] appended to
5281    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
5282    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
5283    /// and vote must never be rewritten to look like a manufactured blocking
5284    /// verdict.
5285    ///
5286    /// Only meaningful once review has actually concluded: `Ready` (handed
5287    /// off with findings still open, or simply concluded clean while minor
5288    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
5289    /// gate failed). Everything else is refused: a run still in progress
5290    /// should simply be resumed, and a `Merged` run's branch has already
5291    /// landed — reopening *this* run's own record cannot change that, so the
5292    /// answer there is a fresh `magi review <branch>`.
5293    ///
5294    /// A real commit here re-verifies through a fresh, ordinary review-only
5295    /// run on the same branch ([`Self::review`]) rather than reopening this
5296    /// run's own `review_loop`: once any round in this run's history went
5297    /// clean, `review_conclusion` treats that as permanent by design (the
5298    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
5299    /// way to force one more genuine reviewer wave out of *this* run without
5300    /// either rewriting history or weakening that guarantee for every other
5301    /// caller. A review-only run costs nothing extra — no implementation, no
5302    /// judging, no vote — and exercises the exact same review → verify →
5303    /// gate → (human) merge path, unmodified.
5304    pub async fn fix_selected(
5305        &mut self,
5306        ids: &[String],
5307        reason: &str,
5308        allow_stale: bool,
5309    ) -> Result<()> {
5310        let reason = reason.trim();
5311        if reason.is_empty() {
5312            bail!("a fix request needs a reason — that is the operator's own record of why");
5313        }
5314        if ids.is_empty() {
5315            bail!("no finding id given");
5316        }
5317        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5318            bail!(
5319                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5320                 has already concluded — can be given a targeted fix. A run still \
5321                 in progress should simply be resumed; a `merged` run's branch has \
5322                 already landed, so its answer is a fresh `magi review <branch>`, \
5323                 not reopening this run's own record",
5324                self.state.id,
5325                self.state.status.as_str()
5326            );
5327        }
5328        let Some(winner) = self.state.winner().cloned() else {
5329            bail!("run {} has no winning candidate to fix", self.state.id);
5330        };
5331        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5332            bail!(
5333                "branch `{}` no longer exists; this run cannot be extended",
5334                winner.branch
5335            );
5336        }
5337        let home = crate::run::home();
5338        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5339            bail!(
5340                "run {} is currently being worked on by another magi process",
5341                self.state.id
5342            );
5343        }
5344        // Held for the rest of this call, including the follow-up review
5345        // below: two `magi fix` invocations against the same run must not
5346        // both reach the worktree manipulation further down, which would
5347        // otherwise race to remove and recreate the same directory — see
5348        // [`FixClaim`]'s own doc.
5349        let _claim = FixClaim::acquire(&self.state.dir())?;
5350
5351        // Resolve every id before spending anything — an unknown id refuses
5352        // the whole request rather than silently dropping it — and dedup
5353        // while keeping the operator's own order.
5354        let mut seen = BTreeSet::new();
5355        let mut findings = Vec::new();
5356        let mut missing = Vec::new();
5357        for id in ids {
5358            if !seen.insert(id.clone()) {
5359                continue;
5360            }
5361            match self.state.finding(id) {
5362                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5363                    id: f.id.clone(),
5364                    severity: f.severity,
5365                    reviewer_vote: rec.vote,
5366                    round: round.round,
5367                    round_head: round.head.clone(),
5368                    reviewer: rec.reviewer,
5369                    agent: rec.agent.clone(),
5370                    file: f.file.clone(),
5371                    line: f.line,
5372                    title: f.title.clone(),
5373                    detail: f.detail.clone(),
5374                    outcome: OperatorFixOutcome::Pending,
5375                }),
5376                None => missing.push(id.clone()),
5377            }
5378        }
5379        if !missing.is_empty() {
5380            bail!(
5381                "unknown finding id(s): {}; nothing was changed",
5382                missing.join(", ")
5383            );
5384        }
5385
5386        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5387        let stale_details: Vec<(String, String)> = findings
5388            .iter()
5389            .filter(|f| f.round_head != head_at_request)
5390            .map(|f| (f.id.clone(), f.round_head.clone()))
5391            .collect();
5392        let stale = !stale_details.is_empty();
5393        if stale && !allow_stale {
5394            bail!(
5395                "the branch has moved since some finding(s) were raised — {} — now \
5396                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5397                stale_details
5398                    .iter()
5399                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5400                    .collect::<Vec<_>>()
5401                    .join(", "),
5402                short(&head_at_request)
5403            );
5404        }
5405
5406        let request = OperatorFixRequest {
5407            requested_at: Timestamp::now(),
5408            reason: reason.to_owned(),
5409            findings,
5410            head_at_request: head_at_request.clone(),
5411            allow_stale,
5412            stale,
5413            fix: None,
5414            result_head: None,
5415            follow_up_review_run: None,
5416        };
5417        self.state.event(
5418            "fix",
5419            format!(
5420                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5421                request.findings.len(),
5422                request
5423                    .findings
5424                    .iter()
5425                    .map(|f| f.id.as_str())
5426                    .collect::<Vec<_>>()
5427                    .join(", "),
5428            ),
5429        );
5430        // Recorded now, before any worktree work or the fixer call itself —
5431        // and re-saved at each checkpoint below: a crash at any point after
5432        // this (mid fixer call, mid follow-up review) must not lose the fact
5433        // that this was requested, for which findings, and why. Everything
5434        // past this point reads and writes through `request_index` rather
5435        // than a local variable, since `request` itself is moved here.
5436        self.state.operator_fixes.push(request);
5437        self.state.save()?;
5438        let request_index = self.state.operator_fixes.len() - 1;
5439
5440        // A fresh, dedicated worktree for this one call, never the winner's
5441        // own worktree in place: that one may already be gone (folded away),
5442        // and reusing it in place would leave the branch checked out there
5443        // when the follow-up review below tries to check it out again. Freed
5444        // immediately after, either way — but only once confirmed clean:
5445        // `worktree_remove` is a `git worktree remove --force`, which would
5446        // otherwise discard uncommitted work left there by the operator or
5447        // another process before this had a chance to even look at it.
5448        let repo = self.state.repo.clone();
5449        let coordination = self.coordinate(&repo).await?;
5450        if winner.worktree.exists() {
5451            // Lockfiles a rescue commit withheld stay untracked on purpose and
5452            // are already recorded; they are not the operator's work to protect.
5453            let dirty = git::git(
5454                &winner.worktree,
5455                &["status", "--porcelain", "--untracked-files=all"],
5456            )
5457            .await?;
5458            let only_withheld = dirty.lines().all(|l| {
5459                l.strip_prefix("?? ")
5460                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5461            });
5462            if !only_withheld {
5463                bail!(
5464                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5465                     discard them first",
5466                    winner.worktree.display()
5467                );
5468            }
5469            git::worktree_remove(&self.state.repo, &winner.worktree)
5470                .await
5471                .ok();
5472        }
5473        let fix_worktree = self.state.worktree_root().join("operator-fix");
5474        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5475        git::git(
5476            &self.state.repo,
5477            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5478        )
5479        .await
5480        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5481        if !git::is_clean(&fix_worktree).await? {
5482            git::worktree_remove(&self.state.repo, &fix_worktree)
5483                .await
5484                .ok();
5485            bail!(
5486                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5487                winner.branch
5488            );
5489        }
5490        let repo_for_setup = self.state.repo.clone();
5491        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5492            Ok(()) => {
5493                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5494            }
5495            Err(e) => Err(e),
5496        };
5497        if let Err(e) = setup {
5498            git::worktree_remove(&self.state.repo, &fix_worktree)
5499                .await
5500                .ok();
5501            return Err(e);
5502        }
5503
5504        let run_id = self.state.id.clone();
5505        let prompts = self.state.config.prompts.clone();
5506        let language = self.state.config.graph.language.clone();
5507        let sessions = self.state.config.graph.sessions;
5508        let artifacts = agent::artifacts_dir(&self.state.dir());
5509        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5510            .findings
5511            .iter()
5512            .map(|f| Finding {
5513                id: f.id.clone(),
5514                severity: f.severity,
5515                file: f.file.clone(),
5516                line: f.line,
5517                title: f.title.clone(),
5518                detail: f.detail.clone(),
5519            })
5520            .collect();
5521        let fix_prompt = prompt::operator_fix_in(
5522            &self.state.instruction,
5523            &finding_list,
5524            reason,
5525            &stale_details,
5526            &head_at_request,
5527            &language,
5528            &self.state.config.graph.github_language,
5529        );
5530        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5531        drop(coordination);
5532        let (job, seat, out) = self
5533            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5534                prompt: fix_prompt.clone(),
5535                spec,
5536                seat,
5537                cwd: fix_worktree.clone(),
5538                timeout,
5539                allow_write: true,
5540                sessions,
5541                artifacts: artifacts.clone(),
5542                stem: "operator-fix".to_owned(),
5543                handover: None,
5544            })
5545            .await;
5546        let agent_id = seat.agent.clone();
5547
5548        let mut fix = FixRecord {
5549            agent: agent_id,
5550            addressed: Vec::new(),
5551            rejected: Vec::new(),
5552            notes: String::new(),
5553            committed: false,
5554            failed: None,
5555            duration_ms: 0,
5556            continuation: None,
5557        };
5558        let mut final_seat = seat.clone();
5559        match out {
5560            AgentOutcome::Ok(o) => {
5561                fix.duration_ms = o.duration_ms;
5562                let parsed = verdict::extract_json::<FixReport>(&o.text);
5563                let incomplete_reason = match &parsed {
5564                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5565                        "the reply parsed, but it reported a command whose own CLI \
5566                         never confirmed an exit status"
5567                            .to_owned(),
5568                    ),
5569                    Ok(_) => None,
5570                    Err(e) => Some(e.to_string()),
5571                };
5572                match incomplete_reason {
5573                    None => {
5574                        let report = parsed.expect("checked Ok above");
5575                        fix.addressed = report.addressed;
5576                        fix.rejected = report.rejected;
5577                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5578                    }
5579                    Some(reason) => {
5580                        let (resumed_seat, resolved, failure, cont) = self
5581                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5582                            .await;
5583                        fix.duration_ms += cont.cumulative_wait_ms;
5584                        fix.continuation = Some(cont);
5585                        final_seat = resumed_seat;
5586                        match resolved {
5587                            Some(report) => {
5588                                fix.addressed = report.addressed;
5589                                fix.rejected = report.rejected;
5590                                fix.notes =
5591                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5592                            }
5593                            None => fix.failed = failure,
5594                        }
5595                    }
5596                }
5597            }
5598            AgentOutcome::Dropped(o) => {
5599                fix.duration_ms = o.duration_ms;
5600                let why = o
5601                    .dropped
5602                    .as_ref()
5603                    .map(|d| d.why.as_str())
5604                    .unwrap_or("the CLI ended the stream without delivering its answer");
5605                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5606            }
5607            AgentOutcome::Quota(o) => {
5608                self.state.quota.push(QuotaLoss {
5609                    seat: final_seat.key.clone(),
5610                    node: "fix".to_owned(),
5611                    at: Timestamp::now(),
5612                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5613                });
5614                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5615            }
5616            AgentOutcome::Failed(e) => fix.failed = Some(e),
5617            AgentOutcome::ResourceBlocked(e) => {
5618                fix.failed = Some(format!("the fixer was not started: {e}"));
5619            }
5620        }
5621        if fix.continuation.is_none() {
5622            fix.continuation = Some(ContinuationRecord::not_needed());
5623        }
5624        self.state.seats.insert(final_seat.key.clone(), final_seat);
5625
5626        let rescue_message = format!(
5627            "magi: operator-selected fix ({}) (uncommitted work)",
5628            self.state.operator_fixes[request_index]
5629                .findings
5630                .iter()
5631                .map(|f| f.id.as_str())
5632                .collect::<Vec<_>>()
5633                .join(", ")
5634        );
5635        let repo = self.state.repo.clone();
5636        let coordination = self.coordinate(&repo).await?;
5637        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5638            self.state.note_withheld("fix", &r.withheld);
5639        }
5640        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5641        fix.committed = after != head_at_request;
5642        git::worktree_remove(&self.state.repo, &fix_worktree)
5643            .await
5644            .ok();
5645        drop(coordination);
5646
5647        self.state.event(
5648            "fix",
5649            match &fix.failed {
5650                Some(reason) => format!(
5651                    "operator fix: adoption report was lost ({reason}); {}",
5652                    if fix.committed {
5653                        "committed"
5654                    } else {
5655                        "NO new commit"
5656                    }
5657                ),
5658                None => format!(
5659                    "operator fix: {} addressed, {} rejected, {}",
5660                    fix.addressed.len(),
5661                    fix.rejected.len(),
5662                    if fix.committed {
5663                        "committed"
5664                    } else {
5665                        "NO new commit"
5666                    }
5667                ),
5668            },
5669        );
5670
5671        // Every selected finding gets an outcome — never left `Pending` once
5672        // the fixer's own turn is over. A report that never came back at all
5673        // marks every one of them `Unreported`, not silently "not addressed":
5674        // quota, a dropped stream, or an exhausted continuation are gaps in
5675        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5676        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5677        for f in &mut self.state.operator_fixes[request_index].findings {
5678            f.outcome = if fix.failed.is_some() {
5679                OperatorFixOutcome::Unreported
5680            } else if fix.addressed.contains(&f.id) {
5681                OperatorFixOutcome::Addressed
5682            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5683                OperatorFixOutcome::Rejected { why: r.why.clone() }
5684            } else {
5685                OperatorFixOutcome::Unreported
5686            };
5687        }
5688
5689        let committed = fix.committed;
5690        if committed {
5691            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5692        }
5693        self.state.operator_fixes[request_index].fix = Some(fix);
5694        // Saved again now that the fixer's own outcome is final, on top of
5695        // the save right after the request was first pushed above.
5696        self.state.save()?;
5697
5698        if committed {
5699            self.state.event(
5700                "fix",
5701                format!(
5702                    "operator fix committed {}; opening a follow-up review-only run",
5703                    short(&after)
5704                ),
5705            );
5706            // The operator asked for the fix, and the follow-up serves whatever
5707            // task the run it follows served.
5708            let origin =
5709                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5710            match Self::review(
5711                &self.state.repo,
5712                &winner.branch,
5713                self.state.config.clone(),
5714                origin,
5715            )
5716            .await
5717            {
5718                Ok(mut follow_up) => {
5719                    follow_up.state.event(
5720                        "start",
5721                        format!(
5722                            "requested by an operator fix on run {} for finding(s) {}",
5723                            self.state.id,
5724                            self.state.operator_fixes[request_index]
5725                                .findings
5726                                .iter()
5727                                .map(|f| f.id.as_str())
5728                                .collect::<Vec<_>>()
5729                                .join(", "),
5730                        ),
5731                    );
5732                    follow_up.state.save()?;
5733                    let follow_up_id = follow_up.state.id.clone();
5734                    // The follow-up is a run like any other: it belongs to the
5735                    // task of the run it follows, or to one filed for it.
5736                    let adopted = match crate::direct::adopt(
5737                        &follow_up.state,
5738                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5739                    ) {
5740                        Ok(a) => a,
5741                        Err(e) => {
5742                            // An ownerless run must not spend agent calls; it
5743                            // stays saved, and `magi run --resume` adopts it.
5744                            self.state.event(
5745                                "fix",
5746                                format!(
5747                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5748                                ),
5749                            );
5750                            self.state.save()?;
5751                            return Ok(());
5752                        }
5753                    };
5754                    let executed = follow_up.execute().await;
5755                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5756                    if let Some(a) = adopted {
5757                        a.finish(&follow_up.state, executed);
5758                    }
5759                    if let Some(e) = failure {
5760                        self.state.event(
5761                            "fix",
5762                            format!(
5763                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5764                            ),
5765                        );
5766                    }
5767                    self.state.operator_fixes[request_index].follow_up_review_run =
5768                        Some(follow_up_id);
5769                }
5770                Err(e) => {
5771                    self.state.event(
5772                        "fix",
5773                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5774                    );
5775                }
5776            }
5777            self.state.save()?;
5778        }
5779
5780        Ok(())
5781    }
5782
5783    // --------------------------------------------------------------- review
5784
5785    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5786    /// names, in order, each at most once, moving on only when the call
5787    /// advances (an error, a quota hit or nothing usable - the same decision
5788    /// point `agent::chain_advances` is for the other chained roles).
5789    ///
5790    /// Each agent gets its own seat from `Runner::seat`: the same agent
5791    /// continues its conversation (the winner's own implementer seat when it
5792    /// is the winner's author, now that the competition is over), another
5793    /// takes a fresh one so the full prompt is sent again. A handover is
5794    /// recorded under `node`, which is what makes the fallback stick for the
5795    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5796    /// is returned, so an exhausted chain reads exactly like a single failed
5797    /// fixer: one quota loss, the same wording, the same refund. An earlier
5798    /// attempt's edits are left in the tree and judged, with the final
5799    /// attempt's, by what git says afterwards.
5800    ///
5801    /// Returns the job that produced the outcome, for `continue_fix_report`.
5802    async fn ask_fixer(
5803        &mut self,
5804        winner: &Candidate,
5805        node: &'static str,
5806        round: Option<usize>,
5807        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5808    ) -> (SeatJob, SeatState, AgentOutcome) {
5809        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5810        self.ask_fixer_among(attempts, node, round, None, build)
5811            .await
5812    }
5813
5814    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5815    /// worktree and a commit: before the next agent is asked after one that
5816    /// did not satisfy the call, the worktree is put back on that commit, so
5817    /// a failed agent's partial edits are not carried into the next one.
5818    async fn ask_fixer_among(
5819        &mut self,
5820        attempts: Vec<(AgentSpec, String)>,
5821        node: &'static str,
5822        round: Option<usize>,
5823        restore: Option<(&Path, &str)>,
5824        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5825    ) -> (SeatJob, SeatState, AgentOutcome) {
5826        let run_id = self.state.id.clone();
5827        let prompts = self.state.config.prompts.clone();
5828        let cache = self.state.config.cache_dir();
5829        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5830        let mut last = None;
5831        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5832            let seat = self.seat(&key, &spec.id);
5833            let mut job = build(spec.clone(), seat);
5834            if i > 0 {
5835                job.stem = format!("{}-{}", job.stem, spec.id);
5836            }
5837            let ctx = WaveCtx {
5838                carry_seats: false,
5839                pauses: Some((&self.pause, &self.interrupt)),
5840                run: &run_id,
5841                node,
5842                prompts: &prompts,
5843                cache: cache.as_deref(),
5844                round,
5845            };
5846            let (seat, out) =
5847                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5848            // A resource block never advances the chain: the chosen agent
5849            // was not asked, so it must be the one asked again on resume.
5850            let advances = match &out {
5851                AgentOutcome::Ok(o) => agent::output_advances(o),
5852                AgentOutcome::ResourceBlocked(_) => false,
5853                _ => true,
5854            };
5855            if let Some(next) = ids.get(i + 1)
5856                && advances
5857            {
5858                self.state.seats.insert(seat.key.clone(), seat.clone());
5859                let class =
5860                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5861                let reason = match &out {
5862                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5863                    other => fail_reason(other),
5864                };
5865                // A failed agent's edits must not reach the next one, and a
5866                // restore that did not work ends the chain: the call fails
5867                // and the caller's own (confirmed) restore decides.
5868                if let Some((wt, rev)) = restore
5869                    && let Err(e) = self.graft_restore(wt, rev).await
5870                {
5871                    let why =
5872                        format!("could not restore the worktree before the next fixer: {e:#}");
5873                    tracing::warn!("{why}");
5874                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5875                        AgentOutcome::ResourceBlocked(why)
5876                    } else {
5877                        AgentOutcome::Failed(why)
5878                    };
5879                    last = Some((job, seat, out));
5880                    break;
5881                }
5882                record_handover(
5883                    &mut self.state,
5884                    node,
5885                    &seat.key,
5886                    &spec.id,
5887                    next,
5888                    &class,
5889                    &reason,
5890                );
5891                continue;
5892            }
5893            last = Some((job, seat, out));
5894            break;
5895        }
5896        last.expect("the fixer chain always has an entry")
5897    }
5898
5899    /// The one repair a verification-only failure earns (see
5900    /// [`crate::recovery`]), before the first review round of a run that took
5901    /// a branch over.
5902    ///
5903    /// Saved as started *before* the fixer is called, so a resume never
5904    /// repairs twice; a start found on re-entry is rolled back to the commit
5905    /// it began from. A failed or unusable fixer leaves the tree as it was and
5906    /// the review goes on over it, so the first round's own verification
5907    /// reports the failure again. The failure is never carried into a round as
5908    /// a result: verification is re-run on whatever commit this leaves.
5909    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5910        use crate::recovery::{Mode, Repair};
5911        if !self.state.reviews.is_empty() {
5912            return Ok(());
5913        }
5914        let Some(mut rec) = self.state.recovery.clone() else {
5915            return Ok(());
5916        };
5917        if rec.mode != Mode::VerifyRepair {
5918            return Ok(());
5919        }
5920        let wt = winner.worktree.clone();
5921        let mut head = git::rev_parse(&wt, "HEAD").await?;
5922        match rec.repair.clone() {
5923            Repair::Done { .. } => return Ok(()),
5924            Repair::WaitingForRestore { before } => {
5925                self.graft_restore(&wt, &before).await?;
5926                head = before;
5927                rec.repair = Repair::NotStarted;
5928                self.state.recovery = Some(rec.clone());
5929                self.state.save()?;
5930            }
5931            Repair::Started { before } => {
5932                // Interrupted: the unfinished edits are not trusted.
5933                // An unconfirmed rollback blocks the run with `Started` kept,
5934                // so a resume retries it; the tree is never reviewed as is.
5935                if let Err(e) = self.graft_restore(&wt, &before).await {
5936                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5937                        return Err(e);
5938                    }
5939                    return self
5940                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5941                        .await;
5942                }
5943                rec.repair = Repair::Done {
5944                    before: before.clone(),
5945                    after: before,
5946                    committed: false,
5947                    failed: Some("interrupted; rolled back".to_owned()),
5948                };
5949                self.state
5950                    .event("recovery", "the repair was interrupted and rolled back");
5951                self.state.recovery = Some(rec);
5952                self.state.save()?;
5953                return Ok(());
5954            }
5955            Repair::NotStarted => {}
5956        }
5957        if crate::recovery::recheck(&mut rec, &head) {
5958            self.state.event("recovery", rec.summary());
5959            self.state.recovery = Some(rec);
5960            self.state.save()?;
5961            return Ok(());
5962        }
5963        rec.repair = Repair::Started {
5964            before: head.clone(),
5965        };
5966        self.state.recovery = Some(rec.clone());
5967        self.state.save()?;
5968
5969        let language = self.state.config.graph.language.clone();
5970        let sessions = self.state.config.graph.sessions;
5971        let artifacts = agent::artifacts_dir(&self.state.dir());
5972        let fix_prompt = prompt::recovery_fix(
5973            &self.state.instruction,
5974            &rec,
5975            self.state.config.graph.review_rounds,
5976            &language,
5977        );
5978        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5979        self.state.event(
5980            "recovery",
5981            format!(
5982                "repair: the fixer starts from the failed verification on {}",
5983                short(&head)
5984            ),
5985        );
5986        let (_job, seat, out) = self
5987            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5988                prompt: fix_prompt.clone(),
5989                spec,
5990                seat,
5991                cwd: wt.clone(),
5992                timeout,
5993                allow_write: true,
5994                sessions,
5995                artifacts: artifacts.clone(),
5996                stem: "fix-recovery".to_owned(),
5997                handover: None,
5998            })
5999            .await;
6000        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6001            rec.repair = Repair::WaitingForRestore { before: head };
6002            self.state.recovery = Some(rec);
6003            return self.stop_for_resource();
6004        }
6005        let failed = match &out {
6006            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6007            AgentOutcome::Quota(o) => {
6008                self.state.quota.push(QuotaLoss {
6009                    seat: seat.key.clone(),
6010                    node: "fix".to_owned(),
6011                    at: Timestamp::now(),
6012                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6013                });
6014                Some("rate limited (quota); the fixer could not run".to_owned())
6015            }
6016            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6017            AgentOutcome::Failed(e) => Some(e.clone()),
6018        };
6019        self.state.seats.insert(seat.key.clone(), seat);
6020        let coordination = self.coordinate(&wt).await?;
6021        let failed = match failed {
6022            // The agent's partial edits are not kept.
6023            Some(why) => {
6024                if let Err(e) = git::restore_to(&wt, &head).await {
6025                    return self
6026                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6027                        .await;
6028                }
6029                Some(why)
6030            }
6031            None => {
6032                if let Ok(r) = git::rescue_commit(
6033                    &wt,
6034                    "magi: recovery repair of the failed verification (uncommitted work)",
6035                )
6036                .await
6037                {
6038                    self.state.note_withheld("fix", &r.withheld);
6039                }
6040                None
6041            }
6042        };
6043        let after = git::rev_parse(&wt, "HEAD").await?;
6044        drop(coordination);
6045        self.state.event(
6046            "recovery",
6047            match &failed {
6048                Some(why) => format!("repair: nothing kept ({why})"),
6049                None if after != head => {
6050                    format!("repair: committed {}", short(&after))
6051                }
6052                None => "repair: the fixer made no commit".to_owned(),
6053            },
6054        );
6055        rec.repair = Repair::Done {
6056            before: head.clone(),
6057            committed: after != head,
6058            after,
6059            failed,
6060        };
6061        self.state.recovery = Some(rec);
6062        self.state.save()?;
6063        Ok(())
6064    }
6065
6066    /// A repair whose rollback could not be confirmed: the run is blocked and
6067    /// resumable, and the unconfirmed state is never reviewed.
6068    async fn stop_recovery(
6069        &mut self,
6070        mut rec: crate::recovery::RecoveryRecord,
6071        head: &str,
6072        why: String,
6073    ) -> Result<()> {
6074        // `Started` stays recorded: a resume retries the restore.
6075        rec.repair = crate::recovery::Repair::Started {
6076            before: head.to_owned(),
6077        };
6078        self.state.recovery = Some(rec);
6079        self.state
6080            .event("recovery", format!("repair could not be undone: {why}"));
6081        self.state.status = RunStatus::Blocked;
6082        self.state.save()?;
6083        bail!("recovery repair could not be rolled back: {why}")
6084    }
6085
6086    async fn review_loop(&mut self) -> Result<()> {
6087        if self.hold_base_sync().await? {
6088            return Ok(());
6089        }
6090        loop {
6091            if self
6092                .state
6093                .base_sync
6094                .as_ref()
6095                .is_some_and(|s| s.conflict.is_some())
6096            {
6097                return Ok(());
6098            }
6099            if self.catchup_repairable() {
6100                let record = self.state.reviews.last().expect("repairable round").clone();
6101                let winner = self
6102                    .state
6103                    .winner()
6104                    .cloned()
6105                    .context("catch-up repair has no winner")?;
6106                if !self.fix_review_round(&winner, record).await? {
6107                    return Ok(());
6108                }
6109            }
6110            self.review_loop_pass().await?;
6111            if !self.catchup_repairable() {
6112                return Ok(());
6113            }
6114        }
6115    }
6116
6117    /// Derive an owed catch-up repair from durable round evidence, including
6118    /// reentry after a crash before the fixer was dispatched.
6119    fn catchup_repairable(&self) -> bool {
6120        self.state.reviews.len() < self.state.config.graph.review_rounds
6121            && self.state.reviews.last().is_some_and(|r| {
6122                !r.clean
6123                    && r.blocking == 0
6124                    && r.fix.is_none()
6125                    && r.e2e_status() == E2eStatus::Failed
6126            })
6127    }
6128
6129    async fn review_loop_pass(&mut self) -> Result<()> {
6130        // A base that would not rebase is a person's decision, not a review
6131        // round: nothing here would change the answer, and reviewers and a
6132        // fixer would be spending real budget on a tree that cannot land
6133        // regardless of what they find.
6134        if self
6135            .state
6136            .base_sync
6137            .as_ref()
6138            .is_some_and(|s| s.conflict.is_some())
6139        {
6140            return Ok(());
6141        }
6142        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6143        // agent files with `magi task add` name the run that paid for it. The
6144        // prompt overlay is cloned alongside it because the waves borrow it
6145        // while `self` is mutably borrowed by the node's own bookkeeping.
6146        let run_id = self.state.id.clone();
6147        let prompts = self.state.config.prompts.clone();
6148        let Some(winner) = self.state.winner().cloned() else {
6149            return Ok(());
6150        };
6151        let max_rounds = self.state.config.graph.review_rounds;
6152        // A clean round, an exhausted round budget, or a stalled tree (see
6153        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6154        // they are recorded — recomputed here, not read off `status`, so a
6155        // reentry into a run that already stopped restates the identical
6156        // verdict instead of silently handing back whatever an earlier node
6157        // in this same walk clobbered `status` to (a solo-candidate
6158        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6159        // below runs an empty range once the budget is spent, and would
6160        // otherwise fall through without touching `status` at all.
6161        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
6162            // A reentry after a crash between the last round's save and
6163            // `stop_reviewing` reaches the hand-off here, not there.
6164            if status == RunStatus::Gating {
6165                self.record_contested_handoff();
6166            }
6167            self.state.status = status;
6168            self.state.save()?;
6169            return Ok(());
6170        }
6171        self.state.status = RunStatus::Reviewing;
6172        // A last recorded round whose own verification never resolved
6173        // (`ResourceBlocked` — the shared build cache, not the patch) is
6174        // never a concluded round, whatever the round budget says: starting
6175        // a fresh round on top of it would spend a whole new reviewer wave
6176        // re-reading an unchanged patch instead of just retrying the one
6177        // check that actually needs it, and once the budget is spent the
6178        // loop below has nothing left to do at all (its range is empty).
6179        // Retry that check directly instead, exactly the same retry
6180        // `stop_reviewing` already does for its own catch-up case.
6181        if self.state.reviews.last().is_some_and(|r| {
6182            r.e2e_status() == E2eStatus::ResourceBlocked
6183                || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
6184        }) {
6185            let shell = self.state.config.shell();
6186            return self
6187                .stop_unresolved_review(
6188                    "the last round's own verification never resolved",
6189                    &shell,
6190                    &winner.worktree,
6191                )
6192                .await;
6193        }
6194
6195        self.recovery_repair(&winner).await?;
6196        if self.resource_halt()? {
6197            return Ok(());
6198        }
6199
6200        let repo = self.state.repo.clone();
6201        let root = self.state.worktree_root();
6202        let language = self.state.config.graph.language.clone();
6203        let sessions = self.state.config.graph.sessions;
6204        let artifacts = self.merge_repair_artifacts();
6205        let base = self.landing_base();
6206        let base_short = short(&base);
6207        let reviewers = self.roles.reviewers.clone();
6208        let shell = self.state.config.shell();
6209
6210        for round in (self.state.reviews.len() + 1)..=max_rounds {
6211            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6212            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6213            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6214            // The prior round's own record, already persisted — never a
6215            // hand-carried variable of just its failing output: that is
6216            // exactly what let a round's e2e result drift out of sync with
6217            // which commit it was actually about (see `SCHEMA`'s doc for
6218            // schema 8). Judged against `head`, the commit reviewers are
6219            // about to look at now, so the summary always reads as "an
6220            // earlier head" here — this round's own patch has not been
6221            // checked yet.
6222            let prev_verification = self
6223                .state
6224                .reviews
6225                .last()
6226                .and_then(|r| r.verification_summary(&head));
6227            // Only the first round of a run that took a branch over: later
6228            // rounds have this run's own history.
6229            let prior_section = match (&self.state.recovery, round) {
6230                (Some(rec), 1) if self.state.reviews.is_empty() => {
6231                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6232                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6233                            Some(crate::recovery::head_of(
6234                                &git::diff(&winner.worktree, reviewed, "HEAD")
6235                                    .await
6236                                    .unwrap_or_default(),
6237                                crate::recovery::SINCE_DIFF_MAX,
6238                            ))
6239                        }
6240                        _ => None,
6241                    };
6242                    Some(prompt::recovery_review(rec, since.as_deref()))
6243                }
6244                _ => None,
6245            };
6246
6247            // Each reviewer gets its own detached checkout of exactly this
6248            // commit: nobody can perturb the winner's tree, and the fixer can
6249            // keep working without racing a reviewer.
6250            let mut jobs = Vec::new();
6251            let coordination = self.coordinate(&repo).await?;
6252            for (r, spec) in reviewers.iter().cloned().enumerate() {
6253                let wt = root.join(format!("review-{}", r + 1));
6254                // `clean -fdx` in a reset wipes setup's products, so every
6255                // round sets the seat up again.
6256                if wt.exists() {
6257                    git::reset_detached(&wt, &head).await?;
6258                } else {
6259                    git::worktree_add_detached(&repo, &wt, &head).await?;
6260                }
6261                self.setup_seat_worktree(&repo, &wt, false).await?;
6262                let seat_key = format!("review-{}", r + 1);
6263                // The seat starts the round on whoever answered it last, not
6264                // on the agent the spec names, so a failure is not re-paid.
6265                let spec = pick_start_spec(
6266                    &self.roles.reviewer_roster,
6267                    spec,
6268                    self.state.seat_history.get(&seat_key),
6269                );
6270                let seat = self.seat(&seat_key, &spec.id);
6271                jobs.push(SeatJob {
6272                    prompt: prompt::review(&prompt::ReviewCtx {
6273                        instruction: &self.state.instruction,
6274                        branch: &winner.branch,
6275                        base_short: &base_short,
6276                        stat: &stat,
6277                        patch: &patch,
6278                        verification: prev_verification.as_ref(),
6279                        prior: prior_section.as_deref(),
6280                        reviewers: reviewers.len(),
6281                        round,
6282                        rounds: max_rounds,
6283                        // A review-only run has no rankings, so nothing
6284                        // competed for this patch and the reviewer is told so.
6285                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6286                        lens: Lens::for_seat(r),
6287                        language: &language,
6288                        github_language: &self.state.config.graph.github_language,
6289                    }),
6290                    spec,
6291                    seat,
6292                    cwd: wt,
6293                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6294                    allow_write: false,
6295                    sessions,
6296                    artifacts: artifacts.clone(),
6297                    stem: format!("review-{round}-{}", r + 1),
6298                    handover: None,
6299                });
6300            }
6301
6302            drop(coordination);
6303            self.state.event(
6304                "review",
6305                format!(
6306                    "round {round}: {} reviewers on {}",
6307                    jobs.len(),
6308                    short(&head)
6309                ),
6310            );
6311            let mut quota_losses = Vec::new();
6312            let review_retries = self.state.config.graph.retries;
6313            let review_cache = self.state.config.cache_dir();
6314            let ctx = WaveCtx {
6315                carry_seats: true,
6316                pauses: Some((&self.pause, &self.interrupt)),
6317                run: &run_id,
6318                node: "review",
6319                prompts: &prompts,
6320                cache: review_cache.as_deref(),
6321                round: Some(round),
6322            };
6323            let results = ask_json_wave::<Review>(
6324                jobs,
6325                Arc::clone(&self.sem),
6326                review_retries,
6327                &self.roles.reviewer_roster,
6328                &ctx,
6329                &mut quota_losses,
6330                &mut self.state,
6331                &|_: &Review| Ok(()),
6332            )
6333            .await;
6334            if self.state.resource_block.is_some() {
6335                return self.stop_for_resource();
6336            }
6337            // Counted before the move below: how many of *this* round's
6338            // reviewer seats were lost to their own rate limit, as opposed to
6339            // a crash, a timeout, or unparsable output — see `round_is_clean`.
6340            let round_quota_missing = quota_losses.len();
6341            self.state.quota.extend(quota_losses);
6342
6343            let mut records = Vec::new();
6344            let mut all_findings = Vec::new();
6345            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6346                let agent_id = seat.agent.clone();
6347                self.state.seats.insert(seat.key.clone(), seat);
6348                let mut record = ReviewRecord {
6349                    reviewer: r + 1,
6350                    agent: agent_id,
6351                    summary: String::new(),
6352                    findings: Vec::new(),
6353                    vote: None,
6354                    failed: None,
6355                    duration_ms: 0,
6356                    // Set for both outcomes: `failed: Some(_)` with
6357                    // `attempts > 0` is a seat every retry still lost, not a
6358                    // recovered one — only `failed: None` with `attempts > 0`
6359                    // reads as "answered after a nudge" (see this field's own
6360                    // doc).
6361                    attempts,
6362                };
6363                match res {
6364                    Ok((review, out)) => {
6365                        // Sanitized here, at the point every other piece of
6366                        // agent prose in this file is (candidate summaries,
6367                        // deliberation turns, vote reasons): a reviewer's own
6368                        // words are the one thing about it that could name
6369                        // it, and reconsideration below broadcasts this same
6370                        // summary and these same findings to every other
6371                        // seat on the panel.
6372                        record.summary =
6373                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6374                        record.vote = Some(review.vote);
6375                        record.duration_ms = out.duration_ms;
6376                        for (n, mut f) in review.findings.into_iter().enumerate() {
6377                            // ids are magi's, never the agent's: the fixer's
6378                            // adoption report is keyed by them.
6379                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6380                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6381                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6382                            // `file` is agent-supplied prose too, never
6383                            // checked against the real tree — the same
6384                            // exposure `title`/`detail` above have, just in
6385                            // a field easy to forget because it looks like a
6386                            // path rather than free text.
6387                            f.file = f
6388                                .file
6389                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6390                            all_findings.push(f.clone());
6391                            record.findings.push(f);
6392                        }
6393                        self.state.event(
6394                            "review",
6395                            format!(
6396                                "round {round}: reviewer {} voted {} with {} finding(s)",
6397                                r + 1,
6398                                review.vote.label(),
6399                                record.findings.len()
6400                            ),
6401                        );
6402                    }
6403                    Err(e) => {
6404                        record.failed = Some(e.to_string());
6405                        self.state.event(
6406                            "review",
6407                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6408                        );
6409                    }
6410                }
6411                records.push(record);
6412            }
6413
6414            // Tally the round's votes and, if they split, spend the one
6415            // round of reconsideration the split -> deliberate -> revote
6416            // shape `judge`/`vote` use for the panel, sized down to what a
6417            // read-only review round can afford: one round, and a revote
6418            // rather than an argument, because the panel already wrote its
6419            // reasoning down as findings the first time around.
6420            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6421            let vote_split =
6422                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6423            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6424            if vote_split {
6425                self.state.event(
6426                    "review",
6427                    format!(
6428                        "round {round}: votes split ({}) — one round of reconsideration",
6429                        initial_votes
6430                            .iter()
6431                            .map(|v| v.label())
6432                            .collect::<Vec<_>>()
6433                            .join(", ")
6434                    ),
6435                );
6436                // Seats read every seat's findings and votes, still numbered
6437                // and never named — the same anonymity `review` itself keeps.
6438                let panel: Vec<ReviewSeatReport<'_>> = records
6439                    .iter()
6440                    .filter_map(|r| {
6441                        r.vote.map(|vote| ReviewSeatReport {
6442                            reviewer: r.reviewer,
6443                            vote,
6444                            summary: &r.summary,
6445                            findings: &r.findings,
6446                        })
6447                    })
6448                    .collect();
6449
6450                let mut jobs = Vec::new();
6451                let mut seats_at = Vec::new();
6452                for (r, spec) in reviewers.iter().cloned().enumerate() {
6453                    // A seat with no initial vote has nothing to reconsider
6454                    // from and stays absent, the same as it stayed absent
6455                    // from `panel` above.
6456                    if records[r].vote.is_none() {
6457                        continue;
6458                    }
6459                    let wt = root.join(format!("review-{}", r + 1));
6460                    let seat_key = format!("review-{}", r + 1);
6461                    let spec = self.occupant(&seat_key, spec);
6462                    let seat = self.seat(&seat_key, &spec.id);
6463                    // A seat with no live session has already forgotten the
6464                    // initial review's prompt — restate the patch it is
6465                    // voting on, the same as `deliberate`/`vote` do for a
6466                    // judge in the same position.
6467                    // The panel already carries this seat's own review and
6468                    // vote, so restating the patch makes the prompt whole for
6469                    // a seat handed to another agent.
6470                    let build = |with_patch: bool| {
6471                        prompt::review_reconsider(&ReviewReconsiderCtx {
6472                            instruction: &self.state.instruction,
6473                            reviewer: r + 1,
6474                            lens: Lens::for_seat(r),
6475                            panel: &panel,
6476                            patch: with_patch.then_some(ReviewPatch {
6477                                branch: &winner.branch,
6478                                base_short: &base_short,
6479                                stat: &stat,
6480                                patch: &patch,
6481                            }),
6482                            round,
6483                            rounds: max_rounds,
6484                            language: &language,
6485                        })
6486                    };
6487                    let full = build(true);
6488                    let prompt = if has_context(&spec, &seat, sessions) {
6489                        build(false)
6490                    } else {
6491                        full.clone()
6492                    };
6493                    jobs.push(SeatJob {
6494                        prompt,
6495                        spec,
6496                        seat,
6497                        cwd: wt,
6498                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6499                        allow_write: false,
6500                        sessions,
6501                        artifacts: artifacts.clone(),
6502                        stem: format!("review-{round}-reconsider-{}", r + 1),
6503                        handover: Some(full),
6504                    });
6505                    seats_at.push(r);
6506                }
6507
6508                let mut recon_quota_losses = Vec::new();
6509                let recon_cache = self.state.config.cache_dir();
6510                let recon_ctx = WaveCtx {
6511                    carry_seats: true,
6512                    pauses: Some((&self.pause, &self.interrupt)),
6513                    run: &run_id,
6514                    node: "review",
6515                    prompts: &prompts,
6516                    cache: recon_cache.as_deref(),
6517                    round: Some(round),
6518                };
6519                let recon_results = ask_json_wave::<ReviewRevote>(
6520                    jobs,
6521                    Arc::clone(&self.sem),
6522                    review_retries,
6523                    &self.roles.reviewer_roster,
6524                    &recon_ctx,
6525                    &mut recon_quota_losses,
6526                    &mut self.state,
6527                    &|_: &ReviewRevote| Ok(()),
6528                )
6529                .await;
6530                if self.state.resource_block.is_some() {
6531                    return self.stop_for_resource();
6532                }
6533                self.state.quota.extend(recon_quota_losses);
6534
6535                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6536                    let agent_id = seat.agent.clone();
6537                    self.state.seats.insert(seat.key.clone(), seat);
6538                    let mut rec = ReviewRevoteRecord {
6539                        reviewer: r + 1,
6540                        agent: agent_id,
6541                        vote: None,
6542                        reason: String::new(),
6543                        failed: None,
6544                    };
6545                    match res {
6546                        Ok((rv, _)) => {
6547                            rec.vote = Some(rv.vote);
6548                            rec.reason =
6549                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6550                            self.state.event(
6551                                "review",
6552                                format!(
6553                                    "round {round}: reviewer {} revoted {}",
6554                                    r + 1,
6555                                    rv.vote.label()
6556                                ),
6557                            );
6558                        }
6559                        Err(e) => {
6560                            rec.failed = Some(e.to_string());
6561                            self.state.event(
6562                                "review",
6563                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6564                            );
6565                        }
6566                    }
6567                    reconsideration.push(rec);
6568                }
6569            } else if initial_votes.len() > 1 {
6570                self.state.event(
6571                    "review",
6572                    format!(
6573                        "round {round}: votes agreed ({}) — no reconsideration",
6574                        initial_votes[0].label()
6575                    ),
6576                );
6577            }
6578
6579            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6580            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6581            // A round that already has a blocking finding and a round left to
6582            // try is going back to the fixer no matter what `verify.e2e`
6583            // says, so running it first only spends the loop's slowest step
6584            // (minutes, for a Rust repo's full test suite) on a head about
6585            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6586            // runs once a round has no blocking findings left (see
6587            // `round_is_clean`, which a deferred — empty — `e2e` can never
6588            // satisfy since `blocking` is nonzero whenever this branch is
6589            // taken), and `stop_reviewing` forces a real run before it will
6590            // ever read a deferred round as green.
6591            let defer_e2e =
6592                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6593            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6594                let reason =
6595                    format!("{blocking} blocking finding(s) already required a fix this round");
6596                self.state.event(
6597                    "verify",
6598                    format!(
6599                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6600                         {}); it will run once a round has none left",
6601                        short(&head)
6602                    ),
6603                );
6604                (Vec::new(), false, true, Some(reason))
6605            } else {
6606                let e2e_commands = self.state.config.verify.e2e.clone();
6607                let cache_dir = self.state.config.cache_dir();
6608                let context = format!("round {round}");
6609                let wait = self.repository_wait();
6610                let (e2e, verify_retried) = with_cache_lease(
6611                    &mut self.state,
6612                    cache_dir.as_deref(),
6613                    "e2e",
6614                    "e2e",
6615                    &winner.worktree,
6616                    &head,
6617                    verify_timeout,
6618                    &context,
6619                    &wait,
6620                    |state, budget| {
6621                        let shell = shell.clone();
6622                        let e2e_commands = e2e_commands.clone();
6623                        let worktree = winner.worktree.clone();
6624                        let context = context.clone();
6625                        async move {
6626                            run_e2e_with_retry(
6627                                state,
6628                                &shell,
6629                                &e2e_commands,
6630                                &worktree,
6631                                budget,
6632                                &context,
6633                            )
6634                            .await
6635                        }
6636                    },
6637                )
6638                .await?;
6639                (e2e, verify_retried, false, None)
6640            };
6641
6642            let expected = records.len();
6643            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6644            let incomplete = answered < expected;
6645            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6646            let policy = self.state.config.graph.incomplete_review;
6647            let clean = round_is_clean(
6648                blocking,
6649                e2e_ok,
6650                answered,
6651                expected,
6652                round_quota_missing,
6653                policy,
6654            );
6655
6656            let mut round_record = ReviewRound {
6657                round,
6658                head: head.clone(),
6659                verified_head: None,
6660                verified_at: None,
6661                reviews: records,
6662                e2e,
6663                verify_retried,
6664                e2e_deferred,
6665                e2e_defer_reason,
6666                fix: None,
6667                blocking,
6668                answered,
6669                expected,
6670                clean,
6671                progressed: false,
6672                vote_split,
6673                reconsideration,
6674                verdict: None,
6675            };
6676            // The final vote per seat is its revote where reconsideration
6677            // ran and answered, its initial vote otherwise — the same
6678            // fallback `tally` uses for a judge whose private vote failed.
6679            round_record.verdict = ReviewVote::worst(
6680                round_record
6681                    .final_votes()
6682                    .into_iter()
6683                    .map(|(_, _, vote)| vote),
6684            );
6685            // Which commit and when magi actually attempted to check —
6686            // known the moment a command was dispatched against `head`,
6687            // whether or not it finished: a resource-blocked attempt still
6688            // targeted a specific commit at a specific time, and leaving
6689            // that unrecorded is exactly what made `verification_summary`
6690            // report a fresh attempt as "commit unknown ... recorded before
6691            // this was tracked", indistinguishable from a genuinely old,
6692            // untracked record. Only a deferred or unconfigured round never
6693            // ran at all and has nothing to record — see
6694            // `ReviewRound::verified_head`'s own doc.
6695            if !matches!(
6696                round_record.e2e_status(),
6697                E2eStatus::Deferred | E2eStatus::NotConfigured
6698            ) {
6699                round_record.verified_head = Some(head.clone());
6700                round_record.verified_at = Some(Timestamp::now());
6701            }
6702
6703            if incomplete {
6704                let missing: Vec<String> = round_record
6705                    .reviews
6706                    .iter()
6707                    .filter(|r| r.failed.is_some())
6708                    .map(|r| format!("review-{}", r.reviewer))
6709                    .collect();
6710                self.state.event(
6711                    "review",
6712                    format!(
6713                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6714                        missing.join(", ")
6715                    ),
6716                );
6717            }
6718
6719            if clean {
6720                self.state.event(
6721                    "review",
6722                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6723                        format!(
6724                            "round {round}: clean (warn policy, incomplete panel) — no \
6725                             blocking findings from the seats that answered, verification green"
6726                        )
6727                    } else if incomplete {
6728                        format!(
6729                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6730                             quorum) — no blocking findings from the seats that answered, \
6731                             verification green",
6732                            expected - answered
6733                        )
6734                    } else {
6735                        format!("round {round}: clean — no blocking findings, verification green")
6736                    },
6737                );
6738                self.state.reviews.push(round_record);
6739                self.state.status = RunStatus::Gating;
6740                self.state.save()?;
6741                return Ok(());
6742            }
6743
6744            // Nothing was raised and verification passed, but not every seat
6745            // answered and `round_is_clean` still refused to call it clean —
6746            // either a seat is missing for a reason other than its own quota
6747            // (a crash, a timeout, unparsable output — worth another try), or
6748            // every seat that could have answered lost its quota and nobody
6749            // is left to decide on: re-review rather than send the fixer
6750            // after a round with nothing to fix.
6751            if incomplete && blocking == 0 && e2e_ok {
6752                self.state.reviews.push(round_record);
6753                self.state.save()?;
6754                if round == max_rounds {
6755                    self.state.status = RunStatus::Blocked;
6756                    self.state.event(
6757                        "review",
6758                        format!(
6759                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6760                             refusing to call it clean",
6761                            expected - answered
6762                        ),
6763                    );
6764                    return Ok(());
6765                }
6766                continue;
6767            }
6768
6769            // Nothing for the fixer to act on (`blocking == 0`) and the only
6770            // reason this round is not clean is that magi itself never got
6771            // a command to run — the shared build cache, not the patch (see
6772            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6773            // the fixer would invite a change to appease contention that has
6774            // nothing to do with the diff, and would leave this attempt
6775            // sitting in the next round's prompt as if it were about an
6776            // earlier, superseded commit rather than what it actually is:
6777            // the same head, still waiting to be checked. Wait for it the
6778            // same way the final round's own contention is already handled,
6779            // whatever round this happens to be.
6780            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6781                self.state.reviews.push(round_record);
6782                return self
6783                    .stop_unresolved_review(
6784                        "the round's own verification could not run",
6785                        &shell,
6786                        &winner.worktree,
6787                    )
6788                    .await;
6789            }
6790
6791            if round == max_rounds {
6792                self.state.reviews.push(round_record);
6793                return self
6794                    .stop_reviewing(
6795                        &format!(
6796                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6797                        ),
6798                        &shell,
6799                        &winner.worktree,
6800                    )
6801                    .await;
6802            }
6803
6804            if !self.fix_review_round(&winner, round_record).await? {
6805                return Ok(());
6806            }
6807        }
6808        Ok(())
6809    }
6810
6811    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
6812    /// False means admission or continuation policy stopped the review loop.
6813    async fn fix_review_round(
6814        &mut self,
6815        winner: &Candidate,
6816        mut round_record: ReviewRound,
6817    ) -> Result<bool> {
6818        let round = round_record.round;
6819        let shell = self.state.config.shell();
6820        let step = self.run_fix_round(winner, &mut round_record).await?;
6821        if matches!(step, FixStep::Resource) {
6822            self.stop_for_resource()?;
6823            return Ok(false);
6824        }
6825        if self.state.reviews.len() < round {
6826            self.state.reviews.push(round_record);
6827        } else {
6828            self.state.reviews[round - 1] = round_record;
6829        }
6830        self.state.save()?;
6831
6832        // The fixer's own report never came back this round, even after
6833        // `continue_fix_report`'s own budget was spent on it — not an
6834        // ordinary "no report" (dropped stream, quota, plain failure),
6835        // which already reads that way and is left to the existing round
6836        // budget. Stopping here, rather than opening another round, is
6837        // what keeps a next reviewer/fixer wave from ever being
6838        // dispatched onto `winner.worktree` while whatever the seat's
6839        // last call may still have running there is unaccounted for: no
6840        // process liveness check exists (and none is being added — see
6841        // AGENTS.md/this task's own scope), so the only way to honour
6842        // "nothing starts before a valid report returns" is to not start
6843        // anything further on this worktree from this run at all.
6844        if matches!(step, FixStep::Recorded { report_lost: true }) {
6845            self.stop_reviewing(
6846                "the fixer's adoption report never came back, even after resuming its \
6847                         own seat; refusing to start another round against the same worktree \
6848                         while that is unresolved",
6849                &shell,
6850                &winner.worktree,
6851            )
6852            .await?;
6853            return Ok(false);
6854        }
6855
6856        let streak = self
6857            .state
6858            .reviews
6859            .iter()
6860            .rev()
6861            .take_while(|r| !r.progressed)
6862            .count();
6863        if streak >= STAGNANT_LIMIT {
6864            self.stop_reviewing(
6865                &format!("the tree has not moved against base for {streak} round(s) in a row"),
6866                &shell,
6867                &winner.worktree,
6868            )
6869            .await?;
6870            return Ok(false);
6871        }
6872        Ok(true)
6873    }
6874
6875    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
6876    async fn run_fix_round(
6877        &mut self,
6878        winner: &Candidate,
6879        round_record: &mut ReviewRound,
6880    ) -> Result<FixStep> {
6881        let round = round_record.round;
6882        let max_rounds = self.state.config.graph.review_rounds;
6883        let run_id = self.state.id.clone();
6884        let prompts = self.state.config.prompts.clone();
6885        let language = self.state.config.graph.language.clone();
6886        let sessions = self.state.config.graph.sessions;
6887        let artifacts = self.merge_repair_artifacts();
6888        let base = self.landing_base();
6889        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6890        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6891        let this_round_verification = round_record.verification_summary(&head);
6892        let all_findings: Vec<_> = round_record
6893            .reviews
6894            .iter()
6895            .flat_map(|r| r.findings.iter().cloned())
6896            .collect();
6897        // Fix. The winner's own implementer seat continues its conversation:
6898        // the competition is over, so context is pure benefit now.
6899        let blocking_findings: Vec<_> = all_findings
6900            .iter()
6901            .filter(|f| f.severity.blocks())
6902            .cloned()
6903            .collect();
6904        let fix_prompt = prompt::fix_in(
6905            &self.state.instruction,
6906            &blocking_findings,
6907            this_round_verification.as_ref(),
6908            round,
6909            max_rounds,
6910            &language,
6911            &self.state.config.graph.github_language,
6912        );
6913        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6914        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6915        let (job, seat, out) = self
6916            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6917                prompt: fix_prompt.clone(),
6918                spec,
6919                seat,
6920                cwd: winner.worktree.clone(),
6921                timeout,
6922                allow_write: true,
6923                sessions,
6924                artifacts: artifacts.clone(),
6925                stem: format!("fix-{round}"),
6926                handover: None,
6927            })
6928            .await;
6929        // Not asked: the round is not recorded (so a resume repeats it),
6930        // no fixer was handed over to, and the seat is left as it was.
6931        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6932            return Ok(FixStep::Resource);
6933        }
6934        let agent_id = seat.agent.clone();
6935
6936        let mut fix = FixRecord {
6937            agent: agent_id,
6938            addressed: Vec::new(),
6939            rejected: Vec::new(),
6940            notes: String::new(),
6941            committed: false,
6942            failed: None,
6943            duration_ms: 0,
6944            continuation: None,
6945        };
6946        let mut continuation = ContinuationRecord::not_needed();
6947        let mut final_seat = seat.clone();
6948        match out {
6949            AgentOutcome::Ok(o) => {
6950                fix.duration_ms = o.duration_ms;
6951                let parsed = verdict::extract_json::<FixReport>(&o.text);
6952                // A parsed report standing next to a command this same
6953                // reply's own CLI never confirmed the exit status of is
6954                // not a resolved answer — the identical `CommandEvidence`
6955                // `state.jobs` renders, read here instead of only on
6956                // display, per the completion judgment and the shown
6957                // record needing to agree.
6958                let incomplete_reason = match &parsed {
6959                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6960                        "the reply parsed, but it reported a command whose own CLI never \
6961                             confirmed an exit status"
6962                            .to_owned(),
6963                    ),
6964                    Ok(_) => None,
6965                    Err(e) => Some(e.to_string()),
6966                };
6967                match incomplete_reason {
6968                    None => {
6969                        let report = parsed.expect("checked Ok above");
6970                        fix.addressed = report.addressed;
6971                        fix.rejected = report.rejected;
6972                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6973                    }
6974                    Some(reason) => {
6975                        let (resumed_seat, resolved, failure, cont) = self
6976                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6977                            .await;
6978                        fix.duration_ms += cont.cumulative_wait_ms;
6979                        continuation = cont;
6980                        final_seat = resumed_seat;
6981                        match resolved {
6982                            Some(report) => {
6983                                fix.addressed = report.addressed;
6984                                fix.rejected = report.rejected;
6985                                fix.notes =
6986                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6987                            }
6988                            None => fix.failed = failure,
6989                        }
6990                    }
6991                }
6992            }
6993            // The CLI's raw error JSON is not a fix report to parse.
6994            AgentOutcome::Dropped(o) => {
6995                fix.duration_ms = o.duration_ms;
6996                let why = o
6997                    .dropped
6998                    .as_ref()
6999                    .map(|d| d.why.as_str())
7000                    .unwrap_or("the CLI ended the stream without delivering its answer");
7001                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7002            }
7003            AgentOutcome::Quota(o) => {
7004                self.state.quota.push(QuotaLoss {
7005                    seat: final_seat.key.clone(),
7006                    node: "fix".to_owned(),
7007                    at: Timestamp::now(),
7008                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7009                });
7010                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7011            }
7012            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7013        }
7014        fix.continuation = Some(continuation);
7015        self.state.seats.insert(final_seat.key.clone(), final_seat);
7016        let coordination = self.coordinate(&winner.worktree).await?;
7017        if let Ok(r) = git::rescue_commit(
7018            &winner.worktree,
7019            &format!("magi: review round {round} fixes (uncommitted work)"),
7020        )
7021        .await
7022        {
7023            self.state.note_withheld("fix", &r.withheld);
7024        }
7025        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7026        fix.committed = after != before;
7027        // Judged by what `git` says moved against base, never by the
7028        // fixer's own `addressed`/`rejected` count — see
7029        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7030        // `patch` snapshot above: swallowing this error would default
7031        // `diff_after` to empty, which almost always differs from a
7032        // non-empty `patch` and reads as "progressed" — exactly backwards
7033        // for a `git` failure the stagnation check cannot see through.
7034        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7035        drop(coordination);
7036        let progressed = diff_after != patch;
7037        let commit_note = if fix.committed {
7038            "committed"
7039        } else {
7040            "NO new commit"
7041        };
7042        let tree_note = if progressed {
7043            "changed vs base"
7044        } else {
7045            "unchanged vs base"
7046        };
7047        self.state.event(
7048            "fix",
7049            match &fix.failed {
7050                // Distinct on purpose from "0 addressed, 0 rejected": the
7051                // fixer's own diff still landed (blocking counts do keep
7052                // falling round over round), only its adoption report did
7053                // not come back, so this must never read like every
7054                // finding was reviewed and declined.
7055                Some(reason) => {
7056                    format!(
7057                        "round {round}: fixer's adoption report was lost ({reason}); \
7058                             {commit_note}, tree {tree_note}"
7059                    )
7060                }
7061                None => format!(
7062                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7063                         {tree_note}{}",
7064                    fix.addressed.len(),
7065                    fix.rejected.len(),
7066                    if continuation.outcome == ContinuationOutcome::Resumed {
7067                        format!(
7068                            " (adoption report recovered after {} continuation(s))",
7069                            continuation.attempts
7070                        )
7071                    } else {
7072                        String::new()
7073                    },
7074                ),
7075            },
7076        );
7077        round_record.fix = Some(fix);
7078        round_record.progressed = progressed;
7079        Ok(FixStep::Recorded {
7080            report_lost: matches!(
7081                continuation.outcome,
7082                ContinuationOutcome::Exhausted
7083                    | ContinuationOutcome::QuotaLost
7084                    | ContinuationOutcome::NoSession
7085            ),
7086        })
7087    }
7088
7089    /// Decide, from the last recorded round's own verification, whether
7090    /// stopping the review loop is a hand-off or a genuine block.
7091    ///
7092    /// Called once the loop has given up trying — the round budget is spent,
7093    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7094    /// findings still open, never while a round is still clean or the
7095    /// incomplete-panel case handled inline above. Gate and e2e are facts
7096    /// about the tree; a lingering review finding is an opinion, and this
7097    /// workload's own `magi stats` puts reviewer precision low enough
7098    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7099    /// must not by itself stand between a green, verified change and the
7100    /// human who decides what to do with it. A red e2e is not an opinion, so
7101    /// that case still blocks, with the failing command and a tail of its
7102    /// output recorded here rather than left in `run.json` for someone to go
7103    /// find.
7104    ///
7105    /// A round that deferred its own e2e (see [`Config::graph`]'s
7106    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7107    /// only because nothing ran, and treating an empty list as a passing one
7108    /// here is exactly the "deferred painted green" bug this function exists
7109    /// to not have. When the last round's own verification never resolved —
7110    /// deferred on purpose, or a real attempt the shared build cache blocked
7111    /// — this makes (or retries) the real run, on the actual worktree this
7112    /// loop is about to stop touching, before deciding anything. A
7113    /// resource-blocked attempt is likewise never read as either green or
7114    /// red: it is evidence about the machine, not the patch (see
7115    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7116    /// blocked cache leaves this call without deciding rather than guessing
7117    /// — the caller retries on a later reentry.
7118    /// Record, once, that the review loop handed off over a blocking finding
7119    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7120    /// `land` asks the owner even with `land_approval` off. Called from every
7121    /// path that concludes `Gating`; a reentry keeps the first record.
7122    fn record_contested_handoff(&mut self) {
7123        if self.state.contested_handoff.is_some() {
7124            return;
7125        }
7126        let Some(contested) = self
7127            .state
7128            .reviews
7129            .last()
7130            .and_then(ReviewRound::contested_handoff)
7131        else {
7132            return;
7133        };
7134        self.state.event(
7135            "review",
7136            format!(
7137                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7138                 wait for the owner's approval",
7139                contested.findings.len(),
7140                contested.rejecters.len()
7141            ),
7142        );
7143        self.state.contested_handoff = Some(contested);
7144    }
7145
7146    async fn stop_unresolved_review(
7147        &mut self,
7148        why: &str,
7149        shell: &[String],
7150        worktree: &Path,
7151    ) -> Result<()> {
7152        self.stop_reviewing_inner(why, shell, worktree, true).await
7153    }
7154
7155    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
7156        self.stop_reviewing_inner(why, shell, worktree, false).await
7157    }
7158
7159    async fn stop_reviewing_inner(
7160        &mut self,
7161        why: &str,
7162        shell: &[String],
7163        worktree: &Path,
7164        repairable: bool,
7165    ) -> Result<()> {
7166        let round_idx = self.state.reviews.len() - 1;
7167        // A deferred round and a resource-blocked one are the same shape
7168        // here: neither has a real result yet, and both get one more
7169        // attempt. Read off `e2e_status` — the single source for this —
7170        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
7171        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
7172        // still retries instead of being read as a settled result the
7173        // instant it stops being empty.
7174        let needs_catchup_run = matches!(
7175            self.state.reviews[round_idx].e2e_status(),
7176            E2eStatus::Deferred | E2eStatus::ResourceBlocked
7177        );
7178        if needs_catchup_run {
7179            let round = self.state.reviews[round_idx].round;
7180            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7181            let commands = self.state.config.verify.e2e.clone();
7182            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
7183            let cache_dir = self.state.config.cache_dir();
7184            let context = format!(
7185                "round {round}: verification unresolved, catching up before the final decision"
7186            );
7187            let wait = self.repository_wait();
7188            let (outcomes, verify_retried) = with_cache_lease(
7189                &mut self.state,
7190                cache_dir.as_deref(),
7191                "e2e",
7192                "e2e",
7193                worktree,
7194                &attempted_head,
7195                timeout,
7196                &context,
7197                &wait,
7198                |state, budget| {
7199                    let shell = shell.to_vec();
7200                    let commands = commands.clone();
7201                    let context = context.clone();
7202                    async move {
7203                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7204                            .await
7205                    }
7206                },
7207            )
7208            .await?;
7209            let last = &mut self.state.reviews[round_idx];
7210            last.e2e = outcomes;
7211            last.verify_retried = verify_retried;
7212            // Always the commit and time this attempt actually targeted,
7213            // whether or not it happens to equal the reviewed `head` and
7214            // whether or not a command finished — see
7215            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
7216            // attempt is recorded too, so a later reader sees "attempted
7217            // again at T2" rather than silence.
7218            last.verified_head = Some(attempted_head);
7219            last.verified_at = Some(Timestamp::now());
7220            if verify_inconclusive(&last.e2e) {
7221                // Still not a real result: `e2e_deferred` is left exactly
7222                // as it was, so `needs_catchup_run` above reads
7223                // `ResourceBlocked` (via `e2e_status`, which checks
7224                // `resource_blocked` before `e2e_deferred`) and retries
7225                // again on the next reentry, rather than recording
7226                // contention as a red e2e and blocking the run on it.
7227                self.state.save()?;
7228                return Ok(());
7229            }
7230            last.e2e_deferred = false;
7231        }
7232        let last = &self.state.reviews[round_idx];
7233        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7234
7235        match last.e2e_status() {
7236            E2eStatus::Failed => {
7237                let red: Vec<String> = last
7238                    .e2e
7239                    .iter()
7240                    .filter(|o| !o.ok())
7241                    .map(|o| {
7242                        format!(
7243                            "`{}` -> {:?}\n{}",
7244                            o.command,
7245                            o.code,
7246                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7247                        )
7248                    })
7249                    .collect();
7250                let can_repair = last.blocking == 0 && last.fix.is_none();
7251                self.state.reviews[round_idx].clean = false;
7252                self.state
7253                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7254                self.state.status = RunStatus::Blocked;
7255                if repairable
7256                    && can_repair
7257                    && self.state.reviews.len() < self.state.config.graph.review_rounds
7258                {
7259                    self.state.status = RunStatus::Reviewing;
7260                }
7261            }
7262            // `needs_catchup_run` above already retried once this call; if
7263            // it is still blocked, this is magi's own admission it could
7264            // not get a command to run, never a verdict on the patch — the
7265            // run is left exactly where a later reentry can retry again.
7266            E2eStatus::ResourceBlocked => {
7267                self.state.event(
7268                    "review",
7269                    format!(
7270                        "{why}; e2e could not run (resource unavailable); not \
7271                         deciding yet"
7272                    ),
7273                );
7274            }
7275            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7276                self.state.event(
7277                    "review",
7278                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7279                );
7280                self.record_contested_handoff();
7281                self.state.status = RunStatus::Gating;
7282            }
7283        }
7284        self.state.save()?;
7285        Ok(())
7286    }
7287
7288    // ----------------------------------------------------------------- gate
7289
7290    async fn gate(&mut self) -> Result<()> {
7291        if self.hold_base_sync().await? {
7292            return Ok(());
7293        }
7294        // Judged by the review record itself, not by `status`: a solo
7295        // candidate's `judge`/`deliberate` skip rewrites `status` on every
7296        // reentry (see `judge`), and trusting it here is exactly how a run
7297        // that exhausted its review budget got gated and merged a second
7298        // time around. `review_conclusion` recomputes the review loop's own
7299        // verdict from the round records themselves — `Gating` for a clean
7300        // round or a hand-off (see `stop_reviewing`), anything else means the
7301        // loop is still going or genuinely blocked.
7302        // A base the winner could not be replayed onto is a decision, not a
7303        // round: there is no landing tree to gate. Read as its own record for
7304        // the same reason the review verdict is.
7305        if self.state.status == RunStatus::Failed
7306            || self
7307                .state
7308                .base_sync
7309                .as_ref()
7310                .is_some_and(|s| s.conflict.is_some())
7311            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7312                != Some(RunStatus::Gating)
7313        {
7314            return Ok(());
7315        }
7316        if self.state.gate_ran {
7317            // `review_loop` derives its conclusion from the clean review
7318            // record on every reentry and therefore puts a completed run back
7319            // in `Gating`. A recorded gate is a stronger, terminal fact:
7320            // retain its original command output (or lack of any, for a repo
7321            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
7322            // doc) and restore `Blocked` on a real failure rather than
7323            // pretending the command is still running or running it a second
7324            // time. `gate_ran == false` remains the only shape — unattempted,
7325            // or a resource-blocked retry — that may still need to execute a
7326            // command.
7327            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7328                self.state.status = RunStatus::Blocked;
7329                self.state.save()?;
7330            }
7331            return Ok(());
7332        }
7333        let Some(winner) = self.state.winner().cloned() else {
7334            return Ok(());
7335        };
7336        self.state.status = RunStatus::Gating;
7337        let mut outcomes = self.run_gate(&winner).await?;
7338        loop {
7339            // A resource-blocked outcome means the gate command never actually
7340            // ran - the shared build cache could not be acquired or confirmed
7341            // fresh in time - which is evidence about the machine, not about
7342            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
7343            // Recording it as a red gate would mark a run `Blocked` on nothing
7344            // but contention magi has already logged; leaving `self.state.gate`
7345            // empty and `self.state.gate_ran` false instead keeps the shape
7346            // this function already treats as "still needs to run" (see the
7347            // early-return above), so the next call retries the command
7348            // rather than concluding anything.
7349            if verify_inconclusive(&outcomes) {
7350                self.state.save()?;
7351                return Ok(());
7352            }
7353            if outcomes.iter().all(CommandOutcome::ok) {
7354                break;
7355            }
7356            match self.gate_fix_round(&winner, &outcomes).await? {
7357                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7358                GateFix::Stop => break,
7359                GateFix::Defer => {
7360                    self.state.save()?;
7361                    return Ok(());
7362                }
7363            }
7364        }
7365        let passed = outcomes.iter().all(CommandOutcome::ok);
7366        self.state.gate = outcomes;
7367        self.state.gate_ran = true;
7368        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7369        if !passed {
7370            self.state.status = RunStatus::Blocked;
7371            let spent = self.state.gate_fixes.len();
7372            self.state.event(
7373                "gate",
7374                if spent == 0 {
7375                    "gate failed; not merging".to_owned()
7376                } else {
7377                    format!("gate failed after {spent} gate-fix round(s); not merging")
7378                },
7379            );
7380        }
7381        self.state.save()?;
7382        Ok(())
7383    }
7384
7385    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
7386    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
7387    /// after review is clean and never on a candidate awaiting judging.
7388    ///
7389    /// Never fails the run: a non-zero exit or timeout is a warning and a
7390    /// recorded outcome, and the gate remains the single arbiter. Nothing
7391    /// configured means nothing happens - no event, no commit. `commit_all`
7392    /// commits any leftover change under the neutral identity and returns
7393    /// `false` when the tree is clean, so no empty commit is ever made.
7394    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7395        let commands = state.config.verify.pre_gate.clone();
7396        if commands.is_empty() {
7397            return;
7398        }
7399        let shell = state.config.shell();
7400        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7401        let (outcomes, _) = run_commands(
7402            state,
7403            "pre_gate",
7404            "pre_gate",
7405            0,
7406            &shell,
7407            &commands,
7408            &winner.worktree,
7409            timeout,
7410        )
7411        .await;
7412        for o in &outcomes {
7413            if !o.ok() {
7414                tracing::warn!(
7415                    "pre_gate `{}` failed ({:?}); the gate decides",
7416                    o.command,
7417                    o.code
7418                );
7419            }
7420            state.event(
7421                "pre_gate",
7422                format!(
7423                    "`{}` -> {}",
7424                    o.command,
7425                    if o.ok() {
7426                        "pass".to_owned()
7427                    } else {
7428                        format!(
7429                            "FAIL ({:?})\n{}",
7430                            o.code,
7431                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7432                        )
7433                    }
7434                ),
7435            );
7436        }
7437        state.pre_gate = outcomes;
7438        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7439            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7440                Ok(head) => {
7441                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7442                    state.pre_gate_commit = Some(head);
7443                }
7444                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7445            },
7446            Ok(false) => {}
7447            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7448        }
7449        if let Err(e) = state.save() {
7450            tracing::warn!("could not persist the pre_gate record: {e:#}");
7451        }
7452    }
7453
7454    /// Run `verify.gate` once against the winner's current tree, logging one
7455    /// event per command. Empty when nothing is configured.
7456    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7457        let wait = self.repository_wait();
7458        run_landing_gate(&mut self.state, winner, &wait).await
7459    }
7460
7461    /// One bounded fix round for a failing gate.
7462    ///
7463    /// The fixer is told the failure came from the gate itself, not from a
7464    /// reviewer, and is shown the failed commands, their exit codes and a tail
7465    /// of their output - whatever `[verify].gate` holds, nothing here knows
7466    /// what those commands run. Only a normal non-zero exit that printed
7467    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7468    /// command or a full disk says nothing about the code, and a fixer sent
7469    /// after it can only appease the machine. The round is judged by what git
7470    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7471    /// again before the gate does, so a fix cannot trade a green gate for a
7472    /// red e2e unnoticed.
7473    async fn gate_fix_round(
7474        &mut self,
7475        winner: &Candidate,
7476        outcomes: &[CommandOutcome],
7477    ) -> Result<GateFix> {
7478        let cap = self.state.config.graph.gate_fix_rounds;
7479        let spent = self.state.gate_fixes.len();
7480        if spent >= cap {
7481            if cap > 0 {
7482                self.state.event(
7483                    "gate",
7484                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7485                );
7486            }
7487            return Ok(GateFix::Stop);
7488        }
7489        if !gate_fixable(outcomes) {
7490            self.state.event(
7491                "gate",
7492                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7493                 command or similar); not spending a fix round on it",
7494            );
7495            return Ok(GateFix::Stop);
7496        }
7497        let min_free = self.state.config.disk.min_free_bytes;
7498        if min_free > 0 {
7499            match crate::disk::free_bytes(&winner.worktree) {
7500                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7501                Ok(free) => {
7502                    self.state.event(
7503                        "gate",
7504                        format!(
7505                            "only {free} bytes free ({min_free} required by `[disk] \
7506                             min_free_bytes`); not spending a fix round on a failure the disk \
7507                             may explain"
7508                        ),
7509                    );
7510                    return Ok(GateFix::Stop);
7511                }
7512                Err(e) => {
7513                    self.state.event(
7514                        "gate",
7515                        format!("free disk space could not be measured ({e:#}); no fix round"),
7516                    );
7517                    return Ok(GateFix::Stop);
7518                }
7519            }
7520        }
7521
7522        let attempt = spent + 1;
7523        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7524        let base = self.landing_base();
7525        let fix_prompt = prompt::gate_fix_in(
7526            &self.state.instruction,
7527            &failed,
7528            attempt,
7529            cap,
7530            &self.state.config.graph.language,
7531            &self.state.config.graph.github_language,
7532        );
7533        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7534        let sessions = self.state.config.graph.sessions;
7535        let artifacts = self.merge_repair_artifacts();
7536        self.state.event(
7537            "gate",
7538            format!("gate failed; gate-fix round {attempt} of {cap}"),
7539        );
7540        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7541        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7542        // Persist before invoking the fixer: it may commit and then be
7543        // interrupted before its result (or the following e2e) is saved.
7544        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7545            head: before.clone(),
7546            passed: false,
7547            outcomes: Vec::new(),
7548        });
7549        self.state.save()?;
7550        let (_, seat, out) = self
7551            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7552                prompt: fix_prompt.clone(),
7553                spec,
7554                seat,
7555                cwd: winner.worktree.clone(),
7556                timeout,
7557                allow_write: true,
7558                sessions,
7559                artifacts: artifacts.clone(),
7560                stem: format!("gate-fix-{attempt}"),
7561                handover: None,
7562            })
7563            .await;
7564        // The fixer was never asked: no round is spent (nothing is recorded)
7565        // and a later reentry asks the same agent again.
7566        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7567            return Ok(GateFix::Defer);
7568        }
7569        let mut record = GateFixRecord {
7570            agent: seat.agent.clone(),
7571            failed,
7572            notes: String::new(),
7573            committed: false,
7574            error: None,
7575        };
7576        match out {
7577            AgentOutcome::Ok(o) => {
7578                // A missing report is not a failed fix: the round is judged
7579                // by the tree below, and the report only carries prose.
7580                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7581                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7582                }
7583            }
7584            AgentOutcome::Dropped(_) => {
7585                record.error = Some("the CLI dropped the stream".to_owned());
7586            }
7587            AgentOutcome::Quota(o) => {
7588                self.state.quota.push(QuotaLoss {
7589                    seat: seat.key.clone(),
7590                    node: "gate-fix".to_owned(),
7591                    at: Timestamp::now(),
7592                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7593                });
7594                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7595            }
7596            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7597        }
7598        self.state.seats.insert(seat.key.clone(), seat);
7599        let coordination = self.coordinate(&winner.worktree).await?;
7600        if let Ok(r) = git::rescue_commit(
7601            &winner.worktree,
7602            &format!("magi: gate fix {attempt} (uncommitted work)"),
7603        )
7604        .await
7605        {
7606            self.state.note_withheld("gate-fix", &r.withheld);
7607        }
7608        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7609        record.committed = after != before;
7610        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7611        let note = record.error.clone();
7612        self.state.gate_fixes.push(record);
7613        self.state.save()?;
7614        if !changed {
7615            self.state.event(
7616                "gate",
7617                match note {
7618                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7619                    None => format!("gate-fix round {attempt}: the tree did not change"),
7620                },
7621            );
7622            return Ok(GateFix::Stop);
7623        }
7624        self.state.event(
7625            "gate",
7626            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7627        );
7628
7629        drop(coordination);
7630        let wait = self.repository_wait();
7631        check_gate_repair_e2e(&mut self.state, winner, &wait).await
7632    }
7633
7634    // ---------------------------------------------------------------- merge
7635
7636    /// One read-only rewrite by the summary's author; a text the gate still
7637    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7638    /// applies on their word or on silence. `None` means the run is parked on
7639    /// that question and nothing may be pushed or opened yet.
7640    async fn guarded_pr_message(
7641        &mut self,
7642        winner: &Candidate,
7643        facts: Option<&BranchFacts>,
7644        posting: bool,
7645    ) -> Result<Option<PrMessage>> {
7646        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7647        let head = match final_head.as_deref() {
7648            Some(h) => crate::pr_outcome::FinalHead::At(h),
7649            None => crate::pr_outcome::FinalHead::Unknown,
7650        };
7651        // Only a pull request that is about to be posted pays for a summary.
7652        let summarized = if posting {
7653            Some(self.summarize_change(winner, final_head.as_deref()).await)
7654        } else {
7655            None
7656        };
7657        // A busy shared resource is not a permanent cause: publishing an
7658        // "unavailable" description would stick (a later head move never
7659        // edits the body). Stop before anything is pushed or opened; the
7660        // run is `Blocked` and a resume asks the summary again.
7661        if self.state.resource_block.is_some() {
7662            self.state.event(
7663                "pr-summary",
7664                "a shared resource was busy; nothing was published, resume to summarize again",
7665            );
7666            return Ok(None);
7667        }
7668        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7669            anyhow::bail!(
7670                "the final commit moved during public description generation; retry before publishing"
7671            );
7672        }
7673        let note = match (&summarized, &self.state.change_summary) {
7674            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7675            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7676            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7677            (None, _) => ChangeNote::NotAsked,
7678        };
7679        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7680        if !posting {
7681            let identity = crate::scrub::Identity::current();
7682            return Ok(Some(PrMessage {
7683                title: crate::scrub::scrub(&pr.title, &identity),
7684                body: crate::scrub::scrub(&pr.body, &identity),
7685            }));
7686        }
7687        // This very text was already rejected and asked about: no second
7688        // judge call, rewrite or question, just carry the record on.
7689        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7690        if let Some(known) = self
7691            .state
7692            .github_text
7693            .clone()
7694            .filter(|g| g.fingerprint == raw_fp)
7695        {
7696            return self.settle_withheld(winner, &pr, known).await;
7697        }
7698        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7699        let mut violations =
7700            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7701        let identity = crate::scrub::Identity::current();
7702        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7703            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7704            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7705        {
7706            violations.push(crate::github_text::Violation::SensitiveData);
7707        }
7708        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7709        // only a language violation starts one; sensitive data alone goes
7710        // straight to the owner, with the redacted text as the fallback.
7711        let has_language = violations
7712            .iter()
7713            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7714        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7715            self.state.event(
7716                "github-text",
7717                if has_language {
7718                    format!("description rejected: {violations:?}; requesting one rewrite")
7719                } else {
7720                    format!("description rejected: {violations:?}; asking the owner")
7721                },
7722            );
7723            let mut rewritten = false;
7724            let mut rejected_rewrite = None;
7725            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7726            {
7727                let output = text;
7728                if let Some(title) = summary_title(&output) {
7729                    // Judge the redacted candidate that will actually be posted.
7730                    // Sensitive data alone must not discard a useful rewrite.
7731                    let raw_body = summary_without_title(&output);
7732                    let clean_title = crate::scrub::scrub(&title, &identity);
7733                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7734                    let redacted = clean_title != title || body != raw_body;
7735                    let title = clean_title;
7736                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7737                    if !complete {
7738                        self.state.event(
7739                            "github-text",
7740                            "description rewrite omitted required findings",
7741                        );
7742                    }
7743                    if !body.trim().is_empty() {
7744                        body.push_str("\n\n");
7745                        body.push_str(&public_pr_evidence(&self.state, head));
7746                    }
7747                    let marker = format!("magi:run/{}", self.state.id);
7748                    if !body.trim().is_empty() && !body.contains(&marker) {
7749                        body.push_str(&format!("\n\n{marker}"));
7750                    }
7751                    let rewrite = PrMessage {
7752                        title: title.clone(),
7753                        body: body.clone(),
7754                    };
7755                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7756                    let mut failed = crate::github_text::check_for(
7757                        &self.state.config.graph,
7758                        &title,
7759                        &body,
7760                        redecision,
7761                    );
7762                    if (!crate::github_text::shareable(&title)
7763                        || !crate::github_text::shareable(&body))
7764                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7765                    {
7766                        failed.push(crate::github_text::Violation::SensitiveData);
7767                    }
7768                    if complete
7769                        && !title.trim().is_empty()
7770                        && !body.trim().is_empty()
7771                        && !failed.is_empty()
7772                    {
7773                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7774                            title: title.clone(),
7775                            body: body.clone(),
7776                            failed: failed
7777                                .iter()
7778                                .map(|v| crate::github_text::category(*v).to_owned())
7779                                .collect(),
7780                        });
7781                    }
7782                    if complete
7783                        && !title.trim().is_empty()
7784                        && !raw_body.trim().is_empty()
7785                        && failed.is_empty()
7786                    {
7787                        decision = redecision;
7788                        pr = PrMessage { title, body };
7789                        rewritten = true;
7790                        self.state
7791                            .event("github-text", "description rewrite passed");
7792                        if redacted {
7793                            self.state.event(
7794                                "github-text",
7795                                "sensitive data removed from description rewrite",
7796                            );
7797                        }
7798                    }
7799                }
7800            }
7801            if !rewritten {
7802                self.state.event(
7803                    "github-text",
7804                    "description rewrite unavailable or rejected; asking the owner",
7805                );
7806                if let Some(w) =
7807                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7808                {
7809                    // A different text than the one asked about before: that
7810                    // question no longer describes anything.
7811                    self.retire_github_text_question("the withheld text changed");
7812                    let record = crate::run::GithubTextAsk {
7813                        rewrite: rejected_rewrite,
7814                        fingerprint: raw_fp,
7815                        title: w.title,
7816                        body: w.body,
7817                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7818                        question: None,
7819                        asks: 0,
7820                        resolved: false,
7821                        chosen_title: None,
7822                        choice: None,
7823                    };
7824                    return self.settle_withheld(winner, &pr, record).await;
7825                }
7826            }
7827        }
7828        let (title, body) =
7829            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7830        Ok(Some(PrMessage { title, body }))
7831    }
7832
7833    /// Describe the whole change from its final diff in either run mode.
7834    ///
7835    /// The summary is tied to the final commit and to the merge base of the
7836    /// resolved remote base; a record for any other pair is dropped and the
7837    /// agent asked again, so a description never outlives the diff it was
7838    /// written from. The seat is read-only and the prompt tells it to read the
7839    /// diff and the code around it. `[roles] synthesizer` selects the agents
7840    /// (unset: the default pick), each asked at most once, advancing on an
7841    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7842    /// carries the reason it is unavailable; nothing is invented then.
7843    async fn summarize_change(
7844        &mut self,
7845        winner: &Candidate,
7846        head: Option<&str>,
7847    ) -> std::result::Result<(), String> {
7848        let Some(head) = head else {
7849            return Err("the final commit could not be read".to_owned());
7850        };
7851        let remote = self.state.config.merge.remote.clone();
7852        let tracking = format!("{remote}/{}", self.state.base_branch);
7853        if !git::rev_exists(&winner.worktree, &tracking).await {
7854            return Err(format!("the remote base `{tracking}` could not be read"));
7855        }
7856        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7857            Ok(b) if !b.is_empty() => b,
7858            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7859        };
7860        let evidence = crate::pr_outcome::summary_evidence(
7861            &self.state,
7862            crate::pr_outcome::FinalHead::At(head),
7863        );
7864        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7865        if self
7866            .state
7867            .change_summary
7868            .as_ref()
7869            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7870        {
7871            return Ok(());
7872        }
7873        if self.state.change_summary.take().is_some() {
7874            self.state.event(
7875                "pr-summary",
7876                "the final diff or evidence changed; the earlier change summary was discarded",
7877            );
7878        }
7879        // Save the source snapshot even if no generation agent is available.
7880        if let Err(e) = write_artifact(
7881            &self.state,
7882            &format!("public-description-{fingerprint}.md"),
7883            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7884        ) {
7885            tracing::warn!("could not save public description evidence: {e:#}");
7886        }
7887        let chain = match agent::pick_chain(
7888            &self.state.config.agents,
7889            self.state.config.roles.synthesizer.as_ref(),
7890            &agent::installed,
7891            "synthesizer",
7892        ) {
7893            Ok(chain) => chain,
7894            Err(e) => {
7895                tracing::warn!("no agent for the change summary: {e:#}");
7896                self.state.event("pr-summary", "no summary agent available");
7897                return Err("no agent was available to summarize it".to_owned());
7898            }
7899        };
7900        let subjects = git::subjects(&winner.worktree, &base, head)
7901            .await
7902            .unwrap_or_default();
7903        let stat = git::diff_stat(&winner.worktree, &base, head)
7904            .await
7905            .unwrap_or_default();
7906        let stat: String = stat.chars().take(6_000).collect();
7907        let mut prompt = crate::prompt::change_summary_in(
7908            &base,
7909            head,
7910            &subjects,
7911            stat.trim(),
7912            &self.state.config.graph.github_language,
7913        );
7914        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7915        let sessions = self.state.config.graph.sessions;
7916        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7917        let prompts = self.state.config.prompts.clone();
7918        let cache = self.state.config.cache_dir();
7919        let run = self.state.id.clone();
7920        for (i, spec) in chain.into_iter().enumerate() {
7921            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7922            let github_spec = match agent::for_github_text(
7923                &spec,
7924                &self.state.config.graph.github_language,
7925                &winner.worktree,
7926            ) {
7927                Ok(spec) => spec,
7928                Err(e) => {
7929                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7930                    continue;
7931                }
7932            };
7933            let job = SeatJob {
7934                spec: github_spec,
7935                seat,
7936                cwd: winner.worktree.clone(),
7937                prompt: prompt.clone(),
7938                timeout,
7939                allow_write: false,
7940                sessions,
7941                artifacts: agent::artifacts_dir(&self.state.dir()),
7942                stem: if i == 0 {
7943                    "change-summary".to_owned()
7944                } else {
7945                    format!("change-summary-{}", spec.id)
7946                },
7947                handover: None,
7948            };
7949            let ctx = WaveCtx {
7950                run: &run,
7951                node: "pr-summary",
7952                prompts: &prompts,
7953                cache: cache.as_deref(),
7954                round: None,
7955                carry_seats: false,
7956                pauses: Some((&self.pause, &self.interrupt)),
7957            };
7958            let (seat, outcome) =
7959                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7960            self.state.seats.insert(seat.key.clone(), seat);
7961            let text = match outcome {
7962                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7963                AgentOutcome::ResourceBlocked(why) => {
7964                    // No agent was asked: the run stops resumably before
7965                    // anything is published (see `guarded_pr_message`).
7966                    if self.state.resource_block.is_none() {
7967                        self.state.resource_block = Some(crate::run::ResourceBlock {
7968                            cause: crate::run::ResourceBlockCause::Other,
7969                            node: "pr-summary".to_owned(),
7970                            kind: "shared resource".to_owned(),
7971                            detail: why,
7972                            at: Timestamp::now(),
7973                        });
7974                    }
7975                    return Err("a shared resource was busy".to_owned());
7976                }
7977                _ => continue,
7978            };
7979            let blind = &self.state.config.blind;
7980            let Some(title) = summary_title(&text) else {
7981                continue;
7982            };
7983            let body: String = summary_without_title(&text)
7984                .lines()
7985                .filter(|l| !l.contains("magi:run/"))
7986                .collect::<Vec<_>>()
7987                .join("\n");
7988            let body = blind::sanitize_prose(body.trim(), blind);
7989            let title = blind::sanitize_prose(&title, blind);
7990            if body.trim().is_empty() || title.trim().is_empty() {
7991                continue;
7992            }
7993            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7994                continue;
7995            }
7996            // The branch must still be the one the agent was shown.
7997            if git::rev_parse(&winner.worktree, "HEAD")
7998                .await
7999                .ok()
8000                .as_deref()
8001                != Some(head)
8002            {
8003                return Err("the branch moved while the summary was written".to_owned());
8004            }
8005            if git::merge_base(&winner.worktree, &tracking, head)
8006                .await
8007                .ok()
8008                .as_deref()
8009                != Some(base.as_str())
8010                || crate::pr_outcome::summary_evidence(
8011                    &self.state,
8012                    crate::pr_outcome::FinalHead::At(head),
8013                ) != evidence
8014            {
8015                return Err("the summary evidence changed while it was written".to_owned());
8016            }
8017            self.state.change_summary = Some(crate::run::ChangeSummary {
8018                head: head.to_owned(),
8019                base,
8020                evidence: fingerprint,
8021                title,
8022                body,
8023            });
8024            self.state.event(
8025                "pr-summary",
8026                format!("change summary written by {} for the final diff", spec.id),
8027            );
8028            if let Err(e) = self.state.save() {
8029                tracing::warn!("could not save the change summary: {e:#}");
8030            }
8031            return Ok(());
8032        }
8033        self.state
8034            .event("pr-summary", "no agent produced a change summary");
8035        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8036    }
8037
8038    /// One read-only rewrite request for a withheld description. The
8039    /// implementer's own conversation is tried first when it exists and can be
8040    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8041    /// writes it in a separate conversation, skipping the implementer's agent.
8042    /// The prompt carries the whole text, so no agent needs any memory. Each
8043    /// agent is asked at most once and the chain advances on an error, quota,
8044    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8045    /// that has run out of quota must not end the rewrite. Nothing is recorded
8046    /// as a handover.
8047    async fn rewrite_pr_text(
8048        &mut self,
8049        winner: &Candidate,
8050        pr: &PrMessage,
8051        violations: &[crate::github_text::Violation],
8052    ) -> Option<String> {
8053        let prompt = format!(
8054            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8055            pr.title, pr.body
8056        );
8057        let sessions = self.state.config.graph.sessions;
8058        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8059        // (spec, the resumed implementer's seat). A fallback seat is created
8060        // only when its turn comes, so an unused fallback leaves no record.
8061        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8062        let key = format!("impl-{}", winner.label);
8063        if let Some(spec) = self
8064            .state
8065            .config
8066            .agents
8067            .iter()
8068            .find(|a| a.id == winner.agent)
8069            .cloned()
8070            && let Some(existing) = self.state.seats.get(&key)
8071            && existing.agent == spec.id
8072            && agent::has_session(spec.kind, existing, sessions)
8073        {
8074            attempts.push((spec, Some(existing.clone())));
8075        }
8076        match agent::pick_chain(
8077            &self.state.config.agents,
8078            self.state.config.roles.language_judge.as_ref(),
8079            &agent::installed,
8080            "language_judge",
8081        ) {
8082            Ok(chain) => {
8083                for spec in chain {
8084                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8085                        continue;
8086                    }
8087                    attempts.push((spec, None));
8088                }
8089            }
8090            Err(e) if attempts.is_empty() => {
8091                tracing::warn!("no agent for the description rewrite: {e:#}");
8092                self.state
8093                    .event("github-text", "no rewrite agent available");
8094                return None;
8095            }
8096            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
8097        }
8098        let prompts = self.state.config.prompts.clone();
8099        let cache = self.state.config.cache_dir();
8100        let run = self.state.id.clone();
8101        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
8102            let resumed = resumed_seat.is_some();
8103            let seat = resumed_seat.unwrap_or_else(|| {
8104                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
8105            });
8106            let github_spec = match agent::for_github_text(
8107                &spec,
8108                &self.state.config.graph.github_language,
8109                &winner.worktree,
8110            ) {
8111                Ok(spec) => spec,
8112                Err(e) => {
8113                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8114                    continue;
8115                }
8116            };
8117            let job = SeatJob {
8118                spec: github_spec,
8119                seat,
8120                cwd: winner.worktree.clone(),
8121                prompt: prompt.clone(),
8122                timeout: retry_budget(timeout_full, resumed),
8123                allow_write: false,
8124                sessions,
8125                artifacts: agent::artifacts_dir(&self.state.dir()),
8126                stem: if i == 0 {
8127                    "github-text-rewrite".to_owned()
8128                } else {
8129                    format!("github-text-rewrite-{}", spec.id)
8130                },
8131                handover: None,
8132            };
8133            let ctx = WaveCtx {
8134                run: &run,
8135                node: "github-text",
8136                prompts: &prompts,
8137                cache: cache.as_deref(),
8138                round: None,
8139                carry_seats: false,
8140                pauses: Some((&self.pause, &self.interrupt)),
8141            };
8142            let (seat, outcome) =
8143                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8144            self.state.seats.insert(seat.key.clone(), seat);
8145            match outcome {
8146                AgentOutcome::Ok(o)
8147                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
8148                {
8149                    return Some(o.text);
8150                }
8151                AgentOutcome::ResourceBlocked(_) => return None,
8152                _ => {}
8153            }
8154        }
8155        None
8156    }
8157
8158    /// Abandon the standing posting-gate question, if any.
8159    fn retire_github_text_question(&mut self, why: &str) {
8160        let Some(id) = self
8161            .state
8162            .github_text
8163            .as_ref()
8164            .and_then(|g| g.question.clone())
8165        else {
8166            return;
8167        };
8168        let _ = ask::Questions::open().update(&id, |q| {
8169            q.abandon(why);
8170            Ok(())
8171        });
8172    }
8173
8174    /// Carry a withheld text to its end: file the question, park while it is
8175    /// open, and on an answer, silence or timeout pick the text to post. The
8176    /// decision is saved before it is used, so it applies once.
8177    async fn settle_withheld(
8178        &mut self,
8179        winner: &Candidate,
8180        pr: &PrMessage,
8181        mut g: crate::run::GithubTextAsk,
8182    ) -> Result<Option<PrMessage>> {
8183        use crate::github_text as gt;
8184        let store = ask::Questions::open();
8185        let timeout = self.state.config.graph.answer_timeout;
8186        if !g.resolved {
8187            let mut standing = g
8188                .question
8189                .as_ref()
8190                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
8191            if let Some(q) = standing.as_ref()
8192                && q.status.open()
8193                && gt::expired(q, timeout)
8194            {
8195                let why = format!("no answer within {}s of asking", timeout.max(1));
8196                standing = store
8197                    .update(&q.id, |q| {
8198                        q.abandon(&why);
8199                        Ok(())
8200                    })
8201                    .ok()
8202                    .map(|(q, ())| q);
8203            }
8204            let mut ask_again: Option<Option<Vec<String>>> = None;
8205            match standing {
8206                // Recorded but never written (a stop between the two saves),
8207                // or removed: ask again without spending the retry.
8208                None if g.asks <= 1 => {
8209                    g.asks = 0;
8210                    ask_again = Some(None);
8211                }
8212                None => {
8213                    g.resolved = true;
8214                    g.chosen_title = None;
8215                    g.choice = Some(GithubTextChoice::Fallback);
8216                }
8217                Some(q) if q.status.open() => {
8218                    self.park_on_github_text(&g)?;
8219                    return Ok(None);
8220                }
8221                Some(q) => match gt::read_reply(&q) {
8222                    gt::Reply::Fallback => {
8223                        g.resolved = true;
8224                        g.chosen_title = None;
8225                        g.choice = Some(GithubTextChoice::Fallback);
8226                    }
8227                    gt::Reply::Original => {
8228                        g.resolved = true;
8229                        g.chosen_title = None;
8230                        g.choice = Some(GithubTextChoice::Original);
8231                    }
8232                    gt::Reply::Rewrite => {
8233                        g.resolved = true;
8234                        g.chosen_title = None;
8235                        g.choice = Some(if g.rewrite.is_some() {
8236                            GithubTextChoice::Rewrite
8237                        } else {
8238                            GithubTextChoice::Fallback
8239                        });
8240                    }
8241                    gt::Reply::Title(text) => {
8242                        let first = text
8243                            .lines()
8244                            .map(str::trim)
8245                            .find(|l| !l.is_empty())
8246                            .unwrap_or("");
8247                        let decision = if first.is_empty() {
8248                            None
8249                        } else {
8250                            let candidate = PrMessage {
8251                                title: first.to_owned(),
8252                                body: String::new(),
8253                            };
8254                            self.judge_pr_language(&winner.worktree, &candidate).await
8255                        };
8256                        match gt::vet_title(
8257                            &text,
8258                            decision,
8259                            self.state.config.graph.github_english(),
8260                        ) {
8261                            Ok(title) => {
8262                                g.resolved = true;
8263                                g.chosen_title = Some(title);
8264                                g.choice = Some(GithubTextChoice::MyTitle);
8265                            }
8266                            Err(cats) if g.asks < 2 => {
8267                                ask_again = Some(Some(
8268                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8269                                ))
8270                            }
8271                            Err(_) => {
8272                                g.resolved = true;
8273                                g.chosen_title = None;
8274                                g.choice = Some(GithubTextChoice::Fallback);
8275                            }
8276                        }
8277                    }
8278                },
8279            }
8280            if let Some(rejected) = ask_again {
8281                // A retry (after a rejected replacement title) is still about
8282                // the original text: the same ORIGINAL / FALLBACK comparison.
8283                let w = gt::Withheld {
8284                    title: g.title,
8285                    body: g.body,
8286                    categories: g.categories.clone(),
8287                };
8288                // Named by the original text's fingerprint alone, so a resume
8289                // rewrites the same file and the detail stays identical.
8290                let artifact = match crate::run::write_artifact(
8291                    &self.state,
8292                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
8293                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8294                ) {
8295                    Ok(p) => Some(p),
8296                    Err(e) => {
8297                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
8298                        None
8299                    }
8300                };
8301                let (detail, comparison) = gt::question_content(
8302                    &w,
8303                    &pr.title,
8304                    &pr.body,
8305                    &self.state.id,
8306                    rejected.as_deref(),
8307                    artifact.as_deref(),
8308                    g.rewrite.as_ref(),
8309                );
8310                let mut q = ask::Question::new(
8311                    self.state.id.clone(),
8312                    gt::ASK_NODE.to_owned(),
8313                    gt::ASK_SEAT.to_owned(),
8314                    gt::question_summary(&self.state.config.graph.language, &w),
8315                    detail,
8316                    gt::question_choices(&w, g.rewrite.as_ref()),
8317                );
8318                q.posting_comparison = Some(comparison);
8319                q.answer_timeout = timeout;
8320                // Recorded before it is written: a stop in between leaves a
8321                // record naming a question the resume then files again.
8322                g.question = Some(q.id.clone());
8323                g.asks += 1;
8324                self.state.github_text = Some(g.clone());
8325                self.state.event(
8326                    "github-text",
8327                    format!("asking the owner about the withheld text ({})", q.short()),
8328                );
8329                self.state.save()?;
8330                store
8331                    .put_executable(&mut q)
8332                    .context("file the posting-gate question")?;
8333                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8334                    tracing::warn!(
8335                        "could not notify about posting-gate question {}: {e:#}",
8336                        q.short()
8337                    );
8338                }
8339                self.park_on_github_text(&g)?;
8340                return Ok(None);
8341            }
8342        }
8343        let resolved = g.chosen_title.clone();
8344        let original = g.choice == Some(GithubTextChoice::Original);
8345        // Field names only: the event log never records the text.
8346        self.state.event(
8347            "github-text",
8348            if original {
8349                format!(
8350                    "posting the original text for: {}",
8351                    match (g.title, g.body) {
8352                        (true, true) => "title, description",
8353                        (true, false) => "title",
8354                        _ => "description",
8355                    }
8356                )
8357            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8358                "posting the saved rewrite title and description".to_owned()
8359            } else if resolved.is_some() {
8360                "posting the owner's replacement title".to_owned()
8361            } else {
8362                "posting the neutral text for the withheld fields".to_owned()
8363            },
8364        );
8365        self.state.github_text = Some(g.clone());
8366        self.state.save()?;
8367        if g.choice == Some(GithubTextChoice::Rewrite)
8368            && let Some(r) = g.rewrite.as_ref()
8369        {
8370            return Ok(Some(PrMessage {
8371                title: r.title.clone(),
8372                body: r.body.clone(),
8373            }));
8374        }
8375        let w = gt::Withheld {
8376            title: g.title,
8377            body: g.body,
8378            categories: g.categories.clone(),
8379        };
8380        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8381        if let Some(t) = resolved {
8382            title = t;
8383        }
8384        // Every withheld field is now neutral or vetted English.
8385        let english = gt::LanguageDecision {
8386            title_english: true,
8387            body_english: true,
8388            body_complete: true,
8389        };
8390        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8391        if original {
8392            // The owner's explicit choice overrides the gate for the fields it
8393            // was made for; the others keep what the gate produced.
8394            if g.title {
8395                title = pr.title.clone();
8396            }
8397            if g.body {
8398                body = pr.body.clone();
8399            }
8400        }
8401        Ok(Some(PrMessage { title, body }))
8402    }
8403
8404    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8405        self.state.github_text = Some(g.clone());
8406        self.state.end_pass_parked(ParkCause::Question);
8407        self.state.parked = true;
8408        self.state.event(
8409            "github-text",
8410            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8411        );
8412        self.state.save()
8413    }
8414
8415    /// Ask `[roles] language_judge` about this text and record which source
8416    /// decided; the text itself is never recorded.
8417    async fn judge_pr_language(
8418        &mut self,
8419        cwd: &Path,
8420        pr: &PrMessage,
8421    ) -> Option<crate::github_text::LanguageDecision> {
8422        self.state.config.roles.language_judge.as_ref()?;
8423        if !self.state.config.graph.language_gate() {
8424            return None;
8425        }
8426        let decision =
8427            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8428        self.state.event(
8429            "github-text",
8430            if decision.is_some() {
8431                "language decided by the language judge"
8432            } else {
8433                "language judge unavailable; using built-in heuristics"
8434            },
8435        );
8436        decision
8437    }
8438
8439    async fn merge(&mut self) -> Result<()> {
8440        if self.hold_base_sync().await? {
8441            return Ok(());
8442        }
8443        // Same reasoning as `gate`: ask the review and gate records directly
8444        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8445        // skip can rewrite on reentry to something that no longer says
8446        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8447        // so a hand-off (open findings, green verification) reaches merge
8448        // exactly like a genuinely clean round does.
8449        //
8450        // A run resumed mid-`land` never reaches here at all: `execute`
8451        // recognises `RunStatus::Landing` before it even calls `prep`, and
8452        // routes straight to `run_land` instead. That has to happen a level
8453        // up from this function, not with a check in here, because
8454        // `review_loop`'s own status recomputation (see its doc) runs
8455        // *before* `merge` on every reentry and would otherwise overwrite
8456        // the `Landing` marker with `Gating` before this node ever saw it.
8457        if self
8458            .state
8459            .base_sync
8460            .as_ref()
8461            .is_some_and(|s| s.conflict.is_some())
8462            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8463                != Some(RunStatus::Gating)
8464            // `gate_ran == false` is not "passed" - `gate` leaves it false
8465            // both before it has ever run and when its last attempt was
8466            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8467            // permission to merge on nothing but the review record. Only a
8468            // gate that actually ran may proceed; an intentional local
8469            // review can also finish with verification explicitly not configured.
8470            // `RunState::gate_status` computes that distinction.
8471            || !self.state.gate_allows_progress()
8472        {
8473            return Ok(());
8474        }
8475        // This node's own record, not `status`: `status == Ready` is not
8476        // unique to the harmless `MergeMode::None` path this line was
8477        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8478        // run's PR was closed without merging — and on that run `mode` is
8479        // still `Pr`, so a reentry that fell through here would push and
8480        // open a second pull request. `self.state.merge` is set exactly once
8481        // this node (or `land`) has already produced a verdict, under every
8482        // mode, which is what "already done" actually means here.
8483        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8484            return Ok(());
8485        }
8486        let Some(winner) = self.state.winner().cloned() else {
8487            return Ok(());
8488        };
8489        let repo = self.state.repo.clone();
8490        let base = self.state.base_branch.clone();
8491        let mode = self.state.config.merge.mode;
8492        let style = self.state.config.merge.style;
8493        let facts = if is_review_run(&self.state) {
8494            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8495            let start = review_base(
8496                &repo,
8497                &self.state.config.merge.remote,
8498                &base,
8499                &self.state.base_commit,
8500                &winner.branch,
8501            )
8502            .await;
8503            branch_facts(&repo, &start, &winner.branch).await
8504        } else {
8505            None
8506        };
8507        // `None` when the base could not be freshly read: then an adopted
8508        // pull request's title is left alone.
8509        let leaked = if is_review_run(&self.state) {
8510            leaked_subjects(&self.state, &winner.branch).await
8511        } else {
8512            Some(Vec::new())
8513        };
8514        // `None`: parked on the owner's word about a withheld title or body.
8515        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8516        // the resume arrives here again.
8517        let Some(pr) = self
8518            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8519            .await?
8520        else {
8521            return Ok(());
8522        };
8523        let message = pr.commit_message();
8524
8525        let outcome = match mode {
8526            MergeMode::None => MergeOutcome {
8527                mode,
8528                ok: true,
8529                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8530                empty: false,
8531            },
8532            MergeMode::Pr | MergeMode::Local
8533                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8534            {
8535                MergeOutcome {
8536                    mode,
8537                    ok: false,
8538                    detail: empty_candidate_detail(&self.state, &base),
8539                    empty: true,
8540                }
8541            }
8542            MergeMode::Local => {
8543                let on = git::current_branch(&repo).await?;
8544                if on.as_deref() != Some(base.as_str()) {
8545                    MergeOutcome {
8546                        mode,
8547                        ok: false,
8548                        detail: format!(
8549                            "{} has {} checked out, not the base branch {base}",
8550                            repo.display(),
8551                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8552                        ),
8553                        empty: false,
8554                    }
8555                } else if !git::is_clean(&repo).await? {
8556                    MergeOutcome {
8557                        mode,
8558                        ok: false,
8559                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8560                        empty: false,
8561                    }
8562                } else {
8563                    let out = match style {
8564                        MergeStyle::Merge => {
8565                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8566                        }
8567                        MergeStyle::Squash => {
8568                            git::merge_squash(&repo, &winner.branch, &message).await?
8569                        }
8570                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8571                    };
8572                    MergeOutcome {
8573                        mode,
8574                        ok: out.ok(),
8575                        detail: if out.ok() { out.stdout } else { out.stderr },
8576                        empty: false,
8577                    }
8578                }
8579            }
8580            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8581                self.state.pr_publication_pending = true;
8582                self.state.save()?;
8583                let out =
8584                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8585                self.state.pr_publication_pending = out.is_err();
8586                MergeOutcome {
8587                    mode,
8588                    ok: out.is_ok(),
8589                    detail: match out {
8590                        Ok(url) => url,
8591                        Err(e) => format!("{e:#}"),
8592                    },
8593                    empty: false,
8594                }
8595            }
8596            MergeMode::Pr => {
8597                let remote = self.state.config.merge.remote.clone();
8598                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8599                if !pushed.ok() {
8600                    MergeOutcome {
8601                        mode,
8602                        ok: false,
8603                        detail: pushed.stderr,
8604                        empty: false,
8605                    }
8606                } else {
8607                    // A retry or resume of a run whose branch already has an
8608                    // open pull request adopts it rather than failing on a
8609                    // duplicate. Only this winner branch into this base:
8610                    // `branch_for` derives the name from the run id, so a
8611                    // different run's pull request never matches.
8612                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8613                    let out = match pr_merge_plan(found) {
8614                        PrPlan::Create => {
8615                            gh_pr_create(
8616                                &winner.worktree,
8617                                &base,
8618                                &winner.branch,
8619                                &pr.title,
8620                                &pr.body,
8621                            )
8622                            .await
8623                        }
8624                        PrPlan::Adopt { url, title } => {
8625                            self.state
8626                                .event("merge", format!("Pr: adopted open pull request {url}"));
8627                            if self.state.github_text.as_ref().is_some_and(|g| {
8628                                g.resolved
8629                                    && g.choice == Some(GithubTextChoice::Rewrite)
8630                                    && g.rewrite.is_some()
8631                            }) {
8632                                // The owner chose the entire pair, even if the
8633                                // existing title already matches or was edited.
8634                                land::set_pr_text_verbatim(
8635                                    &winner.worktree,
8636                                    &url,
8637                                    &pr.title,
8638                                    &pr.body,
8639                                )
8640                                .await
8641                                .map(|()| url)
8642                            } else {
8643                                if title != pr.title
8644                                    && (!is_review_run(&self.state)
8645                                        || leaked
8646                                            .as_deref()
8647                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8648                                    && let Err(e) =
8649                                        if self.state.github_text.as_ref().is_some_and(|g| {
8650                                            g.resolved
8651                                                && g.title
8652                                                && g.choice == Some(GithubTextChoice::Original)
8653                                        }) {
8654                                            // The owner chose to post this title unmodified.
8655                                            land::set_pr_title_verbatim(
8656                                                &winner.worktree,
8657                                                &url,
8658                                                &pr.title,
8659                                            )
8660                                            .await
8661                                        } else {
8662                                            land::set_pr_title(
8663                                                &mut self.state,
8664                                                &winner.worktree,
8665                                                &url,
8666                                                &pr.title,
8667                                            )
8668                                            .await
8669                                        }
8670                                {
8671                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8672                                    self.state
8673                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8674                                }
8675                                Ok(url)
8676                            }
8677                        }
8678                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8679                    };
8680                    match out {
8681                        Ok(url) => MergeOutcome {
8682                            mode,
8683                            ok: true,
8684                            detail: url,
8685                            empty: false,
8686                        },
8687                        Err(e) => MergeOutcome {
8688                            mode,
8689                            ok: false,
8690                            detail: e.to_string(),
8691                            empty: false,
8692                        },
8693                    }
8694                }
8695            }
8696        };
8697
8698        self.state.status = match (mode, outcome.ok) {
8699            (MergeMode::None, _) => RunStatus::Ready,
8700            (_, true) if outcome.empty => RunStatus::Ready,
8701            (MergeMode::Pr, true) => RunStatus::Ready,
8702            (_, true) => RunStatus::Merged,
8703            (_, false) => RunStatus::Blocked,
8704        };
8705        self.state.event(
8706            "merge",
8707            format!(
8708                "{:?}: {}",
8709                mode,
8710                outcome.detail.lines().next().unwrap_or("")
8711            ),
8712        );
8713        self.state.merge = Some(outcome);
8714        self.state.save()?;
8715
8716        // The PR is open and the run would historically stop here, leaving the
8717        // operator to watch checks, feed review comments back to a fixer, and
8718        // merge. That was done by hand six times in one session before this
8719        // existed. Opt-in, because merging is the one irreversible thing magi
8720        // can do to a repository.
8721        if self.state.config.graph.land
8722            && mode == MergeMode::Pr
8723            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8724        {
8725            self.run_land().await?;
8726        }
8727        // `run_land` may have left `status` at `Landing` - still waiting on
8728        // CI or the owner's approval, not actually settled - so this has to
8729        // read whatever `status` ended up as here, not the `Merged` this
8730        // function set a few lines up.
8731        self.settle_questions();
8732        Ok(())
8733    }
8734
8735    /// Enter `land`.
8736    ///
8737    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8738    /// a resumed run's re-entry. `land::land` itself is what serialises the
8739    /// two git-mutating moments inside the loop — the rebase push and
8740    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8741    /// to hold a lock across the whole call, and doing so would serialise
8742    /// this run's CI wait against a *different* run's land-approval resume
8743    /// in the same repository, which is exactly the "must not wait on
8744    /// another task" property the daemon's slot-freeing exists to give.
8745    async fn run_land(&mut self) -> Result<()> {
8746        crate::pr_head::ensure_owner(&self.state)?;
8747        let url = self
8748            .state
8749            .pr
8750            .as_ref()
8751            .map(|pr| pr.url.clone())
8752            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8753            .unwrap_or_default();
8754        let url = url.lines().next().unwrap_or("").trim().to_owned();
8755        if !url.starts_with("http") {
8756            return Ok(());
8757        }
8758        // A land failure is not a lost run: the work is on a branch and the
8759        // pull request is open, which is exactly where a human takes over.
8760        loop {
8761            match land::land(&mut self.state, &url).await {
8762                Ok(_)
8763                    if self
8764                        .state
8765                        .merge_repairs
8766                        .last()
8767                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8768                {
8769                    if self.repair_merge_review().await? {
8770                        continue;
8771                    }
8772                }
8773                Ok(pr) if self.state.parked => {
8774                    // `land` already saved the parked marker; nothing here
8775                    // overrides `status` back to a terminal value while an
8776                    // approval is still outstanding.
8777                    let _ = pr;
8778                }
8779                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8780                    self.state.status = RunStatus::Superseded;
8781                    self.state.save()?;
8782                }
8783                Ok(pr) => {
8784                    self.state.status = match pr.state {
8785                        land::PrLifecycle::Merged => RunStatus::Merged,
8786                        _ => RunStatus::Blocked,
8787                    };
8788                    // Downstream of a confirmed merge only - see
8789                    // `bump::should_release_bump`'s own doc for why this one
8790                    // check covers all three of `land`'s success paths.
8791                    // Best-effort: the run already landed, so a failure here
8792                    // (the decision call, `gh`, `cargo`) is recorded and never
8793                    // turns a landed run into a failed one.
8794                    if bump::should_release_bump(self.state.status)
8795                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8796                    {
8797                        // The event is the run's own record. Not-eligible cases
8798                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8799                        // `Err` is a bump that was tried and failed:
8800                        // `after_merge` itself raises the operator notice for
8801                        // that, whether or not a release PR exists yet.
8802                        self.state
8803                            .event("bump", format!("release bump skipped: {e:#}"));
8804                    }
8805                    // Independent of the bump, and best-effort in the same way:
8806                    // findings the merge left open become follow-up tasks.
8807                    if self.state.status == RunStatus::Merged {
8808                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8809                    }
8810                    self.state.save()?;
8811                }
8812                Err(e) => {
8813                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8814                        return Err(e);
8815                    }
8816                    self.state.status = RunStatus::Blocked;
8817                    self.state.event("land", format!("gave up: {e}"));
8818                    self.state.save()?;
8819                }
8820            }
8821            break;
8822        }
8823        Ok(())
8824    }
8825
8826    /// One repair cycle stays under the existing driver/task claim. An
8827    /// interrupted fixer is rolled back and held rather than run twice.
8828    async fn repair_merge_review(&mut self) -> Result<bool> {
8829        use crate::run::MergeRepairPhase as Phase;
8830        crate::pr_head::ensure_owner(&self.state)?;
8831        let i = self.state.merge_repairs.len() - 1;
8832        let request = self.state.merge_repairs[i].clone();
8833        if request.phase == Phase::Fixing {
8834            let restored = match self.state.winner().cloned() {
8835                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8836                None => Err(anyhow::anyhow!("repair has no winning branch")),
8837            };
8838            let why = match restored {
8839                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8840                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8841                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8842            };
8843            self.hold_merge_repair(i, why)?;
8844            return Ok(false);
8845        }
8846        let failure = if i >= self.state.config.graph.land_rounds
8847            || self.state.config.graph.review_rounds == 0
8848        {
8849            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8850        } else {
8851            None
8852        };
8853        if let Some(why) = failure {
8854            self.hold_merge_repair(i, why)?;
8855            return Ok(false);
8856        }
8857        if request.phase == Phase::Requested {
8858            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8859                Ok(pr) => pr,
8860                Err(e) => {
8861                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8862                    return Ok(false);
8863                }
8864            };
8865            self.state.merge_repairs[i].phase = Phase::Fixing;
8866            self.state.land_approval = None;
8867            self.state.land_armed_head = None;
8868            self.state.save()?;
8869            // Serialize the exact records, including finding ids/details,
8870            // reconsideration reasons, final votes and verification output.
8871            let evidence = serde_json::to_string_pretty(&request)?;
8872            let reason = format!(
8873                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8874            );
8875            let budget = self.state.config.graph.land_rounds;
8876            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8877            match result {
8878                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8879                Ok(land::Fixed::Failed(why)) => {
8880                    self.hold_merge_repair(
8881                        i,
8882                        format!(
8883                            "repair failed: {why}; inspect the branch and run an explicit review"
8884                        ),
8885                    )?;
8886                    return Ok(false);
8887                }
8888                Err(e) => {
8889                    self.hold_merge_repair(
8890                        i,
8891                        format!(
8892                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8893                        ),
8894                    )?;
8895                    return Ok(false);
8896                }
8897            }
8898            // Even no change gets a real new reviewer wave. The previous
8899            // evidence remains in the request; a former clean/concluded round
8900            // cannot short-circuit the new review or gate.
8901            self.begin_merge_repair_review(i)?;
8902        }
8903        self.review_loop().await?;
8904        self.gate().await?;
8905        if self.resource_halt()? || self.state.parked {
8906            return Ok(false);
8907        }
8908        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8909            != Some(RunStatus::Gating)
8910            || !self.state.gate_allows_progress()
8911        {
8912            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8913            return Ok(false);
8914        }
8915        // Review and gate may have added commits after the initial repair.
8916        // Publish exactly their final head to the existing PR, with a lease.
8917        let winner = self
8918            .state
8919            .winner()
8920            .cloned()
8921            .context("repair has no winning branch")?;
8922        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8923        if self.state.gate_head.as_deref() != Some(after.as_str())
8924            || !git::is_clean(&winner.worktree).await?
8925        {
8926            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8927            return Ok(false);
8928        }
8929        let expected = self
8930            .state
8931            .pr_pushed_head
8932            .as_deref()
8933            .unwrap_or(&request.head)
8934            .to_owned();
8935        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8936        // Push the checked commit itself: a local ref moving after the check
8937        // must not change which commit the forge receives.
8938        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8939        let refspec = format!("{after}:refs/heads/{public}");
8940        let push = git::git_raw(
8941            &winner.worktree,
8942            &["push", &lease, &self.state.config.merge.remote, &refspec],
8943        )
8944        .await;
8945        match push {
8946            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8947            Ok(out) => {
8948                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8949                return Ok(false);
8950            }
8951            Err(e) => {
8952                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8953                return Ok(false);
8954            }
8955        }
8956        self.state.merge_repairs[i].phase = Phase::Complete;
8957        self.state.status = RunStatus::Landing;
8958        self.state.event(
8959            "repair",
8960            "new review and gate completed; fresh owner merge decision required",
8961        );
8962        self.state.save()?;
8963        Ok(true)
8964    }
8965
8966    fn merge_repair_artifacts(&self) -> PathBuf {
8967        let artifacts = agent::artifacts_dir(&self.state.dir());
8968        match self
8969            .state
8970            .merge_repairs
8971            .last()
8972            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8973        {
8974            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8975            None => artifacts,
8976        }
8977    }
8978
8979    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8980        self.state.reviews.clear();
8981        self.state.gate.clear();
8982        self.state.gate_ran = false;
8983        self.state.gate_head = None;
8984        self.state.gate_fixes.clear();
8985        self.state.gate_repair_e2e = None;
8986        self.state.pre_gate.clear();
8987        self.state.pre_gate_commit = None;
8988        self.state.contested_handoff = None;
8989        self.state.land_reruns = None;
8990        self.state.land_approval = None;
8991        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8992        self.state.status = RunStatus::Reviewing;
8993        self.state.save()
8994    }
8995
8996    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8997        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8998        self.state.merge_repairs[index].failure = Some(why.clone());
8999        self.state.status = RunStatus::Blocked;
9000        self.state.event("repair", why);
9001        self.state.save()
9002    }
9003
9004    // -------------------------------------------------------------- helpers
9005
9006    /// Fetch or create a seat, keeping its conversation across nodes.
9007    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9008        if let Some(existing) = self.state.seats.get(key)
9009            && existing.agent == agent
9010        {
9011            return existing.clone();
9012        }
9013        // A seat that changes agent mints its session id from the agent too,
9014        // like a handover: the old agent's uuid is already taken by the CLI.
9015        let fresh = if self.state.seats.contains_key(key) {
9016            handover_seat(key, agent, self.state.next_seat_seed())
9017        } else {
9018            SeatState::new(key, agent, self.state.seed)
9019        };
9020        self.state.seats.insert(key.to_owned(), fresh.clone());
9021        fresh
9022    }
9023
9024    /// The agent now holding seat `key`: `spec`, unless a handover moved the
9025    /// seat to another roster agent, in which case that agent. Nodes that
9026    /// continue a seat's conversation (deliberation, the votes, a reviewer's
9027    /// reconsideration) must keep talking to whoever answered it, not slip
9028    /// back to the agent that failed it.
9029    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
9030        match self.state.seats.get(key) {
9031            Some(s) if s.agent != spec.id => {
9032                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
9033            }
9034            _ => spec,
9035        }
9036    }
9037
9038    /// A candidate rendered for judging, with the leak policy applied.
9039    fn view(&self, c: &Candidate) -> CandidateView {
9040        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
9041            .unwrap_or_default();
9042        let (patch, _) = blind::sanitize_patch(
9043            &format!("candidate {} patch", c.label),
9044            &raw,
9045            &self.state.config.blind,
9046        );
9047        CandidateView {
9048            label: c.label,
9049            branch: c.branch.clone(),
9050            summary: c.summary.clone(),
9051            stat: c.stat.clone(),
9052            patch,
9053        }
9054    }
9055
9056    /// The full candidate set as prompt text, for seats with no live session.
9057    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
9058        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
9059        prompt::judge(
9060            "(see above)",
9061            &views,
9062            self.roles.judges.len(),
9063            base_short,
9064            "en",
9065        )
9066    }
9067
9068    /// The final-vote prompt with everything a seat that has no session of its
9069    /// own needs: the candidates, the seat's own ranking and reasons, and the
9070    /// anonymised deliberation it took part in (only when there was one, so a
9071    /// handed-over seat never sees more than the seat it replaces did). The
9072    /// `Final vote` heading stays first.
9073    fn vote_prompt_full(
9074        &self,
9075        j: usize,
9076        labels: &[char],
9077        language: &str,
9078        candidates: &[Candidate],
9079        base_short: &str,
9080    ) -> String {
9081        let mut text = format!(
9082            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
9083            prompt::final_vote(labels, language),
9084            self.state.instruction,
9085            self.candidate_block(candidates, base_short)
9086        );
9087        if let Some(own) = self
9088            .state
9089            .judgements
9090            .get(j)
9091            .filter(|r| !r.ranking.is_empty())
9092        {
9093            let reasons = own
9094                .reasons
9095                .iter()
9096                .map(|(k, v)| format!("- {k}: {v}"))
9097                .collect::<Vec<_>>()
9098                .join("\n");
9099            text.push_str(&format!(
9100                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
9101                own.ranking.iter().collect::<String>(),
9102                if reasons.is_empty() {
9103                    ""
9104                } else {
9105                    ", because:\n"
9106                }
9107            ));
9108        }
9109        if !self.state.deliberation.is_empty() {
9110            text.push_str("\n# What was argued before this vote\n");
9111            for t in self.transcript(&[], j) {
9112                text.push_str(&format!(
9113                    "\n## {}{}\n\n{}\n",
9114                    t.who,
9115                    if t.is_self { " (you)" } else { "" },
9116                    t.body.trim()
9117                ));
9118            }
9119        }
9120        text
9121    }
9122
9123    /// Anonymised transcript for judge `self_idx`.
9124    ///
9125    /// The initial rankings are always the opening statements. Seeding them
9126    /// only when no turn had been taken yet meant every judge after the first
9127    /// argued against a single voice instead of against the actual split — the
9128    /// disagreement is the information, so it is always on the table.
9129    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
9130        let mut turns = Vec::new();
9131        for j in &self.state.judgements {
9132            if j.ranking.is_empty() {
9133                continue;
9134            }
9135            let reasons = j
9136                .reasons
9137                .iter()
9138                .map(|(k, v)| format!("- {k}: {v}"))
9139                .collect::<Vec<_>>()
9140                .join("\n");
9141            turns.push(Turn {
9142                who: format!("Judge {} (opening ranking)", j.judge),
9143                is_self: j.judge == self_idx + 1,
9144                body: format!(
9145                    "Ranked {}{}{reasons}",
9146                    j.ranking.iter().collect::<String>(),
9147                    if reasons.is_empty() {
9148                        ""
9149                    } else {
9150                        ", because:\n"
9151                    }
9152                ),
9153            });
9154        }
9155        for t in self
9156            .state
9157            .deliberation
9158            .iter()
9159            .flat_map(|r| r.turns.iter())
9160            .chain(current)
9161        {
9162            turns.push(Turn {
9163                who: format!("Judge {}", t.judge),
9164                is_self: t.judge == self_idx + 1,
9165                body: t.body.clone(),
9166            });
9167        }
9168        turns
9169    }
9170}
9171
9172/// Does this seat still hold the context a follow-up prompt would rely on?
9173fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
9174    agent::has_session(spec.kind, seat, sessions)
9175}
9176
9177/// The next entry in `roster` after `start`, never wrapping back to the
9178/// front, whose id is not in `tried` yet.
9179///
9180/// Starts one past `start` rather than at the front of `roster`: `start` is
9181/// the seat's own original position, and a seat whose candidate slot already
9182/// sits on the roster's second entry must fall through to the third next, not
9183/// restart at the first — which is very likely a different candidate's own
9184/// agent already. Never wraps back past `start`, for the same reason: an
9185/// entry earlier in the roster than the seat's own position is almost
9186/// certainly some *other* candidate slot's own agent, and once the tail of
9187/// the roster is exhausted there are no more untried agents for *this* seat
9188/// to fall through to — the caller's fallback chain ends there, exactly as
9189/// "no further untried agents remain in the list for that seat" asks for.
9190///
9191/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
9192/// the same id twice (an operator's `roles.implementers` typo, or a
9193/// `[[agents]]` list reused across roles) must not let
9194/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
9195/// over `roster` either finds an untried id or runs out, so this always
9196/// terminates regardless of duplicates.
9197fn next_untried_in_roster<'a>(
9198    roster: &'a [AgentSpec],
9199    start: usize,
9200    tried: &BTreeSet<String>,
9201) -> Option<&'a AgentSpec> {
9202    roster
9203        .get(start + 1..)?
9204        .iter()
9205        .find(|s| !tried.contains(&s.id))
9206}
9207
9208/// The successor for a seat, shared by all four seat kinds (implement, judge,
9209/// review, advise). `others` holds the ids that *currently* occupy the other
9210/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
9211/// sees them), so roster entries beyond the seat count act as spares: a failed
9212/// seat goes to an agent no other seat holds whenever the roster permits.
9213///
9214/// `carried` is `Some` only for the review loop's carried failure history.
9215/// Order: (1) forward from `start`, never wrapping, untried, not a carried
9216/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
9217/// whole roster: untried and not another seat's occupant; (3) the plain walk
9218/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
9219/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
9220/// a better outcome than an empty seat, and it keeps the answer to "is there
9221/// a successor at all" exactly what it was before occupants were considered,
9222/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
9223/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
9224/// carried failure is a cheaper bet than doubling an agent on the panel, and
9225/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
9226/// the same round are handled one at a time, so a seat later in the batch
9227/// sees an earlier one's new occupant but not yet its own freed agent.
9228fn pick_successor<'a>(
9229    roster: &'a [AgentSpec],
9230    start: usize,
9231    tried: &BTreeSet<String>,
9232    carried: Option<&BTreeSet<String>>,
9233    others: &BTreeSet<String>,
9234) -> Option<&'a AgentSpec> {
9235    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9236    roster
9237        .get(start + 1..)
9238        .and_then(|tail| {
9239            tail.iter()
9240                .filter(free)
9241                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9242        })
9243        .or_else(|| {
9244            carried
9245                .is_some()
9246                .then(|| roster.iter().find(free))
9247                .flatten()
9248        })
9249        .or_else(|| match carried {
9250            Some(c) => next_for_seat(roster, start, tried, c),
9251            None => next_untried_in_roster(roster, start, tried),
9252        })
9253}
9254
9255/// The next agent for a seat that carries its failure history across rounds
9256/// (the review loop). `round_tried` is this round's own bound and starts
9257/// empty every round; `carried_failed` only decides priority.
9258///
9259/// First: an id walking forward from `start`, never wrapping, that is neither
9260/// tried this round nor failed in an earlier one. Only when that is exhausted
9261/// does it rescue: the first roster id (in roster order, so this one *does*
9262/// look before `start`) not yet tried this round, which is by then a carried
9263/// failure. Each id is rescued at most once per round, so it cannot loop.
9264fn next_for_seat<'a>(
9265    roster: &'a [AgentSpec],
9266    start: usize,
9267    round_tried: &BTreeSet<String>,
9268    carried_failed: &BTreeSet<String>,
9269) -> Option<&'a AgentSpec> {
9270    roster
9271        .get(start + 1..)?
9272        .iter()
9273        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9274        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9275}
9276
9277/// Where a reviewer seat starts a round: the agent that last answered it when
9278/// it is still on the roster and not marked failed, else the spec's own agent
9279/// unless it failed, else the next roster agent that has not failed, else the
9280/// spec's own agent again (the whole roster failed). Ids no longer on the
9281/// roster are ignored. An empty roster has no handover, so the spec stands.
9282fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9283    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9284        return spec;
9285    };
9286    let ok = |id: &str| !h.failed.contains(id);
9287    if let Some(last) = h.last_ok.as_deref()
9288        && ok(last)
9289        && let Some(s) = roster.iter().find(|s| s.id == last)
9290    {
9291        return s.clone();
9292    }
9293    if ok(&spec.id) {
9294        return spec;
9295    }
9296    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9297    roster
9298        .get(from + 1..)
9299        .into_iter()
9300        .flatten()
9301        .chain(roster.iter())
9302        .find(|s| ok(&s.id))
9303        .cloned()
9304        .unwrap_or(spec)
9305}
9306
9307/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
9308/// seed so a CLI that mints its session id up front (`--session-id`) never
9309/// reuses the uuid the previous agent already opened under the same seat key.
9310pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9311    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9312}
9313
9314/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
9315/// so the classifier below and the code that builds the message cannot drift.
9316const TIMED_OUT: &str = "timed out";
9317
9318impl FailClass {
9319    /// `None` for an answer; otherwise how the turn failed.
9320    fn of(out: &AgentOutcome) -> Option<Self> {
9321        match out {
9322            AgentOutcome::Ok(_) => None,
9323            AgentOutcome::Quota(_) => Some(Self::Quota),
9324            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9325            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9326            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9327            // Not a failure of the agent. Callers check for it first; `None`
9328            // here means "no class to hand over on".
9329            AgentOutcome::ResourceBlocked(_) => None,
9330        }
9331    }
9332
9333    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
9334    fn stem_word(&self) -> &'static str {
9335        match self {
9336            Self::Quota => "quota",
9337            _ => "handover",
9338        }
9339    }
9340}
9341
9342/// The message's first line with its variable parts removed — digit runs and
9343/// path-like tokens — so "exit code 2" and "exit code 7" read
9344/// as one kind of failure.
9345fn failure_signature(msg: &str) -> String {
9346    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9347    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
9348    // the historical machine key while the process diagnosis uses readable prose.
9349    if let Some(code) = line
9350        .strip_prefix("process exited with no usable output (exit code ")
9351        .and_then(|s| s.strip_suffix(')'))
9352        .and_then(|s| s.parse::<i32>().ok())
9353    {
9354        line = format!("exited with some({code}) and no usable output");
9355    } else if matches!(
9356        line.as_str(),
9357        "process exited with no usable output (no exit code reported)"
9358            | "process exited with no usable output (exit code unavailable)"
9359    ) {
9360        line = "exited with none and no usable output".to_owned();
9361    }
9362    let mut out = Vec::new();
9363    for word in line.split_whitespace() {
9364        if word.contains('/') || word.contains('\\') {
9365            out.push("<path>".to_owned());
9366            continue;
9367        }
9368        let mut w = String::new();
9369        let mut in_digits = false;
9370        for c in word.chars() {
9371            if c.is_ascii_digit() {
9372                if !in_digits {
9373                    w.push('#');
9374                }
9375                in_digits = true;
9376            } else {
9377                in_digits = false;
9378                w.push(c);
9379            }
9380        }
9381        out.push(w);
9382    }
9383    out.join(" ").chars().take(120).collect()
9384}
9385
9386/// Whether a seat that just failed with `cur` may go to the next roster agent.
9387/// A quota or a timeout always may. Any other failure may not when the agent
9388/// before it failed the same way: an error the prompt causes would otherwise
9389/// walk the whole roster. `prev` is the class of the immediately preceding
9390/// agent's failure, so a quota or timeout in between breaks the run of
9391/// identical failures by itself.
9392fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9393    match cur {
9394        FailClass::Quota | FailClass::Timeout => true,
9395        FailClass::Other(_) => prev != Some(cur),
9396    }
9397}
9398
9399/// A short human reason for a failed outcome, for the handover record.
9400fn fail_reason(out: &AgentOutcome) -> String {
9401    match out {
9402        AgentOutcome::Ok(_) => String::new(),
9403        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9404        AgentOutcome::Dropped(o) => format!(
9405            "the CLI dropped the stream ({})",
9406            o.dropped
9407                .as_ref()
9408                .map(|d| d.why.as_str())
9409                .unwrap_or("it ended without delivering its answer")
9410        ),
9411        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9412            e.lines().next().unwrap_or("").chars().take(160).collect()
9413        }
9414    }
9415}
9416
9417/// Note one handover in the run: the structured record and, in the timeline,
9418/// the sentence a person reads. A quota keeps the wording it always had.
9419pub(crate) fn record_handover(
9420    state: &mut RunState,
9421    node: &str,
9422    seat: &str,
9423    from: &str,
9424    to: &str,
9425    class: &FailClass,
9426    reason: &str,
9427) {
9428    let message = if *class == FailClass::Quota {
9429        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9430    } else {
9431        format!("{seat}: handed over {from} -> {to} ({reason})")
9432    };
9433    state.event(node, message);
9434    state.handovers.push(Handover {
9435        at: Timestamp::now(),
9436        node: node.to_owned(),
9437        seat: seat.to_owned(),
9438        from: from.to_owned(),
9439        to: to.to_owned(),
9440        reason: reason.to_owned(),
9441    });
9442}
9443
9444/// Did this reply report running a command whose own CLI never confirmed an
9445/// exit status?
9446///
9447/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9448/// command *finished* (see that type's own doc), so this can only be `true`
9449/// for a command whose completion event carried no readable exit code — not
9450/// for one that simply is not mentioned at all. That is the one signal this
9451/// crate can read, from the same record `state.jobs` renders, about a reply
9452/// standing next to work its own CLI cannot vouch for finishing; it is
9453/// deliberately not a check on the exit code's *value* (a fixer legitimately
9454/// runs a command that fails mid-iteration before it succeeds) and not a
9455/// guess at a command still running in the background (which emits no event
9456/// at all, and so leaves no evidence here to find).
9457fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9458    commands.iter().any(|c| c.exit_code.is_none())
9459}
9460
9461/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9462/// trusted as a verified no-op — the adoption guard's own text-level half.
9463///
9464/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9465/// timed out): a marker only earns the benefit of the doubt from a turn the
9466/// CLI itself vouches for finishing properly, the same house style
9467/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9468/// *fix* report to for `commands`. A candidate that timed out, exited
9469/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9470/// is, whatever prose it wrote — this returns `None` before it ever looks at
9471/// `text`. The remaining guards (the tree really is empty, the evidence is
9472/// non-empty) are the caller's: this only reads what the reply *claimed*.
9473fn verified_noop_claim(
9474    usable: bool,
9475    commands: &[agent::CommandEvidence],
9476    text: &str,
9477) -> Option<String> {
9478    (usable && !has_unconfirmed_command(commands))
9479        .then(|| verdict::verified_noop(text))
9480        .flatten()
9481}
9482
9483fn short(commit: &str) -> String {
9484    commit.chars().take(7).collect()
9485}
9486
9487fn make_executable(path: &Path) -> Result<()> {
9488    #[cfg(unix)]
9489    {
9490        use std::os::unix::fs::PermissionsExt as _;
9491        let mut perms = std::fs::metadata(path)?.permissions();
9492        perms.set_mode(0o755);
9493        std::fs::set_permissions(path, perms)?;
9494    }
9495    #[cfg(not(unix))]
9496    {
9497        let _ = path;
9498    }
9499    Ok(())
9500}
9501
9502/// What every seat in one batch shares: where the answers are attributed, the
9503/// prompt overlay they inherit, and the build cache they are told to use.
9504///
9505/// A struct rather than four more parameters: `wave` also needs the run's
9506/// state (to record who is answering right now) and the attempt number, and
9507/// eight positional arguments is both unreadable and a clippy error.
9508struct WaveCtx<'a> {
9509    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9510    /// paid for it.
9511    run: &'a str,
9512    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9513    node: &'a str,
9514    prompts: &'a Prompts,
9515    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9516    cache: Option<&'a Path>,
9517    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9518    /// `JobRecord::round`. `None` for every other node.
9519    round: Option<usize>,
9520    /// Carry each seat's failed-agent history across waves (the review loop
9521    /// only): start-of-round priority and handover choice read
9522    /// [`RunState::seat_history`], and every answer or failure writes it.
9523    carry_seats: bool,
9524    /// The run's park / interrupt requests, so a wait for a shared resource
9525    /// can be abandoned the moment the operator asks to park.
9526    pauses: Option<(&'a Pause, &'a Pause)>,
9527}
9528
9529/// Shared scheduling policy for operations and spawned seat invocations.
9530fn repository_wait(
9531    state: &RunState,
9532    pauses: Option<(&Pause, &Pause)>,
9533) -> crate::repo_lock::WaitPolicy {
9534    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9535    let inherited = crate::repo_lock::WaitPolicy::current();
9536    crate::repo_lock::WaitPolicy::new(
9537        Duration::from_secs(state.config.graph.timeout_resource_wait),
9538        move || {
9539            inherited.cancelled()
9540                || pauses
9541                    .as_ref()
9542                    .is_some_and(|(a, b)| a.parked() || b.parked())
9543        },
9544    )
9545}
9546
9547fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9548    if error.is::<crate::repo_lock::ResourceBlocked>() {
9549        state.resource_block = Some(crate::run::ResourceBlock {
9550            cause: crate::run::ResourceBlockCause::Other,
9551            node: "repository".into(),
9552            kind: if error.to_string().contains("disk pressure") {
9553                "disk pressure"
9554            } else {
9555                "repository coordination"
9556            }
9557            .into(),
9558            detail: error.to_string(),
9559            at: Timestamp::now(),
9560        });
9561        state.event("resource", error.to_string());
9562    }
9563}
9564
9565async fn coordinate_operation(
9566    state: &mut RunState,
9567    cwd: &Path,
9568    wait: &crate::repo_lock::WaitPolicy,
9569) -> Result<Option<crate::repo_lock::SeatLease>> {
9570    let result =
9571        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9572            .await;
9573    if let Err(e) = &result {
9574        record_repository_block(state, e);
9575    }
9576    result
9577}
9578
9579/// Run one job, honouring the parallelism budget.
9580async fn run_one(
9581    job: SeatJob,
9582    sem: Arc<Semaphore>,
9583    ctx: &WaveCtx<'_>,
9584    state: &mut RunState,
9585    attempt: usize,
9586) -> (SeatState, AgentOutcome) {
9587    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9588        .await
9589        .pop()
9590        .expect("one job in, one result out");
9591    (seat, out)
9592}
9593
9594/// Run every job concurrently, capped by the semaphore, preserving order.
9595///
9596/// Every seat in the batch is recorded into [`RunState::active`] before the
9597/// wave starts and cleared as each answer lands, so the run's own record says
9598/// who is still being waited on rather than only who finished.
9599async fn wave(
9600    jobs: Vec<SeatJob>,
9601    sem: Arc<Semaphore>,
9602    ctx: &WaveCtx<'_>,
9603    state: &mut RunState,
9604    attempt: usize,
9605) -> Vec<(usize, SeatState, AgentOutcome)> {
9606    let WaveCtx {
9607        run,
9608        node,
9609        prompts,
9610        cache,
9611        round,
9612        carry_seats: _,
9613        pauses,
9614    } = *ctx;
9615    for job in &jobs {
9616        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9617    }
9618    if let Err(e) = state.save() {
9619        // A failed persist of "who is answering right now" must not abort the
9620        // wave: the seats are already being asked, and the alternative is
9621        // losing the answers to save a status line nobody may even be
9622        // watching.
9623        tracing::warn!("could not persist in-progress seats: {e:#}");
9624    }
9625    // Hold the shared build cache's lease for the whole batch, not per job:
9626    // several candidates (an implement wave) or a fixer legitimately share
9627    // one cache concurrently within this run, and that stays untouched — a
9628    // single lease taken once for the whole wave and released once it is
9629    // done is what stops a *different* borrower (another run's own wave, its
9630    // e2e/gate, a human's `magi review`) from interleaving a build into the
9631    // same directory while this one is in flight. Best-effort, not
9632    // all-or-nothing: a wave that cannot get the lease within its own
9633    // longest job's budget still runs — an hour of paid implementer calls is
9634    // not thrown away over cache contention — but every write-allowed seat
9635    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9636    // below), the same fallback a read-only seat always gets, rather than
9637    // building into a directory this run was never granted. The identity
9638    // record is still invalidated below either way, so the next tracked
9639    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9640    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9641    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9642    // never borrowed from a job's agent budget (and never charged to it either,
9643    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9644    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9645    let cache_guard = if let Some(cache_dir) = cache {
9646        if jobs_had_a_writer {
9647            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9648            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9649            match acquire_cache_lease(
9650                state,
9651                cache_dir,
9652                &owner,
9653                resource_limit,
9654                node,
9655                node,
9656                &cancel,
9657            )
9658            .await
9659            {
9660                Ok(Some(g)) => Some(g),
9661                // Parking: leave without spawning anything. The run parks at
9662                // the next boundary and the seats are asked afresh on resume.
9663                Ok(None) => {
9664                    if let Some(block) = &mut state.resource_block {
9665                        block.cause = crate::run::ResourceBlockCause::Cancelled {
9666                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9667                        };
9668                    }
9669                    return block_wave(
9670                        jobs,
9671                        state,
9672                        node,
9673                        attempt,
9674                        "parked while waiting for the shared build cache",
9675                    );
9676                }
9677                // An unreadable lease is not contention: keep the old
9678                // fallback (no shared cache for this wave's seats).
9679                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9680                Err(e) => {
9681                    // Writing seats may not build into a cache this run does
9682                    // not hold, and an agent that cannot be given one is not
9683                    // asked at all: the run stops resumably instead.
9684                    let why = format!("{e:#}");
9685                    state.event(
9686                        "resource",
9687                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9688                    );
9689                    return block_wave(jobs, state, node, attempt, &why);
9690                }
9691            }
9692        } else {
9693            None
9694        }
9695    } else {
9696        None
9697    };
9698    // Every job keeps its whole budget: the wait above was not the agent's.
9699    // Restart the seats' clocks so "started" means "being spawned now".
9700    state.seats_spawning(node, attempt);
9701    if let Err(e) = state.save() {
9702        tracing::warn!("could not persist the start of a wave: {e:#}");
9703    }
9704    // Queueing behind this wave's own seats for a parallelism slot is normal
9705    // (each of them is bounded by its own agent timeout); only a wait beyond
9706    // all of them plus the resource limit means the slots never came back.
9707    let slot_limit = resource_limit
9708        + jobs
9709            .iter()
9710            .map(|j| j.timeout)
9711            .max()
9712            .unwrap_or_default()
9713            .saturating_mul(jobs.len() as u32);
9714    let mut set = tokio::task::JoinSet::new();
9715    // Seats announce themselves here the moment they hold a parallelism
9716    // permit, i.e. just before their CLI is spawned.
9717    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9718    let overlay = prompts.overlay(node);
9719    let repository_wait = repository_wait(state, pauses);
9720    for (i, mut job) in jobs.into_iter().enumerate() {
9721        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9722        if cache.is_some() {
9723            job.prompt.push('\n');
9724            job.prompt
9725                .push_str(&prompt::build_cache_note(node, job.allow_write));
9726        }
9727        let sem = Arc::clone(&sem);
9728        let run = run.to_owned();
9729        let node = node.to_owned();
9730        // Only implementers were told about the task's attachments, so only
9731        // their seats get the directory widened for reading.
9732        let attachments = if node == "implement" {
9733            state.attachments.clone()
9734        } else {
9735            Vec::new()
9736        };
9737        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9738        // `prompt::build_cache_note`'s doc for why setting it anyway is
9739        // exactly how a sandboxed reviewer's write refusal got reported as a
9740        // defect in the patch, not a property of its own seat. And a
9741        // write-allowed one is handed it only when the lease above was
9742        // actually acquired: a wave that could not get it (`cache_guard` is
9743        // `None`, see its own comment) must not send seats to build into a
9744        // directory this run does not hold - that is the exact concurrent,
9745        // unmanaged-write race this module exists to prevent, not something
9746        // "proceeding anyway" is allowed to reintroduce.
9747        let cache = cache
9748            .filter(|_| job.allow_write && cache_guard.is_some())
9749            .map(Path::to_path_buf);
9750        let spawned_tx = spawned_tx.clone();
9751        let repository_wait = repository_wait.clone();
9752        let coordination = state.config.coordination.clone();
9753        let fetched_base = state.fetched_base.clone();
9754        set.spawn(async move {
9755            // A wait for a parallelism slot is scheduling, not the agent: it
9756            // is bounded by its own limit and ends as a resource block, never
9757            // as a timeout of the agent that was not started.
9758            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9759                Ok(p) => p,
9760                Err(_) => {
9761                    return (
9762                        i,
9763                        job.seat,
9764                        AgentOutcome::ResourceBlocked(format!(
9765                            "no parallel slot became free within {}s",
9766                            slot_limit.as_secs()
9767                        )),
9768                        false,
9769                    );
9770                }
9771            };
9772            spawned_tx.send(job.seat.key.clone()).ok();
9773            let mut seat = job.seat;
9774            let out = repository_wait
9775                .scope(agent::invoke(
9776                    &job.spec,
9777                    &mut seat,
9778                    &Invocation {
9779                        coordination: coordination.as_ref(),
9780                        fetched_base: fetched_base.as_ref(),
9781                        cwd: &job.cwd,
9782                        prompt: &job.prompt,
9783                        timeout: job.timeout,
9784                        allow_write: job.allow_write,
9785                        sessions: job.sessions,
9786                        artifacts: &job.artifacts,
9787                        stem: &job.stem,
9788                        run: &run,
9789                        node: &node,
9790                        cache_dir: cache.as_deref(),
9791                        attachments: &attachments,
9792                        writable: &[],
9793                    },
9794                ))
9795                .await;
9796            let interrupted = out
9797                .as_ref()
9798                .err()
9799                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9800            let out = match out {
9801                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9802                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9803                // Billed work the CLI failed to hand over is not an ordinary
9804                // failure, but its text is the CLI's raw error JSON, not an
9805                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9806                // read it as one by forgetting to check. `usable()` is always
9807                // false here (dropped implies an empty response), so this has
9808                // to be checked before the catch-all `Failed` below or the
9809                // one shape this exists for is lost with the rest.
9810                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9811                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9812                Ok(o) => AgentOutcome::Failed(format!(
9813                    "process exited with no usable output ({})",
9814                    crate::notices::exit_status(o.exit_code)
9815                )),
9816                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9817                    AgentOutcome::ResourceBlocked(e.to_string())
9818                }
9819                Err(e) => AgentOutcome::Failed(e.to_string()),
9820            };
9821            (i, seat, out, interrupted)
9822        });
9823    }
9824    drop(spawned_tx);
9825    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9826    loop {
9827        let joined = tokio::select! {
9828            Some(key) = spawned_rx.recv() => {
9829                // The clock of this seat starts now, not when it queued.
9830                state.seat_spawned(&key);
9831                if let Err(e) = state.save() {
9832                    tracing::warn!("could not persist a seat's start: {e:#}");
9833                }
9834                continue;
9835            }
9836            joined = set.join_next() => joined,
9837        };
9838        let Some(joined) = joined else {
9839            break;
9840        };
9841        let (i, seat, out, interrupted) = match joined {
9842            Ok(v) => v,
9843            // No seat to clear: a panicked task never reported which one it
9844            // was. The defensive sweep below this loop is what stops that
9845            // seat's `active` entry from surviving forever.
9846            Err(e) => {
9847                tracing::error!("agent task panicked: {e}");
9848                continue;
9849            }
9850        };
9851        state.seat_finished(&seat.key);
9852        if interrupted
9853            && node == "implement"
9854            && let Some(label) = state
9855                .candidates
9856                .iter()
9857                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9858                .map(|candidate| candidate.label)
9859        {
9860            state.candidate_preservation.insert(
9861                label,
9862                "interrupted implementation awaiting persistence".into(),
9863            );
9864            state.interrupted_implementations.insert(label);
9865        }
9866        if let AgentOutcome::ResourceBlocked(why) = &out
9867            && state.resource_block.is_none()
9868        {
9869            state.resource_block = Some(crate::run::ResourceBlock {
9870                cause: crate::run::ResourceBlockCause::Other,
9871                node: node.to_owned(),
9872                kind: if why.contains("disk pressure") {
9873                    "disk pressure"
9874                } else if why.contains("repository coordination") {
9875                    "repository coordination"
9876                } else {
9877                    "parallel slot"
9878                }
9879                .to_owned(),
9880                detail: why.clone(),
9881                at: Timestamp::now(),
9882            });
9883            let activity = if interrupted {
9884                "agent execution interrupted"
9885            } else {
9886                "no agent was started"
9887            };
9888            state.event("resource", format!("{node}: {activity} - {why}"));
9889        }
9890        record_jobs(state, node, round, &seat.key, &out);
9891        if let Err(e) = state.save() {
9892            tracing::warn!("could not persist a seat's completion: {e:#}");
9893        }
9894        if collected.len() <= i {
9895            collected.resize_with(i + 1, || None);
9896        }
9897        collected[i] = Some((i, seat, out));
9898    }
9899    // Belt-and-braces for the panic branch above: every seat this exact batch
9900    // started shares this `(node, attempt)` pair, and every seat that finished
9901    // normally already cleared itself, so anything left tagged with it here
9902    // can only be a panicked task's leftover. Cleared unconditionally rather
9903    // than left to read as still answering forever.
9904    if state
9905        .active
9906        .values()
9907        .any(|a| a.node == node && a.attempt == attempt)
9908    {
9909        state
9910            .active
9911            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9912        if let Err(e) = state.save() {
9913            tracing::warn!("could not persist the end of a wave: {e:#}");
9914        }
9915    }
9916    // Whether or not the lease above was actually held, several worktrees
9917    // may just have built into the cache with nothing here able to name one
9918    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9919    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9920    // clean it might not have strictly needed; trusting a stale match would
9921    // cost it a wrong answer.
9922    if let Some(cache_dir) = cache
9923        && jobs_had_a_writer
9924    {
9925        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9926    }
9927    if let Some(guard) = cache_guard {
9928        guard.release();
9929    }
9930    collected.into_iter().flatten().collect()
9931}
9932
9933/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9934/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9935/// this is data collection, not the fix-specific completion contract in
9936/// [`Runner::continue_fix_report`], and applies regardless of which node
9937/// asked.
9938///
9939/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9940/// evidence from; `Failed` does not, and correctly contributes nothing — a
9941/// timeout or crash is not itself evidence about a command the seat may have
9942/// started.
9943fn record_jobs(
9944    state: &mut RunState,
9945    node: &str,
9946    round: Option<usize>,
9947    seat: &str,
9948    out: &AgentOutcome,
9949) {
9950    let commands: &[agent::CommandEvidence] = match out {
9951        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9952        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9953    };
9954    let checked_at = Timestamp::now();
9955    for c in commands {
9956        state.jobs.push(JobRecord {
9957            node: node.to_owned(),
9958            round,
9959            seat: seat.to_owned(),
9960            id: c.id.clone(),
9961            description: c.description.clone(),
9962            checked_at,
9963            status: match c.exit_code {
9964                Some(0) => JobStatus::Completed,
9965                Some(_) => JobStatus::Failed,
9966                None => JobStatus::Unknown,
9967            },
9968            exit_code: c.exit_code,
9969            result_summary: c.result_summary.clone(),
9970            source: c.source.clone(),
9971        });
9972    }
9973}
9974
9975/// Is a review round clean, given how many reviewer seats answered against
9976/// how many the round expected?
9977///
9978/// A seat that never answered (timeout, crash, unparsable output) is not a
9979/// seat that read the patch and found nothing — treating it as such is
9980/// exactly the bug this function exists to close. Under the default `block`
9981/// policy a missing seat can never be clean; `warn` still requires the seats
9982/// that *did* answer to have found nothing blocking and verification to be
9983/// green.
9984///
9985/// `quota_missing` narrows that `block` default for exactly one cause of
9986/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9987/// a session limit lifts by the very next round buys nothing — the seat is
9988/// asked again with the same quota — so once every missing seat is accounted
9989/// for by a quota loss (and at least one seat *did* answer, so a decision has
9990/// something to rest on) the round is decided on the panel that could answer,
9991/// same as `warn` would. A panel that lost every seat to quota is not
9992/// decided here: `answered == 0` falls through to the existing `block`
9993/// fallback so a fully collapsed panel still waits rather than landing on no
9994/// review at all.
9995fn round_is_clean(
9996    blocking: usize,
9997    e2e_ok: bool,
9998    answered: usize,
9999    expected: usize,
10000    quota_missing: usize,
10001    policy: IncompleteReviewPolicy,
10002) -> bool {
10003    if blocking != 0 || !e2e_ok {
10004        return false;
10005    }
10006    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10007        return true;
10008    }
10009    answered > 0 && expected - answered <= quota_missing
10010}
10011
10012/// The review loop's own conclusion, derived entirely from its persisted
10013/// round records and the round budget that produced them — never from
10014/// `status`, so a reentry (or `gate`/`merge` reading it independently)
10015/// recomputes the identical answer regardless of what an earlier node in the
10016/// same walk, or a previous walk, did to `status`.
10017///
10018/// `None` while more rounds remain to try, including when review never ran
10019/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
10020/// gone clean, or the budget is spent, or the tree has stopped moving (see
10021/// [`STAGNANT_LIMIT`]), the answer is one of two things:
10022///
10023/// - An incomplete panel that raised nothing is missing input, not a
10024///   verified tree — never a hand-off candidate, whatever verification said
10025///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
10026/// - Otherwise, green e2e on the last round hands off (see
10027///   [`Runner::stop_reviewing`]); red e2e blocks.
10028///
10029/// A last round whose own verification is still `ResourceBlocked` — magi
10030/// itself never got a command to run, not evidence the patch is broken —
10031/// is neither: this returns `None` for it too, the same as "more rounds
10032/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
10033/// handling of that shape) instead of this cheap recomputation guessing a
10034/// verdict a real attempt never produced.
10035pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
10036    if max_rounds == 0
10037        || reviews.last().is_some_and(|r| {
10038            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
10039        })
10040    {
10041        return Some(RunStatus::Gating);
10042    }
10043    let last = reviews.last()?;
10044    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
10045    if reviews.len() < max_rounds && !stagnant {
10046        return None;
10047    }
10048    if last.incomplete() && last.blocking == 0 {
10049        return Some(RunStatus::Blocked);
10050    }
10051    if last.e2e_status() == E2eStatus::ResourceBlocked
10052        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
10053    {
10054        return None;
10055    }
10056    Some(if last.e2e.iter().all(CommandOutcome::ok) {
10057        RunStatus::Gating
10058    } else {
10059        RunStatus::Blocked
10060    })
10061}
10062
10063/// How long a re-ask may take, given the budget the first attempt had.
10064///
10065/// A `nudged` retry is a request to restate an answer the seat has already
10066/// worked out: it carries no new work, so it does not deserve the original
10067/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
10068/// 133 seconds while a third sat for over ten minutes on a resumed session
10069/// holding 410 KB of prior output - and because the retry had inherited the
10070/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
10071/// judging round whose other seats were long finished.
10072///
10073/// A quarter of the budget, with a floor so that a deliberately short timeout
10074/// does not collapse to nothing. A retry that re-sends the whole prompt
10075/// (because the seat kept no context) is the original job again, and keeps the
10076/// original budget.
10077pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
10078    if nudged {
10079        (full / 4).max(Duration::from_secs(120)).min(full)
10080    } else {
10081        full
10082    }
10083}
10084
10085/// Run a wave and parse each reply, re-asking the seats whose reply was
10086/// unusable.
10087///
10088/// The re-ask is a nudge rather than the whole prompt again when the seat still
10089/// holds its conversation, which is the difference between a cheap retry and
10090/// paying for the entire candidate set twice.
10091///
10092/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
10093/// successor in `roster` is handed to it instead of being re-asked: the
10094/// handover is the retry. A seat with no successor left (a single-agent
10095/// roster, the roster's tail) is nudged as before, up to `retries` times. So
10096/// the asks to one seat in one node number at most
10097/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
10098/// is asked once (a dropped stream is nudged first), and only the last agent
10099/// of the chain gets the `retries` same-agent nudges. Each roster agent is
10100/// tried at most once per seat, walking forward from the seat's own position and never wrapping
10101/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
10102/// other failure stops the chain when the previous agent failed the same way
10103/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
10104/// [`has_context`] is false and the job's own full prompt and full budget are
10105/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
10106/// intermediate ones are not losses) and is returned as a failure like any
10107/// other absent seat — the caller decides whether the panel still has a
10108/// quorum. An empty `roster` disables handover: failures are nudged as they
10109/// always were, and a quota is simply lost. A reply that fails to parse or
10110/// validate is the prompt's doing and is only ever nudged, never handed over.
10111///
10112/// The returned [`SeatState`] names the agent that answered (or tried last).
10113#[allow(clippy::too_many_arguments)]
10114async fn ask_json_wave<T>(
10115    jobs: Vec<SeatJob>,
10116    sem: Arc<Semaphore>,
10117    retries: usize,
10118    roster: &[AgentSpec],
10119    ctx: &WaveCtx<'_>,
10120    losses: &mut Vec<QuotaLoss>,
10121    state: &mut RunState,
10122    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
10123) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10124where
10125    T: serde::de::DeserializeOwned + Send + 'static,
10126{
10127    ask_wave_with(
10128        jobs,
10129        sem,
10130        retries,
10131        roster,
10132        ctx,
10133        losses,
10134        state,
10135        &|text: &str| {
10136            let v = verdict::extract_json::<T>(text)?;
10137            validate(&v)?;
10138            Ok(v)
10139        },
10140    )
10141    .await
10142}
10143
10144/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
10145/// a node whose answer is prose (deliberation) shares the same handover,
10146/// failure classification, quota bookkeeping and bounds instead of a copy.
10147///
10148/// A seat handed to another roster agent is sent the job's `handover` prompt
10149/// (when it has one) rather than `prompt`: the new agent has no session, so a
10150/// resume-style prompt would be incomplete. That holds for the handover ask
10151/// and for every nudge to that agent whose `has_context` is false.
10152#[allow(clippy::too_many_arguments)]
10153async fn ask_wave_with<T>(
10154    jobs: Vec<SeatJob>,
10155    sem: Arc<Semaphore>,
10156    retries: usize,
10157    roster: &[AgentSpec],
10158    ctx: &WaveCtx<'_>,
10159    losses: &mut Vec<QuotaLoss>,
10160    state: &mut RunState,
10161    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
10162) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
10163where
10164    T: Send + 'static,
10165{
10166    let n = jobs.len();
10167    let originals: Vec<SeatJob> = jobs;
10168    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
10169    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
10170    // Nudges each seat's *current* agent has taken — 0 for a first-ask
10171    // answer, N once it has gone through N nudges. Read back once this
10172    // returns, so a caller building a history record (`ReviewRecord`) can
10173    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
10174    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
10175    // field's own doc.
10176    let mut nudges: Vec<usize> = vec![0; n];
10177    // The agent now occupying each seat, the ids it has already been through,
10178    // where in the roster the walk began, the class of the last failure, and
10179    // the stem word of a handover not yet asked (full prompt, full budget).
10180    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
10181    let mut tried: Vec<BTreeSet<String>> = specs
10182        .iter()
10183        .map(|s| BTreeSet::from([s.id.clone()]))
10184        .collect();
10185    let starts: Vec<usize> = specs
10186        .iter()
10187        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
10188        .collect();
10189    let carry = ctx.carry_seats && !roster.is_empty();
10190    // Carried across rounds: ids that failed the seat earlier, and how the
10191    // last failure went (so a repeat of it is not handed over again).
10192    let carried: Vec<BTreeSet<String>> = originals
10193        .iter()
10194        .map(|j| {
10195            state
10196                .seat_history
10197                .get(&j.seat.key)
10198                .filter(|_| carry)
10199                .map(|h| h.failed.clone())
10200                .unwrap_or_default()
10201        })
10202        .collect();
10203    let mut prev: Vec<Option<FailClass>> = originals
10204        .iter()
10205        .map(|j| {
10206            state
10207                .seat_history
10208                .get(&j.seat.key)
10209                .filter(|_| carry)
10210                .and_then(|h| h.last_fail.clone())
10211        })
10212        .collect();
10213    let next_agent =
10214        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10215            let others: BTreeSet<String> = specs
10216                .iter()
10217                .enumerate()
10218                .filter(|(j, _)| *j != i)
10219                .map(|(_, s)| s.id.clone())
10220                .collect();
10221            pick_successor(
10222                roster,
10223                starts[i],
10224                tried,
10225                carry.then(|| &carried[i]),
10226                &others,
10227            )
10228            .cloned()
10229        };
10230    let mut fresh: Vec<Option<String>> = vec![None; n];
10231    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10232    let mut pending: Vec<usize> = (0..n).collect();
10233
10234    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
10235    // asks: an agent with a successor is asked once and handed over, and only
10236    // a seat with no successor spends `retries` nudges on the same agent. This
10237    // only guarantees the loop's own termination whatever those say.
10238    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10239    for round in 0..max_rounds {
10240        if pending.is_empty() {
10241            break;
10242        }
10243        let mut batch = Vec::with_capacity(pending.len());
10244        let mut renudged: Vec<&str> = Vec::new();
10245        for &i in &pending {
10246            let src = &originals[i];
10247            // A seat now held by another agent than the job named has no
10248            // session of its own: it gets the full-context prompt whenever
10249            // it is asked in full (the handover ask, a nudge it cannot
10250            // resume).
10251            let full: &str = match &src.handover {
10252                Some(h) if specs[i].id != src.spec.id => h,
10253                _ => &src.prompt,
10254            };
10255            // The prompt and the budget are one decision: a nudge restates
10256            // finished work, a re-sent prompt redoes it.
10257            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10258                (
10259                    full.to_owned(),
10260                    src.timeout,
10261                    format!("{}-{word}-{}", src.stem, specs[i].id),
10262                )
10263            } else if nudges[i] == 0 {
10264                (full.to_owned(), src.timeout, src.stem.clone())
10265            } else {
10266                renudged.push(src.seat.key.as_str());
10267                let why = done[i]
10268                    .as_ref()
10269                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
10270                    .unwrap_or_else(|| "no parsable answer".to_owned());
10271                let nudge = prompt::nudge(&why);
10272                let nudged = has_context(&specs[i], &seats[i], src.sessions);
10273                let prompt = if nudged {
10274                    nudge
10275                } else {
10276                    format!("{full}\n\n---\n\n{nudge}")
10277                };
10278                (
10279                    prompt,
10280                    retry_budget(src.timeout, nudged),
10281                    format!("{}-retry{}", src.stem, nudges[i]),
10282                )
10283            };
10284            batch.push(SeatJob {
10285                spec: specs[i].clone(),
10286                seat: seats[i].clone(),
10287                cwd: src.cwd.clone(),
10288                prompt,
10289                timeout,
10290                allow_write: src.allow_write,
10291                sessions: src.sessions,
10292                artifacts: src.artifacts.clone(),
10293                stem,
10294                handover: None,
10295            });
10296        }
10297
10298        if !renudged.is_empty() {
10299            state.event(
10300                ctx.node,
10301                format!("retry {round}: re-asking {}", renudged.join(", ")),
10302            );
10303        }
10304        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10305        let mut still = Vec::new();
10306        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10307            seats[i] = seat;
10308            // No agent was asked: nothing to hand over, nudge, remember or
10309            // charge. The caller sees `resource_block` and stops the node.
10310            if let AgentOutcome::ResourceBlocked(why) = &out {
10311                done[i] = Some(Err(anyhow::anyhow!("{why}")));
10312                continue;
10313            }
10314            let class = FailClass::of(&out);
10315            // A dropped stream is nudged first (the conversation is still
10316            // there to pick up); only a seat whose nudges are spent hands over.
10317            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10318                && nudges[i] < retries
10319                && !roster.is_empty();
10320            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10321                let next = should_hand_over(prev[i].as_ref(), &cur)
10322                    .then(|| next_agent(i, &tried[i], &specs))
10323                    .flatten();
10324                if carry {
10325                    let h = state
10326                        .seat_history
10327                        .entry(originals[i].seat.key.clone())
10328                        .or_default();
10329                    h.failed.insert(specs[i].id.clone());
10330                    h.last_fail = Some(cur.clone());
10331                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10332                        h.last_ok = None;
10333                    }
10334                    // Saved before the next agent is asked, so a restart in
10335                    // between does not forget who failed.
10336                    if let Err(e) = state.save() {
10337                        tracing::warn!("could not persist a seat's failure history: {e:#}");
10338                    }
10339                }
10340                if let Some(next) = next {
10341                    record_handover(
10342                        state,
10343                        ctx.node,
10344                        &originals[i].seat.key,
10345                        &specs[i].id,
10346                        &next.id,
10347                        &cur,
10348                        &fail_reason(&out),
10349                    );
10350                    tried[i].insert(next.id.clone());
10351                    prev[i] = Some(cur.clone());
10352                    seats[i] =
10353                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10354                    specs[i] = next;
10355                    fresh[i] = Some(cur.stem_word().to_owned());
10356                    nudges[i] = 0;
10357                    done[i] = Some(Err(anyhow::anyhow!(
10358                        "handed over after: {}",
10359                        fail_reason(&out)
10360                    )));
10361                    still.push(i);
10362                    continue;
10363                }
10364            }
10365            if carry
10366                && !nudge_first
10367                && let Some(cur) = class.clone()
10368            {
10369                // The chain ended here (no successor, or a repeated failure).
10370                let h = state
10371                    .seat_history
10372                    .entry(originals[i].seat.key.clone())
10373                    .or_default();
10374                h.failed.insert(specs[i].id.clone());
10375                h.last_fail = Some(cur);
10376            }
10377            let parsed = match out {
10378                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10379                AgentOutcome::Quota(o) => {
10380                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10381                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10382                }
10383                // Not a parseable answer: the nudge loop re-asks it, which is
10384                // exactly what a dropped stream needs. Just don't hand its raw
10385                // error JSON to `extract_json`.
10386                AgentOutcome::Dropped(o) => {
10387                    let why = o
10388                        .dropped
10389                        .as_ref()
10390                        .map(|d| d.why.as_str())
10391                        .unwrap_or("the CLI ended the stream without delivering its answer");
10392                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10393                }
10394                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10395                    Err(anyhow::anyhow!(e))
10396                }
10397            };
10398            let quota = class == Some(FailClass::Quota);
10399            let failed = parsed.is_err();
10400            done[i] = Some(parsed);
10401            if carry && !failed {
10402                let h = state
10403                    .seat_history
10404                    .entry(originals[i].seat.key.clone())
10405                    .or_default();
10406                h.failed.remove(&specs[i].id);
10407                h.last_ok = Some(specs[i].id.clone());
10408                h.last_fail = None;
10409            }
10410            // Do not re-ask a rate-limited seat (quota) — a retry is known to
10411            // fail the same way; and never re-ask a seat that already parsed.
10412            // A failed agent that still has a successor is not re-asked
10413            // either: the handover was its remedy and has just been refused
10414            // (the chain stops on a repeated failure class). A seat with no
10415            // successor left (a single-agent roster, the roster's tail, or an
10416            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10417            let agent_failure = class.is_some()
10418                && !nudge_first
10419                && !roster.is_empty()
10420                && next_agent(i, &tried[i], &specs).is_some();
10421            if failed && !quota && !agent_failure && nudges[i] < retries {
10422                nudges[i] += 1;
10423                still.push(i);
10424            }
10425        }
10426        pending = still;
10427    }
10428
10429    // One loss per seat whose chain ended on a quota: the intermediate ones
10430    // were absorbed by a handover and are not losses.
10431    for (i, q) in last_quota.into_iter().enumerate() {
10432        if let Some(reset) = q {
10433            losses.push(QuotaLoss {
10434                seat: originals[i].seat.key.clone(),
10435                node: ctx.node.to_owned(),
10436                at: Timestamp::now(),
10437                reset,
10438            });
10439        }
10440    }
10441
10442    seats
10443        .into_iter()
10444        .zip(done)
10445        .zip(nudges)
10446        .map(|((seat, res), attempts)| {
10447            (
10448                seat,
10449                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10450                attempts,
10451            )
10452        })
10453        .collect()
10454}
10455
10456/// Did the wave end without asking any agent (see [`block_wave`])?
10457fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10458    results
10459        .iter()
10460        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10461}
10462
10463/// Settle a wave that never spawned anything: every job answers
10464/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10465/// else about the seat (session, history, quota) is touched.
10466fn block_wave(
10467    jobs: Vec<SeatJob>,
10468    state: &mut RunState,
10469    node: &str,
10470    attempt: usize,
10471    why: &str,
10472) -> Vec<(usize, SeatState, AgentOutcome)> {
10473    state.resource_wait = None;
10474    state
10475        .active
10476        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10477    if let Err(e) = state.save() {
10478        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10479    }
10480    jobs.into_iter()
10481        .enumerate()
10482        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10483        .collect()
10484}
10485
10486/// Acquire the shared build cache's lease, waiting out contention within
10487/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10488/// unbounded wait is never acceptable).
10489///
10490/// A first, non-blocking check happens before ever waiting; if it finds the
10491/// lease busy, that fact is logged as a `verify` event *and* flushed with
10492/// [`RunState::save`] immediately — not only once the wait finally succeeds
10493/// or gives up — so a `magi show` run by a different process while this one
10494/// is still waiting reads a `run.json` that says so, rather than whatever it
10495/// looked like before the wait started. The same applies to the terminal
10496/// failure: logged and saved before this returns `Err`, so a caller that
10497/// could not get the lease at all still leaves a legible record of why.
10498async fn acquire_cache_lease(
10499    state: &mut RunState,
10500    cache_dir: &Path,
10501    owner: &crate::cache::Owner,
10502    budget: Duration,
10503    node: &str,
10504    context: &str,
10505    cancel: &(dyn Fn() -> bool + Sync),
10506) -> Result<Option<crate::cache::Guard>> {
10507    let home = crate::run::home();
10508    let started = Instant::now();
10509    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10510        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10511        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10512        Err(e) => {
10513            state.event(
10514                "verify",
10515                format!("{context}: could not check the shared build cache: {e:#}"),
10516            );
10517            if let Err(e2) = state.save() {
10518                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10519            }
10520            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10521        }
10522    };
10523    state.event(
10524        "verify",
10525        format!(
10526            "{context}: waiting for the shared build cache at {} ({})",
10527            cache_dir.display(),
10528            busy.describe()
10529        ),
10530    );
10531    // Seats of this node are waiting, not running: say so, so the countdown a
10532    // reader sees only ever measures an agent that has been spawned.
10533    state.resource_wait = Some(crate::run::ResourceWait {
10534        node: context.to_owned(),
10535        kind: "shared build cache".to_owned(),
10536        holder: busy.describe(),
10537        since: Timestamp::now(),
10538        limit_secs: budget.as_secs(),
10539    });
10540    if let Err(e) = state.save() {
10541        tracing::warn!("could not persist a cache wait: {e:#}");
10542    }
10543    let remaining = budget.saturating_sub(started.elapsed());
10544    let waited = crate::cache::wait_for_unless(
10545        &home,
10546        cache_dir,
10547        owner,
10548        remaining,
10549        Duration::from_secs(5),
10550        cancel,
10551    )
10552    .await;
10553    state.resource_wait = None;
10554    let cause = match &waited {
10555        Ok(Some(_)) => None,
10556        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10557        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10558            Some(crate::run::ResourceBlockCause::Timeout {
10559                waited_secs: started.elapsed().as_secs(),
10560                limit_secs: budget.as_secs(),
10561            })
10562        }
10563        Err(_) => Some(crate::run::ResourceBlockCause::Other),
10564    };
10565    if let Some(cause) = cause {
10566        let detail = match &waited {
10567            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10568            Err(error) => format!("{error:#}"),
10569        };
10570        state.resource_block = Some(crate::run::ResourceBlock {
10571            cause,
10572            node: node.to_owned(),
10573            kind: "shared build cache".into(),
10574            detail,
10575            at: Timestamp::now(),
10576        });
10577        state.event(
10578            "resource",
10579            state.resource_block.as_ref().unwrap().description(),
10580        );
10581    }
10582    if let Err(error) = state.save() {
10583        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10584    }
10585    waited
10586}
10587
10588/// The lease state could not be read at all (unreadable lease file, a
10589/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10590/// known to be contended, so a wave degrades to running without a shared cache
10591/// instead of stopping on a hold that [`crate::triage`] could never release.
10592#[derive(Debug)]
10593struct CacheCheckError(String);
10594
10595impl std::fmt::Display for CacheCheckError {
10596    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10597        f.write_str(&self.0)
10598    }
10599}
10600
10601impl std::error::Error for CacheCheckError {}
10602
10603/// Keep a specific acquisition outcome when the caller adds command evidence.
10604fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10605    state
10606        .resource_block
10607        .get_or_insert_with(|| crate::run::ResourceBlock {
10608            cause: crate::run::ResourceBlockCause::Other,
10609            node: node.to_owned(),
10610            kind: "shared build cache".into(),
10611            detail: format!("{error:#}"),
10612            at: Timestamp::now(),
10613        });
10614    if let Err(error) = state.save() {
10615        tracing::warn!("could not persist cache resource block: {error:#}");
10616    }
10617}
10618
10619/// Run `body` — a verify command batch — while holding the shared build
10620/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10621/// never interleave with another borrower's build against the same
10622/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10623/// timeout, or a human's own `magi review`. See the `cache` module doc for
10624/// why this matters more than Cargo's own per-target locking covers — two
10625/// *different* worktrees building the same package name/version into one
10626/// cache directory is a staleness bug, not a lock contention one.
10627///
10628/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10629/// agent waves do. Repository scheduling also has its own bounded,
10630/// cancellable resource-wait policy. Contention must not consume the
10631/// verification budget: `body` receives the full `budget` after both leases
10632/// are acquired and the cache is fresh. Without a shared cache, only
10633/// repository coordination is acquired before handing over the full budget.
10634///
10635/// A lease that cannot be acquired within the resource wait is reported as a single
10636/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10637/// verification — the same shape a spawn failure already takes in
10638/// [`run_commands`], so a caller need not special-case it.
10639#[allow(clippy::too_many_arguments)]
10640async fn with_cache_lease<'s, F, Fut>(
10641    state: &'s mut RunState,
10642    cache_dir: Option<&Path>,
10643    node: &str,
10644    seat: &str,
10645    worktree: &Path,
10646    head: &str,
10647    budget: Duration,
10648    context: &str,
10649    wait: &crate::repo_lock::WaitPolicy,
10650    body: F,
10651) -> Result<(Vec<CommandOutcome>, bool)>
10652where
10653    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10654    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10655{
10656    // Match editing waves: cache first, repository second. In particular,
10657    // pre_gate must never hold the repository while waiting for a wave's cache.
10658    let started = Instant::now();
10659    let home = crate::run::home();
10660    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10661    let guard = if let Some(cache_dir) = cache_dir {
10662        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10663        match acquire_cache_lease(
10664            state,
10665            cache_dir,
10666            &owner,
10667            resource_limit,
10668            node,
10669            context,
10670            &|| wait.cancelled(),
10671        )
10672        .await
10673        .and_then(|g| {
10674            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10675        }) {
10676            Ok(g) => Some(g),
10677            Err(e) => {
10678                record_cache_error(state, node, &e);
10679                return Ok((
10680                    vec![CommandOutcome {
10681                        command: "(waiting for the shared build cache)".into(),
10682                        code: None,
10683                        output_tail: e.to_string(),
10684                        duration_ms: started.elapsed().as_millis() as u64,
10685                        resource_blocked: true,
10686                    }],
10687                    false,
10688                ));
10689            }
10690        }
10691    } else {
10692        None
10693    };
10694    let _coordination = coordinate_operation(state, worktree, wait).await?;
10695    if let Some(cache_dir) = cache_dir {
10696        let identity = crate::cache::Identity::new(worktree, head);
10697        if let Err(e) = crate::cache::ensure_fresh(
10698            &home,
10699            cache_dir,
10700            &identity,
10701            &state.config.verify.cargo_manifests(),
10702        ) {
10703            record_cache_error(state, node, &e);
10704            state.event(
10705                "verify",
10706                format!(
10707                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10708                    worktree.display(),
10709                    short(head)
10710                ),
10711            );
10712            if let Some(guard) = guard {
10713                guard.release();
10714            }
10715            return Ok((
10716                vec![CommandOutcome {
10717                    command: "(confirming the shared build cache is fresh)".into(),
10718                    code: None,
10719                    output_tail: e.to_string(),
10720                    duration_ms: started.elapsed().as_millis() as u64,
10721                    resource_blocked: true,
10722                }],
10723                false,
10724            ));
10725        }
10726    }
10727    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10728    if !timed_out_pids.is_empty() {
10729        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10730    }
10731    // Drop repository coordination before releasing the cache, reversing the
10732    // acquisition order. A cancelled/error future also drops both guards.
10733    drop(_coordination);
10734    if let Some(guard) = guard {
10735        guard.release();
10736    }
10737    Ok((outcomes, retried))
10738}
10739
10740/// Poll `pids` — commands [`run_commands`] reports as still running when its
10741/// own timeout elapsed — until every one is confirmed gone, or
10742/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10743///
10744/// Real confirmation where confirmation is possible, not a substitute for
10745/// full process-tree observation: a grandchild the timed-out process spawned
10746/// and that survives independently of it is invisible to a pid check the
10747/// same way it always was, and continuing to observe and collect *that*
10748/// stays a different piece of work with its own owner. This only narrows a
10749/// fixed blind wait into an actual check of the pids this process does know
10750/// about.
10751async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10752    wait_for_pids_with(
10753        pids,
10754        crate::proc::pid_alive,
10755        LEASE_RELEASE_POLL,
10756        LEASE_RELEASE_MAX_WAIT,
10757    )
10758    .await;
10759}
10760
10761/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10762/// interval and ceiling supplied by the caller, so the polling *logic* -
10763/// returns as soon as every pid reports dead, gives up at the ceiling
10764/// otherwise - is testable on millisecond durations without asking the real
10765/// OS about a pid at all.
10766async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10767    pids: &[u32],
10768    alive: F,
10769    poll: Duration,
10770    max_wait: Duration,
10771) {
10772    let deadline = Instant::now() + max_wait;
10773    loop {
10774        if pids.iter().all(|&pid| !alive(pid)) {
10775            return;
10776        }
10777        if Instant::now() >= deadline {
10778            return;
10779        }
10780        tokio::time::sleep(poll).await;
10781    }
10782}
10783
10784/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10785/// admission that it could not even get a verify command to run, as opposed
10786/// to evidence the command actually produced? A caller that would otherwise
10787/// read a resource-blocked outcome as a red command must check this first:
10788/// see [`Runner::gate`], which retries rather than records `Blocked` when
10789/// this is true.
10790fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10791    outcomes.iter().any(|o| o.resource_blocked)
10792}
10793
10794/// What [`Runner::gate_fix_round`] decided.
10795pub(crate) enum GateFix {
10796    /// The tree changed and `verify.e2e` is still green: run the gate again.
10797    Retry,
10798    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10799    /// last failure stands and the run ends blocked.
10800    Stop,
10801    /// `verify.e2e` could not run after the fix (magi's own contention):
10802    /// decide nothing now, a later reentry retries.
10803    Defer,
10804}
10805
10806/// Is every red command in `outcomes` an ordinary failure the code could
10807/// explain: it ran, exited non-zero, and said something?
10808///
10809/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10810/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10811/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10812/// what the command is stays the gate's business.
10813fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10814    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10815    red.peek().is_some()
10816        && red.all(|o| {
10817            !o.resource_blocked
10818                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10819                && !o.output_tail.trim().is_empty()
10820        })
10821}
10822
10823/// Describe one verify command's outcome for the event log, distinguishing a
10824/// build/link failure — the toolchain never produced a binary to run — from
10825/// an actual test failure, since only the latter is a verdict on the patch.
10826fn e2e_outcome_label(o: &CommandOutcome) -> String {
10827    if o.ok() {
10828        return "pass".to_owned();
10829    }
10830    let reason = if o.build_failed() {
10831        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10832    } else {
10833        format!("FAIL ({:?})", o.code)
10834    };
10835    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10836}
10837
10838/// Run `verify.e2e`, retrying once if the first attempt could not build or
10839/// link — a build/link failure is frequently a race against a shared
10840/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10841/// `verify` event per command, tagged with `context` (normally `"round N"`)
10842/// so the two call sites that need this — the ordinary per-round leg in
10843/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10844/// it will ever call a round green — read identically in the event log.
10845async fn run_e2e_with_retry(
10846    state: &mut RunState,
10847    shell: &[String],
10848    commands: &[String],
10849    worktree: &Path,
10850    timeout: Duration,
10851    context: &str,
10852) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10853    let (mut e2e, mut timed_out_pids) = run_commands(
10854        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10855    )
10856    .await;
10857    for o in &e2e {
10858        state.event(
10859            "verify",
10860            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10861        );
10862    }
10863    // A build/link failure is not a verdict on the patch — it is frequently a
10864    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10865    // one retry before letting a red like that decide the round.
10866    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10867    if verify_retried {
10868        state.event(
10869            "verify",
10870            format!(
10871                "{context}: verify could not build/link, not a test result — retrying once \
10872                 before concluding"
10873            ),
10874        );
10875        let retried = run_commands(
10876            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10877        )
10878        .await;
10879        e2e = retried.0;
10880        // Both attempts' timeouts matter, not just the last one: the first
10881        // attempt's descendants may still be alive alongside the retry's.
10882        timed_out_pids.extend(retried.1);
10883        for o in &e2e {
10884            state.event(
10885                "verify",
10886                format!(
10887                    "{context}: retry `{}` -> {}",
10888                    o.command,
10889                    e2e_outcome_label(o)
10890                ),
10891            );
10892        }
10893    }
10894    (e2e, verify_retried, timed_out_pids)
10895}
10896
10897/// Run configured shell commands in `cwd`, in order. The second element is
10898/// the pid of every command that hit `timeout` and was still running when
10899/// this stopped waiting on it (best-effort: `None` when the platform did not
10900/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10901/// that releases a shared resource afterward needs to know.
10902///
10903/// Records `task` into [`RunState::active`] at every command boundary
10904/// (`RunState::task_command`) and clears it once the whole list has run
10905/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10906/// for minutes with no seat and no output of its own to show for it (see
10907/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10908/// apart from "not yet run" without this), and this is the only place that
10909/// knows which command is running right now and how many are left. Three
10910/// saves per command — start, not per second — matching the same "only at a
10911/// boundary" rule [`wave`] already follows for seats.
10912#[allow(clippy::too_many_arguments)]
10913async fn run_commands(
10914    state: &mut RunState,
10915    node: &str,
10916    task: &str,
10917    attempt: usize,
10918    shell: &[String],
10919    commands: &[String],
10920    cwd: &Path,
10921    timeout: Duration,
10922) -> (Vec<CommandOutcome>, Vec<u32>) {
10923    if commands.is_empty() {
10924        // Nothing to mark as running and nothing to clear — an empty list
10925        // means "not configured", and touching `active` (or the disk) over
10926        // that would be a write for every round of a repo with no
10927        // `verify.e2e` / `verify.gate` commands at all.
10928        return (Vec::new(), Vec::new());
10929    }
10930    let mut out = Vec::new();
10931    let mut timed_out_pids = Vec::new();
10932    let total = commands.len();
10933    for (idx, command) in commands.iter().enumerate() {
10934        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10935        if let Err(e) = state.save() {
10936            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10937        }
10938        let started = Instant::now();
10939        let mut cmd = tokio::process::Command::new(&shell[0]);
10940        cmd.quiet();
10941        // Verify the checkout as a standalone caller. Worker ownership and
10942        // pinned executable selection belong to this driver, not to tests or
10943        // tools it runs. Preserve home/seat context and Running's containment.
10944        for key in crate::worker::INVOCATION_ENV {
10945            cmd.env_remove(key);
10946        }
10947        cmd.args(&shell[1..])
10948            .arg(command)
10949            .current_dir(cwd)
10950            .stdin(std::process::Stdio::null())
10951            .stdout(std::process::Stdio::piped())
10952            .stderr(std::process::Stdio::piped())
10953            .kill_on_drop(true);
10954        let mut outputs = Vec::new();
10955        let protected = (|| -> Result<()> {
10956            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10957                outputs.push(guard);
10958            }
10959            if let Some(cache) = state.config.cache_dir() {
10960                outputs.push(crate::retention::OutputLease::cache(
10961                    &crate::run::home(),
10962                    &cache,
10963                    false,
10964                )?);
10965            }
10966            Ok(())
10967        })();
10968        if let Err(e) = protected {
10969            out.push(CommandOutcome {
10970                command: command.clone(),
10971                code: None,
10972                output_tail: format!("output ownership unavailable: {e:#}"),
10973                duration_ms: 0,
10974                resource_blocked: true,
10975            });
10976            continue;
10977        }
10978        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10979        let (code, body) = match spawned {
10980            Ok(mut running) => {
10981                let pid = running.child().id();
10982                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10983                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10984                let mut paths = vec![cwd.to_owned()];
10985                if let Some(cache) = state.config.cache_dir() {
10986                    paths.push(cache);
10987                }
10988                let mut pressure = None;
10989                let status = tokio::select! {
10990                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10991                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10992                        pressure = Some(reason); None
10993                    }
10994                };
10995                let (code, failure) = match status {
10996                    Some(Ok(Ok(status))) => (
10997                        status.code(),
10998                        status
10999                            .code()
11000                            .is_none()
11001                            .then(|| format!("command terminated: {status}")),
11002                    ),
11003                    Some(Ok(Err(error))) => {
11004                        (None, Some(format!("failed waiting for command: {error}")))
11005                    }
11006                    Some(Err(_)) => {
11007                        // Leased cleanup confirms process-tree death itself.
11008                        // Unleased commands still need the caller's PID check
11009                        // after kill-on-drop, before releasing cache ownership.
11010                        if !running.coordinated()
11011                            && let Some(pid) = pid
11012                        {
11013                            timed_out_pids.push(pid);
11014                        }
11015                        (
11016                            None,
11017                            Some(format!("timed out after {}s", timeout.as_secs())),
11018                        )
11019                    }
11020                    None => (None, None),
11021                };
11022                running.finish().await;
11023                if let Some(reason) = pressure {
11024                    state.resource_block = Some(crate::run::ResourceBlock {
11025                        cause: crate::run::ResourceBlockCause::Other,
11026                        node: node.to_owned(),
11027                        kind: "disk pressure".to_owned(),
11028                        detail: reason.clone(),
11029                        at: Timestamp::now(),
11030                    });
11031                    out.push(CommandOutcome {
11032                        command: command.clone(),
11033                        code: None,
11034                        output_tail: reason,
11035                        duration_ms: started.elapsed().as_millis() as u64,
11036                        resource_blocked: true,
11037                    });
11038                    break;
11039                }
11040                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
11041                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
11042                if let Some(failure) = failure {
11043                    body.push('\n');
11044                    body.push_str(&failure);
11045                }
11046                (code, body)
11047            }
11048            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
11049        };
11050        out.push(CommandOutcome {
11051            command: command.clone(),
11052            code,
11053            output_tail: tail(&body, OUTPUT_TAIL),
11054            duration_ms: started.elapsed().as_millis() as u64,
11055            resource_blocked: false,
11056        });
11057    }
11058    state.task_finished(task);
11059    if let Err(e) = state.save() {
11060        tracing::warn!("could not persist the end of {task}: {e:#}");
11061    }
11062    (out, timed_out_pids)
11063}
11064
11065/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
11066/// section (`report::run`) and in the `merge` event this node records — for
11067/// the operator to run by hand.
11068///
11069/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
11070/// branch whose ruleset forbids merge commits (GitHub's "must not contain
11071/// merge commits", or "require linear history") rejects the push a `--no-ff`
11072/// merge would produce, which is exactly the guidance this function replaces.
11073/// `message`'s first line becomes the squash commit's subject, matching the
11074/// note `report::run` prints alongside this command — see that function for
11075/// why an explicit subject is not optional there.
11076fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
11077    let repo = repo.display();
11078    match style {
11079        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
11080        MergeStyle::Squash => {
11081            // The subject sits inside double quotes, and a title an agent
11082            // wrote may carry the characters that break out of them.
11083            let subject = message
11084                .lines()
11085                .next()
11086                .unwrap_or(branch)
11087                .replace(['\\', '"', '$', '`'], "");
11088            format!(
11089                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
11090            )
11091        }
11092        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
11093    }
11094}
11095
11096/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
11097/// under the hood, rejects a `title` over 256 characters and the whole
11098/// command fails — no PR at all, for a run whose body was otherwise fine
11099/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
11100/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
11101/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
11102/// this bound. It is a margin, not a guarantee — a title packed
11103/// with multi-unit characters could still in principle land close to the
11104/// edge, but a real task title's occasional emoji or accented letter fits
11105/// comfortably inside it.
11106const PR_TITLE_MAX: usize = 240;
11107
11108/// A pull request title from the opening line of `text`, or `None` when that
11109/// line is not English (GitHub text is) or has no letters.
11110///
11111/// A line within `max` is kept as is. A longer one is cut at the end of its
11112/// first sentence when that falls inside `max`, else at a word boundary with a
11113/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
11114/// make every merely-truncated title look non-English). The language check
11115/// runs on the kept text before any mark is added, so only what GitHub will
11116/// show is judged: an English opening followed by non-ASCII far past the cut
11117/// still passes.
11118fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
11119    let line = queue::first_line(text)?;
11120    let chars: Vec<char> = line.chars().collect();
11121    let (kept, mark) = if chars.len() <= max {
11122        (line.to_owned(), "")
11123    } else if let Some(end) = sentence_end(&chars, max) {
11124        (chars[..end].iter().collect::<String>(), "")
11125    } else {
11126        let room = max.saturating_sub(3);
11127        // Cut at the last space inside the room; when the char just past the
11128        // room is a space the room already ends on a word.
11129        let cut = if chars[room].is_whitespace() {
11130            room
11131        } else {
11132            chars[..room]
11133                .iter()
11134                .rposition(|c| c.is_whitespace())
11135                .unwrap_or(room)
11136        };
11137        let head: String = chars[..cut].iter().collect();
11138        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
11139        (head.to_owned(), "...")
11140    };
11141    if kept.is_empty() {
11142        return None;
11143    }
11144    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
11145        return None;
11146    }
11147    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
11148        return None;
11149    }
11150    Some(format!("{kept}{mark}"))
11151}
11152
11153/// The char length of the first sentence of `chars` when it ends within `max`
11154/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
11155/// abbreviations so `e.g. foo` does not end a title early.
11156fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
11157    const MIN: usize = 20;
11158    for i in MIN..max.min(chars.len()) {
11159        if !matches!(chars[i], '.' | '!' | '?') {
11160            continue;
11161        }
11162        let Some(&next) = chars.get(i + 1) else {
11163            continue;
11164        };
11165        if !next.is_whitespace() {
11166            continue;
11167        }
11168        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
11169        if after.is_some_and(|c| c.is_ascii_lowercase()) {
11170            continue;
11171        }
11172        let word: String = chars[..i]
11173            .iter()
11174            .rev()
11175            .take_while(|c| !c.is_whitespace())
11176            .collect::<Vec<_>>()
11177            .into_iter()
11178            .rev()
11179            .collect();
11180        let word = word.to_ascii_lowercase();
11181        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
11182            continue;
11183        }
11184        let end = chars[..i]
11185            .iter()
11186            .rposition(|c| !c.is_whitespace())
11187            .map_or(i, |p| p + 1);
11188        return Some(end);
11189    }
11190    None
11191}
11192
11193/// What `merge = "pr"` (and the merge commit of the other modes) says about a
11194/// change: a title and a body describing what was *implemented*, not the task
11195/// that asked for it. A task reads as a request; a reader of the merged
11196/// history wants the change.
11197struct PrMessage {
11198    title: String,
11199    body: String,
11200}
11201
11202impl PrMessage {
11203    /// Title, blank line, body. The first line is the squash/merge commit
11204    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
11205    /// has to stay one sensible line.
11206    fn commit_message(&self) -> String {
11207        format!("{}\n\n{}", self.title, self.body)
11208    }
11209}
11210
11211/// The text after a leading `TITLE:` (any case) on `line`.
11212fn title_marker(line: &str) -> Option<&str> {
11213    let line = line.trim();
11214    let head = line.get(..6)?;
11215    head.eq_ignore_ascii_case("title:")
11216        .then(|| line[6..].trim())
11217}
11218
11219/// The implementer's own one-line title: the `TITLE:` line the implement
11220/// prompt asks for at the top of its SUMMARY. Candidate commits are all
11221/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
11222/// source, and a title that says as much is refused here too.
11223fn summary_title(summary: &str) -> Option<String> {
11224    let first = summary.lines().find(|l| !l.trim().is_empty())?;
11225    let raw = title_marker(first)?;
11226    if raw.is_empty() {
11227        return None;
11228    }
11229    let title = queue::title_from(raw, PR_TITLE_MAX);
11230    let lower = title.to_ascii_lowercase();
11231    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11232        return None;
11233    }
11234    Some(title)
11235}
11236
11237/// How `open_review`'s instruction begins; see [`landing_title`].
11238const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11239
11240/// Marker `open_review` gives a candidate that nothing in the roster wrote.
11241pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11242
11243/// Does this run review work that already existed, rather than implement a
11244/// task? Runs recorded before `reviewed_commits` existed carry only the
11245/// candidate marker.
11246pub(crate) fn is_review_run(state: &RunState) -> bool {
11247    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11248}
11249
11250/// The title of a review-only run: the subject of the oldest commit under
11251/// review. Later commits are usually fixups, and `instruction` is the review
11252/// prompt, which says nothing about the change. GitHub text is English, so a
11253/// non-ASCII or blank subject yields `None` and the caller's neutral title.
11254fn review_title(state: &RunState) -> Option<String> {
11255    english_subject(
11256        state.reviewed_commits.as_ref()?.first()?,
11257        state.config.graph.github_english(),
11258    )
11259}
11260
11261/// `raw` as a pull request title, or `None` when it is blank, not English
11262/// (GitHub text is), or one of magi's own candidate commit subjects.
11263fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11264    let raw = raw.trim();
11265    if raw.is_empty() {
11266        return None;
11267    }
11268    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11269        return None;
11270    }
11271    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11272        return None;
11273    }
11274    let title = queue::title_from(raw, PR_TITLE_MAX);
11275    let lower = title.to_ascii_lowercase();
11276    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11277        return None;
11278    }
11279    Some(title)
11280}
11281
11282/// What a review-only run's branch says about itself, read at the moment the
11283/// pull request is opened.
11284#[derive(Debug, Clone, PartialEq, Eq)]
11285struct BranchFacts {
11286    /// `(subject, body)` of each commit, oldest first.
11287    commits: Vec<(String, String)>,
11288    /// Trimmed `git diff --stat`.
11289    stat: String,
11290}
11291
11292/// Longest diff stat shown: this many file lines plus the summary line.
11293const STAT_FILE_LINES: usize = 25;
11294/// Cap on the commit list, well inside GitHub's 65536-character body limit.
11295const COMMITS_MAX_CHARS: usize = 20_000;
11296
11297/// Read the commits and diff stat of `base..branch`. `None` when git cannot
11298/// say or finds nothing, so the caller falls back to what the run recorded.
11299async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11300    let commits = git::commit_log(repo, base, branch).await.ok()?;
11301    if commits.is_empty() {
11302        return None;
11303    }
11304    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11305    let lines: Vec<&str> = stat.lines().collect();
11306    let stat = if lines.len() > STAT_FILE_LINES + 1 {
11307        let omitted = lines.len() - 1 - STAT_FILE_LINES;
11308        let more = format!(" ... {omitted} more file(s)");
11309        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11310        kept.push(&more);
11311        kept.push(lines[lines.len() - 1]);
11312        kept.join("\n")
11313    } else {
11314        lines.join("\n")
11315    };
11316    Some(BranchFacts { commits, stat })
11317}
11318
11319/// Defang what would break the surrounding markdown: a closing `</details>`
11320/// and a code fence.
11321fn markdown_safe(text: &str) -> String {
11322    text.replace("</details>", "&lt;/details&gt;")
11323        .replace("\x60\x60\x60", "~~~")
11324}
11325
11326fn neutral_title(state: &RunState, winner: char) -> String {
11327    format!(
11328        "chore: land candidate {} of run {}",
11329        winner.to_ascii_uppercase(),
11330        state.id
11331    )
11332}
11333
11334/// Reuse the graph's bounded fixer without starting another graph or driver.
11335/// Copy the working state back even on errors so spent rounds remain visible.
11336pub(crate) async fn repair_landing_gate(
11337    state: &mut RunState,
11338    winner: &Candidate,
11339    outcomes: &[CommandOutcome],
11340) -> Result<GateFix> {
11341    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11342        return Ok(GateFix::Stop);
11343    }
11344    let mut runner = Runner {
11345        roles: state.config.resolve_roles()?,
11346        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11347        state: state.clone(),
11348        pause: Pause::new(),
11349        interrupt: Pause::new(),
11350    };
11351    let result = runner.gate_fix_round(winner, outcomes).await;
11352    *state = runner.state;
11353    result
11354}
11355
11356/// Recheck a repair's e2e requirement across failures, resource deferrals and
11357/// process interruptions. A successful result only applies to its exact SHA.
11358pub(crate) async fn check_gate_repair_e2e(
11359    state: &mut RunState,
11360    winner: &Candidate,
11361    wait: &crate::repo_lock::WaitPolicy,
11362) -> Result<GateFix> {
11363    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11364        return Ok(GateFix::Retry);
11365    }
11366    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11367    if state
11368        .gate_repair_e2e
11369        .as_ref()
11370        .is_some_and(|r| r.head == head && r.passed)
11371    {
11372        return Ok(GateFix::Retry);
11373    }
11374    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11375        head: head.clone(),
11376        passed: false,
11377        outcomes: Vec::new(),
11378    });
11379    state.save()?;
11380    let commands = state.config.verify.e2e.clone();
11381    let outcomes = if commands.is_empty() {
11382        Vec::new()
11383    } else {
11384        let shell = state.config.shell();
11385        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11386        let cache_dir = state.config.cache_dir();
11387        let (outcomes, _) = with_cache_lease(
11388            state,
11389            cache_dir.as_deref(),
11390            "e2e",
11391            "e2e",
11392            &winner.worktree,
11393            &head,
11394            timeout,
11395            "gate repair e2e",
11396            wait,
11397            |state, budget| async move {
11398                run_e2e_with_retry(
11399                    state,
11400                    &shell,
11401                    &commands,
11402                    &winner.worktree,
11403                    budget,
11404                    "gate repair e2e",
11405                )
11406                .await
11407            },
11408        )
11409        .await?;
11410        outcomes
11411    };
11412    let deferred = verify_inconclusive(&outcomes);
11413    let passed = outcomes.iter().all(CommandOutcome::ok)
11414        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11415    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11416        head,
11417        passed,
11418        outcomes,
11419    });
11420    state.save()?;
11421    if deferred {
11422        Ok(GateFix::Defer)
11423    } else if !passed {
11424        state.event(
11425            "gate",
11426            "verify.e2e failed after gate repair; nothing published",
11427        );
11428        Ok(GateFix::Stop)
11429    } else {
11430        Ok(GateFix::Retry)
11431    }
11432}
11433
11434/// Run the same configured gate for a landing head and a fresh graph.
11435pub(crate) async fn run_landing_gate(
11436    state: &mut RunState,
11437    winner: &Candidate,
11438    wait: &crate::repo_lock::WaitPolicy,
11439) -> Result<Vec<CommandOutcome>> {
11440    let shell = state.config.shell();
11441    let gate_commands = state.config.verify.gate.clone();
11442    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11443    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11444        Vec::new()
11445    } else {
11446        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11447        let cache_dir = state.config.cache_dir();
11448        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11449        let (outcomes, _) = with_cache_lease(
11450            state,
11451            cache_dir.as_deref(),
11452            "gate",
11453            "gate",
11454            &winner.worktree,
11455            &head,
11456            timeout,
11457            "final gate",
11458            wait,
11459            |state, budget| {
11460                let shell = shell.clone();
11461                let gate_commands = gate_commands.clone();
11462                let winner = winner.clone();
11463                async move {
11464                    Runner::run_pre_gate(state, &winner).await;
11465                    let (outcomes, timed_out_pids) = run_commands(
11466                        state,
11467                        "gate",
11468                        "gate",
11469                        0,
11470                        &shell,
11471                        &gate_commands,
11472                        &winner.worktree,
11473                        budget,
11474                    )
11475                    .await;
11476                    (outcomes, false, timed_out_pids)
11477                }
11478            },
11479        )
11480        .await?;
11481        outcomes
11482    };
11483    if outcomes.is_empty() {
11484        // Nothing configured to check — distinct from every other
11485        // silence in this run's event log, since an empty `gate` alone
11486        // no longer says whether the gate ran at all (see
11487        // `RunState::gate_ran`'s own doc).
11488        state.event("gate", "no gate commands configured; not tested");
11489    }
11490    for o in &outcomes {
11491        state.event(
11492            "gate",
11493            format!(
11494                "`{}` -> {}",
11495                o.command,
11496                if o.ok() {
11497                    "pass".to_owned()
11498                } else {
11499                    format!(
11500                        "FAIL ({:?})\n{}",
11501                        o.code,
11502                        tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11503                    )
11504                }
11505            ),
11506        );
11507    }
11508    Ok(outcomes)
11509}
11510
11511/// The pull request title to hand to `land::merge_subject`. A review-only run
11512/// opened by an earlier build titled its pull request with the review prompt;
11513/// that title is dropped (empty, so the fallback applies) rather than landed.
11514/// Any other title, including an operator's rename, passes through untouched,
11515/// and so does every title of a run that implements a task.
11516pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11517    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11518        ""
11519    } else {
11520        pr_title
11521    }
11522}
11523
11524/// What the squash subject falls back to when the pull request title is empty
11525/// or candidate-shaped: for a review-only run the derived title, never the
11526/// review prompt held in `instruction`.
11527pub fn landing_subject_source(state: &RunState) -> String {
11528    if is_review_run(state) {
11529        let winner = state.candidates.first().map_or('A', |c| c.label);
11530        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11531    }
11532    state.instruction.clone()
11533}
11534
11535/// `summary` without its `TITLE:` line, which the pull request title already
11536/// carries.
11537fn summary_without_title(summary: &str) -> String {
11538    let mut lines = summary.trim().lines().peekable();
11539    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11540        lines.next();
11541    }
11542    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11543}
11544
11545/// The pull request title and body for the winning candidate.
11546///
11547/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
11548/// the task's own opening line via [`queue::title_from`] when there is none.
11549/// `state.instruction` can open with blank lines (`task_text` only rejects a
11550/// body that is blank *entirely*), which `title_from` skips.
11551///
11552/// Publishing uses an independent description generated from the final diff
11553/// and structured evidence. Internal summaries and reports remain local.
11554/// Non-publishing previews may display the original task and candidate summary.
11555/// The footer retains the run and candidate provenance.
11556#[cfg(test)]
11557fn pr_message(state: &RunState, winner: char) -> PrMessage {
11558    pr_message_with(state, winner, None)
11559}
11560
11561/// [`pr_message`] with what the branch of a review-only run says about itself.
11562/// `facts` is ignored for a run that implements a task.
11563#[cfg(test)]
11564fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11565    let raw = pr_message_raw(
11566        state,
11567        winner,
11568        facts,
11569        crate::pr_outcome::FinalHead::Unreported,
11570        ChangeNote::NotAsked,
11571    );
11572    let id = crate::scrub::Identity::current();
11573    PrMessage {
11574        title: crate::scrub::scrub(&raw.title, &id),
11575        body: crate::scrub::scrub(&raw.body, &id),
11576    }
11577}
11578
11579/// What the public description may say about the change itself.
11580#[derive(Debug, Clone, Copy)]
11581enum ChangeNote<'a> {
11582    /// A local preview or non-publishing merge mode: no public generation needed.
11583    NotAsked,
11584    /// A summary written from the final diff.
11585    Summary(&'a crate::run::ChangeSummary),
11586    /// A summary was asked for and could not be produced; the text says why.
11587    Unavailable(&'a str),
11588}
11589
11590fn pr_message_raw(
11591    state: &RunState,
11592    winner: char,
11593    facts: Option<&BranchFacts>,
11594    head: crate::pr_outcome::FinalHead<'_>,
11595    change: ChangeNote<'_>,
11596) -> PrMessage {
11597    if !matches!(change, ChangeNote::NotAsked) {
11598        let (title, mut body) = match change {
11599            ChangeNote::Summary(c) => (
11600                c.title.clone(),
11601                format!("## Summary\n\n{}\n\n", c.body.trim()),
11602            ),
11603            ChangeNote::Unavailable(why) => (
11604                neutral_title(state, winner),
11605                format!(
11606                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11607                ),
11608            ),
11609            ChangeNote::NotAsked => unreachable!(),
11610        };
11611        body.push_str(&public_pr_evidence(state, head));
11612        body.push_str(&format!(
11613            "\n---\nmagi:run/{} magi:candidate-{}\n",
11614            state.id,
11615            winner.to_ascii_lowercase()
11616        ));
11617        return PrMessage { title, body };
11618    }
11619    pr_preview(state, winner, facts, head)
11620}
11621
11622/// Append public facts from the run records after every prose generation,
11623/// including posting-gate rewrites. The agent cannot drop this provenance.
11624fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11625    let mut body = crate::pr_outcome::review_rounds(state);
11626    body.push_str(&crate::pr_outcome::final_verification(state, head));
11627    let open = state.last_round_findings();
11628    if !open.is_empty() {
11629        body.push_str("## Open review findings\n\n");
11630        for f in open {
11631            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11632        }
11633        body.push('\n');
11634    }
11635    if let Some(c) = state.change_summary.as_ref()
11636        && head == crate::pr_outcome::FinalHead::At(&c.head)
11637    {
11638        body.push_str(&format!(
11639            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11640            c.base, c.head
11641        ));
11642    }
11643    body
11644}
11645
11646/// Local-only preview of the operator's task and candidate summary.
11647fn pr_preview(
11648    state: &RunState,
11649    winner: char,
11650    facts: Option<&BranchFacts>,
11651    head: crate::pr_outcome::FinalHead<'_>,
11652) -> PrMessage {
11653    let summary = state
11654        .candidates
11655        .iter()
11656        .find(|c| c.label == winner)
11657        .map(|c| c.summary.as_str())
11658        .unwrap_or_default();
11659    // The fallback is the operator's own words and may not be English; GitHub
11660    // text always is, so a non-English task gets a neutral title instead.
11661    let review = is_review_run(state);
11662    let title = if review {
11663        facts
11664            .and_then(|f| {
11665                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11666            })
11667            .or_else(|| review_title(state))
11668            .or_else(|| {
11669                state
11670                    .candidates
11671                    .iter()
11672                    .find(|c| c.label == winner)
11673                    .filter(|c| !c.branch.starts_with("magi/"))
11674                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11675            })
11676            .unwrap_or_else(|| neutral_title(state, winner))
11677    } else {
11678        summary_title(summary).unwrap_or_else(|| {
11679            english_title(
11680                &state.instruction,
11681                PR_TITLE_MAX,
11682                state.config.graph.github_english(),
11683            )
11684            .unwrap_or_else(|| neutral_title(state, winner))
11685        })
11686    };
11687
11688    let mut body = String::new();
11689    let what = summary_without_title(summary);
11690    if !what.is_empty() {
11691        body.push_str("## Summary\n\n");
11692        body.push_str(&what);
11693        body.push_str("\n\n");
11694    }
11695
11696    // Review history uses structured facts across all rounds, never reports.
11697    body.push_str(&crate::pr_outcome::review_rounds(state));
11698    body.push_str(&crate::pr_outcome::final_verification(state, head));
11699    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11700
11701    let open = state.open_findings();
11702    if !open.is_empty() {
11703        body.push_str("## Open review findings\n\n");
11704        for f in &open {
11705            body.push_str(&format!(
11706                "- `{}` [{:?}] {}\n",
11707                f.id, f.severity, "Context retained in local review records."
11708            ));
11709        }
11710        body.push('\n');
11711    }
11712
11713    if let Some(fix) = fix
11714        && !fix.rejected.is_empty()
11715    {
11716        body.push_str("## Declined by the fixer\n\n");
11717        for r in &fix.rejected {
11718            body.push_str(&format!(
11719                "- `{}`: {}\n",
11720                r.id, "Reason retained in local review records."
11721            ));
11722        }
11723        body.push('\n');
11724    }
11725
11726    if review {
11727        // The review prompt is not the task; list what the branch carries.
11728        body.push_str("## Commits under review\n\n");
11729        if let Some(facts) = facts {
11730            let mut left = COMMITS_MAX_CHARS;
11731            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11732                let mut entry = format!("- {}\n", markdown_safe(subject));
11733                for l in markdown_safe(text).lines() {
11734                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11735                }
11736                if left == 0 {
11737                    body.push_str(&format!(
11738                        "- ... {} more commit(s)\n",
11739                        facts.commits.len() - i
11740                    ));
11741                    break;
11742                }
11743                if entry.len() > left {
11744                    // Even the first commit is cut: one huge body must not
11745                    // push the whole description past GitHub's limit.
11746                    let mut end = left;
11747                    while !entry.is_char_boundary(end) {
11748                        end -= 1;
11749                    }
11750                    entry.truncate(end);
11751                    entry.push_str("\n  ... (truncated)\n");
11752                    left = 0;
11753                } else {
11754                    left -= entry.len();
11755                }
11756                body.push_str(&entry);
11757            }
11758            if !facts.stat.trim().is_empty() {
11759                body.push_str(&format!(
11760                    "\n## Diff stat\n\n```\n{}\n```\n",
11761                    markdown_safe(facts.stat.trim())
11762                ));
11763            }
11764        } else {
11765            match &state.reviewed_commits {
11766                Some(subjects) => {
11767                    for s in subjects {
11768                        body.push_str(&format!("- {}\n", s.trim()));
11769                    }
11770                }
11771                None => {
11772                    // An older run kept only the prompt, with the commit list
11773                    // after its first paragraph.
11774                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11775                    body.push_str(rest.trim());
11776                    body.push('\n');
11777                }
11778            }
11779        }
11780    } else {
11781        let task = state.instruction.trim();
11782        let task = if task.is_empty() {
11783            "(empty task)"
11784        } else {
11785            task
11786        };
11787        body.push_str(&format!(
11788            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11789            task.replace("</details>", "&lt;/details&gt;")
11790        ));
11791    }
11792
11793    body.push_str(&format!(
11794        "\n---\nmagi:run/{} magi:candidate-{}\n",
11795        state.id,
11796        winner.to_ascii_lowercase()
11797    ));
11798
11799    PrMessage { title, body }
11800}
11801
11802/// Source imports have already been prepared from structured inputs. Legacy
11803/// inferred references must never instruct a seat to reconstruct those imports.
11804fn seeded_instruction(state: &RunState) -> String {
11805    let mut inputs = Vec::new();
11806    if let Some(start) = &state.sources.start_from {
11807        inputs.push(format!(
11808            "Start from `{}` at `{}` (including ancestry).",
11809            start.token, start.sha
11810        ));
11811    }
11812    for pick in &state.sources.pick_commits {
11813        inputs.push(format!(
11814            "Pick `{}` at `{}` (single commit).",
11815            pick.token, pick.sha
11816        ));
11817    }
11818    if inputs.is_empty() {
11819        state.instruction.clone()
11820    } else {
11821        format!(
11822            "{}\n\n# Explicit source inputs already prepared\n\n{}",
11823            state.instruction,
11824            inputs.join("\n")
11825        )
11826    }
11827}
11828
11829/// Does the winner have no commits ahead of the base it would land on?
11830/// Any failure to find out reads as "not empty": the merge then behaves as it
11831/// always did rather than refusing on a guess.
11832async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11833    let base = &state.base_branch;
11834    let mut against = base.clone();
11835    if mode == MergeMode::Pr {
11836        // A pull request lands on the remote's base, never the local branch
11837        // of the same name: if that cannot be read, "not empty" is the safe
11838        // answer.
11839        let remote = &state.config.merge.remote;
11840        let tracking = format!("{remote}/{base}");
11841        let fetched = git::fetch(repo, remote, base).await;
11842        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11843            against = tracking;
11844        } else {
11845            return false;
11846        }
11847    }
11848    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11849}
11850
11851/// Why nothing was opened for an empty winner, with what the task's own
11852/// references resolved to.
11853fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11854    let mut detail = format!(
11855        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11856         nothing to open a pull request for"
11857    );
11858    detail.push_str(
11859        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11860    );
11861    detail
11862}
11863
11864/// What the `Pr` merge does once it knows whether the branch already has an
11865/// open pull request.
11866#[derive(Debug, PartialEq, Eq)]
11867enum PrPlan {
11868    Create,
11869    Adopt { url: String, title: String },
11870    Stop(String),
11871}
11872
11873/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11874/// a failed lookup -> stop with the real reason. Never guesses.
11875fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11876    match found {
11877        Ok(land::OpenPr::None) => PrPlan::Create,
11878        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11879        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11880            "several open pull requests exist for this branch, not picking one: {}",
11881            urls.join(" ")
11882        )),
11883        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11884    }
11885}
11886
11887/// `gh pr create`, returning the PR url.
11888pub(crate) async fn gh_pr_create(
11889    cwd: &Path,
11890    base: &str,
11891    head: &str,
11892    title: &str,
11893    body: &str,
11894) -> Result<String> {
11895    let out = tokio::process::Command::new("gh")
11896        .args([
11897            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11898        ])
11899        .current_dir(cwd)
11900        .quiet()
11901        .stdin(std::process::Stdio::null())
11902        .output()
11903        .await
11904        .context("spawn gh")?;
11905    if out.status.success() {
11906        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11907    } else {
11908        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11909    }
11910}
11911
11912/// Never force deletion of uncommitted work, including unknown legacy trees.
11913async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11914    match std::fs::symlink_metadata(path) {
11915        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11916            // A previous cleanup may have removed the tree before a branch
11917            // deletion or state save failed. Unregister it if still present.
11918            git::worktree_remove_clean(repo, path).await?;
11919            return Ok(());
11920        }
11921        Err(e) => return Err(e.into()),
11922        Ok(_) => {}
11923    }
11924    if !git::is_clean(path).await? {
11925        bail!(
11926            "worktree has uncommitted work; refusing removal: {}",
11927            path.display()
11928        );
11929    }
11930    if !git::worktree_remove_clean(repo, path).await? {
11931        bail!("worktree removal refused: {}", path.display());
11932    }
11933    Ok(())
11934}
11935
11936/// Tear a run's worktrees and branches down.
11937///
11938/// `home` is where the updated `run.json` is saved (via
11939/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11940/// a housekeeping pass already has its own honest `home` handed to it, and
11941/// falling through to the global here would write back through whichever
11942/// directory some other process or test pinned into that `OnceLock` first,
11943/// not the one the caller actually resolved its `runs` and `state` from.
11944pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11945    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11946        bail!("refusing fold: {reason}");
11947    }
11948    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11949        bail!("candidate preservation incomplete; refusing cleanup");
11950    }
11951    // Serialize ordinary folding with the same starts/resumes as target-only
11952    // eviction. Otherwise a second janitor could remove the whole worktree
11953    // around a live output borrower even though target eviction was fenced.
11954    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11955    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11956    let _driver = crate::driver::maintenance(home, &state.id)?;
11957    let mut output_guards = Vec::new();
11958    for entry in std::fs::read_dir(state.worktree_root())
11959        .into_iter()
11960        .flatten()
11961        .flatten()
11962    {
11963        if entry.path().join(".git").is_file() {
11964            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11965        }
11966    }
11967    let repo = state.repo.clone();
11968    // CLI, Web and automatic cleanup all enter here, independently of
11969    // Runner::fold_losers. Protect deletion and shared config release too.
11970    let wait = repository_wait(state, None);
11971    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11972        Ok(lease) => lease,
11973        Err(e) => {
11974            state.save_under(home)?;
11975            return Err(e);
11976        }
11977    };
11978    // Confirmed merges dispose of their owned sources after durable evidence
11979    // capture. Preflight every source before removing anything.
11980    let _source_outputs = if state.status == RunStatus::Merged
11981        && state.merge.as_ref().is_some_and(|merge| merge.ok)
11982    {
11983        verify_merged_fold_sources(state, drop_winner, home).await?
11984    } else {
11985        Vec::new()
11986    };
11987    let root = state.worktree_root();
11988    let winner = state.tally.as_ref().map(|t| t.winner);
11989    let mut removed = Vec::new();
11990
11991    for i in 0..state.candidates.len() {
11992        let c = state.candidates[i].clone();
11993        let is_winner = Some(c.label) == winner;
11994        if is_winner && !drop_winner {
11995            continue;
11996        }
11997        let existed = c.worktree.exists();
11998        // A vanished directory can still have a Git registration that keeps
11999        // its branch checked out. Remove that registration before the branch.
12000        remove_preserved_worktree(&repo, &c.worktree).await?;
12001        if existed {
12002            removed.push(c.worktree.to_string_lossy().into_owned());
12003        }
12004        // A branch handed to a later run (and its pull request) is not this
12005        // run's to delete.
12006        let handed_over = state.released_branches.contains(&c.branch);
12007        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
12008            if !git::branch_delete(&repo, &c.branch).await? {
12009                bail!("candidate branch removal refused");
12010            }
12011            removed.push(c.branch.clone());
12012        }
12013        state.candidates[i].folded = true;
12014    }
12015
12016    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
12017        let path = name.path();
12018        let keep = !drop_winner
12019            && winner.is_some_and(|w| {
12020                path.file_name()
12021                    .is_some_and(|n| n == format!("cand-{w}").as_str())
12022            });
12023        if keep {
12024            continue;
12025        }
12026        remove_preserved_worktree(&repo, &path).await?;
12027        removed.push(path.to_string_lossy().into_owned());
12028    }
12029
12030    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
12031    // judge worktrees, so once the loop above has cleared all of them out,
12032    // the parent is a bare directory nobody else was ever going to remove -
12033    // git only ever managed what was inside it. Left alone, one of these
12034    // accumulates per fully-folded run; the operator's own machine had 74.
12035    // `remove_if_empty` re-checks rather than assuming: a run whose winner
12036    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
12037    // so does anything a run never claimed that happens to share the bay.
12038    remove_if_empty(&root);
12039
12040    if state.enabled_worktree_config && drop_winner {
12041        // A release, not a raw disable: some sibling run in this repository
12042        // may still hold its own reference (see `git::acquire_worktree_config`),
12043        // and only the last release actually turns the setting back off.
12044        git::release_worktree_config(&repo).await.ok();
12045        state.enabled_worktree_config = false;
12046    }
12047    state.save_under(home)?;
12048    Ok(removed)
12049}
12050
12051async fn verify_merged_fold_sources(
12052    state: &RunState,
12053    drop_winner: bool,
12054    home: &Path,
12055) -> Result<Vec<std::fs::File>> {
12056    let winner = state.tally.as_ref().map(|t| t.winner);
12057    let mut heads = BTreeSet::new();
12058    for candidate in &state.candidates {
12059        if Some(candidate.label) == winner && !drop_winner {
12060            continue;
12061        }
12062        anyhow::ensure!(
12063            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
12064            "candidate recovery source is outside the inventoried run root: {}",
12065            candidate.worktree.display()
12066        );
12067        match std::fs::symlink_metadata(&candidate.worktree) {
12068            Ok(_) => {}
12069            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12070            Err(error) => return Err(error.into()),
12071        }
12072        if !state.released_branches.contains(&candidate.branch)
12073            && git::branch_exists(&state.repo, &candidate.branch).await?
12074        {
12075            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
12076        }
12077    }
12078    let mut cargo_guards = Vec::new();
12079    let entries = match std::fs::read_dir(state.worktree_root()) {
12080        Ok(entries) => Some(entries),
12081        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
12082        Err(error) => return Err(error.into()),
12083    };
12084    for entry in entries.into_iter().flatten() {
12085        let path = entry?.path();
12086        let metadata = std::fs::symlink_metadata(&path)?;
12087        anyhow::ensure!(
12088            metadata.is_dir() && !metadata.file_type().is_symlink(),
12089            "non-directory or symlink recovery source retained: {}",
12090            path.display()
12091        );
12092        if !drop_winner
12093            && winner.is_some_and(|w| {
12094                path.file_name()
12095                    .is_some_and(|name| name == format!("cand-{w}").as_str())
12096            })
12097        {
12098            continue;
12099        }
12100        anyhow::ensure!(
12101            path.join(".git").is_file(),
12102            "unregistered recovery source retained: {}",
12103            path.display()
12104        );
12105        anyhow::ensure!(
12106            git::git(
12107                &path,
12108                &[
12109                    "status",
12110                    "--porcelain",
12111                    "--untracked-files=all",
12112                    "--ignored",
12113                    "--",
12114                    ".",
12115                    ":(exclude)target"
12116                ]
12117            )
12118            .await?
12119            .is_empty(),
12120            "source changes or ignored recovery files retained: {}",
12121            path.display()
12122        );
12123        match std::fs::symlink_metadata(path.join("target")) {
12124            Ok(_) => {
12125                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
12126            }
12127            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
12128            Err(error) => return Err(error.into()),
12129        }
12130        for checkpoint in [
12131            "rebase-merge",
12132            "rebase-apply",
12133            "sequencer",
12134            "CHERRY_PICK_HEAD",
12135            "MERGE_HEAD",
12136            "REBASE_HEAD",
12137            "REVERT_HEAD",
12138            "BISECT_LOG",
12139            "AUTO_MERGE",
12140            "MERGE_MSG",
12141        ] {
12142            let location = git::git(
12143                &path,
12144                &[
12145                    "rev-parse",
12146                    "--path-format=absolute",
12147                    "--git-path",
12148                    checkpoint,
12149                ],
12150            )
12151            .await?;
12152            anyhow::ensure!(
12153                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
12154                if e.kind() == std::io::ErrorKind::NotFound),
12155                "Git recovery checkpoint retained: {checkpoint}"
12156            );
12157        }
12158        heads.insert(git::rev_parse(&path, "HEAD").await?);
12159    }
12160    crate::fold_evidence::preserve(state, home, &heads).await?;
12161
12162    Ok(cargo_guards)
12163}
12164
12165/// Remove `dir` if it exists and has nothing in it.
12166///
12167/// Best-effort and silent by design: a directory that is not empty (a run
12168/// whose winner is still parked there, a stray file some other process left)
12169/// is exactly the case this must refuse, and a directory that is already gone
12170/// is not a failure worth reporting either. `std::fs::remove_dir` itself
12171/// already refuses a non-empty directory, so the emptiness check below is
12172/// belt, not suspenders - it is what keeps this from ever attempting the
12173/// removal in the case that matters, rather than trusting `remove_dir`'s
12174/// error path to have no side effects if it ever changed.
12175fn remove_if_empty(dir: &Path) {
12176    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
12177        std::fs::remove_dir(dir).ok();
12178    }
12179}
12180
12181/// Severity of the worst open finding in the last review round, for reporting.
12182pub fn worst_open(state: &RunState) -> Option<Severity> {
12183    state
12184        .reviews
12185        .last()?
12186        .reviews
12187        .iter()
12188        .flat_map(|r| r.findings.iter())
12189        .map(|f| f.severity)
12190        .max()
12191}
12192
12193/// `[worktree] setup` hides its products per worktree, which needs
12194/// `extensions.worktreeConfig`. Taken once per run and released with the run's
12195/// worktrees, like the hook's; every place that runs setup calls this first.
12196async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12197    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12198        git::acquire_worktree_config(repo).await?;
12199        state.enabled_worktree_config = true;
12200    }
12201    Ok(())
12202}
12203
12204#[cfg(test)]
12205mod tests {
12206    #[tokio::test]
12207    async fn launcher_base_fetch_requires_remote_success() {
12208        let tmp = tempfile::tempdir().unwrap();
12209        let repo = tmp.path().join("repo");
12210        let remote = tmp.path().join("remote");
12211        std::fs::create_dir(&repo).unwrap();
12212        std::fs::create_dir(&remote).unwrap();
12213        git::git(&remote, &["init", "--bare", "-b", "main"])
12214            .await
12215            .unwrap();
12216        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12217        git::git(
12218            &repo,
12219            &[
12220                "-c",
12221                "user.name=Fixture",
12222                "-c",
12223                "user.email=fixture@example.invalid",
12224                "commit",
12225                "--allow-empty",
12226                "-m",
12227                "fixture",
12228            ],
12229        )
12230        .await
12231        .unwrap();
12232        git::git(
12233            &repo,
12234            &["remote", "add", "origin", remote.to_str().unwrap()],
12235        )
12236        .await
12237        .unwrap();
12238        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12239        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12240        assert_eq!(
12241            resolve_base(&repo, "main", "origin", None)
12242                .await
12243                .unwrap()
12244                .commit,
12245            head
12246        );
12247        #[cfg(unix)]
12248        {
12249            let policy: crate::config::Coordination = toml::from_str(
12250                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12251            ).unwrap();
12252            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12253            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12254            let held = crate::repo_lock::SeatLease::operation_with(
12255                &repo,
12256                Some(&policy),
12257                &crate::repo_lock::WaitPolicy::default(),
12258            )
12259            .await
12260            .unwrap();
12261            assert!(held.is_some());
12262            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12263            tokio::pin!(fetch);
12264            assert!(
12265                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12266                    .await
12267                    .is_err()
12268            );
12269            assert!(
12270                !repo.join(".git/FETCH_HEAD").exists(),
12271                "no fetch before admission"
12272            );
12273            drop(held);
12274            let fetched = fetch.await.unwrap();
12275            assert_eq!(fetched.commit, head);
12276            assert_eq!(fetched.remote, "origin");
12277            assert_eq!(fetched.branch, "main");
12278        }
12279        // A surviving tracking ref is not successful fetch evidence.
12280        git::git(
12281            &repo,
12282            &[
12283                "remote",
12284                "set-url",
12285                "origin",
12286                tmp.path().join("missing").to_str().unwrap(),
12287            ],
12288        )
12289        .await
12290        .unwrap();
12291        let error = resolve_base(&repo, "main", "origin", None)
12292            .await
12293            .unwrap_err()
12294            .to_string();
12295        assert!(error.contains("cannot read origin/main"));
12296        assert!(error.contains("refusing to branch off"));
12297    }
12298
12299    #[test]
12300    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12301        let leaked = vec!["chore(deps): update a crate".to_owned()];
12302        let own = "fix(daemon): apply a chosen action";
12303        assert!(should_retitle("", own, &leaked));
12304        assert!(should_retitle(
12305            &format!("{REVIEW_PROMPT_OPENING} `x`"),
12306            own,
12307            &leaked
12308        ));
12309        assert!(should_retitle(
12310            "chore: land candidate A of run 1",
12311            own,
12312            &leaked
12313        ));
12314        assert!(should_retitle(
12315            "magi: candidate A (uncommitted work)",
12316            own,
12317            &leaked
12318        ));
12319        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12320        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12321        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12322    }
12323
12324    #[test]
12325    fn pr_merge_plan_creates_adopts_or_stops() {
12326        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12327        assert_eq!(
12328            pr_merge_plan(Ok(land::OpenPr::One {
12329                url: "u".into(),
12330                title: "t".into()
12331            })),
12332            PrPlan::Adopt {
12333                url: "u".into(),
12334                title: "t".into()
12335            }
12336        );
12337        let PrPlan::Stop(many) =
12338            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12339        else {
12340            panic!("many must stop");
12341        };
12342        assert!(many.contains('a') && many.contains('b'));
12343        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12344            panic!("a failed lookup must stop");
12345        };
12346        assert!(err.contains("bad token"));
12347    }
12348
12349    use super::*;
12350    use crate::run::GateStatus;
12351    use std::collections::BTreeMap;
12352    use std::time::Duration;
12353
12354    fn conductor() -> AgentSpec {
12355        AgentSpec {
12356            id: "conductor".to_owned(),
12357            kind: crate::config::AgentKind::Command,
12358            model: None,
12359            command: vec!["true".to_owned()],
12360            extra_args: Vec::new(),
12361            env: BTreeMap::new(),
12362            prompt_delivery: None,
12363            max_turn_secs: None,
12364        }
12365    }
12366
12367    fn spec(id: &str) -> AgentSpec {
12368        AgentSpec {
12369            id: id.to_owned(),
12370            kind: crate::config::AgentKind::Command,
12371            model: None,
12372            command: vec!["true".to_owned()],
12373            extra_args: Vec::new(),
12374            env: BTreeMap::new(),
12375            prompt_delivery: None,
12376            max_turn_secs: None,
12377        }
12378    }
12379
12380    fn ids(xs: &[&str]) -> BTreeSet<String> {
12381        xs.iter().map(|s| (*s).to_owned()).collect()
12382    }
12383
12384    #[test]
12385    fn pick_successor_skips_an_agent_another_seat_holds() {
12386        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
12387        let roster = [spec("a"), spec("b"), spec("c")];
12388        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12389        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12390    }
12391
12392    #[test]
12393    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12394        // The other seat started on c but was handed to d; c is free again.
12395        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12396        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12397        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12398        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12399        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12400    }
12401
12402    #[test]
12403    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12404        let roster = [spec("a"), spec("b")];
12405        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12406        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12407        let carried = ids(&["b"]);
12408        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12409        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12410    }
12411
12412    #[test]
12413    fn pick_successor_returns_none_without_any_untried_successor() {
12414        let roster = [spec("a"), spec("b")];
12415        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12416        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12417        let carried = ids(&["a"]);
12418        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12419    }
12420
12421    #[test]
12422    fn pick_successor_rescue_avoids_another_seats_occupant() {
12423        // b is a carried failure and free; a is held by the other seat.
12424        let roster = [spec("a"), spec("b"), spec("c")];
12425        let carried = ids(&["b", "c"]);
12426        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12427        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12428    }
12429
12430    #[test]
12431    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12432        let roster = [spec("a"), spec("b"), spec("c")];
12433        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12434        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12435    }
12436
12437    #[test]
12438    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12439        let roster = [spec("a"), spec("b"), spec("c")];
12440        let failed = ids(&["a", "b", "c"]);
12441        // Rescue looks at the whole roster, once per id, then runs out.
12442        let mut tried = ids(&["b"]);
12443        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12444        assert_eq!(first.id, "a");
12445        tried.insert(first.id.clone());
12446        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12447        assert_eq!(second.id, "c");
12448        tried.insert(second.id.clone());
12449        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12450    }
12451
12452    #[test]
12453    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12454        let roster = [spec("a"), spec("b")];
12455        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12456        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12457    }
12458
12459    #[test]
12460    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12461        let roster = [spec("a"), spec("b"), spec("c")];
12462        let h = SeatHistory {
12463            failed: ids(&["a"]),
12464            last_ok: Some("b".to_owned()),
12465            last_fail: None,
12466        };
12467        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12468        // A last answerer that left the roster, or later failed, is ignored.
12469        let gone = SeatHistory {
12470            last_ok: Some("zzz".to_owned()),
12471            ..h.clone()
12472        };
12473        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12474        let failed = SeatHistory {
12475            failed: ids(&["a", "b"]),
12476            last_ok: Some("b".to_owned()),
12477            last_fail: None,
12478        };
12479        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12480    }
12481
12482    #[test]
12483    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12484        let first = SeatState::new("review-1", "alpha", 7);
12485        let next = handover_seat("review-1", "gamma", 7);
12486        assert_ne!(first.claude_session, next.claude_session);
12487    }
12488
12489    #[test]
12490    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12491        let mut state = state_with_summary("x", "y");
12492        let a = handover_seat("review-1", "beta", state.next_seat_seed());
12493        let b = handover_seat("review-1", "beta", state.next_seat_seed());
12494        assert_ne!(a.claude_session, b.claude_session);
12495    }
12496
12497    #[test]
12498    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12499        let roster = [spec("a"), spec("b")];
12500        let h = SeatHistory {
12501            failed: ids(&["a", "b"]),
12502            ..SeatHistory::default()
12503        };
12504        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12505        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12506        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12507    }
12508
12509    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
12510    // fallback loop depends on to terminate: it must walk forward from the
12511    // seat's own position, never restart at the front of the roster, and it
12512    // must never hand back an id already tried, however many times that id
12513    // happens to appear.
12514
12515    #[test]
12516    fn failure_signature_ignores_numbers_and_paths() {
12517        assert_eq!(
12518            failure_signature("exited with Some(2) and no usable output"),
12519            failure_signature("exited with Some(137) and no usable output")
12520        );
12521        assert_eq!(
12522            failure_signature("process exited with no usable output (exit code 2)"),
12523            failure_signature("process exited with no usable output (exit code 137)")
12524        );
12525        for (old, new) in [
12526            (
12527                "exited with Some(2) and no usable output",
12528                "process exited with no usable output (exit code 2)",
12529            ),
12530            (
12531                "exited with None and no usable output",
12532                "process exited with no usable output (exit code unavailable)",
12533            ),
12534        ] {
12535            let previous = FailClass::Other(failure_signature(old));
12536            let current = FailClass::Other(failure_signature(new));
12537            assert_eq!(previous, current, "resumption retains the failure class");
12538            assert!(!should_hand_over(Some(&previous), &current));
12539        }
12540        assert_eq!(
12541            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12542            failure_signature("cannot open /var/x.txt: denied")
12543        );
12544        assert_ne!(failure_signature("boom"), failure_signature("bang"));
12545    }
12546
12547    #[test]
12548    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12549        let other = FailClass::Other("x".into());
12550        assert!(should_hand_over(None, &FailClass::Quota));
12551        assert!(should_hand_over(Some(&other), &FailClass::Quota));
12552        assert!(should_hand_over(
12553            Some(&FailClass::Timeout),
12554            &FailClass::Timeout
12555        ));
12556        assert!(should_hand_over(None, &other));
12557        assert!(!should_hand_over(Some(&other), &other));
12558        assert!(should_hand_over(
12559            Some(&other),
12560            &FailClass::Other("y".into())
12561        ));
12562        // A quota or timeout in between ends the run of identical failures.
12563        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12564        assert!(should_hand_over(Some(&FailClass::Quota), &other));
12565    }
12566
12567    #[test]
12568    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12569        let a = SeatState::new("judge-1", "alpha", 7);
12570        let b = handover_seat("judge-1", "beta", 7);
12571        assert_ne!(a.claude_session, b.claude_session);
12572        assert_eq!(b.turns, 0);
12573    }
12574
12575    #[test]
12576    fn a_timeout_is_classified_apart_from_other_failures() {
12577        assert_eq!(
12578            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12579            Some(FailClass::Timeout)
12580        );
12581        assert!(matches!(
12582            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12583            Some(FailClass::Other(_))
12584        ));
12585    }
12586
12587    #[test]
12588    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12589        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12590        let tried = BTreeSet::from(["beta".to_owned()]);
12591        // beta sits at index 1; the next candidate is gamma, never alpha —
12592        // which is very likely a different candidate slot's own agent.
12593        let next = next_untried_in_roster(&roster, 1, &tried);
12594        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12595    }
12596
12597    #[test]
12598    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12599        let roster = vec![spec("alpha"), spec("beta")];
12600        let tried = BTreeSet::from(["beta".to_owned()]);
12601        // beta is the roster's last entry: nothing follows it, and alpha —
12602        // earlier in the roster, almost certainly a different candidate
12603        // slot's own agent — must not be reached by wrapping back to it.
12604        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12605    }
12606
12607    #[test]
12608    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12609        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12610        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12611        // beta (index 1) and gamma (index 2, the only entry after it) have
12612        // both been tried; alpha (index 0) never has, but it comes before
12613        // beta's own position, so there is nothing further for this seat.
12614        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12615    }
12616
12617    #[test]
12618    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12619        let roster = vec![spec("a"), spec("a"), spec("b")];
12620        let tried = BTreeSet::from(["a".to_owned()]);
12621        let next = next_untried_in_roster(&roster, 0, &tried);
12622        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12623    }
12624
12625    #[test]
12626    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12627        let roster = vec![spec("a"), spec("b")];
12628        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12629        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12630    }
12631
12632    #[test]
12633    fn remove_if_empty_only_ever_takes_a_bare_directory() {
12634        let dir = tempfile::tempdir().unwrap();
12635        let bay = dir.path().join("ffff");
12636
12637        // Not there yet: nothing to do, nothing to panic on.
12638        remove_if_empty(&bay);
12639        assert!(!bay.exists());
12640
12641        // Something still inside - the winner's worktree, or a stray file -
12642        // keeps the directory standing.
12643        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12644        remove_if_empty(&bay);
12645        assert!(bay.exists(), "non-empty directory must survive");
12646
12647        // Once the last entry is gone, so is the directory itself.
12648        std::fs::remove_dir(bay.join("cand-A")).unwrap();
12649        remove_if_empty(&bay);
12650        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12651    }
12652
12653    // `round_is_clean` is the exact decision this task fixed: a round with a
12654    // seat that never answered must not read the same as a round every seat
12655    // actually reviewed. These are deterministic and process-free by design —
12656    // the equivalent end-to-end check (a real reviewer timing out under a
12657    // live graph run) is a genuine race against wall-clock contention, and a
12658    // spawn slow enough to blow even a generous budget under a loaded test
12659    // run must not turn this specific regression check flaky.
12660
12661    #[test]
12662    fn a_full_panel_that_found_nothing_is_clean() {
12663        assert!(round_is_clean(
12664            0,
12665            true,
12666            2,
12667            2,
12668            0,
12669            IncompleteReviewPolicy::Block
12670        ));
12671    }
12672
12673    #[test]
12674    fn a_missing_seat_is_never_clean_under_the_default_policy() {
12675        assert!(!round_is_clean(
12676            0,
12677            true,
12678            1,
12679            2,
12680            0,
12681            IncompleteReviewPolicy::Block
12682        ));
12683    }
12684
12685    #[test]
12686    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12687        assert!(!round_is_clean(
12688            1,
12689            true,
12690            1,
12691            2,
12692            0,
12693            IncompleteReviewPolicy::Warn
12694        ));
12695    }
12696
12697    #[test]
12698    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12699        assert!(round_is_clean(
12700            0,
12701            true,
12702            1,
12703            2,
12704            0,
12705            IncompleteReviewPolicy::Warn
12706        ));
12707    }
12708
12709    #[test]
12710    fn a_full_panel_with_an_open_finding_is_not_clean() {
12711        assert!(!round_is_clean(
12712            1,
12713            true,
12714            2,
12715            2,
12716            0,
12717            IncompleteReviewPolicy::Block
12718        ));
12719    }
12720
12721    #[test]
12722    fn a_full_panel_with_a_red_e2e_is_not_clean() {
12723        assert!(!round_is_clean(
12724            0,
12725            false,
12726            2,
12727            2,
12728            0,
12729            IncompleteReviewPolicy::Block
12730        ));
12731    }
12732
12733    // The stall this task closes: under the default `block` policy, a seat
12734    // missing only because it was rate limited must not force a wait for a
12735    // session limit that will not lift by the next round. `round_is_clean`
12736    // is where that quorum carve-out lives; the review loop around it never
12737    // changes what a reviewer's vote or a finding's severity means.
12738
12739    #[test]
12740    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12741        // 1 of 2 answered, and the one missing was quota'd — the exact
12742        // "review-2 rate limited (quota)" shape from the field report.
12743        assert!(round_is_clean(
12744            0,
12745            true,
12746            1,
12747            2,
12748            1,
12749            IncompleteReviewPolicy::Block
12750        ));
12751    }
12752
12753    #[test]
12754    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12755        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
12756        // not a quota loss (`quota_missing` stays 0) — worth another try.
12757        assert!(!round_is_clean(
12758            0,
12759            true,
12760            1,
12761            2,
12762            0,
12763            IncompleteReviewPolicy::Block
12764        ));
12765    }
12766
12767    #[test]
12768    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12769        assert!(!round_is_clean(
12770            1,
12771            true,
12772            1,
12773            2,
12774            1,
12775            IncompleteReviewPolicy::Block
12776        ));
12777        assert!(!round_is_clean(
12778            0,
12779            false,
12780            1,
12781            2,
12782            1,
12783            IncompleteReviewPolicy::Block
12784        ));
12785    }
12786
12787    #[test]
12788    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12789        // Every seat quota'd, nobody answered: there is no panel to decide
12790        // on, so this must fall through to the existing block-and-retry
12791        // fallback rather than call an unreviewed patch clean.
12792        assert!(!round_is_clean(
12793            0,
12794            true,
12795            0,
12796            2,
12797            2,
12798            IncompleteReviewPolicy::Block
12799        ));
12800    }
12801
12802    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12803        CommandOutcome {
12804            command: "test".to_owned(),
12805            code,
12806            output_tail: String::new(),
12807            duration_ms: 0,
12808            resource_blocked,
12809        }
12810    }
12811
12812    #[test]
12813    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12814        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12815        assert!(
12816            !verify_inconclusive(&[outcome(Some(1), false)]),
12817            "an ordinary failure is still evidence about the patch"
12818        );
12819        assert!(verify_inconclusive(&[outcome(None, true)]));
12820        assert!(
12821            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12822            "one inconclusive outcome taints the whole batch"
12823        );
12824        assert!(!verify_inconclusive(&[]));
12825    }
12826
12827    #[tokio::test]
12828    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12829        // Alive for the first two checks, then dead - confirms the loop
12830        // actually re-polls rather than deciding once and sleeping out the
12831        // ceiling regardless.
12832        let calls = std::sync::atomic::AtomicUsize::new(0);
12833        let started = Instant::now();
12834        wait_for_pids_with(
12835            &[123],
12836            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12837            Duration::from_millis(5),
12838            Duration::from_secs(5),
12839        )
12840        .await;
12841        assert!(
12842            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12843            "must keep checking rather than deciding on the first answer"
12844        );
12845        assert!(
12846            started.elapsed() < Duration::from_secs(1),
12847            "must return the moment it is confirmed dead, not wait out the ceiling"
12848        );
12849    }
12850
12851    #[tokio::test]
12852    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12853        let started = Instant::now();
12854        wait_for_pids_with(
12855            &[123],
12856            |_| true, // never reports dead
12857            Duration::from_millis(5),
12858            Duration::from_millis(30),
12859        )
12860        .await;
12861        let elapsed = started.elapsed();
12862        assert!(
12863            elapsed >= Duration::from_millis(30),
12864            "must not give up before its own ceiling: {elapsed:?}"
12865        );
12866        assert!(
12867            elapsed < Duration::from_secs(1),
12868            "must not wait past its own ceiling either: {elapsed:?}"
12869        );
12870    }
12871
12872    #[tokio::test]
12873    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12874        let started = Instant::now();
12875        wait_for_pids_with(
12876            &[],
12877            |_| true,
12878            Duration::from_secs(5),
12879            Duration::from_secs(5),
12880        )
12881        .await;
12882        assert!(
12883            started.elapsed() < Duration::from_millis(200),
12884            "an empty pid list has nothing to confirm"
12885        );
12886    }
12887
12888    // `review_conclusion` is the exact decision the review hand-off task
12889    // fixed: a round budget spent (or a tree that stopped moving) must not
12890    // collapse into `Blocked` regardless of what verification actually
12891    // said. Deterministic and process-free for the same reason the
12892    // `round_is_clean` family above is.
12893    fn review_round(
12894        clean: bool,
12895        blocking: usize,
12896        answered: usize,
12897        expected: usize,
12898        progressed: bool,
12899        e2e_ok: bool,
12900    ) -> ReviewRound {
12901        ReviewRound {
12902            round: 1,
12903            head: "h".to_owned(),
12904            verified_head: None,
12905            verified_at: None,
12906            reviews: Vec::new(),
12907            e2e: vec![CommandOutcome {
12908                command: "test".to_owned(),
12909                code: Some(if e2e_ok { 0 } else { 1 }),
12910                output_tail: String::new(),
12911                duration_ms: 0,
12912                resource_blocked: false,
12913            }],
12914            verify_retried: false,
12915            e2e_deferred: false,
12916            e2e_defer_reason: None,
12917            fix: None,
12918            blocking,
12919            answered,
12920            expected,
12921            clean,
12922            progressed,
12923            vote_split: false,
12924            reconsideration: Vec::new(),
12925            verdict: None,
12926        }
12927    }
12928
12929    #[test]
12930    fn review_conclusion_is_none_when_nothing_has_run() {
12931        assert_eq!(review_conclusion(&[], 3), None);
12932    }
12933
12934    #[test]
12935    fn review_conclusion_is_none_while_rounds_remain() {
12936        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12937        assert_eq!(review_conclusion(&rounds, 3), None);
12938    }
12939
12940    #[test]
12941    fn review_conclusion_is_gating_once_a_round_is_clean() {
12942        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12943        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12944    }
12945
12946    #[test]
12947    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12948        let rounds = vec![
12949            review_round(false, 1, 2, 2, true, true),
12950            review_round(false, 1, 2, 2, true, true),
12951        ];
12952        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12953    }
12954
12955    #[test]
12956    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12957        let rounds = vec![
12958            review_round(false, 1, 2, 2, true, true),
12959            review_round(false, 1, 2, 2, true, false),
12960        ];
12961        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12962    }
12963
12964    #[test]
12965    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12966        // Magi never got a command to run against this round's own head — a
12967        // resource-blocked attempt, not a red one — so this must never
12968        // settle on `Blocked` the way a genuine e2e failure would. `None`
12969        // here is what tells `Runner::review_loop` to retry the check
12970        // itself rather than trust this cheap recomputation with a verdict
12971        // it cannot actually produce.
12972        let mut blocked = review_round(false, 1, 2, 2, true, false);
12973        blocked.e2e[0].resource_blocked = true;
12974        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12975        assert_eq!(review_conclusion(&rounds, 2), None);
12976    }
12977
12978    #[test]
12979    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12980        // Missing input, not a verified tree — never a hand-off candidate.
12981        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12982        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12983    }
12984
12985    #[test]
12986    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12987        let rounds = vec![
12988            review_round(false, 1, 2, 2, false, true),
12989            review_round(false, 1, 2, 2, false, true),
12990        ];
12991        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12992    }
12993
12994    fn secs(n: u64) -> Duration {
12995        Duration::from_secs(n)
12996    }
12997
12998    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12999    /// to make real (and, if it runs at all, real*ly fail*) git calls.
13000    #[tokio::test]
13001    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
13002        ask_test_home();
13003        let tmp = tempfile::tempdir().unwrap();
13004        let repo = tmp.path().join("repo");
13005        std::fs::create_dir_all(&repo).unwrap();
13006        init_repo(&repo);
13007        let g = |args: &[&str]| {
13008            let out = std::process::Command::new("git")
13009                .args(args)
13010                .current_dir(&repo)
13011                .quiet()
13012                .output()
13013                .unwrap();
13014            assert!(
13015                out.status.success(),
13016                "{}",
13017                String::from_utf8_lossy(&out.stderr)
13018            );
13019        };
13020        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13021        g(&["checkout", "-b", "historical"]);
13022        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
13023        g(&["add", "."]);
13024        g(&["commit", "-m", "old alternative"]);
13025        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
13026        g(&["checkout", "-b", "magi/task-selected", &base]);
13027        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
13028        g(&["add", "."]);
13029        g(&["commit", "-m", "measurement infrastructure"]);
13030        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
13031        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
13032        g(&["add", "."]);
13033        g(&["commit", "-m", "dependent implementation"]);
13034        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13035        g(&["checkout", "main"]);
13036        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
13037        g(&["add", "."]);
13038        g(&["commit", "-m", "unrelated base change"]);
13039        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
13040        let origin = tmp.path().join("origin.git");
13041        g(&[
13042            "clone",
13043            "--bare",
13044            repo.to_str().unwrap(),
13045            origin.to_str().unwrap(),
13046        ]);
13047        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
13048        g(&["fetch", "origin"]);
13049        let selection = crate::pr_recovery::Selection {
13050            url: "https://example.test/project/pull/1".into(),
13051            branch: "magi/task-selected".into(),
13052            head: head.clone(),
13053            source_head: None,
13054            source_run: None,
13055            source_proof: None,
13056            disposition: Some(crate::pr_recovery::Disposition::Review),
13057            instruction: "preserve measured behavior".into(),
13058            approved_at: jiff::Timestamp::now(),
13059        };
13060        let mut template = runner_at(RunStatus::Reviewing);
13061        template.state.repo = repo.clone();
13062        template.state.base_commit = current.clone();
13063        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13064        template.state.origin = Some(crate::run::Origin::queue("selected"));
13065        template.state.pr_head = Some(selection.branch.clone());
13066        template.state.pr_recovery = Some(selection.clone());
13067        let mut runner =
13068            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
13069                .await
13070                .unwrap();
13071        let wt = runner.state.winner().unwrap().worktree.clone();
13072        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
13073        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
13074        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
13075        assert!(runner.state.seeds.is_empty());
13076        assert_eq!(runner.state.instruction, "preserve measured behavior");
13077        runner.sync_to_base().await.unwrap();
13078        assert_eq!(
13079            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
13080            "prerequisite"
13081        );
13082        assert_eq!(
13083            std::fs::read_to_string(wt.join("implementation")).unwrap(),
13084            "measured implementation"
13085        );
13086        assert_eq!(
13087            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
13088            "current main change"
13089        );
13090        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13091        assert_eq!(
13092            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13093            head
13094        );
13095
13096        // A failed selected-source setup stays inspectable, never resumable as a new competition.
13097        let mut partial = runner_at(RunStatus::Reviewing);
13098        partial.state.repo = repo.clone();
13099        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
13100        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
13101            run: Some("exit 1".into()),
13102            ..Default::default()
13103        }];
13104        let selection = runner.state.pr_recovery.clone().unwrap();
13105        partial.state.pr_recovery = Some(selection.clone());
13106        partial.state.pr_head = Some(selection.branch.clone());
13107        partial.state.origin = Some(crate::run::Origin::queue("selected"));
13108        let id = partial.state.id.clone();
13109        let error = match Runner::open_selected(
13110            &repo,
13111            partial.state,
13112            partial.roles,
13113            1,
13114            runner.state.base_commit.clone(),
13115            selection,
13116        )
13117        .await
13118        {
13119            Ok(_) => panic!("setup must fail"),
13120            Err(e) => e,
13121        };
13122        assert_eq!(
13123            error
13124                .downcast_ref::<crate::pr_recovery::StartupFailed>()
13125                .unwrap()
13126                .run,
13127            id
13128        );
13129        let failed = RunState::load(&id).unwrap();
13130        assert_eq!(failed.status, RunStatus::Blocked);
13131        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
13132        assert!(Runner::resume(&id).is_err());
13133        runner.state.status = RunStatus::Ready;
13134        runner.state.driver_exited = true;
13135        runner.state.save().unwrap();
13136        let mut retry = runner_at(RunStatus::Reviewing);
13137        retry.state.repo = repo.clone();
13138        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
13139        let mut reselected = runner.state.pr_recovery.clone().unwrap();
13140        reselected.approved_at = jiff::Timestamp::now();
13141        retry.state.origin = Some(crate::run::Origin::queue("selected"));
13142        retry.state.pr_head = Some(reselected.branch.clone());
13143        retry.state.pr_recovery = Some(reselected.clone());
13144        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
13145            .await
13146            .unwrap();
13147        let restarted = Runner::open_selected(
13148            &repo,
13149            retry.state,
13150            retry.roles,
13151            1,
13152            runner.state.base_commit.clone(),
13153            reselected,
13154        )
13155        .await
13156        .unwrap();
13157        assert_eq!(
13158            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
13159                .await
13160                .unwrap(),
13161            head
13162        );
13163        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
13164        assert_eq!(
13165            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
13166            head
13167        );
13168    }
13169
13170    #[tokio::test]
13171    #[cfg(unix)]
13172    async fn fallback_and_interrupted_restoration_require_coordination() {
13173        use std::os::fd::AsRawFd as _;
13174        ask_test_home();
13175        let tmp = tempfile::tempdir().unwrap();
13176        let mut runner = registered_runner(tmp.path()).await;
13177        runner.state.config.graph.timeout_resource_wait = 1;
13178        let winner = runner.state.candidates[0].clone();
13179        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13180        let leftover = winner.worktree.join("leftover");
13181        std::fs::write(&leftover, "untrusted edits").unwrap();
13182        let next = tmp.path().join("next-fixer");
13183        let mut failed = spec("first");
13184        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
13185        let mut successor = spec("second");
13186        successor.command = vec![
13187            "sh".into(),
13188            "-c".into(),
13189            format!("touch '{}'; echo done", next.display()),
13190        ];
13191        let attempts = vec![
13192            (failed.clone(), "first".into()),
13193            (successor.clone(), "second".into()),
13194        ];
13195        let lock_path = runner.state.repo.join(".git/updater.lock");
13196        let external = std::fs::File::create(&lock_path).unwrap();
13197        assert_eq!(
13198            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13199            0
13200        );
13201        let artifacts = tmp.path().join("artifacts");
13202        let build = |spec, seat| SeatJob {
13203            spec,
13204            seat,
13205            cwd: winner.worktree.clone(),
13206            prompt: "fixture".into(),
13207            timeout: Duration::from_secs(10),
13208            allow_write: true,
13209            sessions: false,
13210            artifacts: artifacts.clone(),
13211            stem: "fix".into(),
13212            handover: None,
13213        };
13214        let (_, _, out) = runner
13215            .ask_fixer_among(
13216                attempts.clone(),
13217                "graft",
13218                None,
13219                Some((&winner.worktree, &head)),
13220                build,
13221            )
13222            .await;
13223        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13224        assert!(leftover.exists(), "contention must prevent clean");
13225        assert!(!next.exists());
13226        assert!(runner.state.handovers.is_empty());
13227        // Early interrupted-repair rollback uses the same protected route.
13228        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13229            from_run: "earlier".into(),
13230            generation: 1,
13231            mode: crate::recovery::Mode::VerifyRepair,
13232            reasons: Vec::new(),
13233            reviewed_head: None,
13234            rounds: Vec::new(),
13235            open_findings: Vec::new(),
13236            evidence: None,
13237            criteria_inherited: true,
13238            repair: crate::recovery::Repair::Started {
13239                before: head.clone(),
13240            },
13241        });
13242        assert!(
13243            runner
13244                .recovery_repair(&winner)
13245                .await
13246                .unwrap_err()
13247                .is::<crate::repo_lock::ResourceBlocked>()
13248        );
13249        assert!(matches!(
13250            runner.state.recovery.as_ref().unwrap().repair,
13251            crate::recovery::Repair::Started { .. }
13252        ));
13253        assert!(leftover.exists());
13254        drop(external);
13255        runner.recovery_repair(&winner).await.unwrap();
13256        assert!(!leftover.exists());
13257        assert!(matches!(
13258            runner.state.recovery.as_ref().unwrap().repair,
13259            crate::recovery::Repair::Done { .. }
13260        ));
13261        // A hard lock-open failure also leaves HEAD and untracked work intact.
13262        std::fs::remove_file(&lock_path).unwrap();
13263        std::fs::create_dir(&lock_path).unwrap();
13264        std::fs::write(&leftover, "untrusted edits").unwrap();
13265        let (_, _, out) = runner
13266            .ask_fixer_among(
13267                attempts,
13268                "graft",
13269                None,
13270                Some((&winner.worktree, &head)),
13271                build,
13272            )
13273            .await;
13274        assert!(matches!(out, AgentOutcome::Failed(_)));
13275        assert!(leftover.exists());
13276        assert!(!next.exists());
13277        assert_eq!(
13278            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13279            head
13280        );
13281    }
13282
13283    #[cfg(unix)]
13284    async fn registered_runner(root: &Path) -> Runner {
13285        ask_test_home();
13286        let repo = root.join("repo");
13287        std::fs::create_dir_all(&repo).unwrap();
13288        init_repo(&repo);
13289        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13290        let mut runner = runner_at(RunStatus::Prep);
13291        runner.state.repo = repo.clone();
13292        runner.state.config.coordination = crate::config::test_coordination();
13293        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13294        runner.state.config.graph.worktree_root = Some(root.join("wt"));
13295        runner.state.config.graph.advise = false;
13296        runner.roles.implementers = vec![spec("implement")];
13297        runner.roles.judges = vec![spec("judge")];
13298        runner.prep().await.unwrap();
13299        runner
13300    }
13301
13302    #[tokio::test]
13303    #[cfg(unix)]
13304    async fn parked_implementer_persists_completed_work() {
13305        for registered in [false, true] {
13306            let tmp = tempfile::tempdir().unwrap();
13307            let mut runner = registered_runner(tmp.path()).await;
13308            if !registered {
13309                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13310            }
13311            let marker = tmp.path().join("started");
13312            let release = tmp.path().join("release");
13313            let mut implementer = spec("implement");
13314            implementer.command = vec![
13315                "sh".into(),
13316                "-c".into(),
13317                format!(
13318                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13319                    marker.display(),
13320                    release.display()
13321                ),
13322            ];
13323            runner.roles.implementers = vec![implementer.clone()];
13324            runner.state.config.agents = vec![implementer];
13325            let pause = runner.pause.clone();
13326            let (result, ()) = tokio::join!(runner.implement(), async {
13327                tokio::time::timeout(Duration::from_secs(10), async {
13328                    while !marker.exists() {
13329                        tokio::time::sleep(Duration::from_millis(10)).await;
13330                    }
13331                })
13332                .await
13333                .unwrap();
13334                pause.park();
13335                std::fs::write(&release, "go").unwrap();
13336            });
13337            result.unwrap();
13338            let candidate = &runner.state.candidates[0];
13339            assert!(candidate.commits > 0, "{candidate:?}");
13340            assert!(candidate.summary.contains("saved-result"));
13341            assert_eq!(candidate.files, 1);
13342            let before = candidate.commits;
13343            runner.implement().await.unwrap();
13344            assert_eq!(runner.state.candidates[0].commits, before);
13345        }
13346    }
13347
13348    #[tokio::test]
13349    #[cfg(unix)]
13350    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13351        let tmp = tempfile::tempdir().unwrap();
13352        let mut runner = registered_runner(tmp.path()).await;
13353        let winner = runner.state.candidates[0].clone();
13354        let cache = tmp.path().join("cache");
13355        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13356        runner.state.config.graph.timeout_resource_wait = 0;
13357        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13358        let crate::cache::AcquireOutcome::Acquired(held) =
13359            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13360        else {
13361            panic!("cache must be free")
13362        };
13363        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13364            from_run: "earlier".into(),
13365            generation: 1,
13366            mode: crate::recovery::Mode::VerifyRepair,
13367            reasons: Vec::new(),
13368            reviewed_head: None,
13369            rounds: Vec::new(),
13370            open_findings: Vec::new(),
13371            evidence: None,
13372            criteria_inherited: true,
13373            repair: crate::recovery::Repair::NotStarted,
13374        });
13375        runner.recovery_repair(&winner).await.unwrap();
13376        assert!(matches!(
13377            runner.state.recovery.as_ref().unwrap().repair,
13378            crate::recovery::Repair::WaitingForRestore { .. }
13379        ));
13380        drop(held);
13381        runner.state.resource_block = None;
13382        runner.state.config.graph.timeout_resource_wait = 1;
13383        runner.recovery_repair(&winner).await.unwrap();
13384        assert!(matches!(
13385            runner.state.recovery.as_ref().unwrap().repair,
13386            crate::recovery::Repair::Done { .. }
13387        ));
13388        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13389    }
13390
13391    #[tokio::test]
13392    #[cfg(unix)]
13393    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13394        use std::os::fd::AsRawFd as _;
13395        for graft in [false, true] {
13396            let tmp = tempfile::tempdir().unwrap();
13397            let mut runner = registered_runner(tmp.path()).await;
13398            runner.state.config.graph.timeout_resource_wait = 1;
13399            let winner = runner.state.candidates[0].clone();
13400            let marker = tmp.path().join("fixer-started");
13401            let mut fixer = spec("fixer");
13402            fixer.command = vec![
13403                "sh".into(),
13404                "-c".into(),
13405                format!("echo started >> '{}'; echo done", marker.display()),
13406            ];
13407            runner.roles.fixer = Some(vec![fixer]);
13408            if graft {
13409                let mut loser = winner.clone();
13410                loser.label = 'B';
13411                loser.index = 1;
13412                runner.state.candidates.push(loser);
13413                runner.state.config.graph.graft_runner_up = true;
13414                runner.state.tally = Some(Tally {
13415                    first_choice: BTreeMap::new(),
13416                    borda: BTreeMap::new(),
13417                    winner: winner.label,
13418                    rankings: 1,
13419                    unanimous_initial: true,
13420                    deliberated: false,
13421                    changed_votes: 0,
13422                    unanimous_final: true,
13423                    tie_break: None,
13424                    judges: 1,
13425                    present: 1,
13426                    quorum: 1,
13427                    met_quorum: true,
13428                    uncontested: None,
13429                });
13430                runner.state.judgements.push(crate::run::Judgement {
13431                    judge: 1,
13432                    seat: "judge-1".into(),
13433                    agent: "judge".into(),
13434                    ranking: vec![winner.label, 'B'],
13435                    reasons: BTreeMap::new(),
13436                    confidence: None,
13437                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13438                    order: vec![0, 1],
13439                    failed: None,
13440                    duration_ms: 0,
13441                });
13442            } else {
13443                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13444                    from_run: "earlier".into(),
13445                    generation: 1,
13446                    mode: crate::recovery::Mode::VerifyRepair,
13447                    reasons: Vec::new(),
13448                    reviewed_head: None,
13449                    rounds: Vec::new(),
13450                    open_findings: Vec::new(),
13451                    evidence: None,
13452                    criteria_inherited: true,
13453                    repair: crate::recovery::Repair::NotStarted,
13454                });
13455            }
13456            let reader = crate::repo_lock::SeatLease::for_checkout(
13457                &winner.worktree,
13458                false,
13459                crate::config::test_coordination().as_ref(),
13460            )
13461            .await
13462            .unwrap();
13463            let pause = runner.pause.clone();
13464            let (result, ()) = tokio::join!(
13465                async {
13466                    if graft {
13467                        runner.graft().await
13468                    } else {
13469                        runner.recovery_repair(&winner).await
13470                    }
13471                },
13472                async {
13473                    tokio::time::sleep(Duration::from_millis(100)).await;
13474                    pause.park();
13475                }
13476            );
13477            result.unwrap();
13478            assert!(!marker.exists(), "a blocked seat must not spawn");
13479            // Exercise the persisted state, as a binary replacement/resume does.
13480            runner.state = RunState::load(&runner.state.id).unwrap();
13481            if graft {
13482                assert_eq!(
13483                    runner.state.graft.as_ref().unwrap().outcome,
13484                    GraftOutcome::WaitingForRestore
13485                );
13486            } else {
13487                assert!(matches!(
13488                    runner.state.recovery.as_ref().unwrap().repair,
13489                    crate::recovery::Repair::WaitingForRestore { .. }
13490                ));
13491            }
13492            drop(reader);
13493            runner.pause = Pause::new();
13494            runner.state.resource_block = None;
13495            // Deferred restoration must still respect an external lock owner.
13496            let external =
13497                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13498            assert_eq!(
13499                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13500                0
13501            );
13502            let leftover = winner.worktree.join("leftover");
13503            std::fs::write(&leftover, "earlier fallback edits").unwrap();
13504            let result = if graft {
13505                runner.graft().await
13506            } else {
13507                runner.recovery_repair(&winner).await
13508            };
13509            assert!(
13510                result
13511                    .unwrap_err()
13512                    .is::<crate::repo_lock::ResourceBlocked>()
13513            );
13514            assert!(leftover.exists());
13515            assert!(!marker.exists());
13516            drop(external);
13517            runner.state.resource_block = None;
13518            let result = if graft {
13519                runner.graft().await
13520            } else {
13521                runner.recovery_repair(&winner).await
13522            };
13523            result.unwrap();
13524            assert!(!leftover.exists(), "restore must precede retry");
13525            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13526            if graft {
13527                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13528            } else {
13529                assert!(matches!(
13530                    runner.state.recovery.as_ref().unwrap().repair,
13531                    crate::recovery::Repair::Done { .. }
13532                ));
13533            }
13534        }
13535    }
13536
13537    #[tokio::test]
13538    #[cfg(unix)]
13539    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13540        use std::os::fd::AsRawFd as _;
13541        let tmp = tempfile::tempdir().unwrap();
13542        let mut runner = registered_runner(tmp.path()).await;
13543        runner.state.config.graph.timeout_resource_wait = 1;
13544        let winner = runner.state.candidates[0].clone();
13545        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13546        runner.state.tally = Some(Tally {
13547            first_choice: BTreeMap::new(),
13548            borda: BTreeMap::new(),
13549            winner: winner.label,
13550            rankings: 0,
13551            unanimous_initial: false,
13552            deliberated: false,
13553            changed_votes: 0,
13554            unanimous_final: false,
13555            tie_break: None,
13556            judges: 0,
13557            present: 0,
13558            quorum: 0,
13559            met_quorum: true,
13560            uncontested: Some("fixture".into()),
13561        });
13562        runner.state.graft = Some(GraftRecord {
13563            pre: pre.clone(),
13564            post: None,
13565            lines: None,
13566            limit: None,
13567            agent: None,
13568            outcome: GraftOutcome::Running,
13569            requested: Vec::new(),
13570            adoption: None,
13571            adoption_error: None,
13572        });
13573        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13574        git::commit_all(&winner.worktree, "unfinished graft")
13575            .await
13576            .unwrap();
13577        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13578        let leftover = winner.worktree.join("leftover");
13579        std::fs::write(&leftover, "untrusted edits").unwrap();
13580        let path = runner.state.repo.join(".git/updater.lock");
13581        let external = std::fs::File::create(&path).unwrap();
13582        assert_eq!(
13583            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13584            0
13585        );
13586        assert!(
13587            runner
13588                .graft()
13589                .await
13590                .unwrap_err()
13591                .is::<crate::repo_lock::ResourceBlocked>()
13592        );
13593        assert!(matches!(
13594            runner.state.graft.as_ref().unwrap().outcome,
13595            GraftOutcome::Running
13596        ));
13597        assert_eq!(
13598            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13599            unfinished
13600        );
13601        assert!(leftover.exists());
13602        drop(external);
13603        std::fs::remove_file(&path).unwrap();
13604        std::fs::create_dir(&path).unwrap();
13605        assert!(runner.graft().await.is_err());
13606        assert!(matches!(
13607            runner.state.graft.as_ref().unwrap().outcome,
13608            GraftOutcome::Running
13609        ));
13610        assert_eq!(
13611            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13612            unfinished
13613        );
13614        assert!(leftover.exists());
13615        std::fs::remove_dir(&path).unwrap();
13616        runner.graft().await.unwrap();
13617        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13618        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13619        assert!(!leftover.exists());
13620    }
13621
13622    #[tokio::test]
13623    #[cfg(unix)]
13624    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13625        use std::os::fd::AsRawFd as _;
13626        let tmp = tempfile::tempdir().unwrap();
13627        let mut runner = registered_runner(tmp.path()).await;
13628        runner.state.prep_complete = Some(false);
13629        runner.state.config.graph.timeout_resource_wait = 1;
13630        runner.state.save().unwrap();
13631        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13632        assert_eq!(
13633            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13634            0
13635        );
13636        runner.execute().await.unwrap();
13637        let disk = RunState::load(&runner.state.id).unwrap();
13638        assert!(disk.driver_exited);
13639        assert_eq!(disk.driver_pid, None);
13640        assert_eq!(runner.state.status, RunStatus::Blocked);
13641        assert_eq!(
13642            runner.state.resource_block.as_ref().unwrap().kind,
13643            "repository coordination"
13644        );
13645        assert!(runner.state.prep_failure.is_none());
13646        assert!(runner.state.seats.is_empty());
13647        runner.pause.park();
13648        // Include headroom for Git subprocesses under parallel test load;
13649        // the repository contention deadline itself remains one second.
13650        tokio::time::timeout(Duration::from_secs(10), runner.execute())
13651            .await
13652            .unwrap()
13653            .unwrap();
13654        assert!(runner.state.parked);
13655        assert!(runner.state.prep_failure.is_none());
13656        let disk = RunState::load(&runner.state.id).unwrap();
13657        assert!(disk.driver_exited);
13658        assert_eq!(disk.driver_pid, None);
13659    }
13660
13661    #[tokio::test]
13662    #[cfg(unix)]
13663    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13664        use std::os::fd::AsRawFd as _;
13665        let tmp = tempfile::tempdir().unwrap();
13666        let mut runner = registered_runner(tmp.path()).await;
13667        runner.state.prep_complete = Some(false);
13668        runner.state.config.graph.timeout_resource_wait = 30;
13669        runner.state.save().unwrap();
13670        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13671        assert_eq!(
13672            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13673            0
13674        );
13675        assert!(
13676            tokio::time::timeout(Duration::from_millis(100), runner.execute())
13677                .await
13678                .is_err()
13679        );
13680        let disk = RunState::load(&runner.state.id).unwrap();
13681        assert!(disk.driver_exited);
13682        assert_eq!(disk.driver_pid, None);
13683        assert!(disk.status.resumable());
13684        assert!(runner.state.driver_exited);
13685        assert_eq!(runner.state.driver_pid, None);
13686        runner.state.save().unwrap();
13687        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13688    }
13689
13690    #[tokio::test]
13691    #[cfg(unix)]
13692    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13693        let tmp = tempfile::tempdir().unwrap();
13694        let mut runner = registered_runner(tmp.path()).await;
13695        let winner = runner.state.candidates[0].clone();
13696        let cache = tmp.path().join("cache");
13697        let started = tmp.path().join("pre-gate-started");
13698        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13699        runner.state.config.verify.gate =
13700            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13701        let home = crate::run::home();
13702        let owner =
13703            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13704        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13705            crate::cache::AcquireOutcome::Acquired(g) => g,
13706            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13707        };
13708        let mut gate = Box::pin(runner.run_gate(&winner));
13709        assert!(
13710            tokio::time::timeout(Duration::from_millis(150), &mut gate)
13711                .await
13712                .is_err()
13713        );
13714        assert!(!started.exists(), "pre-gate must wait for the cache too");
13715        // Simulate the wave that already owns the cache entering invoke.
13716        // It must obtain the editor reservation and finish without waiting
13717        // for the gate that is queued behind its cache.
13718        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13719        let editor = crate::repo_lock::SeatLease::checkout_with(
13720            &winner.worktree,
13721            true,
13722            crate::config::test_coordination().as_ref(),
13723            &wait,
13724        )
13725        .await
13726        .unwrap();
13727        drop(editor);
13728        guard.release();
13729        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13730            .await
13731            .unwrap()
13732            .unwrap();
13733        drop(gate);
13734        assert!(outcomes.iter().all(CommandOutcome::ok));
13735        assert!(started.exists());
13736    }
13737
13738    #[tokio::test]
13739    #[cfg(unix)]
13740    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13741        use std::os::fd::AsRawFd as _;
13742
13743        ask_test_home();
13744        let tmp = tempfile::tempdir().unwrap();
13745        let repo = tmp.path().join("repo");
13746        std::fs::create_dir_all(&repo).unwrap();
13747        init_repo(&repo);
13748        let common = repo.join(".git");
13749        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13750        let mut runner = runner_at(RunStatus::Prep);
13751        runner.state.repo = repo.clone();
13752        runner.state.config.coordination = crate::config::test_coordination();
13753        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13754        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13755        runner.state.config.graph.advise = false;
13756        runner.roles.implementers = vec![spec("implement")];
13757        runner.roles.judges = vec![spec("judge")];
13758        runner.prep().await.unwrap();
13759        let winner = runner.state.candidates[0].clone();
13760        // Emulate edits already left by a completed seat. The fixture CLI
13761        // only reports completion; it performs no repository operations.
13762        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13763        let completed = tmp.path().join("seat-completed");
13764        runner.roles.implementers[0].command = vec![
13765            "sh".into(),
13766            "-c".into(),
13767            format!("touch '{}'; echo completed", completed.display()),
13768        ];
13769        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13770        assert_eq!(
13771            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13772            0
13773        );
13774        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13775        let mut implement = Box::pin(runner.implement());
13776        tokio::select! {
13777            result = &mut implement => panic!("rescue must wait: {result:?}"),
13778            result = tokio::time::timeout(Duration::from_secs(10), async {
13779                while !completed.exists() {
13780                    tokio::time::sleep(Duration::from_millis(20)).await;
13781                }
13782            }) => result.unwrap(),
13783        }
13784        assert!(
13785            tokio::time::timeout(Duration::from_millis(200), &mut implement)
13786                .await
13787                .is_err()
13788        );
13789        assert_eq!(
13790            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13791            before
13792        );
13793        assert_eq!(
13794            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13795            0
13796        );
13797        tokio::time::timeout(Duration::from_secs(10), &mut implement)
13798            .await
13799            .unwrap()
13800            .unwrap();
13801        drop(implement);
13802        assert_ne!(
13803            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13804            before
13805        );
13806        assert!(git::is_clean(&winner.worktree).await.unwrap());
13807
13808        let started = tmp.path().join("pre-gate-started");
13809        let proceed = tmp.path().join("pre-gate-proceed");
13810        let checked = tmp.path().join("gate-ran");
13811        runner.state.config.verify.pre_gate = vec![format!(
13812            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13813            started.display(),
13814            proceed.display()
13815        )];
13816        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13817        assert_eq!(
13818            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13819            0
13820        );
13821        let mut gate = Box::pin(runner.run_gate(&winner));
13822        assert!(
13823            tokio::time::timeout(Duration::from_millis(200), &mut gate)
13824                .await
13825                .is_err()
13826        );
13827        assert!(!started.exists());
13828        assert!(!checked.exists());
13829        assert!(git::is_clean(&winner.worktree).await.unwrap());
13830        assert_eq!(
13831            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13832            0
13833        );
13834        tokio::select! {
13835            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13836            result = tokio::time::timeout(Duration::from_secs(10), async {
13837                while !started.exists() {
13838                    tokio::time::sleep(Duration::from_millis(20)).await;
13839                }
13840            }) => result.unwrap(),
13841        }
13842        assert_eq!(
13843            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13844            -1,
13845            "mechanical edits must hold the real lock"
13846        );
13847        std::fs::write(proceed, "ready").unwrap();
13848        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13849            .await
13850            .unwrap()
13851            .unwrap();
13852        drop(gate);
13853        assert!(outcomes.iter().all(CommandOutcome::ok));
13854        assert!(checked.exists());
13855        assert!(runner.state.pre_gate_commit.is_some());
13856        assert!(git::is_clean(&winner.worktree).await.unwrap());
13857        assert_eq!(
13858            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13859            0,
13860            "gate completion must release coordination"
13861        );
13862        drop(outsider);
13863        std::fs::remove_file(common.join("updater.lock")).unwrap();
13864        std::fs::create_dir(common.join("updater.lock")).unwrap();
13865        std::fs::remove_file(&started).unwrap();
13866        std::fs::remove_file(&checked).unwrap();
13867        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13868        let error = runner.run_gate(&winner).await.unwrap_err();
13869        assert!(
13870            error
13871                .to_string()
13872                .contains("acquire mandatory repository lock")
13873        );
13874        assert!(
13875            !started.exists(),
13876            "failed acquisition must prevent pre-gate"
13877        );
13878        assert!(!checked.exists(), "failed acquisition must prevent gate");
13879        assert!(git::is_clean(&winner.worktree).await.unwrap());
13880        assert_eq!(
13881            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13882            head
13883        );
13884    }
13885
13886    #[tokio::test]
13887    #[cfg(unix)]
13888    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13889        use std::os::fd::AsRawFd as _;
13890        use std::time::Duration;
13891
13892        ask_test_home();
13893        let tmp = tempfile::tempdir().unwrap();
13894        let repo = tmp.path().join("repo");
13895        std::fs::create_dir_all(&repo).unwrap();
13896        init_repo(&repo);
13897        let common = repo.join(".git");
13898        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13899        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13900        assert_eq!(
13901            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13902            0
13903        );
13904        let mut runner = runner_at(RunStatus::Prep);
13905        runner.state.repo = repo.clone();
13906        runner.state.config.coordination = crate::config::test_coordination();
13907        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13908        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13909        runner.state.config.graph.advise = true;
13910        runner.state.config.graph.advisors = 1;
13911        runner.roles.implementers = vec![spec("implement")];
13912        runner.roles.judges = vec![spec("judge")];
13913        let root = runner.state.worktree_root();
13914        let started = tmp.path().join("setup-started");
13915        let proceed = tmp.path().join("setup-proceed");
13916        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13917            run: Some(format!(
13918                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13919                started.display(),
13920                proceed.display()
13921            )),
13922            ..Default::default()
13923        }];
13924        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13925        let config = git::git(&repo, &["config", "--local", "--list"])
13926            .await
13927            .unwrap();
13928        let mut prep = Box::pin(runner.prep());
13929        assert!(
13930            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13931                .await
13932                .is_err()
13933        );
13934        assert!(
13935            !root.exists(),
13936            "contention must prevent even checkout creation"
13937        );
13938        assert!(!started.exists(), "contention must prevent setup");
13939        assert_eq!(
13940            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13941            branches
13942        );
13943        assert_eq!(
13944            git::git(&repo, &["config", "--local", "--list"])
13945                .await
13946                .unwrap(),
13947            config
13948        );
13949        drop(lock);
13950        tokio::select! {
13951            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13952            result = tokio::time::timeout(Duration::from_secs(10), async {
13953                while !started.exists() {
13954                    tokio::time::sleep(Duration::from_millis(20)).await;
13955                }
13956            }) => result.unwrap(),
13957        }
13958        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13959        assert_eq!(
13960            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13961            -1
13962        );
13963        assert!(
13964            tokio::time::timeout(
13965                Duration::from_millis(100),
13966                crate::repo_lock::SeatLease::for_checkout(
13967                    &repo,
13968                    true,
13969                    crate::config::test_coordination().as_ref()
13970                )
13971            )
13972            .await
13973            .is_err(),
13974            "preparation must also exclude local editors"
13975        );
13976        std::fs::write(proceed, "ready").unwrap();
13977        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13978            .await
13979            .unwrap()
13980            .unwrap();
13981        drop(prep);
13982        assert_eq!(runner.state.prep_complete, Some(true));
13983        assert!(root.join("judge-1").exists());
13984        assert!(root.join("advisor-1").exists());
13985        assert_eq!(
13986            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13987            0
13988        );
13989    }
13990
13991    #[tokio::test]
13992    #[cfg(unix)]
13993    async fn preparation_lock_failure_prevents_repository_changes() {
13994        ask_test_home();
13995        let tmp = tempfile::tempdir().unwrap();
13996        let repo = tmp.path().join("repo");
13997        std::fs::create_dir_all(&repo).unwrap();
13998        init_repo(&repo);
13999        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14000        // A directory at the required lock path makes opening the real lock
14001        // fail, rather than merely exercising an invalid registration.
14002        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
14003        let mut runner = runner_at(RunStatus::Prep);
14004        runner.state.repo = repo.clone();
14005        runner.state.config.coordination = crate::config::test_coordination();
14006        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14007        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14008        runner.roles.implementers = vec![spec("implement")];
14009        let error = runner.prep().await.unwrap_err();
14010        assert!(
14011            error
14012                .to_string()
14013                .contains("acquire mandatory repository lock")
14014        );
14015        assert!(!runner.state.worktree_root().exists());
14016        assert!(runner.state.candidates.is_empty());
14017        assert!(!repo.join(".git/config.worktree").exists());
14018        assert_eq!(
14019            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
14020            "* main"
14021        );
14022    }
14023
14024    #[tokio::test]
14025    async fn preparation_records_creation_intent_before_git_can_fail() {
14026        ask_test_home();
14027        let tmp = tempfile::tempdir().unwrap();
14028        let repo = tmp.path().join("repo");
14029        std::fs::create_dir_all(&repo).unwrap();
14030        init_repo(&repo);
14031        let mut runner = runner_at(RunStatus::Prep);
14032        runner.state.repo = repo.clone();
14033        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14034        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14035        runner.state.config.blind.commit_msg_hook = false;
14036        runner.roles.implementers = vec![spec("one")];
14037        let label = blind::assign_labels(1, runner.state.seed)[0];
14038        let branch = runner.state.branch_for(label);
14039        // A conflicting branch makes Git refuse creation before a candidate
14040        // can be registered. The durable intent must still identify it.
14041        git::git(&repo, &["branch", &branch]).await.unwrap();
14042        assert!(runner.prep().await.is_err());
14043        let saved = RunState::load(&runner.state.id).unwrap();
14044        assert!(saved.candidates.is_empty());
14045        let event = saved
14046            .events
14047            .iter()
14048            .find(|e| e.node == "prep-intent")
14049            .unwrap();
14050        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
14051        assert_eq!(intent["branch"], branch);
14052        assert_eq!(intent["start"], runner.state.base_commit);
14053        assert_eq!(
14054            intent["worktree"],
14055            runner
14056                .state
14057                .worktree_root()
14058                .join(format!("cand-{label}"))
14059                .to_string_lossy()
14060                .as_ref()
14061        );
14062        assert_eq!(intent["picks"], serde_json::json!([]));
14063    }
14064
14065    #[tokio::test]
14066    async fn invalid_pinned_sources_create_no_partial_worktrees() {
14067        ask_test_home();
14068        let tmp = tempfile::tempdir().unwrap();
14069        let repo = tmp.path().join("repo");
14070        std::fs::create_dir_all(&repo).unwrap();
14071        init_repo(&repo);
14072        let mut runner = runner_at(RunStatus::Prep);
14073        runner.state.repo = repo.clone();
14074        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14075        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14076        runner.state.sources.pick_commits.push(refs::Input {
14077            token: "missing".into(),
14078            sha: "0".repeat(40),
14079            provenance: refs::Provenance::ExplicitCli,
14080        });
14081        assert!(runner.prep().await.is_err());
14082        assert!(runner.state.candidates.is_empty());
14083        assert!(!runner.state.worktree_root().exists());
14084    }
14085
14086    #[tokio::test]
14087    async fn prose_and_legacy_inference_never_import_into_candidates() {
14088        ask_test_home();
14089        let tmp = tempfile::tempdir().unwrap();
14090        let repo = tmp.path().join("repo");
14091        std::fs::create_dir_all(&repo).unwrap();
14092        init_repo(&repo);
14093        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14094        git::git(&repo, &["checkout", "-b", "magi/history/A"])
14095            .await
14096            .unwrap();
14097        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
14098        git::git(&repo, &["add", "."]).await.unwrap();
14099        git::git(&repo, &["commit", "-m", "historical implementation"])
14100            .await
14101            .unwrap();
14102        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
14103        git::git(&repo, &["checkout", "main"]).await.unwrap();
14104        for instruction in [
14105            format!("Example: magi/history/A at {source}"),
14106            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
14107            format!("Never import magi/history/A or {source}"),
14108            format!("Conversation answer: I mentioned {source} only as background"),
14109        ] {
14110            let mut runner = runner_at(RunStatus::Prep);
14111            runner.state.repo = repo.clone();
14112            runner.state.base_commit = base.clone();
14113            runner.state.instruction = instruction.clone();
14114            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
14115            runner.state.config.blind.commit_msg_hook = false;
14116            runner.state.seeds = vec![refs::Seed {
14117                token: "magi/history/A".into(),
14118                sha: source.clone(),
14119                kind: refs::SeedKind::Unmerged,
14120                branch: true,
14121                detail: String::new(),
14122            }];
14123            runner.roles.implementers = vec![spec("one")];
14124            assert_eq!(seeded_instruction(&runner.state), instruction);
14125            // Model an old record with inferred seeds but no structured input field.
14126            let mut legacy = serde_json::to_value(&runner.state).unwrap();
14127            legacy.as_object_mut().unwrap().remove("sources");
14128            runner.state = serde_json::from_value(legacy).unwrap();
14129            assert_eq!(runner.state.sources, refs::Sources::default());
14130            runner.state.save().unwrap();
14131            runner.state = RunState::load(&runner.state.id).unwrap();
14132            runner.prep().await.unwrap();
14133            for candidate in &runner.state.candidates {
14134                assert_eq!(
14135                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
14136                    base
14137                );
14138                assert!(!candidate.worktree.join("unwanted").exists());
14139            }
14140        }
14141    }
14142
14143    #[tokio::test]
14144    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
14145        ask_test_home();
14146        let tmp = tempfile::tempdir().unwrap();
14147        let repo = tmp.path().join("repo");
14148        std::fs::create_dir_all(&repo).unwrap();
14149        init_repo(&repo);
14150        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14151        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
14152        std::fs::write(repo.join("referenced"), "required commit").unwrap();
14153        git::git(&repo, &["add", "."]).await.unwrap();
14154        git::git(&repo, &["commit", "-m", "required seed"])
14155            .await
14156            .unwrap();
14157        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
14158        git::git(&repo, &["checkout", "main"]).await.unwrap();
14159        let mut runner = runner_at(RunStatus::Prep);
14160        runner.state.repo = repo.clone();
14161        runner.state.base_commit = base.clone();
14162        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14163        runner.state.config.graph.advise = false;
14164        runner.state.config.blind.commit_msg_hook = false;
14165        runner.state.sources =
14166            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
14167                .await
14168                .unwrap();
14169        runner.roles.implementers = vec![spec("one"), spec("two")];
14170        runner.roles.judges = vec![spec("judge")];
14171        let label = blind::assign_labels(2, runner.state.seed)[0];
14172        let branch = runner.state.branch_for(label);
14173        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14174        git::worktree_add_branch(&repo, &worktree, &branch, &base)
14175            .await
14176            .unwrap();
14177        runner.state.candidates.push(Candidate {
14178            index: 0,
14179            label,
14180            agent: "one".into(),
14181            branch,
14182            worktree,
14183            summary: String::new(),
14184            stat: String::new(),
14185            files: 0,
14186            commits: 0,
14187            empty: false,
14188            failed: None,
14189            verified_noop: None,
14190            duration_ms: 0,
14191            folded: false,
14192        });
14193        runner.state.save().unwrap();
14194        // Reload the checkpoint immediately before its first cherry-pick.
14195        runner.state = RunState::load(&runner.state.id).unwrap();
14196        assert_eq!(runner.state.prep_complete, Some(false));
14197        runner.prep().await.unwrap();
14198        assert_eq!(runner.state.prep_complete, Some(true));
14199        assert_eq!(runner.state.candidates.len(), 2);
14200        for candidate in &runner.state.candidates {
14201            assert_eq!(
14202                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14203                git::tree_of(&repo, &seed).await.unwrap()
14204            );
14205            assert_eq!(
14206                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14207                "required commit"
14208            );
14209        }
14210        assert!(runner.state.worktree_root().join("judge-1").exists());
14211        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14212        runner.prep().await.unwrap();
14213        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14214
14215        runner.state.prep_complete = Some(false);
14216        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14217            run: Some(format!(
14218                "test -f '{}'",
14219                tmp.path().join("dependency-ready").display()
14220            )),
14221            ..Default::default()
14222        }];
14223        assert!(runner.prep().await.is_err());
14224        assert!(runner.state.prep_failure.is_some());
14225        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14226        runner.state = RunState::load(&runner.state.id).unwrap();
14227        assert!(
14228            runner.prep().await.is_err(),
14229            "correcting the cause alone must not auto-retry"
14230        );
14231        let failed_at = runner.state.events.last().unwrap().at;
14232        assert!(
14233            runner.retry_preparation(failed_at).is_err(),
14234            "old approvals cannot unlock the hold"
14235        );
14236        let mut task = crate::queue::Task::new(
14237            "retry".into(),
14238            "retry setup".into(),
14239            repo.clone(),
14240            crate::queue::Source::Human,
14241        );
14242        task.start(runner.state.id.clone());
14243        task.hold_machine(Some("setup failed".into()));
14244        task.release_operator("cli", None);
14245        runner
14246            .retry_preparation(task.resumption.as_ref().unwrap().at)
14247            .unwrap();
14248        runner.prep().await.unwrap();
14249        assert!(runner.state.prep_failure.is_none());
14250        assert_eq!(runner.state.prep_complete, Some(true));
14251        assert_eq!(runner.state.candidates.len(), 2);
14252        assert!(runner.state.worktree_root().join("judge-1").exists());
14253        for candidate in &runner.state.candidates {
14254            assert_eq!(
14255                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14256                git::tree_of(&repo, &seed).await.unwrap()
14257            );
14258        }
14259    }
14260
14261    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14262        let mut heads = Vec::new();
14263        for candidate in candidates {
14264            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14265        }
14266        heads
14267    }
14268
14269    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14270        ask_test_home();
14271        let tmp = tempfile::tempdir().unwrap();
14272        let repo = tmp.path().join("repo");
14273        std::fs::create_dir(&repo).unwrap();
14274        init_repo(&repo);
14275        let mut runner = runner_at(RunStatus::Reviewing);
14276        runner.state.repo = repo.clone();
14277        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14278        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14279        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14280            let branch = format!("candidate-{label}");
14281            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14282            git::worktree_add_branch(&repo, &worktree, &branch, "main")
14283                .await
14284                .unwrap();
14285            if label == 'B' {
14286                std::fs::write(worktree.join("change"), "winner").unwrap();
14287                git::rescue_commit(&worktree, "candidate change")
14288                    .await
14289                    .unwrap();
14290            }
14291            runner.state.candidates.push(Candidate {
14292                index,
14293                label,
14294                agent: "alpha".into(),
14295                branch,
14296                worktree,
14297                summary: "completed reply".into(),
14298                stat: String::new(),
14299                files: usize::from(label == 'B'),
14300                commits: usize::from(label == 'B'),
14301                empty: label == 'A',
14302                failed: None,
14303                verified_noop: None,
14304                duration_ms: 1,
14305                folded: false,
14306            });
14307        }
14308        runner.state.judge_skipped = true;
14309        runner.state.tally = Some(Tally {
14310            first_choice: BTreeMap::new(),
14311            borda: BTreeMap::new(),
14312            winner: 'B',
14313            rankings: 0,
14314            unanimous_initial: false,
14315            deliberated: false,
14316            changed_votes: 0,
14317            unanimous_final: false,
14318            tie_break: None,
14319            judges: 0,
14320            present: 0,
14321            quorum: 0,
14322            met_quorum: true,
14323            uncontested: Some("only B produced a change".into()),
14324        });
14325        runner.state.save().unwrap();
14326        (tmp, runner)
14327    }
14328
14329    #[tokio::test]
14330    #[cfg(unix)]
14331    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14332        for (solo, failed_checkpoint) in
14333            [(true, false), (false, false), (true, true), (false, true)]
14334        {
14335            let (_tmp, mut runner) = preservation_resume_fixture().await;
14336            if solo {
14337                runner.state.candidates.retain(|c| c.label == 'A');
14338            }
14339            runner.state.candidates[0].empty = false;
14340            runner.state.candidates[0].summary.clear();
14341            runner.state.tally = None;
14342            runner.state.judge_skipped = false;
14343            runner.state.status = RunStatus::Implementing;
14344            runner.state.config.disk.min_free_bytes = u64::MAX;
14345            let mut implementer = spec("alpha");
14346            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14347            runner.roles.implementers = vec![implementer];
14348            let patch = runner.state.dir().join("artifacts/cand-A.patch");
14349            if failed_checkpoint {
14350                std::fs::create_dir_all(&patch).unwrap();
14351            }
14352            runner.state.save().unwrap();
14353            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14354            runner.state = RunState::load(&runner.state.id).unwrap();
14355            assert_eq!(runner.state.status, RunStatus::Blocked);
14356            assert!(runner.state.interrupted_implementations.contains(&'A'));
14357            assert_eq!(runner.state.candidates[0].commits, 0);
14358            assert!(!runner.state.candidates[0].empty);
14359            assert!(runner.state.candidates[0].failed.is_none());
14360            if failed_checkpoint {
14361                assert!(runner.state.candidate_preservation.contains_key(&'A'));
14362                std::fs::remove_dir(patch).unwrap();
14363            }
14364            runner.state.resource_block = None;
14365            runner.state.config.disk.min_free_bytes = 0;
14366            runner.roles.implementers[0].command = vec![
14367                "sh".into(),
14368                "-c".into(),
14369                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14370            ];
14371            runner.implement().await.unwrap();
14372            assert!(runner.state.interrupted_implementations.is_empty());
14373            assert!(runner.state.candidate_preservation.is_empty());
14374            assert_eq!(runner.state.candidates[0].commits, 1);
14375            assert!(!runner.state.candidates[0].empty);
14376            assert!(
14377                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14378                    .unwrap()
14379                    .contains("retried empty interrupted seat")
14380            );
14381            if !solo {
14382                assert_eq!(runner.state.candidates[1].commits, 1);
14383            }
14384        }
14385    }
14386
14387    #[tokio::test]
14388    #[cfg(unix)]
14389    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14390        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14391            let (_tmp, mut runner) = preservation_resume_fixture().await;
14392            runner.state.candidates.retain(|c| c.label == 'A');
14393            runner.state.candidates[0].empty = false;
14394            runner.state.candidates[0].summary.clear();
14395            runner.state.tally = None;
14396            runner.state.judge_skipped = false;
14397            runner.state.status = RunStatus::Implementing;
14398            // Force the real post-spawn watchdog, independently of available
14399            // disk space. Dirty/committed work predates its first measurement.
14400            runner.state.config.disk.min_free_bytes = u64::MAX;
14401            let wt = runner.state.candidates[0].worktree.clone();
14402            std::fs::write(
14403                wt.join("interrupted-source"),
14404                "retain implementation work\n",
14405            )
14406            .unwrap();
14407            if committed {
14408                git::rescue_commit(&wt, "implementation checkpoint")
14409                    .await
14410                    .unwrap();
14411            }
14412            let lock = git::git(
14413                &wt,
14414                &[
14415                    "rev-parse",
14416                    "--path-format=absolute",
14417                    "--git-path",
14418                    "index.lock",
14419                ],
14420            )
14421            .await
14422            .unwrap();
14423            if block_rescue {
14424                std::fs::write(&lock, "fixture-owned lock").unwrap();
14425            }
14426            let mut implementer = spec("alpha");
14427            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14428            runner.roles.implementers = vec![implementer];
14429            runner.state.save().unwrap();
14430            let result = runner.implement().await;
14431            assert_eq!(result.is_err(), block_rescue);
14432            assert_eq!(
14433                runner.state.resource_block.as_ref().unwrap().kind,
14434                "disk pressure"
14435            );
14436            assert!(
14437                runner
14438                    .state
14439                    .events
14440                    .iter()
14441                    .any(|e| e.message.contains("agent execution interrupted"))
14442            );
14443            assert!(
14444                runner
14445                    .state
14446                    .dir()
14447                    .join("artifacts/candidate-A-recovery/staged.patch")
14448                    .exists()
14449            );
14450            if block_rescue {
14451                let saved = RunState::load(&runner.state.id).unwrap();
14452                assert!(saved.candidate_preservation.contains_key(&'A'));
14453                assert!(!saved.candidates[0].empty);
14454                assert_eq!(
14455                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14456                    "retain implementation work\n"
14457                );
14458                std::fs::remove_file(lock).unwrap();
14459                runner.state = saved;
14460            } else {
14461                assert_eq!(runner.state.status, RunStatus::Blocked);
14462                assert_eq!(runner.state.candidates[0].commits, 1);
14463            }
14464            // Recovery must save the candidate before deciding whether another
14465            // implementation is needed. No implementer is available on retry.
14466            runner.roles.implementers.clear();
14467            runner.state.config.disk.min_free_bytes = 0;
14468            runner.state.resource_block = None;
14469            runner.implement().await.unwrap();
14470            assert!(runner.state.candidate_preservation.is_empty());
14471            assert_eq!(runner.state.candidates[0].commits, 1);
14472            assert!(!runner.state.candidates[0].empty);
14473            assert!(git::is_clean(&wt).await.unwrap());
14474            assert!(
14475                crate::run::read_artifact(&runner.state, "cand-A.patch")
14476                    .unwrap()
14477                    .contains("retain implementation work")
14478            );
14479        }
14480    }
14481
14482    #[tokio::test]
14483    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14484        let (_tmp, mut runner) = preservation_resume_fixture().await;
14485        // Schema 15 already confirmed the unfolded no-change result.
14486        runner.implement().await.unwrap();
14487        assert!(runner.state.candidate_preservation.is_empty());
14488        assert!(
14489            !runner
14490                .state
14491                .dir()
14492                .join("artifacts/candidate-A-recovery")
14493                .exists()
14494        );
14495        runner.fold_losers().await.unwrap();
14496        assert!(!runner.state.candidates[0].worktree.exists());
14497        for schema in [14, crate::run::SCHEMA] {
14498            runner.state.schema = schema;
14499            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14500            runner.state.save().unwrap();
14501            runner.state = RunState::load(&runner.state.id).unwrap();
14502            runner.implement().await.unwrap();
14503            assert!(runner.state.candidate_preservation.is_empty());
14504            assert!(runner.state.candidates[0].folded);
14505            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14506        }
14507    }
14508
14509    #[tokio::test]
14510    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
14511        let (_tmp, mut runner) = preservation_resume_fixture().await;
14512        runner.state.candidates.retain(|c| c.label == 'A');
14513        runner.state.tally = None;
14514        runner.state.judge_skipped = false;
14515        let candidate = &mut runner.state.candidates[0];
14516        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
14517        candidate.failed = Some("deployment validation unavailable".into());
14518        std::fs::write(
14519            candidate.worktree.join("repair"),
14520            "future launcher policy\n",
14521        )
14522        .unwrap();
14523        runner
14524            .state
14525            .candidate_preservation
14526            .insert('A', "awaiting persistence".into());
14527        runner.persist_candidates().await.unwrap();
14528        let candidate = &runner.state.candidates[0];
14529        assert!(!candidate.empty);
14530        assert_eq!(candidate.commits, 1);
14531        assert_eq!(candidate.files, 1);
14532        assert!(candidate.failed.is_none());
14533        assert!(candidate.summary.contains("EPERM"));
14534        assert!(
14535            crate::run::read_artifact(&runner.state, "cand-A.patch")
14536                .unwrap()
14537                .contains("future launcher policy")
14538        );
14539        assert!(runner.state.candidate_preservation.is_empty());
14540    }
14541
14542    #[tokio::test]
14543    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14544        let (_tmp, mut runner) = preservation_resume_fixture().await;
14545        let wt = runner.state.candidates[0].worktree.clone();
14546        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14547        runner.state.schema = 14;
14548        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14549        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14550        let tally = serde_json::to_value(&runner.state.tally).unwrap();
14551        for _ in 0..2 {
14552            assert!(runner.implement().await.is_err());
14553            assert_eq!(runner.state.status, RunStatus::Blocked);
14554            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14555            assert!(
14556                runner.state.candidates[0].empty,
14557                "old verdict must remain unusable"
14558            );
14559            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14560            assert!(runner.state.judge_skipped);
14561            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14562            assert!(
14563                crate::run::read_artifact(&runner.state, "cand-A.patch")
14564                    .unwrap()
14565                    .contains("owned edits")
14566            );
14567            assert!(wt.exists());
14568            assert_eq!(
14569                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14570                    .await
14571                    .unwrap(),
14572                1
14573            );
14574            runner.state = RunState::load(&runner.state.id).unwrap();
14575        }
14576    }
14577
14578    #[tokio::test]
14579    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14580        let (_tmp, mut runner) = preservation_resume_fixture().await;
14581        let wt = runner.state.candidates[0].worktree.clone();
14582        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14583        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14584        let index_lock = PathBuf::from(
14585            git::git(
14586                &wt,
14587                &[
14588                    "rev-parse",
14589                    "--path-format=absolute",
14590                    "--git-path",
14591                    "index.lock",
14592                ],
14593            )
14594            .await
14595            .unwrap(),
14596        );
14597        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14598        runner.state.schema = 14;
14599        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14600        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14601        assert!(runner.implement().await.is_err());
14602        assert!(runner.after_implement().is_err());
14603        assert!(runner.fold_losers().await.is_err());
14604        assert_eq!(
14605            std::fs::read_to_string(wt.join("recovered")).unwrap(),
14606            "failed seat edits"
14607        );
14608        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14609        runner.state = RunState::load(&runner.state.id).unwrap();
14610        std::fs::remove_file(index_lock).unwrap();
14611        // Recovery finds changes that the old uncontested result never judged.
14612        assert!(runner.implement().await.is_err());
14613        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14614        assert!(runner.state.candidates[0].empty);
14615        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14616        assert!(
14617            crate::run::read_artifact(&runner.state, "cand-A.patch")
14618                .unwrap()
14619                .contains("failed seat edits")
14620        );
14621        assert!(wt.exists());
14622    }
14623
14624    #[tokio::test]
14625    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14626        let (_tmp, mut runner) = preservation_resume_fixture().await;
14627        let repo = runner.state.repo.clone();
14628        let common = git::git(
14629            &repo,
14630            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14631        )
14632        .await
14633        .unwrap();
14634        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14635        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14636        assert!(runner.fold_losers().await.is_err());
14637        assert!(!runner.state.candidates[0].worktree.exists());
14638        assert!(!runner.state.candidates[0].folded);
14639        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14640        runner.state = RunState::load(&runner.state.id).unwrap();
14641        std::fs::remove_file(branch_lock).unwrap();
14642        let judge = runner.state.worktree_root().join("judge-1");
14643        git::worktree_add_detached(&repo, &judge, "main")
14644            .await
14645            .unwrap();
14646        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14647        runner.roles.judges = vec![spec("judge")];
14648        assert!(runner.fold_losers().await.is_err());
14649        runner.state = RunState::load(&runner.state.id).unwrap();
14650        assert!(
14651            runner.state.candidates[0].folded,
14652            "candidate progress saved before judge failure"
14653        );
14654        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14655        assert!(judge.join("uncommitted").exists());
14656        std::fs::remove_file(judge.join("uncommitted")).unwrap();
14657        runner.fold_losers().await.unwrap();
14658        assert!(!judge.exists());
14659        runner.fold_losers().await.unwrap();
14660    }
14661
14662    #[tokio::test]
14663    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14664        let (_tmp, mut runner) = preservation_resume_fixture().await;
14665        runner.state.candidates.retain(|c| c.label == 'B');
14666        runner.state.tally = None;
14667        runner.state.judge_skipped = false;
14668        runner.state.prep_complete = Some(true);
14669        runner.state.config.graph.advise = false;
14670        runner.state.config.graph.selected_implementer = Some("alpha".into());
14671        let wt = runner.state.candidates[0].worktree.clone();
14672        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14673        runner
14674            .state
14675            .candidate_preservation
14676            .insert('B', "awaiting persistence".into());
14677        let index_lock = git::git(
14678            &wt,
14679            &[
14680                "rev-parse",
14681                "--path-format=absolute",
14682                "--git-path",
14683                "index.lock",
14684            ],
14685        )
14686        .await
14687        .unwrap();
14688        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14689        runner.state.save().unwrap();
14690        assert!(runner.execute_graph().await.is_err());
14691        assert_eq!(runner.state.status, RunStatus::Blocked);
14692        assert!(runner.state.candidate_preservation.contains_key(&'B'));
14693        std::fs::remove_file(index_lock).unwrap();
14694        // Park at the next graph boundary: this tests the real selected-seat
14695        // hold guard without dispatching reviewers or publishing anything.
14696        for _ in 0..2 {
14697            runner.state = RunState::load(&runner.state.id).unwrap();
14698            runner.pause.park();
14699            runner.execute_graph().await.unwrap();
14700            assert_eq!(runner.state.status, RunStatus::Implementing);
14701            assert!(runner.state.candidate_preservation.is_empty());
14702            assert_eq!(runner.state.candidates[0].commits, 2);
14703            assert!(runner.state.tally.is_none());
14704        }
14705        runner.implement().await.unwrap();
14706        assert_eq!(runner.state.status, RunStatus::Judging);
14707        // A genuine selected-implementer hold must still require release.
14708        runner.state.status = RunStatus::Blocked;
14709        runner.state.candidates[0].failed = Some("implementation failed".into());
14710        runner.state.save().unwrap();
14711        let error = runner.execute_graph().await.unwrap_err();
14712        assert!(
14713            error
14714                .to_string()
14715                .contains("selected implementation is held")
14716        );
14717    }
14718
14719    #[tokio::test]
14720    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14721        let (_tmp, mut runner) = preservation_resume_fixture().await;
14722        let repo = runner.state.repo.clone();
14723        let candidate = runner.state.candidates[0].clone();
14724        // Remove only the disposable fixture directory, leaving Git's linked
14725        // worktree registration and branch intact, as in a vanished checkout.
14726        let registered = git::worktree_holding(&repo, &candidate.branch)
14727            .await
14728            .unwrap()
14729            .unwrap();
14730        assert_eq!(
14731            registered.canonicalize().unwrap(),
14732            candidate.worktree.canonicalize().unwrap()
14733        );
14734        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14735        assert_eq!(
14736            git::worktree_holding(&repo, &candidate.branch)
14737                .await
14738                .unwrap(),
14739            Some(registered)
14740        );
14741        assert!(
14742            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14743            "the stale registration initially keeps the branch checked out"
14744        );
14745        runner.fold_losers().await.unwrap();
14746        assert!(runner.state.candidates[0].folded);
14747        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14748        assert_eq!(
14749            git::worktree_holding(&repo, &candidate.branch)
14750                .await
14751                .unwrap(),
14752            None
14753        );
14754        runner.state = RunState::load(&runner.state.id).unwrap();
14755        runner.fold_losers().await.unwrap();
14756    }
14757
14758    #[tokio::test]
14759    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14760        let (_tmp, mut runner) = preservation_resume_fixture().await;
14761        let repo = runner.state.repo.clone();
14762        let candidate = runner.state.candidates[0].clone();
14763        let winner = runner.state.candidates[1].clone();
14764        let registered = git::worktree_holding(&repo, &candidate.branch)
14765            .await
14766            .unwrap()
14767            .unwrap();
14768        assert_eq!(
14769            registered.canonicalize().unwrap(),
14770            candidate.worktree.canonicalize().unwrap()
14771        );
14772        std::fs::remove_dir_all(&candidate.worktree).unwrap();
14773        assert_eq!(
14774            git::worktree_holding(&repo, &candidate.branch)
14775                .await
14776                .unwrap(),
14777            Some(registered)
14778        );
14779        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14780
14781        for _ in 0..2 {
14782            fold_run(&mut runner.state, false, &crate::run::home())
14783                .await
14784                .unwrap();
14785            runner.state = RunState::load(&runner.state.id).unwrap();
14786            assert!(runner.state.candidates[0].folded);
14787            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14788            assert_eq!(
14789                git::worktree_holding(&repo, &candidate.branch)
14790                    .await
14791                    .unwrap(),
14792                None
14793            );
14794            assert!(!runner.state.candidates[1].folded);
14795            assert!(winner.worktree.exists());
14796            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14797        }
14798    }
14799
14800    #[tokio::test]
14801    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14802        ask_test_home();
14803        let tmp = tempfile::tempdir().unwrap();
14804        let repo = tmp.path().join("repo");
14805        std::fs::create_dir(&repo).unwrap();
14806        init_repo(&repo);
14807        let mut runner = runner_at(RunStatus::Implementing);
14808        runner.state.repo = repo.clone();
14809        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14810        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14811        for (index, label) in ['A', 'B'].into_iter().enumerate() {
14812            let branch = format!("candidate-{label}");
14813            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14814            git::worktree_add_branch(&repo, &wt, &branch, "main")
14815                .await
14816                .unwrap();
14817            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14818            git::git(&wt, &["add", "README.md"]).await.unwrap();
14819            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14820            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14821            runner.state.candidates.push(Candidate {
14822                index,
14823                label,
14824                agent: "alpha".to_owned(),
14825                branch,
14826                worktree: wt,
14827                summary: "completed implementation".to_owned(),
14828                stat: String::new(),
14829                files: 0,
14830                commits: 0,
14831                empty: false,
14832                failed: None,
14833                verified_noop: None,
14834                duration_ms: 42,
14835                folded: false,
14836            });
14837            runner
14838                .state
14839                .candidate_preservation
14840                .insert(label, "awaiting persistence".to_owned());
14841        }
14842        let wt = runner.state.candidates[0].worktree.clone();
14843        let index_lock = git::git(
14844            &wt,
14845            &[
14846                "rev-parse",
14847                "--path-format=absolute",
14848                "--git-path",
14849                "index.lock",
14850            ],
14851        )
14852        .await
14853        .unwrap();
14854        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14855        runner.state.save().unwrap();
14856        assert!(runner.persist_candidates().await.is_err());
14857        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14858        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14859        assert_eq!(
14860            runner.state.candidates[1].commits, 1,
14861            "other results are still collected"
14862        );
14863        assert!(
14864            !runner.state.candidates[0].empty,
14865            "failed rescue is not no change"
14866        );
14867        assert_eq!(
14868            runner.state.candidates[0].summary,
14869            "completed implementation"
14870        );
14871        assert!(runner.after_implement().is_err());
14872        let home = tmp.path().join("fold-home");
14873        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14874        assert!(wt.join("new-file").exists());
14875        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14876        assert!(recovery.join("staged.patch").exists());
14877        assert!(recovery.join("unstaged.patch").exists());
14878        assert_eq!(
14879            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14880            "untracked edits\n"
14881        );
14882        runner.state.begin_pass();
14883        runner.state.end_pass_parked(ParkCause::Upgrade {
14884            from: "1.0".into(),
14885            to: Some("1.1".into()),
14886        });
14887        runner.state.save().unwrap();
14888        runner.state = RunState::load(&runner.state.id).unwrap();
14889        assert!(matches!(
14890            runner.state.passes.last().unwrap().end,
14891            Some(crate::run::PassEnd::Parked {
14892                cause: ParkCause::Upgrade { .. }
14893            })
14894        ));
14895        std::fs::remove_file(index_lock).unwrap();
14896        // No implementer is configured: a persistence retry cannot dispatch a seat.
14897        runner.implement().await.unwrap();
14898        assert!(runner.state.candidate_preservation.is_empty());
14899        assert!(
14900            runner
14901                .state
14902                .candidates
14903                .iter()
14904                .all(|c| c.commits == 1 && !c.empty)
14905        );
14906        assert!(git::is_clean(&wt).await.unwrap());
14907        assert!(
14908            crate::run::read_artifact(&runner.state, "cand-A.patch")
14909                .unwrap()
14910                .contains("later tracked edits")
14911        );
14912    }
14913
14914    #[tokio::test]
14915    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14916        ask_test_home();
14917        let tmp = tempfile::tempdir().unwrap();
14918        let repo = tmp.path().join("repo");
14919        std::fs::create_dir(&repo).unwrap();
14920        init_repo(&repo);
14921        let wt = tmp.path().join("candidate");
14922        git::worktree_add_branch(&repo, &wt, "candidate", "main")
14923            .await
14924            .unwrap();
14925        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14926        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14927        assert!(wt.join("uncommitted").exists());
14928        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14929        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14930        let mut runner = runner_at(RunStatus::Implementing);
14931        runner.state.repo = repo.clone();
14932        runner.state.base_commit = "missing-base".to_owned();
14933        runner.state.candidates.push(Candidate {
14934            index: 0,
14935            label: 'A',
14936            agent: "alpha".into(),
14937            branch: "candidate".into(),
14938            worktree: wt.clone(),
14939            summary: "completed".into(),
14940            stat: String::new(),
14941            files: 0,
14942            commits: 0,
14943            empty: false,
14944            failed: None,
14945            verified_noop: None,
14946            duration_ms: 1,
14947            folded: false,
14948        });
14949        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14950        assert!(
14951            fold_run(&mut runner.state, true, &tmp.path().join("home"))
14952                .await
14953                .is_err()
14954        );
14955        assert!(!runner.state.candidates[0].folded);
14956        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14957        std::fs::remove_file(wt.join("uncommitted")).unwrap();
14958        runner
14959            .state
14960            .candidate_preservation
14961            .insert('A', "pending".into());
14962        assert!(runner.persist_candidates().await.is_err());
14963        assert!(!runner.state.candidates[0].empty);
14964        assert!(!runner.state.candidates[0].folded);
14965        assert!(wt.exists());
14966        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14967        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14968        std::fs::create_dir_all(&patch_path).unwrap();
14969        assert!(
14970            runner.persist_candidates().await.is_err(),
14971            "artifact failure stays pending"
14972        );
14973        assert!(runner.state.candidate_preservation.contains_key(&'A'));
14974        assert!(!runner.state.candidates[0].empty);
14975        std::fs::remove_dir(patch_path).unwrap();
14976        runner.persist_candidates().await.unwrap();
14977        assert!(
14978            runner.state.candidates[0].empty,
14979            "only successful inspection confirms no change"
14980        );
14981    }
14982
14983    fn init_repo(dir: &Path) {
14984        let run = |args: &[&str]| {
14985            let out = std::process::Command::new("git")
14986                .args(args)
14987                .current_dir(dir)
14988                .quiet()
14989                .output()
14990                .expect("spawn git");
14991            assert!(
14992                out.status.success(),
14993                "git {args:?} failed: {}",
14994                String::from_utf8_lossy(&out.stderr)
14995            );
14996        };
14997        run(&["init", "-b", "main"]);
14998        run(&["config", "user.name", "magi test"]);
14999        run(&["config", "user.email", "magi@example.com"]);
15000        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
15001        run(&["add", "-A"]);
15002        run(&["commit", "-m", "init"]);
15003    }
15004
15005    // `settle_questions` is what closes the ghost the phone showed: a run's
15006    // seat asked something, the run then ended, and nothing was left to
15007    // abandon the question it left `open`. `HOME` is a process-wide
15008    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
15009    // first time it runs in the binary — every test below still reaches the
15010    // same directory whichever call won, and each gets its own run id from
15011    // `RunState::new`, so they never collide there.
15012    fn ask_test_home() {
15013        crate::run::pin_test_home();
15014    }
15015
15016    #[cfg(unix)]
15017    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
15018        ask_test_home();
15019        let tmp = tempfile::tempdir().unwrap();
15020        let repo = tmp.path().join("repo");
15021        std::fs::create_dir(&repo).unwrap();
15022        init_repo(&repo);
15023        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15024        let origin = tmp.path().join("origin.git");
15025        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
15026            .await
15027            .unwrap();
15028        git::git(
15029            &repo,
15030            &["remote", "add", "origin", origin.to_str().unwrap()],
15031        )
15032        .await
15033        .unwrap();
15034        git::git(&repo, &["checkout", "-b", "fix/retained"])
15035            .await
15036            .unwrap();
15037        std::fs::write(repo.join("retained"), "retained change").unwrap();
15038        git::git(&repo, &["add", "."]).await.unwrap();
15039        git::git(&repo, &["commit", "-m", "retained implementation"])
15040            .await
15041            .unwrap();
15042        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15043        git::git(&repo, &["checkout", "main"]).await.unwrap();
15044        std::fs::write(repo.join("base-change"), "new base").unwrap();
15045        git::git(&repo, &["add", "."]).await.unwrap();
15046        git::git(&repo, &["commit", "-m", "base moved"])
15047            .await
15048            .unwrap();
15049        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15050        git::git(&repo, &["checkout", "fix/retained"])
15051            .await
15052            .unwrap();
15053        let mut runner = runner_at(RunStatus::Voting);
15054        runner.state.repo = repo.clone();
15055        runner.state.base_commit = base;
15056        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15057        runner.state.config.disk.min_free_bytes = 0;
15058        runner.state.config.graph.review_rounds = 1;
15059        runner.state.config.verify.e2e = vec!["true".into()];
15060        runner.state.config.verify.gate = vec!["true".into()];
15061        runner.state.candidates = vec![Candidate {
15062            index: 0,
15063            label: 'A',
15064            agent: "fixture".into(),
15065            branch: "fix/retained".into(),
15066            worktree: repo.clone(),
15067            summary: String::new(),
15068            stat: String::new(),
15069            files: 1,
15070            commits: 1,
15071            empty: false,
15072            failed: None,
15073            verified_noop: None,
15074            duration_ms: 0,
15075            folded: false,
15076        }];
15077        runner.state.tally = Some(Tally {
15078            first_choice: BTreeMap::new(),
15079            borda: BTreeMap::new(),
15080            winner: 'A',
15081            rankings: 0,
15082            unanimous_initial: true,
15083            deliberated: false,
15084            changed_votes: 0,
15085            unanimous_final: true,
15086            tie_break: None,
15087            judges: 0,
15088            present: 0,
15089            quorum: 0,
15090            met_quorum: true,
15091            uncontested: Some("review-only".into()),
15092        });
15093        let mut task = crate::queue::Task::new(
15094            "fixture".into(),
15095            "fixture".into(),
15096            repo,
15097            crate::queue::Source::Human,
15098        );
15099        task.link_run(&runner.state.id);
15100        crate::queue::Queue::open().put(&mut task).unwrap();
15101        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
15102        runner.state.base_sync = Some(BaseSync {
15103            tip: String::new(),
15104            behind: 1,
15105            attempts: BASE_SYNC_ROUNDS,
15106            conflict: None,
15107            already_in: None,
15108        });
15109        runner.sync_to_base().await.unwrap();
15110        assert_eq!(runner.state.status, RunStatus::Blocked);
15111        (tmp, runner, head, task.id)
15112    }
15113
15114    #[cfg(unix)]
15115    #[tokio::test]
15116    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
15117        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
15118        if std::env::var_os(CHILD).is_none() {
15119            let home = tempfile::tempdir().unwrap();
15120            let output = std::process::Command::new(std::env::current_exe().unwrap())
15121                .args([
15122                    "--exact",
15123                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
15124                    "--nocapture",
15125                ])
15126                .env(CHILD, "1")
15127                .env("MAGI_HOME", home.path())
15128                .env("MAGI_WORKER_HOME", home.path())
15129                .env("MAGI_WORKER_ID", "abcdef")
15130                .output()
15131                .unwrap();
15132            assert!(
15133                output.status.success()
15134                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15135                "{}\n{}",
15136                String::from_utf8_lossy(&output.stdout),
15137                String::from_utf8_lossy(&output.stderr)
15138            );
15139            return;
15140        }
15141        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
15142        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15143        let home = crate::run::home();
15144        let queue = crate::queue::Queue::open();
15145        let mut owner = queue.get(&task).unwrap();
15146        owner.release_operator("test-release", None);
15147        owner.start(runner.state.id.clone());
15148        queue.put(&mut owner).unwrap();
15149        runner.state.driver_exited = true;
15150        runner.state.task_instruction = Some(owner.instruction.clone());
15151        runner.state.save().unwrap();
15152        let _claim = queue.claim(&task).unwrap();
15153        let reserve = |id: &str| {
15154            let directory = home.join("workers/jobs").join(id);
15155            std::fs::create_dir_all(&directory).unwrap();
15156            let job = crate::worker::Job {
15157                id: id.into(),
15158                sequence: 1,
15159                kind: "task".into(),
15160                key: format!("task:{task}"),
15161                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
15162                generation: "fixture-generation".into(),
15163                version: None,
15164                status: crate::worker::Status::Running,
15165                result: None,
15166                pid: Some(std::process::id()),
15167                created_at: 0,
15168                finished_at: None,
15169            };
15170            std::fs::write(
15171                directory.join("job.json"),
15172                serde_json::to_vec(&job).unwrap(),
15173            )
15174            .unwrap();
15175            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
15176        };
15177        let _own = reserve("abcdef");
15178        let other = reserve("fedcba");
15179        assert!(
15180            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15181                .await
15182                .is_err()
15183        );
15184        assert!(
15185            RunState::load(&runner.state.id)
15186                .unwrap()
15187                .base_sync_recovery
15188                .authorizations
15189                .is_empty()
15190        );
15191        drop(other);
15192        assert!(crate::daemon::is_working_on(
15193            &home,
15194            &runner.state.id,
15195            Timestamp::now()
15196        ));
15197        assert!(
15198            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
15199        );
15200        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15201            .await
15202            .unwrap();
15203        assert!(
15204            RunState::load(&runner.state.id)
15205                .unwrap()
15206                .base_recovery_pending()
15207        );
15208    }
15209
15210    #[cfg(unix)]
15211    #[tokio::test]
15212    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
15213        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15214        let home = crate::run::home();
15215        runner.state.driver_pid = Some(std::process::id());
15216        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
15217        runner.state.driver_exited = false;
15218        runner.state.begin_pass();
15219        runner.state.save().unwrap();
15220        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
15221        driver.record(&runner.state).unwrap();
15222        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
15223        // Preserve the durable claim as if the process died before Drop saved.
15224        drop(driver);
15225        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
15226        assert_eq!(
15227            runner.state.liveness_under(&home, false),
15228            crate::run::Liveness::Dead
15229        );
15230        let queue = crate::queue::Queue::open();
15231        let _claim = queue.claim(&task).unwrap();
15232        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15233            .await
15234            .unwrap();
15235        let disk = RunState::load(&runner.state.id).unwrap();
15236        assert!(!disk.driver_exited);
15237        assert!(disk.base_recovery_pending());
15238    }
15239
15240    #[cfg(unix)]
15241    #[tokio::test]
15242    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
15243        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
15244        if std::env::var_os(CHILD).is_none() {
15245            use std::os::unix::fs::PermissionsExt;
15246            let tmp = tempfile::tempdir().unwrap();
15247            let gh = tmp.path().join("gh");
15248            std::fs::write(&gh, r#"#!/bin/sh
15249case "$1 $2" in
15250  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
15251  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
15252  *) exit 2 ;;
15253esac
15254"#).unwrap();
15255            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
15256            let mut paths = vec![tmp.path().to_path_buf()];
15257            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
15258            let output = std::process::Command::new(std::env::current_exe().unwrap())
15259                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
15260                .env(CHILD, "1")
15261                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
15262                .env("PATH", std::env::join_paths(paths).unwrap())
15263                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
15264                .output().unwrap();
15265            assert!(
15266                output.status.success()
15267                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
15268                "{}\n{}",
15269                String::from_utf8_lossy(&output.stdout),
15270                String::from_utf8_lossy(&output.stderr)
15271            );
15272            return;
15273        }
15274        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15275        let repo = runner.state.repo.clone();
15276        git::git(&repo, &["checkout", "main"]).await.unwrap();
15277        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
15278        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15279        git::git(&repo, &["checkout", "fix/retained"])
15280            .await
15281            .unwrap();
15282        runner.state.pr = Some(crate::run::PrRecord {
15283            url: "https://example.test/project/pull/1".into(),
15284            number: 1,
15285            state: "open".into(),
15286            checks: "unknown".into(),
15287            round: 0,
15288            rounds: 1,
15289            red_at_merge: Vec::new(),
15290        });
15291        runner.sync_to_base().await.unwrap();
15292        assert_eq!(runner.state.status, RunStatus::Blocked);
15293        assert_eq!(
15294            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
15295            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
15296        );
15297        assert!(runner.state.base_sync_closure_pending());
15298        assert!(crate::handover::can_resume(&runner.state));
15299        let queue = crate::queue::Queue::open();
15300        assert!(!crate::base_sync_recovery::held_task(
15301            &queue,
15302            &queue.get(&task).unwrap()
15303        ));
15304        assert!(runner.state.authorize_base_sync(&task).is_err());
15305        // Legacy closure stops have no typed stop but must retain the checkpoint.
15306        runner.state.base_sync_recovery.stop = None;
15307        runner.state.status = RunStatus::Voting;
15308        runner.state.gate_ran = true;
15309        runner.state.config.graph.review_rounds = 0;
15310        runner.state.save().unwrap();
15311        assert!(crate::handover::can_resume(&runner.state));
15312        runner.state = RunState::load(&runner.state.id).unwrap();
15313        runner.execute_graph().await.unwrap();
15314        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
15315        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
15316        assert_eq!(
15317            runner.state.base_sync.as_ref().unwrap().attempts,
15318            BASE_SYNC_ROUNDS
15319        );
15320        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
15321        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15322        assert!(runner.state.reviews.is_empty());
15323        assert!(runner.state.gate.is_empty());
15324    }
15325
15326    #[cfg(unix)]
15327    #[tokio::test]
15328    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
15329        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15330        runner.state.driver_exited = true;
15331        runner.state.save().unwrap();
15332        let home = crate::run::home();
15333        let queue = crate::queue::Queue::open();
15334        let _claim = queue.claim(&task).unwrap();
15335        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
15336        assert!(
15337            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15338                .await
15339                .is_err()
15340        );
15341        drop(lease);
15342        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
15343        assert!(
15344            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15345                .await
15346                .is_err()
15347        );
15348        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
15349        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15350            .await
15351            .unwrap();
15352        let disk = RunState::load(&runner.state.id).unwrap();
15353        assert!(disk.base_recovery_pending());
15354        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
15355        assert!(
15356            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15357                .await
15358                .is_err()
15359        );
15360        assert_eq!(
15361            RunState::load(&runner.state.id)
15362                .unwrap()
15363                .base_sync_recovery
15364                .authorizations
15365                .len(),
15366            1
15367        );
15368        let mut owner = queue.get(&task).unwrap();
15369        owner.runs.push("20260101-000000-ffff".into());
15370        queue.put(&mut owner).unwrap();
15371        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
15372            .await
15373            .unwrap_err();
15374        assert!(error.to_string().contains("run task ownership changed"));
15375    }
15376
15377    #[cfg(unix)]
15378    #[tokio::test]
15379    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
15380        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
15381        let mut legacy = serde_json::to_value(&runner.state).unwrap();
15382        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
15383        legacy["status"] = serde_json::json!("voting");
15384        runner.state = serde_json::from_value(legacy).unwrap();
15385        runner.state.save().unwrap();
15386        for _ in 0..3 {
15387            runner.execute_graph().await.unwrap();
15388            assert_eq!(runner.state.status, RunStatus::Blocked);
15389            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
15390            assert!(!crate::handover::can_resume(&runner.state));
15391            assert_eq!(
15392                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15393                head
15394            );
15395            assert!(runner.state.reviews.is_empty());
15396            assert!(!runner.state.gate_ran);
15397        }
15398        let notice = crate::notices::run_ended(&runner.state).unwrap();
15399        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
15400        runner.state.begin_pass();
15401        runner.state.status = RunStatus::Voting;
15402        runner.deliberate().await.unwrap();
15403        runner.vote().await.unwrap();
15404        runner.finish_after_tally().await.unwrap();
15405        assert_eq!(runner.state.status, RunStatus::Blocked);
15406        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
15407        assert_eq!(
15408            crate::notices::run_ended(&runner.state).unwrap().since,
15409            notice.since
15410        );
15411    }
15412
15413    #[cfg(unix)]
15414    #[tokio::test]
15415    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
15416        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15417        for detail in [
15418            "fix/retained conflicts with origin/main and did not rebase: conflict",
15419            "rebased fix/retained locally but origin refused the push",
15420            "unknown failure",
15421        ] {
15422            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
15423            runner.state.record_base_sync_stop(&head);
15424            assert!(runner.state.authorize_base_sync(&task).is_err());
15425            runner.state.status = RunStatus::Voting;
15426            runner.execute_graph().await.unwrap();
15427            assert_eq!(runner.state.status, RunStatus::Blocked);
15428            assert!(!crate::handover::can_resume(&runner.state));
15429            assert_eq!(
15430                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15431                head
15432            );
15433        }
15434    }
15435
15436    #[cfg(unix)]
15437    #[tokio::test]
15438    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
15439        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15440        runner.state.authorize_base_sync(&task).unwrap();
15441        assert!(runner.state.consume_base_recovery(&head));
15442        runner.state.save().unwrap();
15443        runner.state = RunState::load(&runner.state.id).unwrap();
15444        runner.execute_graph().await.unwrap();
15445        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15446        assert_eq!(runner.state.status, RunStatus::Blocked);
15447        assert!(!runner.state.consume_base_recovery(&head));
15448        assert!(!crate::handover::can_resume(&runner.state));
15449        runner.state.authorize_base_sync(&task).unwrap();
15450        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 2);
15451        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15452        assert_eq!(
15453            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
15454            head
15455        );
15456    }
15457
15458    #[cfg(unix)]
15459    #[tokio::test]
15460    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
15461        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
15462        let mut old_review = review_round(true, 0, 1, 1, false, true);
15463        old_review.head = head.clone();
15464        old_review.verified_head = Some(head.clone());
15465        runner.state.reviews = vec![old_review];
15466        runner.state.gate = vec![CommandOutcome {
15467            command: "old gate".into(),
15468            code: None,
15469            output_tail: "timed out after 2400s".into(),
15470            duration_ms: 2_400_000,
15471            resource_blocked: false,
15472        }];
15473        runner.state.gate_ran = true;
15474        runner.state.gate_head = Some(head.clone());
15475        runner.state.land_approval = Some(crate::run::LandApproval {
15476            question: "old-head-approval".into(),
15477            head: head.clone(),
15478        });
15479        runner.state.authorize_base_sync(&task).unwrap();
15480        runner.sync_to_base().await.unwrap();
15481        assert!(!runner.state.base_sync_blocked());
15482        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
15483        assert_ne!(head, fresh);
15484        assert!(runner.state.reviews.is_empty());
15485        assert!(!runner.state.gate_ran);
15486        let saved = &runner.state.base_sync_recovery.authorizations[0];
15487        assert_eq!(saved.reviews[0].head, head);
15488        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
15489        assert!(saved.gate_ran);
15490        assert!(!saved.gate[0].ok());
15491        runner.review_loop().await.unwrap();
15492        runner.gate().await.unwrap();
15493        assert_eq!(runner.state.reviews[0].head, fresh);
15494        assert_eq!(
15495            runner.state.reviews[0].verified_head.as_deref(),
15496            Some(fresh.as_str())
15497        );
15498        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
15499        assert!(runner.state.gate_ran);
15500        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15501        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
15502        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
15503        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15504        assert_eq!(
15505            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
15506            "retained change"
15507        );
15508    }
15509
15510    #[cfg(unix)]
15511    #[tokio::test]
15512    async fn base_recovery_moving_base_stops_again_after_one_authorized_rebase() {
15513        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
15514        let queue = crate::queue::Queue::open();
15515        let mut owner = queue.get(&task).unwrap();
15516        owner.release_operator("test-resume", None);
15517        queue.put(&mut owner).unwrap();
15518        let generation = owner.resumption.as_ref().unwrap().generation;
15519        runner.state.task_instruction = Some(owner.instruction.clone());
15520        runner.state.driver_exited = true;
15521        runner.state.save().unwrap();
15522        let home = crate::run::home();
15523        let _claim = queue.claim(&task).unwrap();
15524        owner.resumption.as_mut().unwrap().at =
15525            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
15526        queue.put(&mut owner).unwrap();
15527        assert!(
15528            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15529                .await
15530                .is_err(),
15531            "an approval from before the stop grants nothing"
15532        );
15533        owner.resumption.as_mut().unwrap().at = Timestamp::now();
15534        queue.put(&mut owner).unwrap();
15535        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15536            .await
15537            .unwrap();
15538        assert!(
15539            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15540                .await
15541                .is_err()
15542        );
15543        runner.state = RunState::load(&runner.state.id).unwrap();
15544        assert_eq!(
15545            runner.state.base_sync_recovery.authorizations[0].generation,
15546            Some(generation)
15547        );
15548        runner.sync_to_base().await.unwrap();
15549        let repo = runner.state.repo.clone();
15550        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
15551        git::git(&repo, &["checkout", "main"]).await.unwrap();
15552        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
15553        git::git(&repo, &["add", "."]).await.unwrap();
15554        git::git(&repo, &["commit", "-m", "move base again"])
15555            .await
15556            .unwrap();
15557        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
15558        git::git(&repo, &["checkout", "fix/retained"])
15559            .await
15560            .unwrap();
15561        runner.finish_after_tally().await.unwrap();
15562        assert_eq!(runner.state.status, RunStatus::Blocked);
15563        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15564        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15565        assert!(!runner.state.base_recovery_pending());
15566        assert!(!crate::handover::can_resume(&runner.state));
15567        assert!(!runner.state.gate_ran);
15568        // Even replaying the generation with a later timestamp grants nothing.
15569        owner.resumption.as_mut().unwrap().at = Timestamp::now();
15570        queue.put(&mut owner).unwrap();
15571        assert!(
15572            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
15573                .await
15574                .is_err()
15575        );
15576        runner.state = RunState::load(&runner.state.id).unwrap();
15577        runner.sync_to_base().await.unwrap();
15578        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
15579        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
15580        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
15581    }
15582
15583    #[cfg(unix)]
15584    #[tokio::test]
15585    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
15586        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
15587        runner.state.base_sync = None;
15588        runner.state.status = RunStatus::Blocked;
15589        let mut second = runner.state.candidates[0].clone();
15590        second.label = 'B';
15591        runner.state.candidates.push(second);
15592        assert!(runner.state.votes.is_empty());
15593        assert!(runner.state.deliberation.is_empty());
15594        runner.deliberate().await.unwrap();
15595        runner.vote().await.unwrap();
15596        assert_eq!(runner.state.status, RunStatus::Blocked);
15597        assert!(runner.state.votes.is_empty());
15598        assert!(runner.state.deliberation.is_empty());
15599    }
15600
15601    /// A minimal, git-free `Runner` at a given status — `settle_questions`
15602    /// reads nothing else off it.
15603    fn runner_at(status: RunStatus) -> Runner {
15604        let mut state = RunState::new(
15605            PathBuf::from("/nonexistent/repo"),
15606            "main".to_owned(),
15607            "deadbeef".to_owned(),
15608            "task".to_owned(),
15609            Config::default(),
15610        );
15611        // These fixtures share one home and save concurrently. Short clock
15612        // IDs can collide, making an ownership test read another test's run.
15613        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
15614        state.id = format!(
15615            "20000101-000000-{:04x}",
15616            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
15617        );
15618        state.status = status;
15619        Runner {
15620            state,
15621            roles: ResolvedRoles {
15622                implementers: Vec::new(),
15623                judges: Vec::new(),
15624                reviewers: Vec::new(),
15625                fixer: None,
15626                conductor: conductor(),
15627                implementer_roster: Vec::new(),
15628                judge_roster: Vec::new(),
15629                reviewer_roster: Vec::new(),
15630            },
15631            sem: Arc::new(Semaphore::new(1)),
15632            pause: Pause::new(),
15633            interrupt: Pause::new(),
15634        }
15635    }
15636
15637    #[tokio::test]
15638    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
15639        crate::run::pin_test_home();
15640        let mut runner = runner_at(RunStatus::Gating);
15641        runner.state.save().unwrap();
15642        let id = runner.state.id.clone();
15643        let passes = runner.state.passes.len();
15644        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15645        let mut claiming = Box::pin(runner.claim_driver());
15646        std::future::poll_fn(|cx| {
15647            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
15648            std::task::Poll::Ready(())
15649        })
15650        .await;
15651        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
15652        drop(owner);
15653        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
15654            .await
15655            .unwrap()
15656            .unwrap();
15657        assert_eq!(execution.runner.state.passes.len(), passes + 1);
15658        assert!(execution.runner.state.resource_block.is_none());
15659        assert_eq!(
15660            execution.runner.state.liveness(false),
15661            crate::run::Liveness::Live
15662        );
15663        drop(execution);
15664        assert!(RunState::load(&id).unwrap().driver_exited);
15665    }
15666
15667    #[tokio::test]
15668    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
15669        crate::run::pin_test_home();
15670        let mut runner = runner_at(RunStatus::Gating);
15671        runner.state.save().unwrap();
15672        // run.json can commit, but the subsequent lease metadata write fails.
15673        // Cleanup must run after releasing the short ownership lock.
15674        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
15675        assert!(runner.claim_driver().await.is_err());
15676        assert!(runner.state.driver_exited);
15677        assert_eq!(runner.state.driver_pid, None);
15678        let disk = RunState::load(&runner.state.id).unwrap();
15679        assert!(disk.driver_exited);
15680        assert_eq!(disk.driver_pid, None);
15681        assert_eq!(disk.status, RunStatus::Gating);
15682    }
15683
15684    #[tokio::test]
15685    async fn an_early_graph_error_releases_its_driver() {
15686        crate::run::pin_test_home();
15687        let mut runner = runner_at(RunStatus::Blocked);
15688        runner.state.config.graph.selected_implementer = Some("selected".into());
15689        runner.state.save().unwrap();
15690        assert!(runner.execute().await.is_err());
15691        let disk = RunState::load(&runner.state.id).unwrap();
15692        assert!(disk.driver_exited);
15693        assert_eq!(disk.driver_pid, None);
15694        assert_eq!(disk.status, RunStatus::Blocked);
15695    }
15696
15697    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
15698        crate::run::MergeRepair {
15699            question: "repair-question".into(),
15700            head: "old-head".into(),
15701            pr: "https://github.com/o/r/pull/42".into(),
15702            reviews: vec![review_round(false, 1, 1, 1, false, true)],
15703            gate: vec![outcome(Some(0), false)],
15704            gate_head: Some("old-head".into()),
15705            gate_fixes: Vec::new(),
15706            pre_gate: Vec::new(),
15707            phase,
15708            failure: None,
15709        }
15710    }
15711
15712    #[tokio::test]
15713    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
15714        crate::run::pin_test_home();
15715        let mut runner = runner_at(RunStatus::Landing);
15716        runner.state.merge_repairs.push(merge_repair_request(
15717            crate::run::MergeRepairPhase::Requested,
15718        ));
15719        runner.state.save().unwrap();
15720        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
15721        assert!(
15722            runner
15723                .execute()
15724                .await
15725                .unwrap_err()
15726                .to_string()
15727                .contains("already running")
15728        );
15729        assert_eq!(
15730            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15731            crate::run::MergeRepairPhase::Requested
15732        );
15733        drop(claim);
15734        let mut disk = runner.state.clone();
15735        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
15736        disk.save().unwrap();
15737        assert!(
15738            runner
15739                .execute()
15740                .await
15741                .unwrap_err()
15742                .to_string()
15743                .contains("decision changed")
15744        );
15745        assert_eq!(
15746            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
15747            crate::run::MergeRepairPhase::Complete
15748        );
15749    }
15750
15751    #[test]
15752    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
15753        crate::run::pin_test_home();
15754        let mut runner = runner_at(RunStatus::Landing);
15755        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
15756        runner.state.reviews = request.reviews.clone();
15757        runner.state.gate = request.gate.clone();
15758        runner.state.gate_ran = true;
15759        runner.state.land_approval = Some(crate::run::LandApproval {
15760            question: "old".into(),
15761            head: "old-head".into(),
15762        });
15763        runner.state.merge_repairs.push(request);
15764        runner.begin_merge_repair_review(0).unwrap();
15765        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
15766        assert!(!runner.state.gate_status().ok());
15767        assert!(runner.state.land_approval.is_none());
15768        let disk = RunState::load(&runner.state.id).unwrap();
15769        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
15770        assert!(disk.merge_repairs[0].gate[0].ok());
15771        assert_eq!(
15772            disk.merge_repairs[0].phase,
15773            crate::run::MergeRepairPhase::Reviewing
15774        );
15775        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
15776        assert!(disk.reviews.is_empty());
15777    }
15778
15779    #[tokio::test]
15780    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
15781        crate::run::pin_test_home();
15782        for phase in [
15783            crate::run::MergeRepairPhase::Fixing,
15784            crate::run::MergeRepairPhase::Requested,
15785        ] {
15786            let mut runner = runner_at(RunStatus::Landing);
15787            runner.state.config.graph.land_rounds = 0;
15788            runner.state.merge_repairs.push(merge_repair_request(phase));
15789            runner.state.save().unwrap();
15790            assert!(!runner.repair_merge_review().await.unwrap());
15791            assert_eq!(runner.state.status, RunStatus::Blocked);
15792            assert_eq!(
15793                runner.state.merge_repairs[0].phase,
15794                crate::run::MergeRepairPhase::Failed
15795            );
15796            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
15797            assert!(
15798                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
15799                    "interrupted"
15800                } else {
15801                    "budget exhausted"
15802                })
15803            );
15804            assert!(runner.state.merge.is_none());
15805        }
15806    }
15807
15808    /// `park_here` folding in the reason `Pause::park_because` recorded -
15809    /// this is what lets an operator reading a run's events tell an
15810    /// interrupt-driven park from an ordinary shutdown park.
15811    #[test]
15812    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15813        crate::run::pin_test_home();
15814        let mut runner = runner_at(RunStatus::Implementing);
15815        let interrupt = Pause::new();
15816        runner.watch_interrupt(interrupt.clone());
15817
15818        interrupt.park_because("task a1b2 asked to run first");
15819
15820        assert!(runner.park_here().expect("park_here"));
15821        assert!(runner.state.parked);
15822        let last = runner.state.events.last().expect("a park event");
15823        assert_eq!(last.node, "park");
15824        assert!(
15825            last.message.contains("task a1b2 asked to run first"),
15826            "expected the interrupt reason in {:?}",
15827            last.message
15828        );
15829    }
15830
15831    /// The cause is stored on the pass when it parks, and the first cause a
15832    /// shared pause was given is the one that stays.
15833    #[test]
15834    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15835        crate::run::pin_test_home();
15836        let mut runner = runner_at(RunStatus::Implementing);
15837        runner.state.begin_pass();
15838        let shutdown = Pause::new();
15839        runner.on_pause(shutdown.clone());
15840        let upgrade = ParkCause::Upgrade {
15841            from: "0.121.0".to_owned(),
15842            to: Some("0.121.1".to_owned()),
15843        };
15844        shutdown.park_for(upgrade.clone());
15845        shutdown.park_for(ParkCause::Operator);
15846
15847        assert!(runner.park_here().expect("park_here"));
15848        assert_eq!(
15849            runner.state.passes[0].end,
15850            Some(crate::run::PassEnd::Parked { cause: upgrade })
15851        );
15852        let last = runner.state.events.last().expect("a park event");
15853        assert!(last.message.contains("0.121.0"), "{}", last.message);
15854    }
15855
15856    #[tokio::test]
15857    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15858        crate::run::pin_test_home();
15859        let mut runner = runner_at(RunStatus::Reviewing);
15860        runner.state.save().unwrap();
15861        let mut execution = runner.claim_driver().await.expect("first execution");
15862        assert_eq!(execution.runner.state.passes.len(), 1);
15863        execution
15864            .driver
15865            .finish(&mut execution.runner.state)
15866            .await
15867            .unwrap();
15868    }
15869
15870    #[tokio::test]
15871    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15872        crate::run::pin_test_home();
15873        let mut runner = runner_at(RunStatus::Reviewing);
15874        runner.state.config.agents = vec![conductor()];
15875        runner.state.config.graph.timeout_resource_wait = 0;
15876        runner.state.save().unwrap();
15877        let id = runner.state.id.clone();
15878        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15879        assert!(runner.claim_driver().await.is_err());
15880        drop(owner);
15881        drop(runner);
15882        let mut resumed = Runner::resume(&id).unwrap();
15883        assert_eq!(resumed.state.driver_claimed, Some(false));
15884        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15885        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15886        let mut execution = resumed
15887            .claim_driver()
15888            .await
15889            .expect("retry initial claim after reload");
15890        assert_eq!(execution.runner.state.driver_claimed, Some(true));
15891        assert_eq!(execution.runner.state.passes.len(), 1);
15892        execution
15893            .driver
15894            .finish(&mut execution.runner.state)
15895            .await
15896            .unwrap();
15897    }
15898
15899    #[tokio::test]
15900    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15901        crate::run::pin_test_home();
15902        let mut runner = runner_at(RunStatus::Reviewing);
15903        runner.state.config.agents = vec![conductor()];
15904        runner.state.save().unwrap();
15905        let id = runner.state.id.clone();
15906        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15907        let mut resumed = Runner::resume(&id).unwrap();
15908        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15909        assert!(resumed.claim_driver().await.is_err());
15910        assert!(RunState::load(&id).unwrap().passes.is_empty());
15911        drop(lease);
15912        let mut execution = resumed.claim_driver().await.unwrap();
15913        execution
15914            .driver
15915            .finish(&mut execution.runner.state)
15916            .await
15917            .unwrap();
15918    }
15919
15920    #[tokio::test]
15921    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15922        crate::run::pin_test_home();
15923        let mut runner = runner_at(RunStatus::Reviewing);
15924        let state = &mut runner.state;
15925        state.config.agents = vec![conductor()];
15926        state.driver_claimed = None;
15927        state.driver_pid = Some(std::process::id());
15928        state.driver_started_at = None;
15929        state.begin_pass();
15930        state.save().unwrap();
15931        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15932        std::fs::write(
15933            state.dir().join("driver.json"),
15934            serde_json::to_vec(&serde_json::json!({
15935                "pid": state.driver_pid, "started_at": null,
15936                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15937            }))
15938            .unwrap(),
15939        )
15940        .unwrap();
15941        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15942        drop(lease);
15943        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15944        let mut resumed = Runner::resume(&state.id).unwrap();
15945        assert!(!resumed.state.driver_exited);
15946        let mut execution = resumed
15947            .claim_driver()
15948            .await
15949            .expect("lease proves previous execution ended");
15950        assert_eq!(execution.runner.state.passes.len(), 2);
15951        execution
15952            .driver
15953            .finish(&mut execution.runner.state)
15954            .await
15955            .unwrap();
15956    }
15957
15958    #[tokio::test]
15959    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15960        crate::run::pin_test_home();
15961        let mut runner = runner_at(RunStatus::Reviewing);
15962        let mut json = serde_json::to_value(&runner.state).unwrap();
15963        json.as_object_mut().unwrap().remove("driver_claimed");
15964        runner.state = serde_json::from_value(json).unwrap();
15965        assert_eq!(runner.state.driver_claimed, None);
15966        runner.state.save().unwrap();
15967        let mut execution = runner
15968            .claim_driver()
15969            .await
15970            .expect("the execution lease, not a missing marker, excludes a second driver");
15971        assert_eq!(execution.runner.state.passes.len(), 1);
15972        execution
15973            .driver
15974            .finish(&mut execution.runner.state)
15975            .await
15976            .unwrap();
15977    }
15978
15979    #[tokio::test]
15980    async fn driver_admission_refuses_a_proven_live_driver() {
15981        crate::run::pin_test_home();
15982        let mut runner = runner_at(RunStatus::Reviewing);
15983        let pid = std::process::id();
15984        runner.state.driver_pid = Some(pid);
15985        runner.state.driver_started_at = crate::proc::process_started_at(pid);
15986        assert!(runner.state.driver_started_at.is_some());
15987        runner.state.save().unwrap();
15988        let error = runner
15989            .claim_driver()
15990            .await
15991            .err()
15992            .expect("a live driver must refuse");
15993        assert!(error.to_string().contains("live driver"), "{error:#}");
15994        let saved = RunState::load(&runner.state.id).unwrap();
15995        assert_eq!(saved.driver_pid, Some(pid));
15996        assert_eq!(saved.status, RunStatus::Reviewing);
15997        assert!(saved.passes.is_empty());
15998    }
15999
16000    #[test]
16001    fn parking_resource_halt_persists_the_pass_once() {
16002        crate::run::pin_test_home();
16003        let mut runner = runner_at(RunStatus::Reviewing);
16004        runner.state.begin_pass();
16005        runner.state.resource_block = Some(crate::run::ResourceBlock {
16006            node: "fix".into(),
16007            kind: "shared build cache".into(),
16008            detail: "cancelled".into(),
16009            at: Timestamp::now(),
16010            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
16011        });
16012        let cause = ParkCause::Upgrade {
16013            from: "old".into(),
16014            to: Some("new".into()),
16015        };
16016        runner.pause.park_for(cause.clone());
16017        // Both direct fixer exits and graph resource boundaries must complete
16018        // parking, rather than returning a stale Reviewing record.
16019        runner.stop_for_resource().unwrap();
16020        assert!(runner.resource_halt().unwrap());
16021        assert!(runner.park_here().unwrap());
16022        let saved = RunState::load(&runner.state.id).unwrap();
16023        assert!(saved.parked);
16024        assert_eq!(saved.status, RunStatus::Reviewing);
16025        assert!(saved.passes[0].ended_at.is_some());
16026        assert_eq!(
16027            saved.passes[0].end,
16028            Some(crate::run::PassEnd::Parked {
16029                cause: cause.clone()
16030            })
16031        );
16032        assert_eq!(
16033            saved.resource_block.unwrap().cause,
16034            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
16035        );
16036        assert_eq!(
16037            saved
16038                .events
16039                .iter()
16040                .filter(|event| event.node == "park")
16041                .count(),
16042            1
16043        );
16044    }
16045
16046    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
16047    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
16048    /// cleared) must not make a *different* run - one only watching its own,
16049    /// unshared interrupt `Pause` - see itself as parked. If a future change
16050    /// ever collapsed these back into one handle, the interrupt scheduler
16051    /// would park every run for the rest of the daemon's life, not just the
16052    /// one it meant to interrupt.
16053    #[test]
16054    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
16055        crate::run::pin_test_home();
16056        let mut runner = runner_at(RunStatus::Implementing);
16057        let shutdown = Pause::new();
16058        runner.on_pause(shutdown.clone());
16059        let interrupt = Pause::new();
16060        runner.watch_interrupt(interrupt.clone());
16061
16062        // Nobody has asked for anything yet.
16063        assert!(!runner.park_here().expect("park_here"));
16064        assert!(!runner.state.parked);
16065
16066        // Only the interrupt handle fires; the shutdown handle stays clear.
16067        interrupt.park_because("test");
16068        assert!(!shutdown.parked());
16069        assert!(runner.park_here().expect("park_here"));
16070    }
16071
16072    /// The property every prior attempt at this feature failed to pin down:
16073    /// asking a run to park while one of its nodes has a real, in-flight
16074    /// async operation running (an agent call, in production) must not cut
16075    /// that operation short. `park_here` is only ever consulted *between*
16076    /// `execute`'s node calls - see its own doc - so nothing inside a node
16077    /// can observe a park request until the node itself returns. This proves
16078    /// that structurally, with real `tokio` concurrency and a channel
16079    /// handshake (never a sleep, which would only prove "usually", not
16080    /// "cannot"): the "node" below reports that it has genuinely started,
16081    /// and only then is the park requested; the node still has to be told to
16082    /// finish before `park_here` is ever called, exactly mirroring every
16083    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
16084    /// in `execute`.
16085    #[tokio::test]
16086    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
16087        crate::run::pin_test_home();
16088        let mut runner = runner_at(RunStatus::Implementing);
16089        let interrupt = Pause::new();
16090        runner.watch_interrupt(interrupt.clone());
16091
16092        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
16093        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
16094
16095        // Stands in for one node's in-flight agent call: it proves it has
16096        // genuinely started, then blocks - exactly as a spawned CLI process
16097        // does - until told to finish.
16098        let node = async move {
16099            started_tx.send(()).expect("send started");
16100            finish_rx.await.expect("recv finish");
16101            "node finished"
16102        };
16103
16104        let interrupter = async move {
16105            started_rx.await.expect("recv started");
16106            // The call is now genuinely in flight. Ask it to park.
16107            interrupt.park_because("higher-priority task waiting");
16108            // Nothing the node does can observe this yet - there is no
16109            // check inside it, by construction - so let the executor run
16110            // anything pending and then let the node finish on its own.
16111            tokio::task::yield_now().await;
16112            finish_tx.send(()).expect("send finish");
16113        };
16114
16115        let (node_result, ()) = tokio::join!(node, interrupter);
16116        assert_eq!(
16117            node_result, "node finished",
16118            "the in-flight call ran to completion"
16119        );
16120
16121        // Only now, at the boundary the real `execute` would check right
16122        // after this node, does the park take effect.
16123        assert!(runner.park_here().expect("park_here"));
16124        assert!(runner.state.parked);
16125    }
16126
16127    /// A run parked mid-competition carries every field it had accumulated
16128    /// through the exact same disk round-trip an ordinary resume uses -
16129    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
16130    /// Nothing about parking for an interrupt is a special case of that path;
16131    /// this is what proves it rather than assuming it.
16132    #[test]
16133    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
16134        crate::run::pin_test_home();
16135        let mut runner = runner_at(RunStatus::Judging);
16136        // `Runner::resume` re-resolves roles from the saved config, which
16137        // refuses an empty roster - give it the same minimal one `conductor`
16138        // itself uses.
16139        runner.state.config.agents = vec![conductor()];
16140        runner.state.candidates = vec![Candidate {
16141            index: 0,
16142            label: 'A',
16143            agent: "alpha".to_owned(),
16144            branch: "magi/x/A".to_owned(),
16145            worktree: PathBuf::from("/nonexistent/worktree"),
16146            summary: "did the thing".to_owned(),
16147            stat: "1 file changed".to_owned(),
16148            files: 1,
16149            commits: 1,
16150            empty: false,
16151            failed: None,
16152            verified_noop: None,
16153            duration_ms: 1234,
16154            folded: false,
16155        }];
16156        let run_id = runner.state.id.clone();
16157
16158        let interrupt = Pause::new();
16159        runner.watch_interrupt(interrupt.clone());
16160        interrupt.park_because("task c3d4 asked to run first");
16161        assert!(runner.park_here().expect("park_here"));
16162
16163        let resumed = Runner::resume(&run_id).expect("resume");
16164        assert_eq!(resumed.state.candidates.len(), 1);
16165        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
16166        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
16167        assert_eq!(resumed.state.status, runner.state.status);
16168        assert!(
16169            resumed.state.parked,
16170            "still parked until `execute` actually walks the graph again"
16171        );
16172        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
16173    }
16174
16175    /// A fresh open question on `run`, stored and handed back for assertions.
16176    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
16177        let mut q = ask::Question::new(
16178            run.to_owned(),
16179            "implement".to_owned(),
16180            "impl-A".to_owned(),
16181            "Which storage backend should the cache use?".to_owned(),
16182            String::new(),
16183            vec!["SQLite".to_owned(), "Redis".to_owned()],
16184        );
16185        store.put(&mut q).unwrap();
16186        q
16187    }
16188
16189    #[test]
16190    fn a_failed_runs_open_question_is_abandoned() {
16191        ask_test_home();
16192        let store = ask::Questions::open();
16193        let mut runner = runner_at(RunStatus::Failed);
16194        let run = runner.state.id.clone();
16195        let q = ask_open_question(&store, &run);
16196
16197        runner.settle_questions();
16198
16199        let back = store.get(&q.id).unwrap();
16200        assert!(
16201            !back.status.open(),
16202            "the seat that asked died with the run; nobody is left to read an answer"
16203        );
16204        assert!(
16205            back.detail.contains(&run) && back.detail.contains("failed"),
16206            "the reason names what the run became, not just that it is gone: {}",
16207            back.detail
16208        );
16209    }
16210
16211    #[test]
16212    fn a_merged_runs_open_question_is_abandoned_too() {
16213        ask_test_home();
16214        let store = ask::Questions::open();
16215        // A run that finishes cleanly still leaves nobody to read an answer -
16216        // this is not only a failure-path cleanup.
16217        for status in [RunStatus::Merged, RunStatus::Ready] {
16218            let mut runner = runner_at(status);
16219            let run = runner.state.id.clone();
16220            let q = ask_open_question(&store, &run);
16221
16222            runner.settle_questions();
16223
16224            let back = store.get(&q.id).unwrap();
16225            assert!(
16226                !back.status.open(),
16227                "{status:?} run's question must not outlive the run"
16228            );
16229        }
16230    }
16231
16232    #[test]
16233    fn a_still_resumable_runs_open_question_is_left_alone() {
16234        ask_test_home();
16235        let store = ask::Questions::open();
16236        // `Blocked` and `Stalled` can still be resumed — the candidates, the
16237        // review round and the seat sessions are all still on disk — so a
16238        // question asked mid-round may yet get a real answer from a real
16239        // resume. Sweeping it here would be exactly the failure mode this
16240        // whole feature exists to avoid on the other side.
16241        for status in [RunStatus::Blocked, RunStatus::Stalled] {
16242            let mut runner = runner_at(status);
16243            let run = runner.state.id.clone();
16244            let q = ask_open_question(&store, &run);
16245
16246            runner.settle_questions();
16247
16248            let back = store.get(&q.id).unwrap();
16249            assert!(
16250                back.status.open(),
16251                "{status:?} is still alive; the question must still be waiting"
16252            );
16253        }
16254    }
16255
16256    #[test]
16257    fn settle_questions_never_touches_an_already_answered_question() {
16258        ask_test_home();
16259        let store = ask::Questions::open();
16260        let mut runner = runner_at(RunStatus::Failed);
16261        let run = runner.state.id.clone();
16262        let mut q = ask_open_question(&store, &run);
16263        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
16264            .unwrap();
16265        store.put(&mut q).unwrap();
16266
16267        // Called twice, the way a crash-recovered daemon reclaim and the
16268        // graph's own cleanup both can for the same run — `abandon_for_run`
16269        // only ever touches what is still open, so this must be inert both
16270        // times, not merely the second.
16271        runner.settle_questions();
16272        runner.settle_questions();
16273
16274        let back = store.get(&q.id).unwrap();
16275        assert_eq!(
16276            back.status,
16277            ask::QuestionStatus::Answered,
16278            "a real answer is a decision on record, never overwritten by a sweep"
16279        );
16280    }
16281
16282    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
16283    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
16284    /// without merging, whose winner is still the operator's answer to read.
16285    /// Nothing previously called `fold_run` itself with a real `tally`, so
16286    /// this is the first test to pin down the one distinction the whole
16287    /// automatic-fold feature depends on: the winner's worktree and branch
16288    /// must survive, everything else sharing the run's worktree bay - a
16289    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
16290    /// second sweep treats every non-winner directory under the bay alike -
16291    /// must not.
16292    #[tokio::test]
16293    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
16294        crate::run::pin_test_home();
16295        let tmp = tempfile::tempdir().expect("tempdir");
16296        let repo = tmp.path().join("repo");
16297        std::fs::create_dir_all(&repo).unwrap();
16298        init_repo(&repo);
16299
16300        let mut config = Config::default();
16301        config.graph.worktree_root = Some(tmp.path().join("wt"));
16302
16303        let mut state = RunState::new(
16304            repo.clone(),
16305            "main".to_owned(),
16306            "deadbeef".to_owned(),
16307            "task".to_owned(),
16308            config,
16309        );
16310        let root = state.worktree_root();
16311        let wt_a = root.join("cand-A");
16312        let wt_b = root.join("cand-B");
16313        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
16314            .await
16315            .expect("worktree A");
16316        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
16317            .await
16318            .expect("worktree B");
16319
16320        state.candidates = vec![
16321            Candidate {
16322                index: 0,
16323                label: 'A',
16324                agent: "alpha".to_owned(),
16325                branch: "magi/x/A".to_owned(),
16326                worktree: wt_a.clone(),
16327                summary: String::new(),
16328                stat: String::new(),
16329                files: 0,
16330                commits: 0,
16331                empty: false,
16332                failed: None,
16333                verified_noop: None,
16334                duration_ms: 0,
16335                folded: false,
16336            },
16337            Candidate {
16338                index: 1,
16339                label: 'B',
16340                agent: "beta".to_owned(),
16341                branch: "magi/x/B".to_owned(),
16342                worktree: wt_b.clone(),
16343                summary: String::new(),
16344                stat: String::new(),
16345                files: 0,
16346                commits: 0,
16347                empty: false,
16348                failed: None,
16349                verified_noop: None,
16350                duration_ms: 0,
16351                folded: false,
16352            },
16353        ];
16354        state.tally = Some(Tally {
16355            first_choice: BTreeMap::from([('A', 1)]),
16356            borda: BTreeMap::new(),
16357            winner: 'A',
16358            rankings: 1,
16359            unanimous_initial: true,
16360            deliberated: false,
16361            changed_votes: 0,
16362            unanimous_final: true,
16363            tie_break: None,
16364            judges: 1,
16365            present: 1,
16366            quorum: 1,
16367            met_quorum: true,
16368            uncontested: None,
16369        });
16370        state.status = RunStatus::Ready;
16371
16372        fold_run(&mut state, false, &crate::run::home())
16373            .await
16374            .expect("fold_run");
16375
16376        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
16377        assert!(
16378            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16379            "the unmerged winner's branch survives"
16380        );
16381        assert!(
16382            !state.candidates[0].folded,
16383            "the winner is not marked folded"
16384        );
16385
16386        assert!(!wt_b.exists(), "the loser's worktree is removed");
16387        assert!(
16388            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
16389            "the loser's branch is removed"
16390        );
16391        assert!(state.candidates[1].folded, "the loser is marked folded");
16392    }
16393
16394    #[tokio::test]
16395    #[cfg(unix)]
16396    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
16397        use std::os::fd::AsRawFd as _;
16398        use std::time::Duration;
16399
16400        let tmp = tempfile::tempdir().unwrap();
16401        let repo = tmp.path().join("repo");
16402        std::fs::create_dir_all(&repo).unwrap();
16403        init_repo(&repo);
16404        let common = repo.join(".git");
16405        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
16406        let home = tmp.path().join("home");
16407        let mut config = Config::default();
16408        config.coordination = crate::config::test_coordination();
16409        config.graph.worktree_root = Some(tmp.path().join("wt"));
16410        config.graph.timeout_resource_wait = 1;
16411        let mut state = RunState::new(
16412            repo.clone(),
16413            "main".into(),
16414            "deadbeef".into(),
16415            "task".into(),
16416            config,
16417        );
16418        let root = state.worktree_root();
16419        let candidate = root.join("cand-A");
16420        let reviewer = root.join("review-1");
16421        let branch = state.branch_for('A');
16422        git::worktree_add_branch(&repo, &candidate, &branch, "main")
16423            .await
16424            .unwrap();
16425        git::worktree_add_detached(&repo, &reviewer, "main")
16426            .await
16427            .unwrap();
16428        git::acquire_worktree_config(&repo).await.unwrap();
16429        state.enabled_worktree_config = true;
16430        state.status = RunStatus::Ready;
16431        state.candidates.push(Candidate {
16432            index: 0,
16433            label: 'A',
16434            agent: "implement".into(),
16435            branch: branch.clone(),
16436            worktree: candidate.clone(),
16437            summary: String::new(),
16438            stat: String::new(),
16439            files: 0,
16440            commits: 0,
16441            empty: false,
16442            failed: None,
16443            verified_noop: None,
16444            duration_ms: 0,
16445            folded: false,
16446        });
16447        state.save_under(&home).unwrap();
16448        let config_before = std::fs::read(common.join("config")).unwrap();
16449        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
16450            .await
16451            .unwrap();
16452        // A separate OS handle bypasses the launcher's local gate, exactly as
16453        // an external updater does, and must still prevent all folding work.
16454        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
16455        assert_eq!(
16456            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
16457            0
16458        );
16459        let mut fold = Box::pin(fold_run(&mut state, true, &home));
16460        assert!(
16461            tokio::time::timeout(Duration::from_millis(200), &mut fold)
16462                .await
16463                .is_err()
16464        );
16465        assert!(candidate.exists());
16466        assert!(reviewer.exists());
16467        assert!(git::branch_exists(&repo, &branch).await.unwrap());
16468        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16469        assert_eq!(
16470            git::git(&repo, &["worktree", "list", "--porcelain"])
16471                .await
16472                .unwrap(),
16473            worktrees_before
16474        );
16475        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
16476            .await
16477            .unwrap()
16478            .unwrap_err();
16479        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
16480        drop(fold);
16481        assert!(candidate.exists() && reviewer.exists());
16482        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
16483        assert_eq!(
16484            state.resource_block.as_ref().unwrap().kind,
16485            "repository coordination"
16486        );
16487        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
16488        drop(external);
16489        let mut fold = Box::pin(fold_run(&mut state, true, &home));
16490        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
16491            .await
16492            .unwrap()
16493            .unwrap();
16494        drop(fold);
16495        assert!(removed.contains(&branch));
16496        assert!(!root.exists());
16497        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
16498        assert!(state.candidates[0].folded);
16499        assert!(!state.enabled_worktree_config);
16500        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
16501        let enabled = git::git_raw(
16502            &repo,
16503            &["config", "--local", "--get", "extensions.worktreeConfig"],
16504        )
16505        .await
16506        .unwrap();
16507        assert!(!enabled.ok(), "the last config reference was released");
16508    }
16509
16510    /// A branch handed to a later run is that run's (and its pull request's):
16511    /// folding the run that released it must not delete it.
16512    #[tokio::test]
16513    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
16514        let tmp = tempfile::tempdir().expect("tempdir");
16515        let repo = tmp.path().join("repo");
16516        std::fs::create_dir_all(&repo).unwrap();
16517        init_repo(&repo);
16518        let home = tmp.path().join("home");
16519
16520        let mut config = Config::default();
16521        config.graph.worktree_root = Some(tmp.path().join("wt"));
16522        let mut state = RunState::new(
16523            repo.clone(),
16524            "main".to_owned(),
16525            "deadbeef".to_owned(),
16526            "task".to_owned(),
16527            config,
16528        );
16529        // The worktree is already gone (released); the branch survives.
16530        git::git(&repo, &["branch", "magi/x/A", "main"])
16531            .await
16532            .expect("branch");
16533        state.candidates = vec![Candidate {
16534            index: 0,
16535            label: 'A',
16536            agent: "alpha".to_owned(),
16537            branch: "magi/x/A".to_owned(),
16538            worktree: state.worktree_root().join("cand-A"),
16539            summary: String::new(),
16540            stat: String::new(),
16541            files: 0,
16542            commits: 0,
16543            empty: false,
16544            failed: None,
16545            verified_noop: None,
16546            duration_ms: 0,
16547            folded: true,
16548        }];
16549        state.released_to = Some("20260901-000000-new1".to_owned());
16550        state.released_branches = vec!["magi/x/A".to_owned()];
16551
16552        fold_run(&mut state, true, &home).await.expect("fold_run");
16553        state.status = RunStatus::Merged;
16554        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
16555        state.merge = Some(crate::run::MergeOutcome {
16556            mode: MergeMode::Pr,
16557            ok: true,
16558            detail: "confirmed merged".into(),
16559            empty: false,
16560        });
16561        fold_run(&mut state, true, &home)
16562            .await
16563            .expect("merged fold");
16564
16565        assert!(
16566            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
16567            "the handed-over branch survives a fold"
16568        );
16569    }
16570
16571    /// A winner with nothing ahead of the base is caught before `gh` is ever
16572    /// asked for a pull request, and the message carries what the task's
16573    /// references resolved to.
16574    #[tokio::test]
16575    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
16576        let tmp = tempfile::tempdir().expect("tempdir");
16577        let repo = tmp.path().join("repo");
16578        std::fs::create_dir_all(&repo).unwrap();
16579        init_repo(&repo);
16580        let run = |args: &[&str]| {
16581            let out = std::process::Command::new("git")
16582                .quiet()
16583                .args(args)
16584                .current_dir(&repo)
16585                .output()
16586                .expect("spawn git");
16587            assert!(out.status.success(), "git {args:?}");
16588        };
16589        run(&["branch", "magi/x/A"]);
16590        run(&["checkout", "-q", "-b", "magi/x/B"]);
16591        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
16592        run(&["add", "-A"]);
16593        run(&["commit", "-q", "-m", "work"]);
16594        run(&["checkout", "-q", "main"]);
16595
16596        // A pull request is compared against the remote's base, so the
16597        // fixture needs one. Before it exists nothing can be read, and the
16598        // answer must be "not empty".
16599        let probe = RunState::new(
16600            repo.clone(),
16601            "main".to_owned(),
16602            "deadbeef".to_owned(),
16603            "task".to_owned(),
16604            Config::default(),
16605        );
16606        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
16607        let bare = tmp.path().join("origin.git");
16608        let out = std::process::Command::new("git")
16609            .quiet()
16610            .args(["init", "-q", "--bare"])
16611            .arg(&bare)
16612            .output()
16613            .expect("spawn git");
16614        assert!(out.status.success(), "git init --bare");
16615        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
16616        run(&["push", "-q", "origin", "main"]);
16617
16618        let mut state = RunState::new(
16619            repo.clone(),
16620            "main".to_owned(),
16621            "deadbeef".to_owned(),
16622            "task".to_owned(),
16623            Config::default(),
16624        );
16625        state.seeds = vec![refs::Seed {
16626            token: "magi/27b2/A".to_owned(),
16627            kind: refs::SeedKind::Unresolved,
16628            sha: String::new(),
16629            branch: true,
16630            detail: "no branch or commit named magi/27b2/A".to_owned(),
16631        }];
16632
16633        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
16634        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
16635        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
16636        let detail = empty_candidate_detail(&state, "main");
16637        assert!(detail.starts_with("empty candidate"), "{detail}");
16638        assert!(!detail.contains("magi/27b2/A"), "{detail}");
16639        assert!(detail.contains("--start-from"), "{detail}");
16640    }
16641
16642    /// `status == Ready` used to be read as "this is the harmless
16643    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
16644    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
16645    /// run's PR was closed without merging — and reentering `merge` with
16646    /// `mode` still `Pr` does not know the difference, so it pushed and
16647    /// opened a second pull request. `mode == Local` reproduces the same
16648    /// blind spot without a network call: reentry must not attempt another
16649    /// git merge once this node has already recorded an outcome.
16650    #[tokio::test]
16651    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
16652        let tmp = tempfile::tempdir().expect("tempdir");
16653        let repo = tmp.path().join("repo");
16654        std::fs::create_dir_all(&repo).unwrap();
16655        init_repo(&repo);
16656
16657        let mut config = Config::default();
16658        config.merge.mode = MergeMode::Local;
16659
16660        let mut state = RunState::new(
16661            repo.clone(),
16662            "main".to_owned(),
16663            "deadbeef".to_owned(),
16664            "task".to_owned(),
16665            config,
16666        );
16667        state.candidates = vec![Candidate {
16668            index: 0,
16669            label: 'A',
16670            agent: "alpha".to_owned(),
16671            branch: "does-not-exist".to_owned(),
16672            worktree: repo.clone(),
16673            summary: String::new(),
16674            stat: String::new(),
16675            files: 0,
16676            commits: 0,
16677            empty: false,
16678            failed: None,
16679            verified_noop: None,
16680            duration_ms: 0,
16681            folded: false,
16682        }];
16683        state.tally = Some(Tally {
16684            first_choice: BTreeMap::from([('A', 1)]),
16685            borda: BTreeMap::new(),
16686            winner: 'A',
16687            rankings: 1,
16688            unanimous_initial: true,
16689            deliberated: false,
16690            changed_votes: 0,
16691            unanimous_final: true,
16692            tie_break: None,
16693            judges: 0,
16694            present: 0,
16695            quorum: 0,
16696            met_quorum: true,
16697            uncontested: Some("only candidate A produced a change".to_owned()),
16698        });
16699        state.reviews = vec![ReviewRound {
16700            round: 1,
16701            head: "deadbeef".to_owned(),
16702            verified_head: None,
16703            verified_at: None,
16704            reviews: Vec::new(),
16705            e2e: Vec::new(),
16706            fix: None,
16707            blocking: 0,
16708            answered: 0,
16709            expected: 0,
16710            clean: true,
16711            verify_retried: false,
16712            e2e_deferred: false,
16713            e2e_defer_reason: None,
16714            progressed: false,
16715            vote_split: false,
16716            reconsideration: Vec::new(),
16717            verdict: None,
16718        }];
16719        state.gate = vec![CommandOutcome {
16720            command: "test".to_owned(),
16721            code: Some(0),
16722            output_tail: String::new(),
16723            duration_ms: 0,
16724            resource_blocked: false,
16725        }];
16726        state.gate_ran = true;
16727        // Reached its conclusion already — e.g. `land` closing the PR without
16728        // merging it, which (like the honest `MergeMode::None` path) leaves
16729        // `status` at `Ready`. The recorded outcome is what actually marks
16730        // this node done.
16731        state.status = RunStatus::Ready;
16732        state.merge = Some(MergeOutcome {
16733            mode: MergeMode::Local,
16734            ok: false,
16735            detail: "already concluded".to_owned(),
16736            empty: false,
16737        });
16738
16739        let mut runner = Runner {
16740            state,
16741            roles: ResolvedRoles {
16742                implementers: Vec::new(),
16743                judges: Vec::new(),
16744                reviewers: Vec::new(),
16745                fixer: None,
16746                conductor: conductor(),
16747                implementer_roster: Vec::new(),
16748                judge_roster: Vec::new(),
16749                reviewer_roster: Vec::new(),
16750            },
16751            sem: Arc::new(Semaphore::new(1)),
16752            pause: Pause::new(),
16753            interrupt: Pause::new(),
16754        };
16755
16756        runner.merge().await.expect("merge");
16757
16758        assert_eq!(
16759            runner.state.status,
16760            RunStatus::Ready,
16761            "a concluded run's status must not change on reentry"
16762        );
16763        assert_eq!(
16764            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16765            Some("already concluded"),
16766            "merge must not run again once the node already recorded an outcome"
16767        );
16768    }
16769
16770    /// `gate` leaves `state.gate_ran` false both before it has ever run and
16771    /// when its last attempt was resource-blocked (the shared build cache
16772    /// could not be acquired or confirmed fresh in time - see
16773    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
16774    /// `Vec` this also leaves behind used to read as "nothing failed" and let
16775    /// a run merge a tree the gate never actually checked - exactly the case
16776    /// a contended cache produces on every retry until it clears. `merge`
16777    /// must refuse until `gate` has actually recorded an attempt.
16778    #[tokio::test]
16779    async fn merge_refuses_a_gate_that_has_not_actually_run() {
16780        let tmp = tempfile::tempdir().expect("tempdir");
16781        let repo = tmp.path().join("repo");
16782        std::fs::create_dir_all(&repo).unwrap();
16783        init_repo(&repo);
16784
16785        let mut config = Config::default();
16786        config.merge.mode = MergeMode::Local;
16787
16788        let mut state = RunState::new(
16789            repo.clone(),
16790            "main".to_owned(),
16791            "deadbeef".to_owned(),
16792            "task".to_owned(),
16793            config,
16794        );
16795        state.candidates = vec![Candidate {
16796            index: 0,
16797            label: 'A',
16798            agent: "alpha".to_owned(),
16799            branch: "does-not-exist".to_owned(),
16800            worktree: repo.clone(),
16801            summary: String::new(),
16802            stat: String::new(),
16803            files: 0,
16804            commits: 0,
16805            empty: false,
16806            failed: None,
16807            verified_noop: None,
16808            duration_ms: 0,
16809            folded: false,
16810        }];
16811        state.tally = Some(Tally {
16812            first_choice: BTreeMap::from([('A', 1)]),
16813            borda: BTreeMap::new(),
16814            winner: 'A',
16815            rankings: 1,
16816            unanimous_initial: true,
16817            deliberated: false,
16818            changed_votes: 0,
16819            unanimous_final: true,
16820            tie_break: None,
16821            judges: 0,
16822            present: 0,
16823            quorum: 0,
16824            met_quorum: true,
16825            uncontested: Some("only candidate A produced a change".to_owned()),
16826        });
16827        state.reviews = vec![ReviewRound {
16828            round: 1,
16829            head: "deadbeef".to_owned(),
16830            verified_head: None,
16831            verified_at: None,
16832            reviews: Vec::new(),
16833            e2e: Vec::new(),
16834            fix: None,
16835            blocking: 0,
16836            answered: 0,
16837            expected: 0,
16838            clean: true,
16839            verify_retried: false,
16840            e2e_deferred: false,
16841            e2e_defer_reason: None,
16842            progressed: false,
16843            vote_split: false,
16844            reconsideration: Vec::new(),
16845            verdict: None,
16846        }];
16847        // The point: `gate` has not recorded anything yet.
16848        state.gate = Vec::new();
16849        state.gate_ran = false;
16850        state.status = RunStatus::Gating;
16851
16852        let mut runner = Runner {
16853            state,
16854            roles: ResolvedRoles {
16855                implementers: Vec::new(),
16856                judges: Vec::new(),
16857                reviewers: Vec::new(),
16858                fixer: None,
16859                conductor: conductor(),
16860                implementer_roster: Vec::new(),
16861                judge_roster: Vec::new(),
16862                reviewer_roster: Vec::new(),
16863            },
16864            sem: Arc::new(Semaphore::new(1)),
16865            pause: Pause::new(),
16866            interrupt: Pause::new(),
16867        };
16868
16869        runner.merge().await.expect("merge");
16870
16871        assert!(
16872            runner.state.merge.is_none(),
16873            "an empty gate must never be read as a passing one: {:?}",
16874            runner.state.merge
16875        );
16876    }
16877
16878    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
16879    /// commands configured and `merge.mode` is `none` (a review-only run).
16880    /// `gate` must still record a real attempt — zero commands, vacuously
16881    /// passed — rather than leaving `state.gate` empty in a way `merge`
16882    /// cannot tell apart from "never ran"; otherwise the run reaches
16883    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
16884    #[tokio::test]
16885    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16886        let tmp = tempfile::tempdir().expect("tempdir");
16887        let repo = tmp.path().join("repo");
16888        std::fs::create_dir_all(&repo).unwrap();
16889        init_repo(&repo);
16890
16891        // Default config: `verify.gate` empty, `merge.mode` is `none`.
16892        let config = Config::default();
16893
16894        let mut state = RunState::new(
16895            repo.clone(),
16896            "main".to_owned(),
16897            "deadbeef".to_owned(),
16898            "task".to_owned(),
16899            config,
16900        );
16901        state.candidates = vec![Candidate {
16902            index: 0,
16903            label: 'A',
16904            agent: "alpha".to_owned(),
16905            branch: "does-not-exist".to_owned(),
16906            worktree: repo.clone(),
16907            summary: String::new(),
16908            stat: String::new(),
16909            files: 0,
16910            commits: 0,
16911            empty: false,
16912            failed: None,
16913            verified_noop: None,
16914            duration_ms: 0,
16915            folded: false,
16916        }];
16917        state.tally = Some(Tally {
16918            first_choice: BTreeMap::from([('A', 1)]),
16919            borda: BTreeMap::new(),
16920            winner: 'A',
16921            rankings: 1,
16922            unanimous_initial: true,
16923            deliberated: false,
16924            changed_votes: 0,
16925            unanimous_final: true,
16926            tie_break: None,
16927            judges: 0,
16928            present: 0,
16929            quorum: 0,
16930            met_quorum: true,
16931            uncontested: Some("only candidate A produced a change".to_owned()),
16932        });
16933        state.reviews = vec![ReviewRound {
16934            round: 1,
16935            head: "deadbeef".to_owned(),
16936            verified_head: None,
16937            verified_at: None,
16938            reviews: Vec::new(),
16939            e2e: Vec::new(),
16940            fix: None,
16941            blocking: 0,
16942            answered: 0,
16943            expected: 0,
16944            clean: true,
16945            verify_retried: false,
16946            e2e_deferred: false,
16947            e2e_defer_reason: None,
16948            progressed: false,
16949            vote_split: false,
16950            reconsideration: Vec::new(),
16951            verdict: None,
16952        }];
16953
16954        let mut runner = Runner {
16955            state,
16956            roles: ResolvedRoles {
16957                implementers: Vec::new(),
16958                judges: Vec::new(),
16959                reviewers: Vec::new(),
16960                fixer: None,
16961                conductor: conductor(),
16962                implementer_roster: Vec::new(),
16963                judge_roster: Vec::new(),
16964                reviewer_roster: Vec::new(),
16965            },
16966            sem: Arc::new(Semaphore::new(1)),
16967            pause: Pause::new(),
16968            interrupt: Pause::new(),
16969        };
16970
16971        runner.gate().await.expect("gate");
16972        assert!(
16973            runner.state.gate_ran,
16974            "zero configured commands is still a real attempt, not an unrun gate"
16975        );
16976        assert!(runner.state.gate.is_empty());
16977        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16978        assert_ne!(
16979            runner.state.status,
16980            RunStatus::Blocked,
16981            "a gate with nothing to check must not read as failed"
16982        );
16983
16984        runner.merge().await.expect("merge");
16985        assert_eq!(
16986            runner.state.status,
16987            RunStatus::Ready,
16988            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16989        );
16990    }
16991
16992    /// `Config::cache_dir` is derived from `verify.e2e` as well as
16993    /// `verify.gate` (so the e2e leg and the final gate never build against
16994    /// different directories). With zero `verify.gate` commands but a
16995    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
16996    /// that lease before discovering it had nothing to run - so a repo with
16997    /// no gate commands could come back `resource_blocked` (and therefore
16998    /// still `gate_ran == false`) on nothing but an unrelated run holding the
16999    /// cache, exactly the contention this run's own zero commands could
17000    /// never have touched. `gate` must recognise there is nothing to check
17001    /// before it ever asks for the lease.
17002    #[tokio::test]
17003    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
17004        crate::run::pin_test_home();
17005        let home = crate::run::home();
17006
17007        let tmp = tempfile::tempdir().expect("tempdir");
17008        let repo = tmp.path().join("repo");
17009        std::fs::create_dir_all(&repo).unwrap();
17010        init_repo(&repo);
17011        // Unique to this test, so holding its lease cannot collide with
17012        // another test sharing the same process-wide `home`.
17013        let cache_dir = tmp.path().join("target");
17014
17015        let mut config = Config::default();
17016        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
17017        // `verify.gate` stays empty (the default). Bounded so a regression
17018        // that does start waiting fails the test in seconds, not hangs it.
17019        config.graph.timeout_verify = Some(2);
17020        config.graph.timeout_resource_wait = 2;
17021
17022        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17023        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17024            .expect("no io error acquiring directly")
17025        {
17026            crate::cache::AcquireOutcome::Acquired(g) => g,
17027            crate::cache::AcquireOutcome::Busy(b) => {
17028                panic!("expected the direct acquire to win the lease first: {b:?}")
17029            }
17030        };
17031
17032        let mut state = RunState::new(
17033            repo.clone(),
17034            "main".to_owned(),
17035            "deadbeef".to_owned(),
17036            "task".to_owned(),
17037            config,
17038        );
17039        state.candidates = vec![Candidate {
17040            index: 0,
17041            label: 'A',
17042            agent: "alpha".to_owned(),
17043            branch: "does-not-exist".to_owned(),
17044            worktree: repo.clone(),
17045            summary: String::new(),
17046            stat: String::new(),
17047            files: 0,
17048            commits: 0,
17049            empty: false,
17050            failed: None,
17051            verified_noop: None,
17052            duration_ms: 0,
17053            folded: false,
17054        }];
17055        state.tally = Some(Tally {
17056            first_choice: BTreeMap::from([('A', 1)]),
17057            borda: BTreeMap::new(),
17058            winner: 'A',
17059            rankings: 1,
17060            unanimous_initial: true,
17061            deliberated: false,
17062            changed_votes: 0,
17063            unanimous_final: true,
17064            tie_break: None,
17065            judges: 0,
17066            present: 0,
17067            quorum: 0,
17068            met_quorum: true,
17069            uncontested: Some("only candidate A produced a change".to_owned()),
17070        });
17071        state.reviews = vec![ReviewRound {
17072            round: 1,
17073            head: "deadbeef".to_owned(),
17074            verified_head: None,
17075            verified_at: None,
17076            reviews: Vec::new(),
17077            e2e: Vec::new(),
17078            fix: None,
17079            blocking: 0,
17080            answered: 0,
17081            expected: 0,
17082            clean: true,
17083            verify_retried: false,
17084            e2e_deferred: false,
17085            e2e_defer_reason: None,
17086            progressed: false,
17087            vote_split: false,
17088            reconsideration: Vec::new(),
17089            verdict: None,
17090        }];
17091
17092        let mut runner = Runner {
17093            state,
17094            roles: ResolvedRoles {
17095                implementers: Vec::new(),
17096                judges: Vec::new(),
17097                reviewers: Vec::new(),
17098                fixer: None,
17099                conductor: conductor(),
17100                implementer_roster: Vec::new(),
17101                judge_roster: Vec::new(),
17102                reviewer_roster: Vec::new(),
17103            },
17104            sem: Arc::new(Semaphore::new(1)),
17105            pause: Pause::new(),
17106            interrupt: Pause::new(),
17107        };
17108
17109        let started = std::time::Instant::now();
17110        runner.gate().await.expect("gate");
17111        assert!(
17112            started.elapsed() < Duration::from_secs(1),
17113            "a gate with nothing to run must never wait on a lease it never needed"
17114        );
17115        assert!(
17116            runner.state.gate_ran,
17117            "zero commands is still a real, immediate attempt"
17118        );
17119        assert!(runner.state.gate.is_empty());
17120        assert_ne!(
17121            runner.state.status,
17122            RunStatus::Blocked,
17123            "must not read as resource-blocked on a lease it never asked for"
17124        );
17125    }
17126
17127    /// A separate process carries a live worker's environment throughout all
17128    /// verification entry points; the parallel harness never changes its env.
17129    #[cfg(unix)]
17130    #[tokio::test]
17131    async fn verification_is_standalone_inside_a_live_worker() {
17132        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
17133        if std::env::var_os(CHILD).is_none() {
17134            let home = tempfile::tempdir().unwrap();
17135            let generation = crate::worker::pin_current(home.path()).unwrap();
17136            let output = std::process::Command::new(std::env::current_exe().unwrap())
17137                .args([
17138                    "--exact",
17139                    "graph::tests::verification_is_standalone_inside_a_live_worker",
17140                    "--nocapture",
17141                ])
17142                .env(CHILD, "1")
17143                .env("MAGI_HOME", home.path())
17144                .env("MAGI_WORKER_HOME", home.path())
17145                .env("MAGI_WORKER_ID", "abcdef")
17146                .env("MAGI_WORKER_GENERATION", &generation.id)
17147                .env("MAGI_WORKER_EXE", &generation.executable)
17148                .env("MAGI_RUN", "verification-parent")
17149                .env("MAGI_NODE", "gate")
17150                .env("MAGI_SEAT", "worker-fixture")
17151                .env("MAGI_PROCESS_OWNER", "fixture-owner")
17152                .env("MAGI_NO_AUTOUPDATE", "0")
17153                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
17154                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
17155                .env(
17156                    "VERIFY_EXPECT_TOOLCHAIN",
17157                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
17158                )
17159                .env(
17160                    "VERIFY_EXPECT_CACHE",
17161                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
17162                )
17163                .output()
17164                .unwrap();
17165            assert!(
17166                output.status.success()
17167                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17168                "{}\n{}",
17169                String::from_utf8_lossy(&output.stdout),
17170                String::from_utf8_lossy(&output.stderr)
17171            );
17172            return;
17173        }
17174        let home = crate::run::home();
17175        let directory = home.join("workers/jobs/abcdef");
17176        std::fs::create_dir_all(&directory).unwrap();
17177        let job = crate::worker::Job {
17178            id: "abcdef".into(),
17179            sequence: 1,
17180            kind: "task".into(),
17181            key: "task:verification".into(),
17182            payload: serde_json::json!({}),
17183            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
17184            version: Some(env!("CARGO_PKG_VERSION").into()),
17185            status: crate::worker::Status::Running,
17186            result: None,
17187            pid: Some(std::process::id()),
17188            created_at: 0,
17189            finished_at: None,
17190        };
17191        std::fs::write(
17192            directory.join("job.json"),
17193            serde_json::to_vec(&job).unwrap(),
17194        )
17195        .unwrap();
17196        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
17197        let assert_worker = || {
17198            assert_eq!(
17199                crate::worker::current_job(&home).unwrap().unwrap().id,
17200                job.id
17201            );
17202            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
17203            assert_eq!(
17204                crate::worker::helper_executable(&home).unwrap(),
17205                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
17206            );
17207            assert!(
17208                crate::updater::Checker::new(&crate::config::Update {
17209                    mode: crate::config::UpdateMode::Notify,
17210                    interval: None,
17211                    auto_restart: false,
17212                })
17213                .is_none(),
17214                "the driver remains pinned, even while tests are standalone"
17215            );
17216        };
17217        assert_worker();
17218        let repo = home.join("repo");
17219        std::fs::create_dir_all(&repo).unwrap();
17220        init_repo(&repo);
17221        // Keep storage, seat/control context, containment and tool lookup.
17222        // No inherited values (or credentials) are printed on success.
17223        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
17224            test -z "${MAGI_WORKER_GENERATION+x}" &&
17225            test -z "${MAGI_WORKER_HOME+x}" &&
17226            test -z "${MAGI_WORKER_EXE+x}" &&
17227            test "$MAGI_HOME/repo" -ef . &&
17228            test "$MAGI_RUN" = verification-parent &&
17229            test "$MAGI_NODE" = gate &&
17230            test "$MAGI_SEAT" = worker-fixture &&
17231            test -n "$MAGI_PROCESS_OWNER" &&
17232            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
17233            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
17234            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
17235            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
17236        // Negative control: an ordinary child inherits the contamination.
17237        assert!(
17238            !std::process::Command::new("sh")
17239                .args(["-c", probe])
17240                .current_dir(&repo)
17241                .status()
17242                .unwrap()
17243                .success()
17244        );
17245        let unrelated_home = tempfile::tempdir().unwrap();
17246        assert!(
17247            crate::worker::current_job(unrelated_home.path()).is_err(),
17248            "an inherited worker id cannot be resolved in a unit fixture's home"
17249        );
17250        let mut runner = runner_at(RunStatus::Gating);
17251        runner.state.repo = repo.clone();
17252        runner.state.config.disk.min_free_bytes = 0;
17253        runner.state.config.coordination = crate::config::test_coordination();
17254        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
17255        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
17256        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
17257        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
17258        let winner = Candidate {
17259            index: 0,
17260            label: 'A',
17261            agent: "fixture".into(),
17262            branch: "main".into(),
17263            worktree: repo.clone(),
17264            summary: String::new(),
17265            stat: String::new(),
17266            files: 0,
17267            commits: 0,
17268            empty: false,
17269            failed: None,
17270            verified_noop: None,
17271            duration_ms: 0,
17272            folded: false,
17273        };
17274        let (e2e, retried, pids) = run_e2e_with_retry(
17275            &mut runner.state,
17276            &["sh".into(), "-c".into()],
17277            &[format!("{probe} && printf initial")],
17278            &repo,
17279            Duration::from_secs(10),
17280            "initial",
17281        )
17282        .await;
17283        assert!(!retried && pids.is_empty());
17284        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
17285        assert_eq!(e2e[0].output_tail, "initial");
17286        let gate = runner.run_gate(&winner).await.unwrap();
17287        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
17288        // An interrupted repair must recheck e2e from saved state. Landing
17289        // reuses that state and must run both pre-gate and gate again.
17290        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
17291            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
17292            passed: false,
17293            outcomes: Vec::new(),
17294        });
17295        runner.state.save().unwrap();
17296        let mut resumed = RunState::load(&runner.state.id).unwrap();
17297        let wait = crate::repo_lock::WaitPolicy::default();
17298        assert!(matches!(
17299            check_gate_repair_e2e(&mut resumed, &winner, &wait)
17300                .await
17301                .unwrap(),
17302            GateFix::Retry
17303        ));
17304        let saved = RunState::load(&resumed.id).unwrap();
17305        let repair = saved.gate_repair_e2e.unwrap();
17306        assert!(repair.passed);
17307        assert_eq!(repair.outcomes.len(), 1);
17308        assert!(repair.outcomes[0].ok());
17309        let landing = run_landing_gate(&mut resumed, &winner, &wait)
17310            .await
17311            .unwrap();
17312        assert_eq!(landing.len(), 1);
17313        assert!(landing[0].ok(), "{landing:?}");
17314        assert_eq!(
17315            std::fs::read_to_string(repo.join("phases")).unwrap(),
17316            "pre\ngate\ne2e\npre\ngate\n"
17317        );
17318        assert_worker();
17319    }
17320
17321    #[cfg(unix)]
17322    #[tokio::test]
17323    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
17324        crate::run::pin_test_home();
17325        let tmp = tempfile::tempdir().unwrap();
17326        let mut config = Config::default();
17327        config.disk.min_free_bytes = 0;
17328        let mut state = RunState::new(
17329            tmp.path().to_owned(),
17330            "main".into(),
17331            "base".into(),
17332            "verification evidence".into(),
17333            config,
17334        );
17335        let (outcomes, pids) = run_commands(
17336            &mut state,
17337            "gate",
17338            "gate",
17339            1,
17340            &["sh".into(), "-c".into()],
17341            &[
17342                "printf before-timeout; exec sleep 30".into(),
17343                "kill -TERM $$".into(),
17344                "exit 7".into(),
17345                "printf success".into(),
17346            ],
17347            tmp.path(),
17348            Duration::from_secs(1),
17349        )
17350        .await;
17351        wait_for_timed_out_children_to_die(&pids).await;
17352        assert_eq!(
17353            pids.len(),
17354            1,
17355            "unleased timeout retains its death-check PID"
17356        );
17357        assert_eq!(outcomes.len(), 4);
17358        assert_eq!(outcomes[0].code, None);
17359        assert!(outcomes[0].output_tail.contains("before-timeout"));
17360        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
17361        assert_eq!(outcomes[1].code, None);
17362        assert!(outcomes[1].output_tail.contains("command terminated:"));
17363        assert!(!outcomes[1].output_tail.contains("timed out"));
17364        assert_eq!(outcomes[2].code, Some(7));
17365        assert!(!outcomes[2].output_tail.contains("timed out"));
17366        assert_eq!(outcomes[3].code, Some(0));
17367        assert_eq!(outcomes[3].output_tail, "success");
17368        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
17369    }
17370
17371    /// The addendum's second gap: a `verify.gate` command running for real
17372    /// wall-clock time had nothing at all to show for it in `active` before
17373    /// `run_commands` learned to record it — a run could sit in `Gating` for
17374    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
17375    /// what was actually happening. Proven with a genuinely still-running
17376    /// command, not just a before/after check on the final state: a poller
17377    /// task reads the same `run.json` `gate()` is writing, the same way the
17378    /// phone or `magi show` would, while the shell command is still blocked
17379    /// on its own release marker.
17380    #[tokio::test]
17381    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
17382        crate::run::pin_test_home();
17383
17384        let tmp = tempfile::tempdir().expect("tempdir");
17385        let repo = tmp.path().join("repo");
17386        std::fs::create_dir_all(&repo).unwrap();
17387        init_repo(&repo);
17388
17389        let mut config = Config::default();
17390        config.verify.gate = vec![
17391            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
17392             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
17393                .to_owned(),
17394        ];
17395
17396        let mut state = RunState::new(
17397            repo.clone(),
17398            "main".to_owned(),
17399            "deadbeef".to_owned(),
17400            "task".to_owned(),
17401            config,
17402        );
17403        let run_id = state.id.clone();
17404        state.candidates = vec![Candidate {
17405            index: 0,
17406            label: 'A',
17407            agent: "alpha".to_owned(),
17408            branch: "does-not-exist".to_owned(),
17409            worktree: repo.clone(),
17410            summary: String::new(),
17411            stat: String::new(),
17412            files: 0,
17413            commits: 0,
17414            empty: false,
17415            failed: None,
17416            verified_noop: None,
17417            duration_ms: 0,
17418            folded: false,
17419        }];
17420        state.tally = Some(Tally {
17421            first_choice: BTreeMap::from([('A', 1)]),
17422            borda: BTreeMap::new(),
17423            winner: 'A',
17424            rankings: 1,
17425            unanimous_initial: true,
17426            deliberated: false,
17427            changed_votes: 0,
17428            unanimous_final: true,
17429            tie_break: None,
17430            judges: 0,
17431            present: 0,
17432            quorum: 0,
17433            met_quorum: true,
17434            uncontested: Some("only candidate A produced a change".to_owned()),
17435        });
17436        state.reviews = vec![ReviewRound {
17437            round: 1,
17438            head: "deadbeef".to_owned(),
17439            verified_head: None,
17440            verified_at: None,
17441            reviews: Vec::new(),
17442            e2e: Vec::new(),
17443            fix: None,
17444            blocking: 0,
17445            answered: 0,
17446            expected: 0,
17447            clean: true,
17448            verify_retried: false,
17449            e2e_deferred: false,
17450            e2e_defer_reason: None,
17451            progressed: false,
17452            vote_split: false,
17453            reconsideration: Vec::new(),
17454            verdict: None,
17455        }];
17456
17457        let mut runner = Runner {
17458            state,
17459            roles: ResolvedRoles {
17460                implementers: Vec::new(),
17461                judges: Vec::new(),
17462                reviewers: Vec::new(),
17463                fixer: None,
17464                conductor: conductor(),
17465                implementer_roster: Vec::new(),
17466                judge_roster: Vec::new(),
17467                reviewer_roster: Vec::new(),
17468            },
17469            sem: Arc::new(Semaphore::new(1)),
17470            pause: Pause::new(),
17471            interrupt: Pause::new(),
17472        };
17473
17474        let started_marker = repo.join("started.marker");
17475        let release_marker = repo.join("release.marker");
17476        let poller = tokio::spawn(async move {
17477            // Bounded so a regression that never records the task entry
17478            // fails this test in seconds instead of hanging the suite —
17479            // the same shape `a_park_requested_while_a_seat_is_mid_call_
17480            // does_not_cut_it_short` uses for the same reason.
17481            for _ in 0..100 {
17482                if started_marker.exists()
17483                    && let Ok(s) = crate::run::RunState::load(&run_id)
17484                    && let Some(a) = s.active.get("gate")
17485                {
17486                    std::fs::write(&release_marker, b"go").expect("release marker");
17487                    return Some(a.clone());
17488                }
17489                tokio::time::sleep(Duration::from_millis(50)).await;
17490            }
17491            None
17492        });
17493
17494        runner.gate().await.expect("gate");
17495        let captured = poller.await.expect("poller task");
17496        let captured = captured.expect(
17497            "the poller never saw a `gate` task entry in run.json while the command was \
17498             still blocked on its own release marker",
17499        );
17500
17501        assert_eq!(captured.task.as_deref(), Some("gate"));
17502        assert_eq!(captured.node, "gate");
17503        assert_eq!(captured.index, Some(1));
17504        assert_eq!(captured.total, Some(1));
17505        assert!(
17506            captured
17507                .command
17508                .as_deref()
17509                .is_some_and(|c| c.contains("started.marker")),
17510            "{captured:?}"
17511        );
17512
17513        assert!(
17514            runner.state.active.is_empty(),
17515            "the entry must be cleared once the command actually finished: {:?}",
17516            runner.state.active
17517        );
17518        assert!(runner.state.gate_ran);
17519        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
17520    }
17521
17522    /// The hand-off over a blocking finding a reviewer rejected on leaves a
17523    /// record for `land`; one with only a Minor, or no reject, leaves none.
17524    #[tokio::test]
17525    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
17526        use crate::verdict::{Finding, ReviewVote, Severity};
17527        crate::run::pin_test_home();
17528        let tmp = tempfile::tempdir().expect("tempdir");
17529        let repo = tmp.path().join("repo");
17530        std::fs::create_dir_all(&repo).unwrap();
17531        init_repo(&repo);
17532
17533        for (severity, vote, expect) in [
17534            (Severity::Major, ReviewVote::Reject, true),
17535            (Severity::Minor, ReviewVote::Reject, false),
17536            (Severity::Major, ReviewVote::Approve, false),
17537        ] {
17538            let mut round = review_round(false, 1, 1, 1, false, true);
17539            round.reviews = vec![ReviewRecord {
17540                reviewer: 1,
17541                agent: "alpha".to_owned(),
17542                summary: String::new(),
17543                findings: vec![Finding {
17544                    id: "R1-1-1".to_owned(),
17545                    severity,
17546                    file: None,
17547                    line: None,
17548                    title: "t".to_owned(),
17549                    detail: String::new(),
17550                }],
17551                vote: Some(vote),
17552                failed: None,
17553                duration_ms: 0,
17554                attempts: 0,
17555            }];
17556            let mut state = RunState::new(
17557                repo.clone(),
17558                "main".to_owned(),
17559                "deadbeef".to_owned(),
17560                "task".to_owned(),
17561                Config::default(),
17562            );
17563            state.reviews = vec![round];
17564            let mut runner = Runner {
17565                state,
17566                roles: ResolvedRoles {
17567                    implementers: Vec::new(),
17568                    judges: Vec::new(),
17569                    reviewers: Vec::new(),
17570                    fixer: None,
17571                    conductor: conductor(),
17572                    implementer_roster: Vec::new(),
17573                    judge_roster: Vec::new(),
17574                    reviewer_roster: Vec::new(),
17575                },
17576                sem: Arc::new(Semaphore::new(1)),
17577                pause: Pause::new(),
17578                interrupt: Pause::new(),
17579            };
17580            let shell = runner.state.config.shell();
17581            runner
17582                .stop_reviewing("round budget spent", &shell, &repo)
17583                .await
17584                .expect("stop_reviewing");
17585            assert_eq!(runner.state.status, RunStatus::Gating);
17586            assert_eq!(
17587                runner.state.contested_handoff.is_some(),
17588                expect,
17589                "{severity:?} + {vote:?}"
17590            );
17591        }
17592    }
17593
17594    /// The shape the incident this whole fix responds to actually had: the
17595    /// round budget spent, the last round's own e2e blocked on the shared
17596    /// build cache (held here by a live pid — this test process — exactly
17597    /// `cache`'s own unit tests' pattern for "another owner, still alive"
17598    /// without forking a process). `stop_reviewing` must retry it — not
17599    /// silently leave the round looking untouched (the catch-up-only half of
17600    /// the bug), and not read the contention as a red `e2e` and block the
17601    /// run on it (the other half). Called directly, the same way
17602    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
17603    /// above exercises `gate`, so this never needs a real cargo build to
17604    /// reach: the lease is never released, so `with_cache_lease` never gets
17605    /// past acquiring it into anything that would need a real workspace.
17606    #[tokio::test]
17607    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
17608        crate::run::pin_test_home();
17609        let home = crate::run::home();
17610
17611        let tmp = tempfile::tempdir().expect("tempdir");
17612        let repo = tmp.path().join("repo");
17613        std::fs::create_dir_all(&repo).unwrap();
17614        init_repo(&repo);
17615        let head = crate::git::rev_parse(&repo, "HEAD")
17616            .await
17617            .expect("rev-parse");
17618        // Unique to this test, so holding its lease cannot collide with
17619        // another test sharing the same process-wide `home`.
17620        let cache_dir = tmp.path().join("target");
17621
17622        let mut config = Config::default();
17623        config.verify.e2e = vec![format!(
17624            "CARGO_TARGET_DIR='{}' test -f README.md",
17625            cache_dir.display()
17626        )];
17627        config.graph.review_rounds = 1;
17628        // Bounded so a regression that does start waiting fails the test in
17629        // seconds, not hangs it.
17630        config.graph.timeout_verify = Some(2);
17631        config.graph.timeout_resource_wait = 2;
17632
17633        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17634        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17635            .expect("no io error acquiring directly")
17636        {
17637            crate::cache::AcquireOutcome::Acquired(g) => g,
17638            crate::cache::AcquireOutcome::Busy(b) => {
17639                panic!("expected the direct acquire to win the lease first: {b:?}")
17640            }
17641        };
17642
17643        let mut state = RunState::new(
17644            repo.clone(),
17645            "main".to_owned(),
17646            head.clone(),
17647            "task".to_owned(),
17648            config,
17649        );
17650        state.candidates = vec![Candidate {
17651            index: 0,
17652            label: 'A',
17653            agent: "alpha".to_owned(),
17654            branch: "does-not-exist".to_owned(),
17655            worktree: repo.clone(),
17656            summary: String::new(),
17657            stat: String::new(),
17658            files: 0,
17659            commits: 0,
17660            empty: false,
17661            failed: None,
17662            verified_noop: None,
17663            duration_ms: 0,
17664            folded: false,
17665        }];
17666        state.tally = Some(Tally {
17667            first_choice: BTreeMap::from([('A', 1)]),
17668            borda: BTreeMap::new(),
17669            winner: 'A',
17670            rankings: 1,
17671            unanimous_initial: true,
17672            deliberated: false,
17673            changed_votes: 0,
17674            unanimous_final: true,
17675            tie_break: None,
17676            judges: 0,
17677            present: 0,
17678            quorum: 0,
17679            met_quorum: true,
17680            uncontested: Some("only candidate A produced a change".to_owned()),
17681        });
17682        // The round budget's last round, deferred: `needs_catchup_run`'s
17683        // other trigger. `stop_reviewing`'s retry machinery must treat this
17684        // exactly like a resource-blocked attempt once it actually runs.
17685        state.reviews = vec![ReviewRound {
17686            round: 1,
17687            head: head.clone(),
17688            verified_head: None,
17689            verified_at: None,
17690            reviews: Vec::new(),
17691            e2e: Vec::new(),
17692            fix: None,
17693            blocking: 1,
17694            answered: 1,
17695            expected: 1,
17696            clean: false,
17697            verify_retried: false,
17698            e2e_deferred: true,
17699            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
17700            progressed: false,
17701            vote_split: false,
17702            reconsideration: Vec::new(),
17703            verdict: None,
17704        }];
17705
17706        let mut runner = Runner {
17707            state,
17708            roles: ResolvedRoles {
17709                implementers: Vec::new(),
17710                judges: Vec::new(),
17711                reviewers: Vec::new(),
17712                fixer: None,
17713                conductor: conductor(),
17714                implementer_roster: Vec::new(),
17715                judge_roster: Vec::new(),
17716                reviewer_roster: Vec::new(),
17717            },
17718            sem: Arc::new(Semaphore::new(1)),
17719            pause: Pause::new(),
17720            interrupt: Pause::new(),
17721        };
17722
17723        let shell = runner.state.config.shell();
17724        runner
17725            .stop_reviewing("round budget spent", &shell, &repo)
17726            .await
17727            .expect("stop_reviewing");
17728
17729        let last = runner.state.reviews.last().expect("round record");
17730        assert_eq!(
17731            last.e2e_status(),
17732            E2eStatus::ResourceBlocked,
17733            "the shared cache is still held; the attempt must read as blocked, not deferred or \
17734             failed: {last:?}"
17735        );
17736        assert_eq!(
17737            last.verified_head.as_deref(),
17738            Some(head.as_str()),
17739            "which commit this attempt targeted is known even though nothing finished checking \
17740             it"
17741        );
17742        let first_attempt_at = last
17743            .verified_at
17744            .expect("when this attempt ran is known too");
17745        assert_ne!(
17746            runner.state.status,
17747            RunStatus::Blocked,
17748            "contention is evidence about the machine, not the patch — it must not settle the \
17749             run as blocked: {:?}",
17750            runner.state.status
17751        );
17752        assert!(
17753            !runner
17754                .state
17755                .events
17756                .iter()
17757                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
17758            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
17759            runner.state.events
17760        );
17761
17762        // The cache is still held: a later reentry must retry the same
17763        // round's verification again — not leave it looking exactly as
17764        // untouched as the first blocked attempt, which is indistinguishable
17765        // from never having tried again at all.
17766        runner
17767            .stop_reviewing("round budget spent", &shell, &repo)
17768            .await
17769            .expect("stop_reviewing retry");
17770        assert_eq!(
17771            runner.state.reviews.len(),
17772            1,
17773            "no new round was started: {:?}",
17774            runner.state.reviews
17775        );
17776        let last = runner.state.reviews.last().expect("round record");
17777        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
17778        assert!(
17779            last.verified_at.expect("still known") > first_attempt_at,
17780            "a second reentry must be a fresh attempt, not a stale copy of the first"
17781        );
17782        assert_ne!(runner.state.status, RunStatus::Blocked);
17783
17784        held.release();
17785    }
17786
17787    #[cfg(unix)]
17788    #[tokio::test]
17789    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
17790        ask_test_home();
17791        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
17792            let tmp = tempfile::tempdir().unwrap();
17793            let repo = tmp.path().join("repo");
17794            std::fs::create_dir(&repo).unwrap();
17795            init_repo(&repo);
17796            let run_git = |args: &[&str]| {
17797                let out = std::process::Command::new("git")
17798                    .args(args)
17799                    .current_dir(&repo)
17800                    .output()
17801                    .unwrap();
17802                assert!(
17803                    out.status.success(),
17804                    "{args:?}: {}",
17805                    String::from_utf8_lossy(&out.stderr)
17806                );
17807            };
17808            let origin = tmp.path().join("origin.git");
17809            run_git(&["init", "--bare", origin.to_str().unwrap()]);
17810            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17811            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17812            run_git(&["add", "spec.rs"]);
17813            run_git(&["commit", "-m", "initial spec"]);
17814            run_git(&["push", "origin", "main"]);
17815            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17816            run_git(&["checkout", "-b", "fix/review"]);
17817            std::fs::write(
17818                repo.join("check.rs"),
17819                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17820            )
17821            .unwrap();
17822            run_git(&["add", "check.rs"]);
17823            run_git(&["commit", "-m", "use spec"]);
17824            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17825            run_git(&["checkout", "main"]);
17826            std::fs::write(
17827                repo.join("spec.rs"),
17828                "struct AgentSpec { max_turn_secs: u64 }\n",
17829            )
17830            .unwrap();
17831            run_git(&["add", "spec.rs"]);
17832            run_git(&["commit", "-m", "extend spec"]);
17833            run_git(&["push", "origin", "main"]);
17834            run_git(&["checkout", "fix/review"]);
17835
17836            let mut runner = runner_at(RunStatus::Reviewing);
17837            runner.state.repo = repo.clone();
17838            runner.state.base_commit = base;
17839            runner.state.pr_head = Some("fix/review".into());
17840            runner.state.config.graph.review_rounds = max_rounds;
17841            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17842            runner.state.config.graph.sessions = false;
17843            runner.state.config.disk.min_free_bytes = 0;
17844            runner.state.config.verify.e2e =
17845                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17846            runner.state.candidates = vec![Candidate {
17847                index: 0,
17848                label: 'A',
17849                agent: "fixture".into(),
17850                branch: "fix/review".into(),
17851                worktree: repo.clone(),
17852                summary: String::new(),
17853                stat: String::new(),
17854                files: 1,
17855                commits: 1,
17856                empty: false,
17857                failed: None,
17858                verified_noop: None,
17859                duration_ms: 0,
17860                folded: false,
17861            }];
17862            runner.state.tally = Some(Tally {
17863                first_choice: BTreeMap::new(),
17864                borda: BTreeMap::new(),
17865                winner: 'A',
17866                rankings: 0,
17867                unanimous_initial: true,
17868                deliberated: false,
17869                changed_votes: 0,
17870                unanimous_final: true,
17871                tie_break: None,
17872                judges: 0,
17873                present: 0,
17874                quorum: 0,
17875                met_quorum: true,
17876                uncontested: Some("review-only".into()),
17877            });
17878            let mut last = review_round(false, 0, 1, 1, false, false);
17879            last.head = reviewed.clone();
17880            last.e2e_deferred = true;
17881            last.e2e[0].resource_blocked = true;
17882            last.e2e[0].output_tail = "waiting for the shared build cache".into();
17883            if max_rounds == 1 {
17884                last.e2e.clear();
17885            }
17886            runner.state.reviews = vec![last];
17887            let calls = tmp.path().join("fix-calls");
17888            let mut fixer = spec("repair");
17889            fixer
17890                .env
17891                .insert("FIX_CALLS".into(), calls.display().to_string());
17892            fixer.command = vec!["sh".into(), "-c".into(), r#"
17893                echo fix >> "$FIX_CALLS"
17894                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17895                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17896                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17897                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17898            "#.into()];
17899            let mut reviewer = spec("reviewer");
17900            reviewer.command = vec![
17901                "sh".into(),
17902                "-c".into(),
17903                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17904                    .into(),
17905            ];
17906            runner.roles.fixer = Some(vec![fixer]);
17907            runner.roles.reviewers = vec![reviewer];
17908
17909            runner.sync_to_base().await.unwrap();
17910            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17911            assert_ne!(rebased, reviewed);
17912            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17913            if reentry {
17914                // Simulate a crash after catch-up evidence was saved but before
17915                // dispatching the fixer. No dedicated pending-repair field exists.
17916                let shell = runner.state.config.shell();
17917                runner
17918                    .stop_unresolved_review("deferred verification", &shell, &repo)
17919                    .await
17920                    .unwrap();
17921                assert!(runner.catchup_repairable());
17922                assert!(!calls.exists());
17923                runner.state = RunState::load(&runner.state.id).unwrap();
17924                assert!(runner.catchup_repairable());
17925            }
17926            runner.review_loop().await.unwrap();
17927            let first = &runner.state.reviews[0];
17928            assert_eq!(first.head, reviewed);
17929            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17930            assert!(
17931                first.verify_retried,
17932                "a reproduced build failure gets exactly one retry: {:?}",
17933                first.e2e
17934            );
17935            assert!(first.e2e[0].output_tail.contains("E0063"));
17936            assert!(!runner.catchup_repairable());
17937            if max_rounds == 2 {
17938                assert_eq!(
17939                    runner.state.status,
17940                    RunStatus::Gating,
17941                    "{}",
17942                    crate::report::run(&runner.state)
17943                );
17944                assert_eq!(runner.state.reviews.len(), 2);
17945                assert!(first.fix.as_ref().unwrap().committed);
17946                assert!(runner.state.reviews[1].clean);
17947                assert_eq!(
17948                    runner
17949                        .state
17950                        .reviews
17951                        .iter()
17952                        .filter(|r| r.fix.is_some())
17953                        .count(),
17954                    1
17955                );
17956                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17957                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17958            } else {
17959                assert_eq!(runner.state.status, RunStatus::Blocked);
17960                assert!(first.fix.is_none());
17961                assert!(!calls.exists());
17962                let recovery = crate::recovery::assess(
17963                    "prior",
17964                    Some(&runner.state),
17965                    &runner.landing_base(),
17966                    crate::recovery::Relation::Diverged,
17967                );
17968                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17969                let reasons = recovery.reasons.join("\n");
17970                assert!(
17971                    reasons.contains(&reviewed) && reasons.contains(&rebased),
17972                    "{reasons}"
17973                );
17974                assert!(
17975                    reasons.contains("base sync to")
17976                        && reasons.contains("additional review is required"),
17977                    "{reasons}"
17978                );
17979            }
17980        }
17981    }
17982
17983    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
17984    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
17985    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
17986    /// own top-of-function fast path (`review_conclusion`) correctly reads
17987    /// this shape as `None` rather than guessing `Blocked`, and the loop's
17988    /// own `for` range is empty once the round budget is spent, so
17989    /// `review_loop` must retry the check itself rather than silently doing
17990    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
17991    /// above exercises directly, but through `review_loop`'s own entry point
17992    /// this time, proving the wiring between the two rather than just the
17993    /// retry logic in isolation.
17994    #[tokio::test]
17995    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17996        crate::run::pin_test_home();
17997        let home = crate::run::home();
17998
17999        let tmp = tempfile::tempdir().expect("tempdir");
18000        let repo = tmp.path().join("repo");
18001        std::fs::create_dir_all(&repo).unwrap();
18002        init_repo(&repo);
18003        let head = crate::git::rev_parse(&repo, "HEAD")
18004            .await
18005            .expect("rev-parse");
18006        let cache_dir = tmp.path().join("target");
18007
18008        let mut config = Config::default();
18009        config.verify.e2e = vec![format!(
18010            "CARGO_TARGET_DIR='{}' test -f README.md",
18011            cache_dir.display()
18012        )];
18013        config.graph.review_rounds = 1;
18014        config.graph.timeout_verify = Some(2);
18015        config.graph.timeout_resource_wait = 2;
18016
18017        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
18018        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
18019            .expect("no io error acquiring directly")
18020        {
18021            crate::cache::AcquireOutcome::Acquired(g) => g,
18022            crate::cache::AcquireOutcome::Busy(b) => {
18023                panic!("expected the direct acquire to win the lease first: {b:?}")
18024            }
18025        };
18026
18027        let mut state = RunState::new(
18028            repo.clone(),
18029            "main".to_owned(),
18030            head.clone(),
18031            "task".to_owned(),
18032            config,
18033        );
18034        state.candidates = vec![Candidate {
18035            index: 0,
18036            label: 'A',
18037            agent: "alpha".to_owned(),
18038            branch: "does-not-exist".to_owned(),
18039            worktree: repo.clone(),
18040            summary: String::new(),
18041            stat: String::new(),
18042            files: 0,
18043            commits: 0,
18044            empty: false,
18045            failed: None,
18046            verified_noop: None,
18047            duration_ms: 0,
18048            folded: false,
18049        }];
18050        state.tally = Some(Tally {
18051            first_choice: BTreeMap::from([('A', 1)]),
18052            borda: BTreeMap::new(),
18053            winner: 'A',
18054            rankings: 1,
18055            unanimous_initial: true,
18056            deliberated: false,
18057            changed_votes: 0,
18058            unanimous_final: true,
18059            tie_break: None,
18060            judges: 0,
18061            present: 0,
18062            quorum: 0,
18063            met_quorum: true,
18064            uncontested: Some("only candidate A produced a change".to_owned()),
18065        });
18066        // The exact shape a prior process's `stop_reviewing` would have left
18067        // on disk: the round budget's last round, a real attempt already
18068        // made and already resource-blocked.
18069        state.reviews = vec![ReviewRound {
18070            round: 1,
18071            head: head.clone(),
18072            verified_head: Some(head.clone()),
18073            verified_at: Some(jiff::Timestamp::now()),
18074            reviews: Vec::new(),
18075            e2e: vec![CommandOutcome {
18076                command: format!(
18077                    "CARGO_TARGET_DIR='{}' test -f README.md",
18078                    cache_dir.display()
18079                ),
18080                code: None,
18081                output_tail: "waiting for the shared build cache".to_owned(),
18082                duration_ms: 0,
18083                resource_blocked: true,
18084            }],
18085            fix: None,
18086            blocking: 1,
18087            answered: 1,
18088            expected: 1,
18089            clean: false,
18090            verify_retried: false,
18091            e2e_deferred: false,
18092            e2e_defer_reason: None,
18093            progressed: false,
18094            vote_split: false,
18095            reconsideration: Vec::new(),
18096            verdict: None,
18097        }];
18098
18099        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
18100        let mut runner = Runner {
18101            state,
18102            roles: ResolvedRoles {
18103                implementers: Vec::new(),
18104                judges: Vec::new(),
18105                reviewers: Vec::new(),
18106                fixer: None,
18107                conductor: conductor(),
18108                implementer_roster: Vec::new(),
18109                judge_roster: Vec::new(),
18110                reviewer_roster: Vec::new(),
18111            },
18112            sem: Arc::new(Semaphore::new(1)),
18113            pause: Pause::new(),
18114            interrupt: Pause::new(),
18115        };
18116
18117        // The lease is still held throughout, so this reentry's own retry is
18118        // also contended — proving `review_loop` actually tried again (not
18119        // that it happened to succeed) is what the timestamp comparison
18120        // below is for.
18121        runner.review_loop().await.expect("review_loop");
18122
18123        assert_eq!(
18124            runner.state.reviews.len(),
18125            1,
18126            "no new round was started on top of the unresolved one: {:?}",
18127            runner.state.reviews
18128        );
18129        let last = &runner.state.reviews[0];
18130        assert_eq!(
18131            last.e2e_status(),
18132            E2eStatus::ResourceBlocked,
18133            "still contended: {last:?}"
18134        );
18135        assert!(
18136            last.verified_at.expect("still known") > first_attempt_at,
18137            "review_loop must have actually retried the check, not left it exactly as found"
18138        );
18139        assert_ne!(
18140            runner.state.status,
18141            RunStatus::Blocked,
18142            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
18143            runner.state.status
18144        );
18145
18146        held.release();
18147    }
18148
18149    #[tokio::test]
18150    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
18151        crate::run::pin_test_home();
18152        for status in [RunStatus::Blocked, RunStatus::Gating] {
18153            let mut runner = runner_at(status);
18154            runner.state.config.graph.land = true;
18155            runner.state.gate_ran = true;
18156            runner.state.merge = Some(MergeOutcome {
18157                mode: MergeMode::Pr,
18158                ok: false,
18159                detail: "the fixer produced no commit".into(),
18160                empty: false,
18161            });
18162            let url = "https://example.invalid/o/r/pull/1";
18163            runner.state.pr = Some(crate::run::PrRecord {
18164                url: url.into(),
18165                number: 1,
18166                state: "open".into(),
18167                checks: "red".into(),
18168                round: 1,
18169                rounds: 1,
18170                red_at_merge: Vec::new(),
18171            });
18172            runner.state.save().unwrap();
18173            // The nonexistent repository makes observation fail before a forge
18174            // command can run. We still must reach land, rather than silently
18175            // return from the graph with its old gate and merge records.
18176            runner.execute_graph().await.unwrap();
18177            assert_eq!(runner.state.status, RunStatus::Blocked);
18178            assert!(runner.state.land_sync_pending);
18179            assert!(runner.state.events.iter().any(|event|
18180                event.node == "land" && event.message == format!("watching {url}")));
18181            assert_eq!(
18182                runner.state.merge.as_ref().unwrap().detail,
18183                "the fixer produced no commit"
18184            );
18185        }
18186    }
18187
18188    #[tokio::test]
18189    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
18190        crate::run::pin_test_home();
18191        let tmp = tempfile::tempdir().expect("tempdir");
18192        let repo = tmp.path().join("repo");
18193        std::fs::create_dir_all(&repo).unwrap();
18194        init_repo(&repo);
18195
18196        let mut config = Config::default();
18197        config.merge.mode = MergeMode::Pr;
18198        config.graph.land = true;
18199        config.graph.land_approval = false;
18200
18201        let mut state = RunState::new(
18202            repo.clone(),
18203            "main".to_owned(),
18204            "deadbeef".to_owned(),
18205            "task".to_owned(),
18206            config,
18207        );
18208        state.candidates = vec![Candidate {
18209            index: 0,
18210            label: 'A',
18211            agent: "alpha".to_owned(),
18212            branch: "does-not-exist".to_owned(),
18213            worktree: repo.clone(),
18214            summary: String::new(),
18215            stat: String::new(),
18216            files: 0,
18217            commits: 0,
18218            empty: false,
18219            failed: None,
18220            verified_noop: None,
18221            duration_ms: 0,
18222            folded: false,
18223        }];
18224        state.tally = Some(Tally {
18225            first_choice: BTreeMap::from([('A', 1)]),
18226            borda: BTreeMap::new(),
18227            winner: 'A',
18228            rankings: 1,
18229            unanimous_initial: true,
18230            deliberated: false,
18231            changed_votes: 0,
18232            unanimous_final: true,
18233            tie_break: None,
18234            judges: 0,
18235            present: 0,
18236            quorum: 0,
18237            met_quorum: true,
18238            uncontested: Some("only candidate A produced a change".to_owned()),
18239        });
18240        state.reviews = vec![ReviewRound {
18241            round: 1,
18242            head: "deadbeef".to_owned(),
18243            verified_head: None,
18244            verified_at: None,
18245            reviews: Vec::new(),
18246            e2e: Vec::new(),
18247            fix: None,
18248            blocking: 0,
18249            answered: 0,
18250            expected: 0,
18251            clean: true,
18252            verify_retried: false,
18253            e2e_deferred: false,
18254            e2e_defer_reason: None,
18255            progressed: false,
18256            vote_split: false,
18257            reconsideration: Vec::new(),
18258            verdict: None,
18259        }];
18260        state.gate = vec![CommandOutcome {
18261            command: "test".to_owned(),
18262            code: Some(0),
18263            output_tail: String::new(),
18264            duration_ms: 0,
18265            resource_blocked: false,
18266        }];
18267        state.gate_ran = true;
18268        // A first pass through `merge` already pushed and opened this pull
18269        // request; `status` is `Landing` because a previous call into `land`
18270        // parked or was interrupted before it reached a terminal outcome.
18271        state.status = RunStatus::Landing;
18272        state.merge = Some(MergeOutcome {
18273            mode: MergeMode::Pr,
18274            ok: true,
18275            detail: "https://example.invalid/x/y/pull/1".to_owned(),
18276            empty: false,
18277        });
18278
18279        // The Landing-resume shortcut calls `run_land` directly rather than
18280        // through `merge`, which is exactly the call site that used to skip
18281        // `settle_questions` - see the fixture below.
18282        ask_test_home();
18283        // A real resume loads a persisted run; driver ownership is checked
18284        // against that record before the Landing shortcut is entered.
18285        state.save().expect("save landing run");
18286        let store = ask::Questions::open();
18287        let q = ask_open_question(&store, &state.id);
18288
18289        let mut runner = Runner {
18290            state,
18291            roles: ResolvedRoles {
18292                implementers: Vec::new(),
18293                judges: Vec::new(),
18294                reviewers: Vec::new(),
18295                fixer: None,
18296                conductor: conductor(),
18297                implementer_roster: Vec::new(),
18298                judge_roster: Vec::new(),
18299                reviewer_roster: Vec::new(),
18300            },
18301            sem: Arc::new(Semaphore::new(1)),
18302            pause: Pause::new(),
18303            interrupt: Pause::new(),
18304        };
18305
18306        // `execute`, not `merge` directly: the Landing-resume shortcut lives
18307        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
18308        // exactly because `review_loop` would otherwise clobber the marker
18309        // first.
18310        runner.execute().await.expect("execute");
18311
18312        assert_eq!(
18313            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
18314            Some("https://example.invalid/x/y/pull/1"),
18315            "reentry must not push again or open a second pull request over the \
18316             one `land` is already watching"
18317        );
18318        assert_ne!(
18319            runner.state.status,
18320            RunStatus::Landing,
18321            "land could not actually reach the fake pull request, so it must \
18322             have given up rather than left the run silently parked forever"
18323        );
18324        // `land` could not reach the fake pull request, so it gave up into
18325        // `Blocked` - still resumable, so the question must not have been
18326        // swept just because this branch now also calls `settle_questions`.
18327        assert_eq!(runner.state.status, RunStatus::Blocked);
18328        assert!(
18329            store.get(&q.id).unwrap().status.open(),
18330            "Blocked is still alive; settle_questions must have been a no-op here"
18331        );
18332    }
18333
18334    fn state_with_round(round: ReviewRound) -> RunState {
18335        let mut s = RunState::new(
18336            PathBuf::from("/repo"),
18337            "main".to_owned(),
18338            "abc1234".to_owned(),
18339            "add retries".to_owned(),
18340            Config::default(),
18341        );
18342        s.reviews = vec![round];
18343        s
18344    }
18345
18346    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
18347        crate::verdict::Finding {
18348            id: id.to_owned(),
18349            severity,
18350            file: None,
18351            line: None,
18352            title: title.to_owned(),
18353            detail: String::new(),
18354        }
18355    }
18356
18357    #[test]
18358    fn pr_body_names_open_findings_and_declined_ones() {
18359        let round = ReviewRound {
18360            round: 2,
18361            head: "deadbee".to_owned(),
18362            verified_head: None,
18363            verified_at: None,
18364            reviews: vec![ReviewRecord {
18365                attempts: 0,
18366                reviewer: 1,
18367                agent: "alpha".to_owned(),
18368                summary: String::new(),
18369                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
18370                vote: None,
18371                failed: None,
18372                duration_ms: 0,
18373            }],
18374            e2e: vec![CommandOutcome {
18375                command: "cargo test".to_owned(),
18376                code: Some(0),
18377                output_tail: String::new(),
18378                duration_ms: 0,
18379                resource_blocked: false,
18380            }],
18381            verify_retried: false,
18382            e2e_deferred: false,
18383            e2e_defer_reason: None,
18384            fix: Some(FixRecord {
18385                agent: "alpha".to_owned(),
18386                addressed: Vec::new(),
18387                rejected: vec![crate::verdict::Rejection {
18388                    id: "R1-1-1".to_owned(),
18389                    why: "not reachable from any caller".to_owned(),
18390                }],
18391                notes: String::new(),
18392                committed: true,
18393                failed: None,
18394                duration_ms: 0,
18395                continuation: None,
18396            }),
18397            blocking: 0,
18398            answered: 1,
18399            expected: 1,
18400            clean: false,
18401            progressed: true,
18402            vote_split: false,
18403            reconsideration: Vec::new(),
18404            verdict: None,
18405        };
18406        let state = state_with_round(round);
18407        let body = pr_message(&state, 'A').body;
18408
18409        assert!(body.contains("add retries"), "the task must still be there");
18410        assert!(body.contains("R2-1-1"), "{body}");
18411        assert!(!body.contains("unused import"), "{body}");
18412        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
18413        assert!(
18414            !body.contains("not reachable from any caller"),
18415            "the reason it was declined: {body}"
18416        );
18417    }
18418
18419    #[test]
18420    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
18421        let round = ReviewRound {
18422            round: 1,
18423            head: "deadbee".to_owned(),
18424            verified_head: None,
18425            verified_at: None,
18426            reviews: vec![ReviewRecord {
18427                attempts: 0,
18428                reviewer: 1,
18429                agent: "alpha".to_owned(),
18430                summary: String::new(),
18431                findings: Vec::new(),
18432                vote: None,
18433                failed: None,
18434                duration_ms: 0,
18435            }],
18436            e2e: Vec::new(),
18437            verify_retried: false,
18438            e2e_deferred: false,
18439            e2e_defer_reason: None,
18440            fix: None,
18441            blocking: 0,
18442            answered: 1,
18443            expected: 1,
18444            clean: true,
18445            progressed: false,
18446            vote_split: false,
18447            reconsideration: Vec::new(),
18448            verdict: None,
18449        };
18450        let state = state_with_round(round);
18451        let body = pr_message(&state, 'A').body;
18452        assert!(!body.contains("Open review findings"), "{body}");
18453        assert!(!body.contains("Declined"), "{body}");
18454    }
18455
18456    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
18457        let mut state = RunState::new(
18458            PathBuf::from("/repo"),
18459            "main".to_owned(),
18460            "abc1234".to_owned(),
18461            instruction.to_owned(),
18462            Config::default(),
18463        );
18464        state.candidates.push(Candidate {
18465            index: 0,
18466            label: 'A',
18467            agent: "alpha".to_owned(),
18468            branch: "magi/x/A".to_owned(),
18469            worktree: PathBuf::from("/wt"),
18470            summary: summary.to_owned(),
18471            stat: String::new(),
18472            files: 1,
18473            commits: 1,
18474            empty: false,
18475            failed: None,
18476            verified_noop: None,
18477            folded: false,
18478            duration_ms: 0,
18479        });
18480        state
18481    }
18482
18483    fn review_state(subjects: &[&str]) -> RunState {
18484        let mut state = state_with_summary(
18485            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
18486            "",
18487        );
18488        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18489        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
18490        state
18491    }
18492
18493    /// A branch rebased onto a main that moved past the recorded
18494    /// `base_commit` is titled from its own first commit, never main's.
18495    #[tokio::test]
18496    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
18497        ask_test_home();
18498        let tmp = tempfile::tempdir().unwrap();
18499        let repo = tmp.path().join("repo");
18500        std::fs::create_dir_all(&repo).unwrap();
18501        init_repo(&repo);
18502        let origin = tmp.path().join("origin.git");
18503        let g = |dir: &Path, args: &[&str]| {
18504            let out = std::process::Command::new("git")
18505                .args(args)
18506                .current_dir(dir)
18507                .quiet()
18508                .output()
18509                .expect("spawn git");
18510            assert!(
18511                out.status.success(),
18512                "git {args:?}: {}",
18513                String::from_utf8_lossy(&out.stderr)
18514            );
18515        };
18516        g(
18517            tmp.path(),
18518            &[
18519                "clone",
18520                "--bare",
18521                "-q",
18522                repo.to_str().unwrap(),
18523                origin.to_str().unwrap(),
18524            ],
18525        );
18526        g(
18527            &repo,
18528            &["remote", "add", "origin", origin.to_str().unwrap()],
18529        );
18530        let c1 = git::rev_parse(&repo, "main").await.unwrap();
18531
18532        // Main moves on; the branch is built on top of the new main.
18533        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
18534        g(&repo, &["add", "-A"]);
18535        g(
18536            &repo,
18537            &["commit", "-q", "-m", "chore(deps): update a crate"],
18538        );
18539        g(&repo, &["push", "-q", "origin", "main"]);
18540        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
18541        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18542        g(&repo, &["add", "-A"]);
18543        g(
18544            &repo,
18545            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
18546        );
18547        g(&repo, &["checkout", "-q", "main"]);
18548
18549        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
18550        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
18551        // Without a readable tracking ref the recorded base's merge base is used.
18552        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
18553        assert_eq!(fallback, c1);
18554
18555        let mut state = review_state(&[
18556            "chore(deps): update a crate",
18557            "fix(daemon): apply a chosen action",
18558        ]);
18559        state.repo = repo.clone();
18560        state.base_branch = "main".to_owned();
18561        state.base_commit = c1;
18562        refresh_reviewed_commits(&mut state, "feat/own").await;
18563        assert_eq!(
18564            state.reviewed_commits,
18565            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
18566        );
18567        assert_eq!(
18568            review_title(&state).as_deref(),
18569            Some("fix(daemon): apply a chosen action")
18570        );
18571        assert_eq!(
18572            leaked_subjects(&state, "feat/own").await,
18573            Some(vec!["chore(deps): update a crate".to_owned()])
18574        );
18575        // A stale tracking ref is still used when the fetch fails, but the
18576        // leak list is withheld.
18577        state.config.merge.remote = "nowhere".to_owned();
18578        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
18579    }
18580
18581    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
18582    /// with `origin/main` one commit behind it, and a summarizing agent that
18583    /// prints whatever `reply` holds.
18584    struct SummaryFixture {
18585        _tmp: tempfile::TempDir,
18586        repo: PathBuf,
18587        reply: PathBuf,
18588        runner: Runner,
18589    }
18590
18591    fn summary_fixture() -> SummaryFixture {
18592        ask_test_home();
18593        let tmp = tempfile::tempdir().unwrap();
18594        let repo = tmp.path().join("repo");
18595        std::fs::create_dir_all(&repo).unwrap();
18596        init_repo(&repo);
18597        let origin = tmp.path().join("origin.git");
18598        let g = |args: &[&str]| {
18599            let out = std::process::Command::new("git")
18600                .args(args)
18601                .current_dir(&repo)
18602                .quiet()
18603                .output()
18604                .expect("spawn git");
18605            assert!(
18606                out.status.success(),
18607                "git {args:?}: {}",
18608                String::from_utf8_lossy(&out.stderr)
18609            );
18610        };
18611        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
18612        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
18613        g(&["push", "-q", "origin", "main"]);
18614        g(&["checkout", "-q", "-b", "feat/own"]);
18615        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
18616        g(&["add", "-A"]);
18617        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
18618        let reply = tmp.path().join("reply.txt");
18619        std::fs::write(
18620            &reply,
18621            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
18622        )
18623        .unwrap();
18624        let mut runner = runner_at(RunStatus::Gating);
18625        let mut state = review_state(&["feat(own): add the own file"]);
18626        state.repo = repo.clone();
18627        state.base_branch = "main".to_owned();
18628        state.candidates[0].branch = "feat/own".to_owned();
18629        state.candidates[0].worktree = repo.clone();
18630        let mut author = spec("alpha");
18631        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
18632        author
18633            .env
18634            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
18635        state.config.agents = vec![author];
18636        runner.state = state;
18637        SummaryFixture {
18638            _tmp: tmp,
18639            repo,
18640            reply,
18641            runner,
18642        }
18643    }
18644
18645    async fn posted(f: &mut SummaryFixture) -> PrMessage {
18646        let winner = f.runner.state.candidates[0].clone();
18647        f.runner
18648            .guarded_pr_message(&winner, None, true)
18649            .await
18650            .unwrap()
18651            .expect("an English description never parks")
18652    }
18653
18654    fn clean_round_with_minor_findings() -> ReviewRound {
18655        let mut round = passing_round(2, None);
18656        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
18657        round.reviews = vec![ReviewRecord {
18658            attempts: 0,
18659            reviewer: 1,
18660            agent: "alpha".into(),
18661            summary: "内部レビュー".into(),
18662            findings: vec![
18663                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
18664                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
18665            ],
18666            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
18667            failed: None,
18668            duration_ms: 0,
18669        }];
18670        round
18671    }
18672
18673    #[tokio::test]
18674    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
18675        for review_only in [false, true] {
18676            let mut f = summary_fixture();
18677            if !review_only {
18678                f.runner.state.candidates[0].agent = "alpha".into();
18679            }
18680            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
18681            assert!(f.runner.state.reviews[0].clean);
18682            assert!(f.runner.state.reviews[0].fix.is_none());
18683            assert!(f.runner.state.open_findings().is_empty());
18684            assert_eq!(
18685                crate::pr_outcome::required_finding_ids(&f.runner.state),
18686                ["R2-1-1", "R2-1-2"]
18687            );
18688            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18689            let evidence = crate::pr_outcome::summary_evidence(
18690                &f.runner.state,
18691                crate::pr_outcome::FinalHead::At(&head),
18692            );
18693            assert!(
18694                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
18695            );
18696            // The ordinary reply omits both IDs: no cached description is adopted.
18697            let omitted = posted(&mut f).await;
18698            assert!(omitted.body.contains("No description"));
18699            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
18700            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
18701            assert!(f.runner.state.change_summary.is_none());
18702            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
18703            let described = posted(&mut f).await;
18704            assert!(described.body.contains("improve documentation for callers"));
18705            assert!(described.body.contains("remaining edge case"));
18706            assert!(described.body.contains("## Open review findings"));
18707            assert!(!described.body.contains("未修正"));
18708        }
18709    }
18710
18711    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
18712        let mut f = summary_fixture();
18713        if !review_only {
18714            f.runner.state.candidates[0].agent = "alpha".into();
18715        }
18716        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18717        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
18718        fixed
18719            .fix
18720            .as_mut()
18721            .unwrap()
18722            .rejected
18723            .push(crate::verdict::Rejection {
18724                id: "R1-1-1".into(),
18725                why: "呼び出されないコードのため辞退".into(),
18726            });
18727        let mut clean = clean_round_with_minor_findings();
18728        clean.verified_head = Some(head.clone());
18729        f.runner.state.reviews = vec![fixed, clean];
18730        ok_gate(&mut f.runner.state, Some(&head));
18731        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
18732        f.runner.state.config.agents.push(scripted("beta", reply));
18733        f.runner.state.config.roles.language_judge =
18734            Some(crate::config::AgentChoice::One("beta".into()));
18735        f
18736    }
18737
18738    #[tokio::test]
18739    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
18740        for review_only in [false, true] {
18741            let mut f = rewrite_fixture_with_findings(review_only,
18742                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
18743            let m = posted(&mut f).await;
18744            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18745            let expected =
18746                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
18747            assert!(m.body.contains(&expected));
18748            assert!(m.body.contains("## Review rounds"));
18749            assert!(
18750                m.body
18751                    .contains("Gate (`verify.gate`): passed on the final commit")
18752            );
18753            assert!(
18754                m.body
18755                    .contains("Tests (`verify.e2e`): passed on the final commit")
18756            );
18757            let c = f.runner.state.change_summary.as_ref().unwrap();
18758            assert!(
18759                m.body
18760                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
18761            );
18762            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18763            assert!(crate::pr_outcome::covers_required_findings(
18764                &f.runner.state,
18765                &m.body
18766            ));
18767            assert!(m.body.contains("documentation remains incomplete"));
18768            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
18769            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
18770            assert!(f.runner.state.github_text.is_none());
18771        }
18772    }
18773
18774    #[tokio::test]
18775    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
18776        for review_only in [false, true] {
18777            for reply in [
18778                "TITLE: fix: public description\nA short English description.",
18779                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
18780            ] {
18781                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
18782                let winner = f.runner.state.candidates[0].clone();
18783                let posted = f
18784                    .runner
18785                    .guarded_pr_message(&winner, None, true)
18786                    .await
18787                    .unwrap();
18788                assert!(posted.is_none());
18789                assert!(f.runner.state.parked);
18790                let withheld = f.runner.state.github_text.as_ref().unwrap();
18791                assert!(
18792                    withheld.rewrite.is_none(),
18793                    "an incomplete rewrite must not be selectable"
18794                );
18795                let q = ask::Questions::open()
18796                    .get(withheld.question.as_ref().unwrap())
18797                    .unwrap();
18798                assert!(
18799                    !q.choices
18800                        .contains(&crate::github_text::USE_REWRITE.to_owned())
18801                );
18802                assert!(
18803                    q.detail.contains("R1-1-1")
18804                        && q.detail.contains("R2-1-1")
18805                        && q.detail.contains("R2-1-2")
18806                );
18807            }
18808        }
18809    }
18810
18811    #[tokio::test]
18812    async fn japanese_reports_stay_local_in_both_public_run_modes() {
18813        for review_only in [false, true] {
18814            let mut f = summary_fixture();
18815            if !review_only {
18816                f.runner.state.candidates[0].agent = "alpha".into();
18817            }
18818            f.runner.state.instruction = "内部タスクの説明".into();
18819            f.runner.state.candidates[0].summary = "内部の候補報告".into();
18820            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18821            f.runner.state.reviews = vec![
18822                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18823                round_with_notes(2, Some("二回目の内部修正報告です。")),
18824                passing_round(3, Some(&head)),
18825            ];
18826            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18827            ok_gate(&mut f.runner.state, Some(&head));
18828            let captured = f.repo.join("public-prompt.txt");
18829            let author = &mut f.runner.state.config.agents[0];
18830            author.command = vec![
18831                "sh".into(),
18832                "-c".into(),
18833                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18834            ];
18835            author.env.insert(
18836                "PROMPT_FILE".into(),
18837                captured.to_string_lossy().into_owned(),
18838            );
18839            let m = posted(&mut f).await;
18840            assert!(m.body.contains("Adds `own.txt`"));
18841            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18842            assert!(m.body.contains("R1-1-1"));
18843            assert!(
18844                m.body
18845                    .contains("Gate (`verify.gate`): passed on the final commit")
18846            );
18847            assert!(
18848                m.body
18849                    .contains("Tests (`verify.e2e`): passed on the final commit")
18850            );
18851            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18852            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18853            let prompt = std::fs::read_to_string(captured).unwrap();
18854            assert!(prompt.contains(&format!(
18855                "git diff {} {head}",
18856                f.runner.state.change_summary.as_ref().unwrap().base
18857            )));
18858            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18859            assert_eq!(
18860                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18861                "二回目の内部修正報告です。"
18862            );
18863        }
18864    }
18865
18866    #[tokio::test]
18867    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18868        for review_only in [false, true] {
18869            let mut f = summary_fixture();
18870            if !review_only {
18871                f.runner.state.candidates[0].agent = "alpha".into();
18872            }
18873            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18874            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18875            std::fs::write(&f.reply, "no usable description").unwrap();
18876            let failed = posted(&mut f).await;
18877            assert!(
18878                failed
18879                    .body
18880                    .contains("No description of what this change does could be generated")
18881            );
18882            assert!(failed.body.is_ascii());
18883            assert!(f.runner.state.github_text.is_none());
18884            assert!(f.runner.state.change_summary.is_none());
18885            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18886            let evidence = crate::pr_outcome::summary_evidence(
18887                &f.runner.state,
18888                crate::pr_outcome::FinalHead::At(&head),
18889            );
18890            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18891            assert!(
18892                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18893                    .exists()
18894            );
18895            f.runner.state =
18896                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18897            std::fs::write(
18898                &f.reply,
18899                "TITLE: feat(own): add the own file\nAdds the committed own file.",
18900            )
18901            .unwrap();
18902            let retried = posted(&mut f).await;
18903            assert!(retried.body.contains("Adds the committed own file."));
18904            assert_eq!(
18905                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18906                "日本語の修正報告"
18907            );
18908        }
18909    }
18910
18911    #[tokio::test]
18912    async fn same_head_verification_and_findings_invalidate_public_prose() {
18913        let mut f = summary_fixture();
18914        posted(&mut f).await;
18915        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18916        let original = f
18917            .runner
18918            .state
18919            .change_summary
18920            .as_ref()
18921            .unwrap()
18922            .evidence
18923            .clone();
18924        ok_gate(&mut f.runner.state, Some(&head));
18925        std::fs::write(
18926            &f.reply,
18927            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18928        )
18929        .unwrap();
18930        let verified = posted(&mut f).await;
18931        assert!(verified.body.contains("now verified"));
18932        assert_ne!(
18933            f.runner.state.change_summary.as_ref().unwrap().evidence,
18934            original
18935        );
18936        let mut round = round_with_notes(1, Some("内部報告"));
18937        round.reviews.push(ReviewRecord {
18938            attempts: 0,
18939            reviewer: 1,
18940            agent: "alpha".into(),
18941            summary: "内部レビュー".into(),
18942            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18943            vote: None,
18944            failed: None,
18945            duration_ms: 0,
18946        });
18947        round
18948            .fix
18949            .as_mut()
18950            .unwrap()
18951            .rejected
18952            .push(crate::verdict::Rejection {
18953                id: "R1-1-1".into(),
18954                why: "呼び出し元から到達できない".into(),
18955            });
18956        f.runner.state.reviews = vec![round];
18957        // A description omitting the required ID is unusable, even with a title.
18958        let omitted = posted(&mut f).await;
18959        assert!(omitted.body.contains("No description"));
18960        assert!(omitted.body.contains("R1-1-1"));
18961        assert!(!omitted.body.contains("到達"));
18962        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18963        let described = posted(&mut f).await;
18964        assert!(described.body.contains("unreachable from callers"));
18965        assert!(described.body.contains("R1-1-1"));
18966        assert!(!described.body.contains("到達"));
18967        let evidence = f
18968            .runner
18969            .state
18970            .change_summary
18971            .as_ref()
18972            .unwrap()
18973            .evidence
18974            .clone();
18975        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18976        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18977        assert!(posted(&mut f).await.body.contains("rationale updated"));
18978        assert_ne!(
18979            f.runner.state.change_summary.as_ref().unwrap().evidence,
18980            evidence
18981        );
18982        // Old serialized summaries have no evidence fingerprint and must regenerate.
18983        f.runner
18984            .state
18985            .change_summary
18986            .as_mut()
18987            .unwrap()
18988            .evidence
18989            .clear();
18990        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18991        assert!(posted(&mut f).await.body.contains("current specification"));
18992    }
18993
18994    #[tokio::test]
18995    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18996        let mut f = summary_fixture();
18997        let m = posted(&mut f).await;
18998        assert_eq!(m.title, "feat(own): add the own file");
18999        assert!(
19000            m.body
19001                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
19002            "{}",
19003            m.body
19004        );
19005        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
19006        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
19007        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
19008        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
19009        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
19010        let kept = f.runner.state.change_summary.as_ref().unwrap();
19011        assert_eq!(
19012            (kept.head.as_str(), kept.base.as_str()),
19013            (head.as_str(), base.as_str())
19014        );
19015        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
19016    }
19017
19018    #[tokio::test]
19019    async fn review_only_summary_receives_the_configured_github_language() {
19020        let mut f = summary_fixture();
19021        f.runner.state.config.graph.github_language = "ja".into();
19022        let captured = f.repo.join("summary-prompt.txt");
19023        let author = &mut f.runner.state.config.agents[0];
19024        author.command = vec![
19025            "sh".into(),
19026            "-c".into(),
19027            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
19028        ];
19029        author.env.insert(
19030            "PROMPT_FILE".into(),
19031            captured.to_string_lossy().into_owned(),
19032        );
19033        std::fs::write(
19034            &f.reply,
19035            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
19036        )
19037        .unwrap();
19038        let m = posted(&mut f).await;
19039        let prompt = std::fs::read_to_string(captured).unwrap();
19040        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
19041        assert!(!prompt.contains("in English"));
19042        assert_eq!(m.title, "feat(own): ファイルを追加");
19043        assert!(m.body.contains("own.txt を追加します。"));
19044    }
19045
19046    #[tokio::test]
19047    async fn review_fixes_come_after_the_whole_change_summary() {
19048        let mut f = summary_fixture();
19049        f.runner.state.reviews = vec![
19050            round_with_notes(1, Some("handled the PATH case")),
19051            round_with_notes(2, None),
19052        ];
19053        let m = posted(&mut f).await;
19054        let summary = m.body.find("## Summary").expect("summary");
19055        let fixes = m.body.find("## Review rounds").expect("review rounds");
19056        assert!(summary < fixes, "{}", m.body);
19057        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
19058        assert!(m.body.contains("## Final verification"), "{}", m.body);
19059    }
19060
19061    #[tokio::test]
19062    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
19063        let mut f = summary_fixture();
19064        let first = posted(&mut f).await;
19065        let first_head = f.runner.state.change_summary.clone().unwrap().head;
19066        // The same head reuses the record: a different reply is not read.
19067        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
19068        let again = posted(&mut f).await;
19069        assert_eq!((again.title, again.body), (first.title, first.body));
19070
19071        let g = |args: &[&str]| {
19072            let out = std::process::Command::new("git")
19073                .args(args)
19074                .current_dir(&f.repo)
19075                .quiet()
19076                .output()
19077                .expect("spawn git");
19078            assert!(out.status.success(), "git {args:?}");
19079        };
19080        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
19081        g(&["add", "-A"]);
19082        g(&["commit", "-q", "-m", "fix(own): handle more"]);
19083        let second = posted(&mut f).await;
19084        let kept = f.runner.state.change_summary.clone().unwrap();
19085        assert_ne!(kept.head, first_head);
19086        assert!(second.body.contains("Not read yet."), "{}", second.body);
19087        assert!(!second.body.contains("so the first summary is read"));
19088        assert_eq!(second.title, "feat(own): other");
19089    }
19090
19091    #[tokio::test]
19092    async fn an_unavailable_summary_is_stated_and_never_invented() {
19093        let mut f = summary_fixture();
19094        std::fs::write(&f.reply, "no title line here").unwrap();
19095        let m = posted(&mut f).await;
19096        assert!(
19097            m.body
19098                .contains("No description of what this change does could be generated"),
19099            "{}",
19100            m.body
19101        );
19102        assert!(m.body.contains("internal reports remain in local records"));
19103        assert!(f.runner.state.change_summary.is_none());
19104        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
19105        // A preview asks nobody and claims nothing.
19106        let winner = f.runner.state.candidates[0].clone();
19107        let preview = f
19108            .runner
19109            .guarded_pr_message(&winner, None, false)
19110            .await
19111            .unwrap()
19112            .unwrap();
19113        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
19114    }
19115
19116    #[test]
19117    fn pr_message_review_single_commit_uses_its_subject() {
19118        let state = review_state(&["feat(nats): per-role user"]);
19119        let m = pr_message(&state, 'A');
19120        assert_eq!(m.title, "feat(nats): per-role user");
19121        assert!(!m.body.contains("Review the work already"), "{}", m.body);
19122        assert!(m.body.contains("## Commits under review"), "{}", m.body);
19123    }
19124
19125    #[test]
19126    fn pr_message_review_multi_commit_takes_the_oldest() {
19127        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
19128        let m = pr_message(&state, 'A');
19129        assert_eq!(m.title, "feat: the change");
19130        for s in ["feat: the change", "fix: typo", "fix: again"] {
19131            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
19132        }
19133    }
19134
19135    #[test]
19136    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
19137        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
19138            let mut state = review_state(&[first, "fix: later fixup"]);
19139            state.candidates[0].branch = "機能/ログイン".to_owned();
19140            let m = pr_message(&state, 'A');
19141            assert!(
19142                m.title.starts_with("chore: land candidate A of run"),
19143                "{}",
19144                m.title
19145            );
19146        }
19147    }
19148
19149    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
19150        BranchFacts {
19151            commits: commits
19152                .iter()
19153                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
19154                .collect(),
19155            stat: stat.to_owned(),
19156        }
19157    }
19158
19159    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
19160        let mut r = review_round(true, 0, 1, 1, true, true);
19161        r.round = round;
19162        r.fix = notes.map(|n| FixRecord {
19163            agent: "fixer".to_owned(),
19164            addressed: Vec::new(),
19165            rejected: Vec::new(),
19166            notes: n.to_owned(),
19167            committed: true,
19168            failed: None,
19169            duration_ms: 0,
19170            continuation: None,
19171        });
19172        r
19173    }
19174
19175    #[test]
19176    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
19177        let state = review_state(&["ignored"]);
19178        let f = facts(
19179            &[
19180                (
19181                    "fix(login): resolve PATH on macOS",
19182                    "Login shells skip rc files.",
19183                ),
19184                ("fix: address review", ""),
19185            ],
19186            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
19187        );
19188        let m = pr_message_with(&state, 'A', Some(&f));
19189        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
19190        assert!(
19191            m.body
19192                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
19193            "{}",
19194            m.body
19195        );
19196        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
19197        assert!(m.body.contains("## Diff stat"), "{}", m.body);
19198        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
19199        for banned in [
19200            "Review the work already",
19201            "no task statement",
19202            "Original task",
19203        ] {
19204            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
19205        }
19206        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
19207    }
19208
19209    #[test]
19210    fn pr_message_review_truncates_a_huge_first_commit_body() {
19211        let state = review_state(&["ignored"]);
19212        let f = facts(
19213            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
19214            "s",
19215        );
19216        let m = pr_message_with(&state, 'A', Some(&f));
19217        assert!(m.body.len() < 30_000, "{}", m.body.len());
19218        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
19219        assert!(
19220            m.body.contains("- ... 1 more commit(s)"),
19221            "{}",
19222            m.body.len()
19223        );
19224        assert!(m.body.ends_with("magi:candidate-a\n"));
19225    }
19226
19227    #[test]
19228    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
19229        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
19230        assert_eq!(m.title, "feat: x");
19231        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
19232        assert!(!m.body.contains("Diff stat"), "{}", m.body);
19233        assert!(!m.body.contains("no task statement"), "{}", m.body);
19234    }
19235
19236    #[test]
19237    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
19238        let mut state = review_state(&["日本語の件名"]);
19239        state.candidates[0].branch = "fix/macos-login-path".to_owned();
19240        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
19241        state.candidates[0].branch = "機能/ログイン".to_owned();
19242        assert!(
19243            pr_message(&state, 'A')
19244                .title
19245                .starts_with("chore: land candidate A")
19246        );
19247    }
19248
19249    #[test]
19250    fn pr_message_review_fixes_survive_a_clean_final_round() {
19251        let mut state = review_state(&["feat: x"]);
19252        state.reviews = vec![
19253            round_with_notes(1, Some("handled the PATH case")),
19254            round_with_notes(2, None),
19255        ];
19256        let body = pr_message(&state, 'A').body;
19257        assert!(
19258            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
19259            "{body}"
19260        );
19261        assert!(body.contains("### Round 1"), "{body}");
19262
19263        state.reviews = vec![
19264            round_with_notes(1, Some("first fix")),
19265            round_with_notes(2, Some("")),
19266            round_with_notes(3, Some("second fix")),
19267            round_with_notes(4, None),
19268        ];
19269        let body = pr_message(&state, 'A').body;
19270        assert!(
19271            body.contains("### Round 1") && !body.contains("first fix"),
19272            "{body}"
19273        );
19274        assert!(
19275            body.contains("### Round 3") && !body.contains("second fix"),
19276            "{body}"
19277        );
19278        assert!(body.contains("### Round 2"), "{body}");
19279    }
19280
19281    const FINAL: &str = "abcdef1234567890";
19282
19283    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
19284        let mut r = round_with_notes(round, None);
19285        r.verified_head = verified.map(str::to_owned);
19286        r
19287    }
19288
19289    fn ok_gate(state: &mut RunState, head: Option<&str>) {
19290        state.gate = vec![CommandOutcome {
19291            command: "gate".to_owned(),
19292            code: Some(0),
19293            output_tail: String::new(),
19294            duration_ms: 0,
19295            resource_blocked: false,
19296        }];
19297        state.gate_ran = true;
19298        state.gate_head = head.map(str::to_owned);
19299    }
19300
19301    fn body_at(state: &RunState) -> String {
19302        pr_message_raw(
19303            state,
19304            'A',
19305            None,
19306            crate::pr_outcome::FinalHead::At(FINAL),
19307            ChangeNote::NotAsked,
19308        )
19309        .body
19310    }
19311
19312    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
19313
19314    #[test]
19315    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
19316        let mut state = review_state(&["feat: x"]);
19317        state.reviews = vec![
19318            round_with_notes(1, Some(SANDBOXED)),
19319            passing_round(2, Some(FINAL)),
19320        ];
19321        ok_gate(&mut state, Some(FINAL));
19322        let body = body_at(&state);
19323        assert!(!body.contains("Handled the PATH case."), "{body}");
19324        assert!(!body.contains("could not commit"), "{body}");
19325        assert!(!body.contains("could not run"), "{body}");
19326        assert!(
19327            body.contains("The fix is committed on the branch"),
19328            "{body}"
19329        );
19330        assert!(!body.contains("sandbox"), "{body}");
19331        assert!(
19332            body.contains("Tests (`verify.e2e`): passed on the final commit"),
19333            "{body}"
19334        );
19335        assert!(
19336            body.contains("Gate (`verify.gate`): passed on the final commit"),
19337            "{body}"
19338        );
19339        // The raw report is untouched in the record.
19340        assert!(
19341            state.reviews[0]
19342                .fix
19343                .as_ref()
19344                .unwrap()
19345                .notes
19346                .contains("could not commit")
19347        );
19348    }
19349
19350    #[test]
19351    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
19352        let mut state = review_state(&["feat: x"]);
19353        let mut r1 = round_with_notes(1, Some(SANDBOXED));
19354        r1.fix.as_mut().unwrap().committed = false;
19355        let mut r2 = passing_round(2, Some(FINAL));
19356        r2.e2e[0].code = Some(1);
19357        state.reviews = vec![r1, r2];
19358        let body = body_at(&state);
19359        assert!(!body.contains("could not commit"), "{body}");
19360        assert!(!body.contains("could not run"), "{body}");
19361        assert!(body.contains("This round produced no commit"), "{body}");
19362        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
19363        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
19364        assert!(!body.contains("passed"), "{body}");
19365    }
19366
19367    #[test]
19368    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
19369        let mut state = review_state(&["feat: x"]);
19370        state.reviews = vec![
19371            round_with_notes(1, Some(SANDBOXED)),
19372            passing_round(2, Some("0123456789abcdef")),
19373        ];
19374        ok_gate(&mut state, Some("0123456789abcdef"));
19375        let body = body_at(&state);
19376        assert!(!body.contains("HTTP tests could not run"), "{body}");
19377        // The commit claim is still settled by the structured record.
19378        assert!(!body.contains("could not commit"), "{body}");
19379        assert!(
19380            body.contains("passed on an earlier commit (`0123456`)"),
19381            "{body}"
19382        );
19383        assert!(!body.contains("passed on the final commit"), "{body}");
19384
19385        // A gate recorded before `gate_head` existed is "not recorded".
19386        ok_gate(&mut state, None);
19387        let body = body_at(&state);
19388        assert!(
19389            body.contains("passed on a commit that was not recorded"),
19390            "{body}"
19391        );
19392    }
19393
19394    #[test]
19395    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
19396        let state = state_with_summary(
19397            "add retries to the client",
19398            "TITLE: feat: retries\n\nDid it.",
19399        );
19400        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
19401        assert_eq!(m.title, "feat: retries");
19402        assert!(
19403            m.body.contains("<summary>Original task</summary>"),
19404            "{}",
19405            m.body
19406        );
19407        assert!(!m.body.contains("Commits under review"), "{}", m.body);
19408        assert!(
19409            m.body
19410                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
19411            "{}",
19412            m.body
19413        );
19414    }
19415
19416    #[test]
19417    fn pr_message_review_bounds_a_long_english_subject() {
19418        let long = format!("feat: {}", "word ".repeat(100));
19419        let m = pr_message(&review_state(&[&long]), 'A');
19420        assert!(m.title.starts_with("feat: word"), "{}", m.title);
19421        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19422    }
19423
19424    #[test]
19425    fn pr_message_implementation_run_is_unchanged_by_review_support() {
19426        let state = state_with_summary("add retries\n\ndetails", "- did some things");
19427        let m = pr_message(&state, 'A');
19428        assert_eq!(m.title, "add retries");
19429        assert!(m.body.contains("<summary>Original task</summary>"));
19430        assert!(!m.body.contains("Commits under review"));
19431        assert_eq!(landing_subject_source(&state), state.instruction);
19432    }
19433
19434    #[test]
19435    fn review_run_squash_subject_is_the_change_not_the_prompt() {
19436        let state = review_state(&["feat: the change", "fix: typo"]);
19437        let source = landing_subject_source(&state);
19438        assert_eq!(land::merge_subject("", &source), "feat: the change");
19439        assert_eq!(
19440            land::merge_subject("magi: candidate A (uncommitted work)", &source),
19441            "feat: the change"
19442        );
19443        // An operator's rename still wins.
19444        assert_eq!(
19445            land::merge_subject("feat: renamed by hand", &source),
19446            "feat: renamed by hand"
19447        );
19448        let blank = review_state(&["日本語"]);
19449        assert!(
19450            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
19451        );
19452    }
19453
19454    #[test]
19455    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
19456        let state = review_state(&["feat: the change"]);
19457        let source = landing_subject_source(&state);
19458        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
19459        assert_eq!(
19460            land::merge_subject(landing_title(&state, old), &source),
19461            "feat: the change"
19462        );
19463        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
19464        let task = state_with_summary("add retries", "");
19465        assert_eq!(landing_title(&task, old), old);
19466    }
19467
19468    #[test]
19469    fn pr_message_describes_the_change_not_the_task() {
19470        let state = state_with_summary(
19471            "今回やってほしいこと: results projector を直す",
19472            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
19473        );
19474        let m = pr_message(&state, 'A');
19475        assert_eq!(m.title, "fix(web): batch the runs list reads");
19476        assert!(
19477            m.body.starts_with("## Summary\n\n- reads run.json once"),
19478            "{}",
19479            m.body
19480        );
19481        assert!(!m.body.contains("TITLE:"), "{}", m.body);
19482        let task_at = m.body.find("今回やってほしいこと").unwrap();
19483        let details_at = m.body.find("<details>").unwrap();
19484        assert!(
19485            details_at < task_at,
19486            "the task lives inside <details>: {}",
19487            m.body
19488        );
19489        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
19490        assert!(m.body.contains("magi:candidate-a"));
19491    }
19492
19493    #[test]
19494    fn pr_message_falls_back_to_the_task_without_a_title_line() {
19495        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
19496        let m = pr_message(&state, 'A');
19497        assert_eq!(m.title, "add retries");
19498        assert!(
19499            m.body.contains("## Summary\n\n- did some things"),
19500            "{}",
19501            m.body
19502        );
19503
19504        let none = RunState::new(
19505            PathBuf::from("/repo"),
19506            "main".to_owned(),
19507            "abc1234".to_owned(),
19508            "add retries".to_owned(),
19509            Config::default(),
19510        );
19511        let m = pr_message(&none, 'A');
19512        assert_eq!(m.title, "add retries");
19513        assert!(!m.body.contains("## Summary"), "{}", m.body);
19514    }
19515
19516    #[test]
19517    fn pr_message_refuses_the_candidate_commit_subject() {
19518        for bad in [
19519            "TITLE: magi: candidate A (uncommitted work)",
19520            "TITLE: chore: stuff (uncommitted work)",
19521            "TITLE:   ",
19522        ] {
19523            let state = state_with_summary("add retries", bad);
19524            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
19525        }
19526    }
19527
19528    #[test]
19529    fn pr_message_bounds_a_very_long_task_and_title() {
19530        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
19531        let state = state_with_summary(&long, "- nothing");
19532        let m = pr_message(&state, 'A');
19533        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19534        assert!(!m.title.contains('\n'));
19535
19536        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
19537        let m = pr_message(&state, 'A');
19538        assert!(m.title.starts_with("feat: "));
19539        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
19540        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
19541    }
19542
19543    fn long_title_of(instruction: &str) -> String {
19544        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
19545    }
19546
19547    #[test]
19548    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
19549        let first = "Make the landing path keep a readable title for long tasks";
19550        let line = format!(
19551            "{first}. {}",
19552            "And then keep going with more words ".repeat(20)
19553        );
19554        let t = long_title_of(&line);
19555        assert_eq!(t, first);
19556        assert!(!t.starts_with("chore: land"));
19557    }
19558
19559    #[test]
19560    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
19561        let line = "word ".repeat(200);
19562        let t = long_title_of(&line);
19563        assert!(t.ends_with("word..."), "{t}");
19564        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
19565    }
19566
19567    #[test]
19568    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
19569        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
19570            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
19571        }
19572    }
19573
19574    #[test]
19575    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
19576        let mut state = state_with_summary("再試行を追加する", "- nothing");
19577        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
19578        state.config.graph.github_language = "ja".into();
19579        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
19580    }
19581
19582    #[test]
19583    fn pr_message_title_limit_is_exact() {
19584        let at = "a".repeat(PR_TITLE_MAX);
19585        assert_eq!(long_title_of(&at), at);
19586        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
19587        assert!(over.ends_with("..."), "{over}");
19588        assert_eq!(over.chars().count(), PR_TITLE_MAX);
19589    }
19590
19591    #[test]
19592    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
19593        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
19594        let t = long_title_of(&line);
19595        assert!(t.ends_with("..."), "{t}");
19596        assert!(t.is_ascii(), "{t}");
19597    }
19598
19599    #[test]
19600    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
19601        let line = format!(
19602            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
19603            "plus more filler words ".repeat(20)
19604        );
19605        let t = long_title_of(&line);
19606        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
19607    }
19608
19609    #[test]
19610    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
19611        let line = format!("\n\n# {}", "title words ".repeat(40));
19612        let state = state_with_summary(&line, "- nothing");
19613        let m = pr_message(&state, 'A');
19614        assert!(m.title.starts_with("title words"), "{}", m.title);
19615        assert_eq!(
19616            land::merge_subject(&m.title, &landing_subject_source(&state)),
19617            m.title
19618        );
19619        // An operator's rename wins untouched.
19620        assert_eq!(
19621            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
19622            "feat: renamed by hand"
19623        );
19624    }
19625
19626    #[tokio::test]
19627    async fn github_text_rewrite_is_bounded_and_falls_back() {
19628        crate::run::pin_test_home();
19629        for (reply, accepted) in [
19630            (
19631                "TITLE: fix: retries\nAdd retries for failed requests.",
19632                true,
19633            ),
19634            (
19635                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
19636                false,
19637            ),
19638            (
19639                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
19640                true,
19641            ),
19642            (
19643                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
19644                true,
19645            ),
19646            (
19647                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
19648                true,
19649            ),
19650            (
19651                "TITLE: fix: retries\n日本語の説明です。 token=secret",
19652                false,
19653            ),
19654            ("TITLE: fix: retries\n", false),
19655        ] {
19656            let dir = tempfile::tempdir().unwrap();
19657            let mut runner = runner_at(RunStatus::Gating);
19658            runner.state = state_with_summary(
19659                "add retries",
19660                "TITLE: fix: retries\n日本語の説明を書きます。",
19661            );
19662            runner.state.repo = dir.path().to_owned();
19663            runner.state.candidates[0].worktree = dir.path().to_owned();
19664            let mut author = spec("alpha");
19665            author.command = vec![
19666                "sh".into(),
19667                "-c".into(),
19668                "printf '%s' \"$REWRITE_REPLY\"".into(),
19669            ];
19670            author.env.insert("REWRITE_REPLY".into(), reply.into());
19671            runner.state.config.agents = vec![author];
19672            seed_public_summary(&mut runner);
19673            let winner = runner.state.candidates[0].clone();
19674            let first = runner
19675                .guarded_pr_message(&winner, None, true)
19676                .await
19677                .unwrap();
19678            let message = if accepted {
19679                first.expect("a passing rewrite never asks")
19680            } else {
19681                // Withheld: the run parks on a question and posts nothing yet.
19682                assert!(first.is_none());
19683                assert!(runner.state.parked);
19684                let store = ask::Questions::open();
19685                let asked: Vec<_> = store
19686                    .list()
19687                    .into_iter()
19688                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19689                    .collect();
19690                assert_eq!(asked.len(), 1);
19691                assert!(!asked[0].detail.contains("secret"));
19692                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
19693                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
19694                assert_eq!(
19695                    asked[0]
19696                        .choices
19697                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
19698                    rewrite.is_some()
19699                );
19700                // Silence: the question lapses and the neutral text is used,
19701                // without a second rewrite call or a second question.
19702                store
19703                    .update(&asked[0].id, |q| {
19704                        q.abandon("test");
19705                        Ok(())
19706                    })
19707                    .unwrap();
19708                let again = runner
19709                    .guarded_pr_message(&winner, None, true)
19710                    .await
19711                    .unwrap()
19712                    .expect("an abandoned question falls back");
19713                let n = store
19714                    .list()
19715                    .into_iter()
19716                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19717                    .count();
19718                assert_eq!(n, 1, "asked once per rejected text");
19719                again
19720            };
19721            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
19722            assert!(!message.body.contains("secret"));
19723            assert!(!message.body.contains("prefix"));
19724            assert!(!message.title.contains("secret"));
19725            assert_eq!(
19726                message.body.contains("Add retries for failed requests."),
19727                accepted
19728            );
19729            assert_eq!(
19730                runner.state.seats["github-text-rewrite"].turns, 1,
19731                "only one rewrite invocation"
19732            );
19733            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
19734            if accepted {
19735                assert!(runner.state.github_text.is_none());
19736                assert!(!runner.state.parked);
19737                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
19738                assert!(message.body.contains("## Final verification"));
19739                assert!(message.body.contains("Evidence: final diff"));
19740                assert!(
19741                    message
19742                        .body
19743                        .contains(&format!("magi:run/{}", runner.state.id))
19744                );
19745            }
19746            if !accepted {
19747                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19748                assert!(
19749                    message
19750                        .body
19751                        .contains(&format!("magi:run/{}", runner.state.id))
19752                );
19753            }
19754        }
19755    }
19756
19757    fn scripted(id: &str, reply: &str) -> AgentSpec {
19758        let mut a = spec(id);
19759        a.command = vec![
19760            "sh".into(),
19761            "-c".into(),
19762            "printf '%s' \"$REWRITE_REPLY\"".into(),
19763        ];
19764        a.env.insert("REWRITE_REPLY".into(), reply.into());
19765        a
19766    }
19767
19768    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
19769    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
19770
19771    /// Gate tests inject generated public output, not internal reports.
19772    fn seed_public_summary(runner: &mut Runner) {
19773        let repo = &runner.state.repo;
19774        let g = |args: &[&str]| {
19775            let out = std::process::Command::new("git")
19776                .args(args)
19777                .current_dir(repo)
19778                .quiet()
19779                .output()
19780                .unwrap();
19781            assert!(out.status.success(), "git {args:?}");
19782            String::from_utf8_lossy(&out.stdout).trim().to_owned()
19783        };
19784        g(&["init", "-q", "-b", "main"]);
19785        g(&[
19786            "-c",
19787            "user.name=Test",
19788            "-c",
19789            "user.email=test@example.invalid",
19790            "commit",
19791            "-q",
19792            "--allow-empty",
19793            "-m",
19794            "feat: fixture",
19795        ]);
19796        let head = g(&["rev-parse", "HEAD"]);
19797        g(&["update-ref", "refs/remotes/origin/main", &head]);
19798        let evidence = crate::pr_outcome::summary_evidence(
19799            &runner.state,
19800            crate::pr_outcome::FinalHead::At(&head),
19801        );
19802        let generated = &runner.state.candidates[0].summary;
19803        runner.state.change_summary = Some(crate::run::ChangeSummary {
19804            base: head.clone(),
19805            head,
19806            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
19807            title: summary_title(generated).unwrap(),
19808            body: summary_without_title(generated),
19809        });
19810    }
19811
19812    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19813        let mut runner = runner_at(RunStatus::Gating);
19814        runner.state = state_with_summary("add retries", JA_SUMMARY);
19815        if review_only {
19816            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19817        }
19818        runner.state.repo = dir.to_owned();
19819        runner.state.candidates[0].worktree = dir.to_owned();
19820        runner.state.config.agents = agents;
19821        seed_public_summary(&mut runner);
19822        runner
19823    }
19824
19825    fn gate_questions(runner: &Runner) -> usize {
19826        ask::Questions::open()
19827            .list()
19828            .into_iter()
19829            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19830            .count()
19831    }
19832
19833    #[tokio::test]
19834    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19835        crate::run::pin_test_home();
19836        let dir = tempfile::tempdir().unwrap();
19837        let mut runner = rewrite_runner(
19838            dir.path(),
19839            true,
19840            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19841        );
19842        runner.state.config.roles.language_judge =
19843            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19844        let winner = runner.state.candidates[0].clone();
19845        let msg = runner
19846            .guarded_pr_message(&winner, None, true)
19847            .await
19848            .unwrap()
19849            .expect("an English rewrite posts without asking");
19850        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19851        assert!(msg.body.contains("<details>kept</details>"));
19852        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19853        assert_eq!(gate_questions(&runner), 0);
19854        assert!(runner.state.github_text.is_none());
19855        assert!(!runner.state.seats.contains_key("impl-A"));
19856        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19857    }
19858
19859    #[tokio::test]
19860    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19861        crate::run::pin_test_home();
19862        let dir = tempfile::tempdir().unwrap();
19863        let mut runner = rewrite_runner(
19864            dir.path(),
19865            false,
19866            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19867        );
19868        runner.state.config.roles.language_judge =
19869            Some(crate::config::AgentChoice::One("beta".into()));
19870        runner.state.seats.insert(
19871            "impl-A".into(),
19872            SeatState::new("impl-A", "alpha", runner.state.seed),
19873        );
19874        let winner = runner.state.candidates[0].clone();
19875        let msg = runner
19876            .guarded_pr_message(&winner, None, true)
19877            .await
19878            .unwrap()
19879            .expect("rewritten");
19880        assert!(msg.body.contains("<details>kept</details>"));
19881        assert_eq!(runner.state.seats["impl-A"].turns, 0);
19882        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19883        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19884    }
19885
19886    #[tokio::test]
19887    async fn rewrite_reuses_a_resumable_implementer_session() {
19888        crate::run::pin_test_home();
19889        let dir = tempfile::tempdir().unwrap();
19890        let mut runner = rewrite_runner(
19891            dir.path(),
19892            false,
19893            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19894        );
19895        runner.state.config.roles.language_judge =
19896            Some(crate::config::AgentChoice::One("beta".into()));
19897        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19898        seat.turns = 1;
19899        runner.state.seats.insert("impl-A".into(), seat);
19900        let winner = runner.state.candidates[0].clone();
19901        runner
19902            .guarded_pr_message(&winner, None, true)
19903            .await
19904            .unwrap()
19905            .expect("rewritten");
19906        assert_eq!(runner.state.seats["impl-A"].turns, 2);
19907        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19908    }
19909
19910    #[tokio::test]
19911    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19912        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
19913        // session, was out of quota, and nothing else was asked.
19914        crate::run::pin_test_home();
19915        let dir = tempfile::tempdir().unwrap();
19916        let mut exhausted = spec("alpha");
19917        exhausted.command = vec!["false".into()];
19918        let mut runner = rewrite_runner(
19919            dir.path(),
19920            false,
19921            vec![exhausted, scripted("beta", EN_REWRITE)],
19922        );
19923        // The implementer's own agent is in the role too: it is not asked twice.
19924        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19925            "alpha".into(),
19926            "beta".into(),
19927        ]));
19928        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19929        seat.turns = 1;
19930        runner.state.seats.insert("impl-A".into(), seat);
19931        let winner = runner.state.candidates[0].clone();
19932        let msg = runner
19933            .guarded_pr_message(&winner, None, true)
19934            .await
19935            .unwrap()
19936            .expect("the role rewrites after the implementer fails");
19937        assert!(msg.body.contains("`src/net.rs`"));
19938        assert_eq!(
19939            runner.state.seats["impl-A"].turns, 2,
19940            "the implementer was tried once"
19941        );
19942        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19943        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19944    }
19945
19946    #[tokio::test]
19947    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19948        crate::run::pin_test_home();
19949        let dir = tempfile::tempdir().unwrap();
19950        let mut broken = spec("bad");
19951        broken.command = vec!["false".into()];
19952        let mut runner = rewrite_runner(
19953            dir.path(),
19954            true,
19955            vec![
19956                broken,
19957                scripted("noformat", "Add retries."),
19958                scripted("good", EN_REWRITE),
19959            ],
19960        );
19961        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19962            "bad".into(),
19963            "noformat".into(),
19964            "good".into(),
19965        ]));
19966        let winner = runner.state.candidates[0].clone();
19967        let msg = runner
19968            .guarded_pr_message(&winner, None, true)
19969            .await
19970            .unwrap()
19971            .expect("the third agent rewrites");
19972        assert!(msg.body.contains("`src/net.rs`"));
19973        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19974        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19975    }
19976
19977    #[tokio::test]
19978    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19979        crate::run::pin_test_home();
19980        // (agents, role): still Japanese; no TITLE; nothing resolvable.
19981        for (agents, role, saved) in [
19982            (
19983                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19984                Some("beta"),
19985                true,
19986            ),
19987            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19988            (Vec::new(), None, false),
19989        ] {
19990            let dir = tempfile::tempdir().unwrap();
19991            let mut runner = rewrite_runner(dir.path(), true, agents);
19992            runner.state.config.roles.language_judge =
19993                role.map(|r| crate::config::AgentChoice::One(r.into()));
19994            let winner = runner.state.candidates[0].clone();
19995            let first = runner
19996                .guarded_pr_message(&winner, None, true)
19997                .await
19998                .unwrap();
19999            assert!(first.is_none(), "withheld text parks the run");
20000            assert!(runner.state.parked);
20001            assert_eq!(gate_questions(&runner), 1);
20002            let g = runner.state.github_text.as_ref().unwrap();
20003            assert_eq!(g.rewrite.is_some(), saved);
20004            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
20005        }
20006    }
20007
20008    #[tokio::test]
20009    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
20010        crate::run::pin_test_home();
20011        let dir = tempfile::tempdir().unwrap();
20012        let mut runner = runner_at(RunStatus::Gating);
20013        runner.state =
20014            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
20015        runner.state.repo = dir.path().to_owned();
20016        runner.state.candidates[0].worktree = dir.path().to_owned();
20017        let mut author = spec("alpha");
20018        author.command = vec![
20019            "sh".into(),
20020            "-c".into(),
20021            "printf '%s' \"$REWRITE_REPLY\"".into(),
20022        ];
20023        author.env.insert(
20024            "REWRITE_REPLY".into(),
20025            format!(
20026                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
20027                "English sentence. ".repeat(600)
20028            ),
20029        );
20030        runner.state.config.agents = vec![author];
20031        seed_public_summary(&mut runner);
20032        let winner = runner.state.candidates[0].clone();
20033        assert!(
20034            runner
20035                .guarded_pr_message(&winner, None, true)
20036                .await
20037                .unwrap()
20038                .is_none()
20039        );
20040        let record = runner.state.github_text.clone().unwrap();
20041        let rewrite = record.rewrite.clone().unwrap();
20042        assert_eq!(rewrite.failed, ["body-language"]);
20043        assert!(
20044            rewrite
20045                .body
20046                .ends_with(&format!("magi:run/{}", runner.state.id))
20047        );
20048        let store = ask::Questions::open();
20049        let q = store.get(record.question.as_ref().unwrap()).unwrap();
20050        assert!(
20051            q.choices
20052                .contains(&crate::github_text::USE_REWRITE.to_owned())
20053        );
20054        for label in [
20055            "ORIGINAL",
20056            "FALLBACK",
20057            "REWRITE",
20058            "Rules still failed: body-language",
20059            "truncated",
20060        ] {
20061            assert!(q.detail.contains(label), "{label}");
20062        }
20063        let artifact = std::fs::read_to_string(crate::run::artifact_path(
20064            &runner.state,
20065            &crate::github_text::artifact_name(&record.fingerprint),
20066        ))
20067        .unwrap();
20068        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
20069        // Restart from serialized state, then retry after a bad owner title.
20070        runner.state =
20071            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
20072        store
20073            .update(&q.id, |q| {
20074                q.say("まだ日本語")?;
20075                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
20076            })
20077            .unwrap();
20078        assert!(
20079            runner
20080                .guarded_pr_message(&winner, None, true)
20081                .await
20082                .unwrap()
20083                .is_none()
20084        );
20085        let retry = runner.state.github_text.as_ref().unwrap();
20086        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
20087        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
20088        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
20089        assert!(
20090            q.posting_comparison
20091                .as_ref()
20092                .unwrap()
20093                .matches_detail(&q.detail)
20094        );
20095        store
20096            .update(&q.id, |q| {
20097                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20098            })
20099            .unwrap();
20100        let message = runner
20101            .guarded_pr_message(&winner, None, true)
20102            .await
20103            .unwrap()
20104            .unwrap();
20105        assert_eq!(message.title, rewrite.title);
20106        assert_eq!(message.body, rewrite.body);
20107        assert_eq!(
20108            runner.state.github_text.as_ref().unwrap().choice,
20109            Some(GithubTextChoice::Rewrite)
20110        );
20111        let again = runner
20112            .guarded_pr_message(&winner, None, true)
20113            .await
20114            .unwrap()
20115            .unwrap();
20116        assert_eq!(again.title, message.title);
20117        assert_eq!(again.body, message.body);
20118        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
20119        assert!(
20120            !runner
20121                .state
20122                .events
20123                .iter()
20124                .any(|e| e.message.contains(&rewrite.title)
20125                    || e.message.contains("English sentence."))
20126        );
20127    }
20128
20129    #[tokio::test]
20130    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
20131        crate::run::pin_test_home();
20132        let dir = tempfile::tempdir().unwrap();
20133        let mut runner = runner_at(RunStatus::Gating);
20134        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
20135        runner.state.repo = dir.path().to_owned();
20136        runner.state.candidates[0].worktree = dir.path().to_owned();
20137        seed_public_summary(&mut runner);
20138        let winner = runner.state.candidates[0].clone();
20139        assert!(
20140            runner
20141                .guarded_pr_message(&winner, None, true)
20142                .await
20143                .unwrap()
20144                .is_none()
20145        );
20146        let record = runner.state.github_text.as_ref().unwrap();
20147        assert!(record.rewrite.is_none());
20148        // Records written before the additive field deserialize with no rewrite.
20149        let mut old = serde_json::to_value(record).unwrap();
20150        old.as_object_mut().unwrap().remove("rewrite");
20151        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
20152        assert!(old.rewrite.is_none());
20153        let id = record.question.clone().unwrap();
20154        ask::Questions::open()
20155            .update(&id, |q| {
20156                q.choices.push(crate::github_text::USE_REWRITE.into());
20157                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
20158            })
20159            .unwrap();
20160        let message = runner
20161            .guarded_pr_message(&winner, None, true)
20162            .await
20163            .unwrap()
20164            .unwrap();
20165        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
20166        assert_eq!(
20167            runner.state.github_text.as_ref().unwrap().choice,
20168            Some(GithubTextChoice::Fallback)
20169        );
20170    }
20171
20172    #[tokio::test]
20173    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
20174        crate::run::pin_test_home();
20175        let dir = tempfile::tempdir().unwrap();
20176        let mut runner = runner_at(RunStatus::Gating);
20177        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
20178        runner.state.repo = dir.path().to_owned();
20179        runner.state.candidates[0].worktree = dir.path().to_owned();
20180        seed_public_summary(&mut runner);
20181        let winner = runner.state.candidates[0].clone();
20182        assert!(
20183            runner
20184                .guarded_pr_message(&winner, None, true)
20185                .await
20186                .unwrap()
20187                .is_none()
20188        );
20189        let store = ask::Questions::open();
20190        let find = |run: &str| {
20191            store
20192                .list()
20193                .into_iter()
20194                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
20195                .unwrap()
20196        };
20197        let run = runner.state.id.clone();
20198        let q = find(&run);
20199        let fp = runner
20200            .state
20201            .github_text
20202            .as_ref()
20203            .unwrap()
20204            .fingerprint
20205            .clone();
20206        let name = crate::github_text::artifact_name(&fp);
20207        let path = crate::run::artifact_path(&runner.state, &name);
20208        assert!(q.detail.contains(&path.display().to_string()));
20209        assert!(q.detail.contains("title-language: title"));
20210        let saved = std::fs::read_to_string(&path).expect("artifact written");
20211        assert!(saved.contains("日本語のタイトル"));
20212        assert!(q.cwd.is_none());
20213        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
20214        // A bad replacement is refused and asked about once more.
20215        let say = |id: &str, text: &str, choice: &str| {
20216            store
20217                .update(id, |q| {
20218                    q.thread.push(ask::Turn {
20219                        who: ask::Who::Operator,
20220                        body: text.to_owned(),
20221                        at: jiff::Timestamp::now(),
20222                        note: None,
20223                    });
20224                    q.answer(ask::Answer::Choice(choice.to_owned()))
20225                })
20226                .unwrap();
20227        };
20228        say(&q.id, "まだ日本語", "use my text");
20229        assert!(
20230            runner
20231                .guarded_pr_message(&winner, None, true)
20232                .await
20233                .unwrap()
20234                .is_none()
20235        );
20236        let second = find(&run);
20237        assert_ne!(second.id, q.id);
20238        assert!(second.detail.contains("did not pass"));
20239        // The retry still compares the original text; the rejected
20240        // replacement is named by category only.
20241        assert!(second.detail.contains("日本語のタイトル"));
20242        assert!(!second.detail.contains("まだ日本語"));
20243        assert!(second.detail.contains(&path.display().to_string()));
20244        say(&second.id, "fix: retry failed requests", "use my text");
20245        let message = runner
20246            .guarded_pr_message(&winner, None, true)
20247            .await
20248            .unwrap()
20249            .expect("a vetted replacement is posted");
20250        assert_eq!(message.title, "fix: retry failed requests");
20251        assert_eq!(
20252            runner
20253                .state
20254                .github_text
20255                .as_ref()
20256                .unwrap()
20257                .chosen_title
20258                .as_deref(),
20259            Some("fix: retry failed requests")
20260        );
20261        assert!(runner.state.github_text.as_ref().unwrap().resolved);
20262        // Resumed again: the saved decision is applied, nothing is re-asked.
20263        let again = runner
20264            .guarded_pr_message(&winner, None, true)
20265            .await
20266            .unwrap()
20267            .unwrap();
20268        assert_eq!(again.title, message.title);
20269        assert_eq!(
20270            store
20271                .list()
20272                .into_iter()
20273                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
20274                .count(),
20275            2
20276        );
20277    }
20278
20279    #[test]
20280    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
20281        // What magi itself writes stays English under any configured language,
20282        // so a future localisation of these headings fails here. (The agents'
20283        // own text is also checked by the posting gate.)
20284        let mut state = state_with_summary(
20285            "add retries",
20286            "TITLE: fix(web): batch reads\n- reads run.json once",
20287        );
20288        state.config.graph.language = "ja".to_owned();
20289        let m = pr_message(&state, 'A');
20290        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
20291        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
20292
20293        // The task is the operator's own text: it goes in untouched, and the
20294        // fallback title (no summary) may be in its language too.
20295        let task = "今回やってほしいこと: results projector を直す";
20296        let mut state = state_with_summary(task, "- no title line");
20297        state.config.graph.language = "ja".to_owned();
20298        let m = pr_message(&state, 'A');
20299        assert_eq!(
20300            m.title,
20301            format!("chore: land candidate A of run {}", state.id)
20302        );
20303        assert!(
20304            m.body.contains(&format!(
20305                "<summary>Original task</summary>\n\n{task}\n\n</details>"
20306            )),
20307            "{}",
20308            m.body
20309        );
20310    }
20311
20312    #[test]
20313    fn pr_message_scrubs_home_paths_and_addresses() {
20314        let state = state_with_summary(
20315            "fix it in /Users/someone/src/x",
20316            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
20317        );
20318        let m = pr_message(&state, 'A');
20319        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
20320            assert!(!m.body.contains(leak), "{}", m.body);
20321        }
20322        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
20323    }
20324
20325    #[test]
20326    fn pr_message_survives_a_task_that_closes_details() {
20327        let state = state_with_summary("a </details> b", "TITLE: fix: x");
20328        let m = pr_message(&state, 'A');
20329        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
20330    }
20331
20332    #[test]
20333    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
20334        let cmd = manual_merge_command(
20335            MergeStyle::Squash,
20336            Path::new("/repo"),
20337            "b",
20338            "fix: \"quoted\" $(x) `y`\n\nbody",
20339        );
20340        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
20341    }
20342
20343    #[test]
20344    fn manual_merge_command_matches_the_configured_style() {
20345        let repo = Path::new("/repo");
20346        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
20347
20348        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
20349        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
20350
20351        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
20352        assert_eq!(
20353            squash,
20354            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
20355             \"Merge magi run 0832 (candidate A)\""
20356        );
20357
20358        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
20359        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
20360    }
20361
20362    #[test]
20363    fn a_nudge_gets_a_quarter_of_the_budget() {
20364        // The judge and implement budgets magi ships with.
20365        assert_eq!(retry_budget(secs(1200), true), secs(300));
20366        assert_eq!(retry_budget(secs(3600), true), secs(900));
20367    }
20368
20369    #[test]
20370    fn a_resent_prompt_keeps_the_whole_budget() {
20371        // The seat kept no context, so the retry is the original job again and
20372        // shortening it would only guarantee a second failure.
20373        assert_eq!(retry_budget(secs(1200), false), secs(1200));
20374        assert_eq!(retry_budget(secs(60), false), secs(60));
20375    }
20376
20377    #[test]
20378    fn the_floor_never_exceeds_the_original_budget() {
20379        // A short configured timeout must not be *raised* by the floor: the
20380        // operator asked for a bound, and a retry may not outlast the attempt
20381        // it is retrying.
20382        assert_eq!(retry_budget(secs(60), true), secs(60));
20383        assert_eq!(retry_budget(secs(480), true), secs(120));
20384        assert_eq!(retry_budget(secs(0), true), secs(0));
20385    }
20386
20387    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
20388        agent::CommandEvidence {
20389            id: "item1".to_owned(),
20390            description: "cargo test".to_owned(),
20391            exit_code,
20392            result_summary: String::new(),
20393            source: "codex".to_owned(),
20394        }
20395    }
20396
20397    #[test]
20398    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
20399        // No evidence is not the same fact as unconfirmed evidence: a
20400        // backend with no adapter, or a reply that ran no commands at all,
20401        // must not be misread as carrying a dangling job.
20402        assert!(!has_unconfirmed_command(&[]));
20403    }
20404
20405    #[test]
20406    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
20407        // Deliberately not a check on the exit code's *value*: a fixer
20408        // legitimately runs something that fails mid-iteration before it
20409        // succeeds, and that must never by itself reopen a valid report.
20410        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
20411        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
20412        assert!(!has_unconfirmed_command(&[
20413            evidence(Some(0)),
20414            evidence(Some(101))
20415        ]));
20416    }
20417
20418    #[test]
20419    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
20420        assert!(has_unconfirmed_command(&[
20421            evidence(Some(0)),
20422            evidence(None)
20423        ]));
20424    }
20425
20426    #[test]
20427    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
20428        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20429        assert_eq!(
20430            verified_noop_claim(true, &[], text).as_deref(),
20431            Some("already fixed by b32cfc4, on main.")
20432        );
20433    }
20434
20435    #[test]
20436    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
20437        // A timeout or a bad exit code reads as the ordinary loss it is,
20438        // whatever the reply's own prose claims.
20439        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20440        assert!(verified_noop_claim(false, &[], text).is_none());
20441    }
20442
20443    #[test]
20444    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
20445        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
20446        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
20447        // A confirmed command alongside the marker is fine.
20448        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
20449    }
20450
20451    #[test]
20452    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
20453        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
20454    }
20455
20456    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
20457    /// pair, in order, labelled A, B, C, ...
20458    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
20459        runner.state.candidates = shape
20460            .iter()
20461            .enumerate()
20462            .map(|(i, &(empty, verified))| Candidate {
20463                index: i,
20464                label: (b'A' + i as u8) as char,
20465                agent: "sonnet".to_owned(),
20466                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
20467                worktree: PathBuf::from(format!("/wt/{i}")),
20468                summary: String::new(),
20469                stat: String::new(),
20470                files: 0,
20471                commits: 0,
20472                empty,
20473                failed: None,
20474                verified_noop: verified.map(str::to_owned),
20475                duration_ms: 0,
20476                folded: false,
20477            })
20478            .collect();
20479    }
20480
20481    #[test]
20482    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
20483        ask_test_home();
20484        let mut runner = runner_at(RunStatus::Implementing);
20485        set_candidates(
20486            &mut runner,
20487            &[
20488                (true, Some("already on main at b32cfc4")),
20489                (true, Some("same fix, see the existing test")),
20490            ],
20491        );
20492
20493        runner
20494            .after_implement()
20495            .expect("a verified no-op is not an error");
20496
20497        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
20498    }
20499
20500    #[test]
20501    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
20502        ask_test_home();
20503        let mut runner = runner_at(RunStatus::Implementing);
20504        // Candidate A declares a verified no-op; candidate B simply wrote
20505        // nothing and said nothing about why. One candidate's claim is not
20506        // the whole run's agreement.
20507        set_candidates(
20508            &mut runner,
20509            &[(true, Some("already on main at b32cfc4")), (true, None)],
20510        );
20511
20512        let err = runner
20513            .after_implement()
20514            .expect_err("an unverified empty candidate must still fail the run");
20515
20516        assert!(
20517            err.to_string().contains("no candidate produced a change"),
20518            "{err}"
20519        );
20520        assert_eq!(runner.state.status, RunStatus::Failed);
20521    }
20522
20523    #[test]
20524    fn after_implement_still_fails_an_ordinary_all_empty_run() {
20525        ask_test_home();
20526        let mut runner = runner_at(RunStatus::Implementing);
20527        set_candidates(&mut runner, &[(true, None), (true, None)]);
20528
20529        let err = runner
20530            .after_implement()
20531            .expect_err("no candidate declared anything; this is an ordinary failure");
20532
20533        assert!(
20534            err.to_string().contains("no candidate produced a change"),
20535            "{err}"
20536        );
20537        assert_eq!(runner.state.status, RunStatus::Failed);
20538    }
20539}
20540
20541/// A graft rollback whose restore could not be confirmed (see
20542/// [`Runner::graft`]).
20543#[derive(Debug)]
20544struct GraftRestoreFailed(String);
20545
20546impl std::fmt::Display for GraftRestoreFailed {
20547    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20548        write!(
20549            f,
20550            "could not restore the winner to its pre-graft commit: {}",
20551            self.0
20552        )
20553    }
20554}
20555
20556impl std::error::Error for GraftRestoreFailed {}
20557
20558/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
20559/// not the agent: they never draw on an agent's own timeout, and running out of
20560/// them never reads as an agent failure.
20561#[cfg(test)]
20562mod resource_wait_tests {
20563    use super::*;
20564    use std::collections::BTreeMap;
20565
20566    fn sh(id: &str, script: &str) -> AgentSpec {
20567        AgentSpec {
20568            id: id.to_owned(),
20569            kind: crate::config::AgentKind::Command,
20570            model: None,
20571            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
20572            extra_args: Vec::new(),
20573            env: BTreeMap::new(),
20574            prompt_delivery: None,
20575            max_turn_secs: None,
20576        }
20577    }
20578
20579    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
20580        SeatJob {
20581            seat: SeatState::new("impl-A", &spec.id, 1),
20582            spec,
20583            cwd: dir.to_path_buf(),
20584            prompt: "do it".to_owned(),
20585            timeout,
20586            allow_write: true,
20587            sessions: false,
20588            artifacts: dir.join("art"),
20589            stem: "impl-A".to_owned(),
20590            handover: None,
20591        }
20592    }
20593
20594    fn state(resource_wait_secs: u64) -> RunState {
20595        let mut config = Config::default();
20596        config.graph.timeout_resource_wait = resource_wait_secs;
20597        RunState::new(
20598            PathBuf::from("/repo"),
20599            "main".to_owned(),
20600            "abc1234".to_owned(),
20601            "task".to_owned(),
20602            config,
20603        )
20604    }
20605
20606    fn hold(cache: &Path) -> crate::cache::Guard {
20607        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
20608        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
20609            crate::cache::AcquireOutcome::Acquired(g) => g,
20610            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
20611        }
20612    }
20613
20614    async fn run(
20615        jobs: Vec<SeatJob>,
20616        sem: Arc<Semaphore>,
20617        cache: Option<&Path>,
20618        st: &mut RunState,
20619    ) -> Vec<(usize, SeatState, AgentOutcome)> {
20620        let prompts = Prompts::default();
20621        let ctx = WaveCtx {
20622            run: "r",
20623            node: "implement",
20624            prompts: &prompts,
20625            cache,
20626            round: None,
20627            carry_seats: false,
20628            pauses: None,
20629        };
20630        wave(jobs, sem, &ctx, st, 0).await
20631    }
20632
20633    #[tokio::test]
20634    async fn verification_cache_wait_preserves_the_command_budget() {
20635        crate::run::pin_test_home();
20636        let tmp = tempfile::tempdir().unwrap();
20637        let cache = tmp.path().join("target");
20638        let guard = hold(&cache);
20639        let release = tokio::spawn(async move {
20640            tokio::time::sleep(Duration::from_millis(100)).await;
20641            guard.release();
20642        });
20643        let mut st = state(10);
20644        let budget = Duration::from_millis(50);
20645        let wait = repository_wait(&st, None);
20646        let (outcomes, _) = with_cache_lease(
20647            &mut st,
20648            Some(&cache),
20649            "gate",
20650            "gate",
20651            tmp.path(),
20652            "abc1234",
20653            budget,
20654            "test gate",
20655            &wait,
20656            |_, available| async move {
20657                assert_eq!(
20658                    available, budget,
20659                    "contention must not shorten verification"
20660                );
20661                (Vec::new(), false, Vec::new())
20662            },
20663        )
20664        .await
20665        .unwrap();
20666        release.await.unwrap();
20667        assert!(outcomes.is_empty(), "{outcomes:?}");
20668        assert!(
20669            st.events
20670                .iter()
20671                .any(|event| event.message.contains("waiting for"))
20672        );
20673        assert!(st.resource_wait.is_none());
20674        // The verification body no longer owns the lease after returning.
20675        hold(&cache).release();
20676    }
20677
20678    #[tokio::test]
20679    async fn verification_cache_wait_expires_without_starting_commands() {
20680        crate::run::pin_test_home();
20681        let tmp = tempfile::tempdir().unwrap();
20682        let cache = tmp.path().join("target");
20683        let _held = hold(&cache);
20684        let mut st = state(0);
20685        let wait = repository_wait(&st, None);
20686        let (outcomes, _) = with_cache_lease(
20687            &mut st,
20688            Some(&cache),
20689            "gate",
20690            "gate",
20691            tmp.path(),
20692            "abc1234",
20693            Duration::from_secs(60),
20694            "test gate",
20695            &wait,
20696            |_, _| async { panic!("commands must not start without the cache lease") },
20697        )
20698        .await
20699        .unwrap();
20700        assert_eq!(outcomes.len(), 1);
20701        assert!(outcomes[0].resource_blocked);
20702        assert!(st.resource_wait.is_none());
20703        assert!(matches!(
20704            st.resource_block.unwrap().cause,
20705            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
20706        ));
20707    }
20708
20709    #[tokio::test]
20710    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
20711        crate::run::pin_test_home();
20712        let tmp = tempfile::tempdir().unwrap();
20713        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
20714        let cache = tmp.path().join("cache");
20715        let mut state = state(21600);
20716        let wait = repository_wait(&state, None);
20717        let (outcomes, _) = with_cache_lease(
20718            &mut state,
20719            Some(&cache),
20720            "gate",
20721            "gate",
20722            tmp.path(),
20723            "head",
20724            Duration::from_secs(60),
20725            "gate",
20726            &wait,
20727            |_, _| async { panic!("freshness failure must not start verification") },
20728        )
20729        .await
20730        .unwrap();
20731        assert_eq!(outcomes.len(), 1);
20732        assert!(outcomes[0].resource_blocked);
20733        let block = state.resource_block.unwrap();
20734        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
20735        assert!(!block.description().contains("timed out"));
20736    }
20737
20738    #[tokio::test]
20739    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
20740        crate::run::pin_test_home();
20741        let tmp = tempfile::tempdir().unwrap();
20742        let cache = tmp.path().join("cache");
20743        let _held = hold(&cache);
20744        let mut state = state(21600);
20745        let pause = Pause::new();
20746        let interrupt = Pause::new();
20747        let cause = ParkCause::Operator;
20748        pause.park_for(cause.clone());
20749        let prompts = Prompts::default();
20750        let ctx = WaveCtx {
20751            run: "test",
20752            node: "fix",
20753            prompts: &prompts,
20754            cache: Some(&cache),
20755            round: None,
20756            carry_seats: false,
20757            pauses: Some((&pause, &interrupt)),
20758        };
20759        let result = wave(
20760            vec![job(
20761                sh("writer", "exit 99"),
20762                tmp.path(),
20763                Duration::from_secs(1),
20764            )],
20765            Arc::new(Semaphore::new(1)),
20766            &ctx,
20767            &mut state,
20768            0,
20769        )
20770        .await;
20771        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
20772        assert_eq!(result[0].1.turns, 0);
20773        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20774        assert_eq!(
20775            state.resource_block.unwrap().cause,
20776            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
20777        );
20778        assert!(state.active.is_empty());
20779    }
20780
20781    #[tokio::test]
20782    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
20783        crate::run::pin_test_home();
20784        let tmp = tempfile::tempdir().unwrap();
20785        let cache = tmp.path().join("target-a");
20786        let guard = hold(&cache);
20787        // Held for longer than the agent's whole 1s budget.
20788        tokio::spawn(async move {
20789            tokio::time::sleep(Duration::from_millis(1500)).await;
20790            guard.release();
20791        });
20792        let mut st = state(60);
20793        let results = run(
20794            vec![job(
20795                sh("alpha", "echo ok"),
20796                tmp.path(),
20797                Duration::from_secs(1),
20798            )],
20799            Arc::new(Semaphore::new(1)),
20800            Some(&cache),
20801            &mut st,
20802        )
20803        .await;
20804        assert!(
20805            matches!(results[0].2, AgentOutcome::Ok(_)),
20806            "the wait must not be charged to the agent"
20807        );
20808        assert!(st.resource_block.is_none());
20809        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20810        assert!(st.resource_wait.is_none() && st.active.is_empty());
20811    }
20812
20813    #[tokio::test]
20814    #[cfg(unix)]
20815    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20816        use std::os::fd::AsRawFd as _;
20817        crate::run::pin_test_home();
20818        let tmp = tempfile::tempdir().unwrap();
20819        crate::git::git(tmp.path(), &["init"]).await.unwrap();
20820        let common = tmp.path().join(".git");
20821        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20822        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20823        assert_eq!(
20824            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20825            0
20826        );
20827        let mut st = state(1);
20828        st.config.coordination = crate::config::test_coordination();
20829        let mut reader = job(
20830            sh("reader", "echo must-not-run"),
20831            tmp.path(),
20832            Duration::from_secs(30),
20833        );
20834        reader.allow_write = false;
20835        let results = run(
20836            vec![reader.clone()],
20837            Arc::new(Semaphore::new(1)),
20838            None,
20839            &mut st,
20840        )
20841        .await;
20842        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20843        assert_eq!(results[0].1.turns, 0);
20844        assert!(FailClass::of(&results[0].2).is_none());
20845        assert_eq!(
20846            st.resource_block.as_ref().unwrap().kind,
20847            "repository coordination"
20848        );
20849        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20850        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20851        drop(external);
20852        let editor = crate::repo_lock::SeatLease::for_checkout(
20853            tmp.path(),
20854            true,
20855            crate::config::test_coordination().as_ref(),
20856        )
20857        .await
20858        .unwrap();
20859        let pause = Pause::new();
20860        let interrupt = Pause::new();
20861        let prompts = Prompts::default();
20862        let ctx = WaveCtx {
20863            run: "r",
20864            node: "review",
20865            prompts: &prompts,
20866            cache: None,
20867            round: None,
20868            carry_seats: false,
20869            pauses: Some((&pause, &interrupt)),
20870        };
20871        let (results, ()) = tokio::join!(
20872            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20873            async {
20874                tokio::time::sleep(Duration::from_millis(50)).await;
20875                pause.park();
20876            }
20877        );
20878        assert!(
20879            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20880        );
20881        assert_eq!(results[0].1.turns, 0);
20882        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20883        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20884        drop(editor);
20885    }
20886
20887    #[tokio::test]
20888    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20889        crate::run::pin_test_home();
20890        let tmp = tempfile::tempdir().unwrap();
20891        let cache = tmp.path().join("target-b");
20892        let _held = hold(&cache);
20893        let mut st = state(1);
20894        let results = run(
20895            vec![job(
20896                sh("alpha", "echo ok"),
20897                tmp.path(),
20898                Duration::from_secs(1),
20899            )],
20900            Arc::new(Semaphore::new(1)),
20901            Some(&cache),
20902            &mut st,
20903        )
20904        .await;
20905        let (_, seat, out) = &results[0];
20906        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20907        assert!(wave_blocked(&results));
20908        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20909        assert_eq!(seat.turns, 0, "nothing was asked");
20910        assert!(
20911            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20912            "no agent was spawned"
20913        );
20914        assert!(FailClass::of(out).is_none(), "not a failure class");
20915        let block = st.resource_block.as_ref().expect("recorded for the resume");
20916        assert_eq!(block.kind, "shared build cache");
20917        assert!(matches!(
20918            block.cause,
20919            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20920        ));
20921        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20922        assert!(st.active.is_empty());
20923    }
20924
20925    #[tokio::test]
20926    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20927        crate::run::pin_test_home();
20928        let tmp = tempfile::tempdir().unwrap();
20929        let mut st = state(1);
20930        let results = run(
20931            vec![job(
20932                sh("alpha", "echo ok"),
20933                tmp.path(),
20934                Duration::from_secs(1),
20935            )],
20936            Arc::new(Semaphore::new(0)),
20937            None,
20938            &mut st,
20939        )
20940        .await;
20941        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20942        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20943        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20944    }
20945
20946    #[tokio::test]
20947    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20948        crate::run::pin_test_home();
20949        let tmp = tempfile::tempdir().unwrap();
20950        let mut st = state(60);
20951        let results = run(
20952            vec![job(
20953                sh("alpha", "sleep 30"),
20954                tmp.path(),
20955                Duration::from_secs(1),
20956            )],
20957            Arc::new(Semaphore::new(1)),
20958            None,
20959            &mut st,
20960        )
20961        .await;
20962        let out = &results[0].2;
20963        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20964        let class = FailClass::of(out).expect("a failure");
20965        assert_eq!(class, FailClass::Timeout);
20966        assert!(should_hand_over(None, &class), "existing handover policy");
20967        assert!(st.resource_block.is_none());
20968    }
20969}