Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        anyhow::ensure!(
1443            state
1444                .base_sync_recovery
1445                .stop
1446                .as_ref()
1447                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449        );
1450        let fetched =
1451            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452        anyhow::ensure!(
1453            fetched.ok(),
1454            "cannot authorize base recovery without a fetched base"
1455        );
1456        let approved_tip = git::rev_parse(
1457            &state.repo,
1458            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459        )
1460        .await?;
1461        if let Some(control) = control.as_deref_mut() {
1462            let queue = crate::queue::Queue::at(home.join("queue"));
1463            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464            if let Some(prior) = state
1465                .base_sync_recovery
1466                .authorizations
1467                .iter()
1468                .find(|a| a.request.as_deref() == Some(&control.key()))
1469            {
1470                anyhow::ensure!(
1471                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472                    "recovery request was consumed or invalidated; it cannot queue again"
1473                );
1474                return control.enqueue(&queue);
1475            }
1476        }
1477        if resumption {
1478            anyhow::ensure!(
1479                crate::daemon::compatible_run_instruction(&state, &queued),
1480                "resumption instruction does not match the retained run"
1481            );
1482            let approval = queued
1483                .resumption
1484                .as_ref()
1485                .context("no operator resumption")?;
1486            anyhow::ensure!(
1487                approval.instruction == queued.instruction,
1488                "resumption approval does not match the current task instruction"
1489            );
1490            state.authorize_base_sync_resumption(task, approval)?;
1491        } else {
1492            state.authorize_base_sync(task)?;
1493        }
1494        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495        state
1496            .base_sync_recovery
1497            .authorizations
1498            .last_mut()
1499            .unwrap()
1500            .scope = Some(crate::base_sync_recovery::Scope {
1501            tip: approved_tip,
1502            instruction: queued.instruction.clone(),
1503            episode,
1504        });
1505        if let Some(control) = control.as_deref_mut() {
1506            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507            authorization.request = Some(control.key());
1508            authorization.generation = control.generation();
1509            authorization.decision = control.decision();
1510            authorization.question = control
1511                .request
1512                .decision
1513                .as_ref()
1514                .map(|_| control.request.request.clone());
1515        }
1516        state.event(
1517            "base-recovery",
1518            "owner authorized a bounded base synchronization interval; fresh verification required",
1519        );
1520        state.save_under(home)?;
1521        if let Some(control) = control {
1522            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523        }
1524        Ok(())
1525    }
1526
1527    /// Reopen an existing run.
1528    pub fn resume(id: &str) -> Result<Self> {
1529        let state = RunState::load(id)?;
1530        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531        if state.startup.as_ref().is_some_and(|s| !s.complete)
1532            || (state.status == RunStatus::Prep
1533                && state.instruction.trim().is_empty()
1534                && state.candidates.is_empty()
1535                && state.prep_complete == Some(false))
1536        {
1537            bail!(
1538                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539            );
1540        }
1541        crate::pr_head::ensure_owner(&state)?;
1542        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543            bail!(
1544                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545            );
1546        }
1547        if let Some(to) = &state.released_to {
1548            bail!(
1549                "run {} cannot be resumed: its worktree was released to run {}",
1550                state.short(),
1551                crate::run::short_of(to)
1552            );
1553        }
1554        let roles = state.config.resolve_roles()?;
1555        if let Some(id) = &state.config.graph.selected_implementer
1556            && state.candidates.iter().any(|c| c.agent != *id)
1557        {
1558            bail!(
1559                "replacement implementer pin does not match the saved candidates; task must stay held"
1560            );
1561        }
1562        let max_parallel = state.config.graph.max_parallel.max(1);
1563        Ok(Self {
1564            state,
1565            roles,
1566            sem: Arc::new(Semaphore::new(max_parallel)),
1567            pause: Pause::new(),
1568            interrupt: Pause::new(),
1569        })
1570    }
1571
1572    /// Walk the graph to a terminal state, skipping nodes already recorded.
1573    ///
1574    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1575    /// the phone - ends here, so this is the one place a run that ended
1576    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1577    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1578    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579        // Allocate before polling: boxing inside an async wrapper would still
1580        // retain its construction frame while the nested graph is polled.
1581        Box::pin(self.execute_inner())
1582    }
1583
1584    async fn execute_inner(&mut self) -> Result<()> {
1585        anyhow::ensure!(
1586            self.state.startup.as_ref().is_none_or(|s| s.complete),
1587            "startup incomplete; inspect retained evidence before explicit recovery"
1588        );
1589        if let Some(selection) = &self.state.pr_recovery
1590            && (selection.source_proof.is_some() || selection.source_run.is_some())
1591        {
1592            Box::pin(selection.verify_source(&self.state.repo)).await?;
1593        }
1594        // Queue claims cover daemon work; this also serializes manual drivers
1595        // consuming a repair answer, including the new review and decision.
1596        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597            || self.state.land_approval.is_some()
1598            || self.state.merge_decision.is_some()
1599            || !self.state.merge_repairs.is_empty()
1600            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601        {
1602            let claim = FixClaim::acquire(&self.state.dir())?;
1603            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604            let disk = RunState::load(&self.state.id)?;
1605            let checkpoint = |s: &RunState| {
1606                (
1607                    s.merge_repairs.len(),
1608                    s.merge_repairs
1609                        .last()
1610                        .map(|r| (r.question.clone(), r.phase)),
1611                    s.land_approval
1612                        .as_ref()
1613                        .map(|a| (a.question.clone(), a.head.clone())),
1614                )
1615            };
1616            if checkpoint(&disk) != checkpoint(&self.state) {
1617                bail!(
1618                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619                );
1620            }
1621            Some(claim)
1622        } else {
1623            None
1624        };
1625        let mut execution = self.claim_driver().await?;
1626        let runner = &mut *execution.runner;
1627        let wait = runner.repository_wait();
1628        // Keep the graph's large future off the callers' stack, including
1629        // standalone task attempts running on a default-sized test thread.
1630        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631        let result = match result {
1632            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633                record_repository_block(&mut runner.state, &e);
1634                runner.park_here()?;
1635                Ok(())
1636            }
1637            other => other,
1638        };
1639        execution.driver.finish(&mut runner.state).await?;
1640        let ended = if result.is_err() {
1641            Some({
1642                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643                if let Err(error) = &result {
1644                    notice.message = format!(
1645                        "Run {} stopped with an error: {}",
1646                        runner.state.id,
1647                        error.to_string().lines().next().unwrap_or("unknown error")
1648                    );
1649                }
1650                notice
1651            })
1652        } else {
1653            crate::notices::run_ended(&runner.state)
1654        };
1655        if let Some(notice) = ended {
1656            crate::notices::raise(notice);
1657        }
1658        result
1659    }
1660
1661    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663    }
1664
1665    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666        let wait = self.repository_wait();
1667        coordinate_operation(&mut self.state, cwd, &wait).await
1668    }
1669
1670    /// Validate ownership and persist the driver claim in one critical section.
1671    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672        let owner = match crate::pr_head::owner_lock_wait(
1673            &crate::run::home(),
1674            &self.state.id,
1675            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676        )
1677        .await
1678        {
1679            Ok(owner) => owner,
1680            Err(error) => {
1681                let detail = format!("worktree maintenance or ownership update: {error:#}");
1682                // Admission did not own run.json: retain this resource outcome
1683                // in memory for queue settlement without overwriting a concurrent
1684                // retirement or publication transaction.
1685                self.state.resource_block = Some(crate::run::ResourceBlock {
1686                    cause: crate::run::ResourceBlockCause::Other,
1687                    node: "start".into(),
1688                    kind: "worktree maintenance".into(),
1689                    detail: detail.clone(),
1690                    at: Timestamp::now(),
1691                });
1692                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693            }
1694        };
1695        let disk = RunState::load(&self.state.id)?;
1696        crate::pr_head::ensure_owner(&disk)?;
1697        let claimed =
1698            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699        // Only proven-live ownership refuses here. Unknown records (legacy runs
1700        // with no claim marker or process identity) fall through to the
1701        // exclusive execution lease taken just below, which is what actually
1702        // keeps a second driver out.
1703        anyhow::ensure!(
1704            disk.liveness(claimed) != crate::run::Liveness::Live,
1705            "run already has a live driver; wait for its execution to end"
1706        );
1707        if let Some(to) = &disk.released_to {
1708            bail!(
1709                "run {} cannot continue: its worktree was released to run {}",
1710                self.state.short(),
1711                crate::run::short_of(to)
1712            );
1713        }
1714        anyhow::ensure!(
1715            serde_json::to_value(&disk.base_sync_recovery)?
1716                == serde_json::to_value(&self.state.base_sync_recovery)?,
1717            "base recovery changed since this run was loaded; reload the saved run"
1718        );
1719        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720        let mut execution = DriverExecution {
1721            runner: self,
1722            driver,
1723        };
1724        // Declare after the execution guard: on any claim error this short
1725        // lock must drop before that guard reloads and releases ownership.
1726        let _owner = owner;
1727        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728            .await?;
1729        let state = &mut execution.runner.state;
1730        // Moving again, so it is no longer parked. Set before the walk rather
1731        // than in `resume`, so every way of re-entering the graph clears it
1732        // and a card cannot claim a run is waiting to be resumed while the
1733        // agents are already working.
1734        state.parked = false;
1735        state.resource_wait = None;
1736        state.resource_block = None;
1737        // Any seat this state still lists as answering belongs to whatever
1738        // process last drove this run — this one included, if it crashed
1739        // mid-wave. Cleared and flushed immediately, before anything else
1740        // runs, so a resume can never show a seat as live when nothing is
1741        // asking it anything yet; the node that actually dispatches the next
1742        // wave repopulates it.
1743        state.clear_active();
1744        // Recorded in the same spot, and flushed together with the clear
1745        // above: this is the pid a reader checks (`RunState::liveness`) when
1746        // no daemon claim exists to answer "is a process still driving this
1747        // run" — a plain `magi run` / `magi review` typed into a terminal
1748        // claims nothing there. Always overwritten, never only-if-absent, so
1749        // a resumed run's stale pid from a previous, possibly-dead process
1750        // can never survive into this one's own report. Unlike
1751        // `clear_active`, this changes on every single `execute()` call, so
1752        // the save below is now unconditional rather than only-if-cleared.
1753        //
1754        // `driver_started_at` is recorded in the same breath, from this same
1755        // pid, so `liveness` can tell a live pid that is genuinely still us
1756        // apart from one the OS has since handed to an unrelated process —
1757        // see that field's own doc for why the pid alone is not enough.
1758        let pid = std::process::id();
1759        state.driver_pid = Some(pid);
1760        state.driver_started_at = crate::proc::process_started_at(pid);
1761        state.driver_exited = false;
1762        state.driver_claimed = Some(true);
1763        state.begin_pass();
1764        execution.driver.bind(state);
1765        state.save()?;
1766        execution.driver.record(state)?;
1767        Ok(execution)
1768    }
1769
1770    /// No conflict checkpoint proves dispatch was never reached. End that
1771    /// consumed generation so an owner can explicitly authorize a fresh one.
1772    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774        if let Err(error) = &result
1775            && !error.is::<crate::repo_lock::ResourceBlocked>()
1776            && self
1777                .state
1778                .repair_grants
1779                .last()
1780                .is_some_and(|g| g.conflict.is_none())
1781        {
1782            self.state.repair_grants.last_mut().unwrap().phase =
1783                crate::repair_budget::Phase::Complete;
1784            self.state.status = RunStatus::Blocked;
1785            self.state.save()?;
1786            bail!(
1787                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788                self.state.passes.last().map_or(0, |p| p.index)
1789            );
1790        }
1791        result
1792    }
1793
1794    /// A grant owns only these repair attempts. Confirmation reviews cannot
1795    /// create another fixer allowance, and every operation is restart-checkpointed.
1796    async fn execute_granted_repair(&mut self) -> Result<()> {
1797        use crate::repair_budget::{Kind, Phase};
1798        let index = self.state.repair_grants.len() - 1;
1799        let winner = self
1800            .state
1801            .winner()
1802            .cloned()
1803            .context("repair winner missing")?;
1804        loop {
1805            let grant = self.state.repair_grants[index].clone();
1806            match grant.phase {
1807                Phase::Pending => {
1808                    if grant.spent >= grant.receipt.request.rounds {
1809                        self.state.repair_grants[index].phase = Phase::Complete;
1810                        self.state.status = RunStatus::Blocked;
1811                        self.state.save()?;
1812                        return Ok(());
1813                    }
1814                    // Reserve before dispatch. A crash can consume an attempt,
1815                    // but can never duplicate it or reset saved fixer history.
1816                    self.state.repair_grants[index].spent += 1;
1817                    self.state.repair_grants[index].phase = Phase::Fixing;
1818                    self.state.save()?;
1819                    match grant.receipt.request.kind {
1820                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821                        Kind::Gate => {
1822                            let failed = self.state.gate.clone();
1823                            let _ = self.gate_fix_round(&winner, &failed).await?;
1824                        }
1825                        Kind::Review => {
1826                            let mut record = self
1827                                .state
1828                                .reviews
1829                                .last()
1830                                .cloned()
1831                                .context("review failure missing")?;
1832                            let round = record.round;
1833                            let step = self.run_fix_round(&winner, &mut record).await?;
1834                            self.state.reviews[round - 1] = record;
1835                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1836                                self.state.repair_grants[index].phase = Phase::Complete;
1837                                self.state.status = RunStatus::Blocked;
1838                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839                                self.state.save()?;
1840                                return Ok(());
1841                            }
1842                            if matches!(step, FixStep::Resource) {
1843                                self.stop_for_resource()?;
1844                                return Ok(());
1845                            }
1846                            self.state.save()?;
1847                        }
1848                    }
1849                    if self.resource_halt()? {
1850                        return Ok(());
1851                    }
1852                }
1853                Phase::Fixing => {
1854                    if grant.receipt.request.kind == Kind::BaseConflict {
1855                        self.repair_base_conflict(false).await?;
1856                    }
1857                    // Interrupted fixers may have left source changes or an
1858                    // unpushed commit. Preserve them before fresh assessment.
1859                    let _coordination = self.coordinate(&winner.worktree).await?;
1860                    let rescued = git::rescue_commit(
1861                        &winner.worktree,
1862                        "magi: preserve interrupted granted repair",
1863                    )
1864                    .await?;
1865                    self.state.note_withheld("repair-grant", &rescued.withheld);
1866                    // Fresh checks must supersede even a same-head failure.
1867                    // Archive rather than erase failure evidence or counters.
1868                    self.state
1869                        .verification_history
1870                        .push(crate::run::VerificationEvidence {
1871                            superseded_by: None,
1872                            archived_at: Some(Timestamp::now()),
1873                            fixes_spent: Some(self.state.gate_fixes.len()),
1874                            gate_head: self.state.gate_head.take(),
1875                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1876                            gate: std::mem::take(&mut self.state.gate),
1877                            e2e: self.state.head_e2e.take(),
1878                        });
1879                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880                        self.state
1881                            .verification_history
1882                            .push(crate::run::VerificationEvidence {
1883                                superseded_by: None,
1884                                archived_at: Some(Timestamp::now()),
1885                                fixes_spent: Some(self.state.gate_fixes.len()),
1886                                gate_head: None,
1887                                gate_ran: false,
1888                                gate: Vec::new(),
1889                                e2e: Some(e2e),
1890                            });
1891                    }
1892                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1893                    self.state.repair_grants[index].head =
1894                        git::rev_parse(&winner.worktree, "HEAD").await?;
1895                    self.state.repair_grants[index].phase = Phase::Reviewing;
1896                    if grant.receipt.request.kind == Kind::BaseConflict {
1897                        self.state.base_sync_recovery.stop = None;
1898                        if let Some(sync) = &mut self.state.base_sync {
1899                            sync.tip = grant
1900                                .receipt
1901                                .request
1902                                .tip
1903                                .clone()
1904                                .context("repair base tip missing")?;
1905                            sync.conflict = None;
1906                            sync.behind = 0;
1907                        }
1908                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909                    }
1910                    self.state.save()?;
1911                }
1912                Phase::Reviewing => {
1913                    self.review_loop_pass().await?;
1914                    if self.resource_halt()? {
1915                        return Ok(());
1916                    }
1917                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918                    let clean = self.state.reviews.len() > grant.review_start
1919                        && self.state.reviews.last().is_some_and(|r| {
1920                            r.head == head
1921                                && r.clean
1922                                && r.expected > 0
1923                                && r.answered == r.expected
1924                                && matches!(
1925                                    r.e2e_status(),
1926                                    E2eStatus::Passed | E2eStatus::NotConfigured
1927                                )
1928                        });
1929                    if !clean {
1930                        // A confirmation review carries no additional fix budget.
1931                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932                            == Kind::Review
1933                            && grant.spent < grant.receipt.request.rounds
1934                            && self.state.reviews.last().is_some_and(|r| {
1935                                r.fix.is_none()
1936                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937                            }) {
1938                            Phase::Pending
1939                        } else {
1940                            Phase::Complete
1941                        };
1942                        self.state.status = RunStatus::Blocked;
1943                        self.state.event(
1944                            "repair-grant",
1945                            "fresh review did not clear the repaired head; no publication",
1946                        );
1947                        self.state.save()?;
1948                        if self.state.repair_grants[index].phase == Phase::Pending {
1949                            continue;
1950                        }
1951                        return Ok(());
1952                    }
1953                    self.state.repair_grants[index].phase = Phase::Gating;
1954                    self.state.save()?;
1955                }
1956                Phase::Gating => {
1957                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958                    self.state.reconcile_verification_head(&head);
1959                    let wait = self.repository_wait();
1960                    if self
1961                        .state
1962                        .reviews
1963                        .last()
1964                        .is_none_or(|r| r.head != head || !r.clean)
1965                    {
1966                        self.state.status = RunStatus::Blocked;
1967                        self.state.repair_grants[index].phase = Phase::Complete;
1968                        self.state.event(
1969                            "repair-grant",
1970                            "head changed since fresh review; repair held",
1971                        );
1972                        self.state.save()?;
1973                        return Ok(());
1974                    }
1975                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976                        return Ok(());
1977                    }
1978                    let outcomes = self.run_gate(&winner).await?;
1979                    if verify_inconclusive(&outcomes) {
1980                        return Ok(());
1981                    }
1982                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983                    if after != head {
1984                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1985                        self.state.repair_grants[index].head = after;
1986                        self.state.repair_grants[index].phase = Phase::Reviewing;
1987                        self.state.status = RunStatus::Blocked;
1988                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989                        self.state.save()?;
1990                        return Ok(());
1991                    }
1992                    let passed = outcomes.iter().all(CommandOutcome::ok);
1993                    // Follow-up grants owe the same guarded checkout delivery
1994                    // as the conflict repair that introduced the new head.
1995                    if passed
1996                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997                    {
1998                        return Ok(());
1999                    }
2000                    self.state.repair_grants[index].succeeded = passed;
2001                    self.state.gate = outcomes;
2002                    self.state.gate_ran = true;
2003                    self.state.status = if passed {
2004                        RunStatus::Gating
2005                    } else {
2006                        RunStatus::Blocked
2007                    };
2008                    self.state.repair_grants[index].phase = if !passed
2009                        && grant.receipt.request.kind == Kind::Gate
2010                        && grant.spent < grant.receipt.request.rounds
2011                    {
2012                        Phase::Pending
2013                    } else {
2014                        Phase::Complete
2015                    };
2016                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2017                        // Persist publication debt with the successful checkpoint.
2018                        self.state.pr_publication_pending = true;
2019                    } else if passed {
2020                        self.state.status = RunStatus::Ready;
2021                    }
2022                    self.state.save()?;
2023                    if !passed {
2024                        continue;
2025                    }
2026                    // Existing PR ownership and push lease stay in place. The
2027                    // normal publisher owes the new reviewed/gated SHA, and
2028                    // land's fresh owner approval remains mandatory.
2029                    if self.state.config.merge.mode != MergeMode::Pr {
2030                        self.state.status = RunStatus::Ready;
2031                        self.state.event(
2032                            "repair-grant",
2033                            "repair verified locally; grant carries no merge authorization",
2034                        );
2035                        self.state.save()?;
2036                        return Ok(());
2037                    }
2038                    return self.merge().await;
2039                }
2040                Phase::Complete => return Ok(()),
2041            }
2042        }
2043    }
2044
2045    async fn execute_graph(&mut self) -> Result<()> {
2046        // Even a terminal run may reenter through an explicit recovery action.
2047        // Restore before prep shortcuts, saved conversations or graph nodes use
2048        // a disposable path; GC may already have removed its original objects.
2049        let archive_home = crate::run::home().canonicalize()?;
2050        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051            let repo = self.state.repo.clone();
2052            let _coordination = self.coordinate(&repo).await?;
2053            ensure_setup_config(&mut self.state, &repo).await?;
2054            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055        }
2056        self.state.configure_recovery_resume()?;
2057        if self
2058            .state
2059            .repair_grants
2060            .last()
2061            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062        {
2063            return self.execute_granted_repair().await;
2064        }
2065        if self
2066            .state
2067            .repair_grants
2068            .last()
2069            .is_some_and(|g| !g.succeeded)
2070        {
2071            self.state.status = RunStatus::Blocked;
2072            self.state.save()?;
2073            return Ok(());
2074        }
2075        // Preservation may invalidate an old empty classification. Resolve it
2076        // before any resume shortcut can reuse a verdict or publish a branch.
2077        self.persist_candidates().await?;
2078        // A saved base stop outranks stage markers, including legacy Voting.
2079        // Recovery never reenters implementation, deliberation or voting.
2080        if self.state.base_sync_blocked() {
2081            self.sync_to_base().await?;
2082            if !self.state.base_sync_blocked()
2083                && self.state.status != RunStatus::AlreadyInBase
2084                && !self.resource_halt()?
2085            {
2086                self.finish_after_tally().await?;
2087            }
2088            return Ok(());
2089        }
2090        if self
2091            .state
2092            .merge_decision
2093            .as_ref()
2094            .is_some_and(|decision| !decision.closed)
2095            && self.state.status != RunStatus::Landing
2096        {
2097            if crate::merge_decision::resumable_hold(&self.state) {
2098                self.state.merge = Some(MergeOutcome {
2099                    mode: MergeMode::Pr,
2100                    ok: false,
2101                    detail: self
2102                        .state
2103                        .merge_decision
2104                        .as_ref()
2105                        .unwrap()
2106                        .target
2107                        .pr
2108                        .clone(),
2109                    empty: false,
2110                });
2111                self.run_land().await?;
2112                self.settle_questions();
2113                return Ok(());
2114            }
2115            bail!(
2116                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117            );
2118        }
2119        if self.state.config.graph.selected_implementer.is_some()
2120            && self.state.status == RunStatus::Blocked
2121            && self.state.tally.is_none()
2122            && self.state.candidate_preservation.is_empty()
2123            && self.state.interrupted_implementations.is_empty()
2124        {
2125            bail!(
2126                "selected implementation is held; explicitly release the owning task to retry it"
2127            );
2128        }
2129
2130        if self.state.merge_repairs.last().is_some_and(|r| {
2131            matches!(
2132                r.phase,
2133                crate::run::MergeRepairPhase::Requested
2134                    | crate::run::MergeRepairPhase::Fixing
2135                    | crate::run::MergeRepairPhase::Reviewing
2136            )
2137        }) {
2138            if self.repair_merge_review().await? {
2139                self.run_land().await?;
2140            }
2141            self.settle_questions();
2142            return Ok(());
2143        }
2144        // Publication already passed review and gate. Resume only that node:
2145        // syncing to a newer base here could change the verified commit.
2146        if self.state.pr_publication_pending {
2147            self.merge().await?;
2148            if self.resource_halt()? {
2149                return Ok(());
2150            }
2151            self.state.save()?;
2152            self.settle_questions();
2153            return Ok(());
2154        }
2155        // A run that already lost its quorum never resumes into the verdict
2156        // machinery: `deliberate` and `vote` would otherwise clobber the
2157        // stalled marker back to Voting and the run would keep going past a
2158        // verdict that is no longer trustworthy. Everything already recorded is
2159        // kept, so the run stays resumable (or foldable) for a human to pick up.
2160        //
2161        // On --resume the run gets one chance to repair itself: the seats a
2162        // rate limit took out are re-asked. If their quota has since reset and
2163        // the quorum is restored, the run picks up and finishes; otherwise it
2164        // stays stale and still-resumable for a later retry. If it does not
2165        // recover, the returned status stays `Stalled` and nothing was
2166        // clobbered (the recovery only mutates entries for the lost seats).
2167        if self.state.status == RunStatus::Stalled {
2168            if self.recover_stall().await? {
2169                self.finish_after_tally().await?;
2170            } else {
2171                // Still below quorum: persist the marker and stay resumable.
2172                self.state.save()?;
2173            }
2174            return Ok(());
2175        }
2176        // A run parked inside `land` - watching CI, mid fix-round, or
2177        // waiting on the owner's merge approval - resumes directly into it,
2178        // never back through `prep`. Everything before `merge` already
2179        // concluded; that is the only way `status` reaches `Landing` in the
2180        // first place. Re-walking `review_loop` first would also be actively
2181        // wrong: its own status recomputation (see its doc) treats any
2182        // clean round as reason to set `status` to `Gating`, which would
2183        // clobber this marker before `merge` ever ran, and this run would
2184        // never find its way back into `land` at all.
2185        if self.state.status == RunStatus::Landing
2186            || crate::handover::landing_checkpoint(&self.state)
2187        {
2188            self.state.land_sync_pending = true;
2189            self.state.save()?;
2190            self.run_land().await?;
2191            // `run_land` may have settled the run right here - CI came back
2192            // green and the PR merged, say - without ever passing back
2193            // through `merge`'s own trailing call. Whatever it left `status`
2194            // as is what this has to read.
2195            self.settle_questions();
2196            return Ok(());
2197        }
2198        // A run parked at `merge` on the owner's word about a withheld PR text
2199        // resumes straight into `merge`. Re-walking the tail would sync to the
2200        // base again (possibly rebasing, possibly spending the fixer) after the
2201        // gate had passed, and then push a tree nobody reviewed or gated.
2202        if self.state.github_text.is_some()
2203            && self.state.merge.is_none()
2204            && self.state.status == RunStatus::Gating
2205        {
2206            self.merge().await?;
2207            if self.resource_halt()? {
2208                return Ok(());
2209            }
2210            self.state.save()?;
2211            return Ok(());
2212        }
2213        if let Err(e) = self.prep().await {
2214            if e.is::<crate::repo_lock::ResourceBlocked>() {
2215                return Err(e);
2216            }
2217            self.state.status = RunStatus::Blocked;
2218            self.state
2219                .prep_failure
2220                .get_or_insert_with(|| format!("{e:#}"));
2221            self.state
2222                .event("prep", format!("preparation stopped: {e:#}"));
2223            self.state.save()?;
2224            return Err(e);
2225        }
2226        if self.park_here()? {
2227            return Ok(());
2228        }
2229        self.advise().await?;
2230        if self.park_here()? {
2231            return Ok(());
2232        }
2233        self.implement().await?;
2234        if self.park_here()? {
2235            return Ok(());
2236        }
2237        // `after_implement` already saved the state and settled any open
2238        // questions when it set this; nothing later in the graph has
2239        // anything to judge.
2240        if self.state.status == RunStatus::VerifiedNoop {
2241            return Ok(());
2242        }
2243        self.judge().await?;
2244        if self.park_here()? {
2245            return Ok(());
2246        }
2247        self.deliberate().await?;
2248        if self.park_here()? {
2249            return Ok(());
2250        }
2251        self.vote().await?;
2252        if self.park_here()? {
2253            return Ok(());
2254        }
2255        self.tally()?;
2256        // A verdict that lost its quorum is not trustworthy: do not review,
2257        // gate, or merge on it. Everything already done is kept, so the run
2258        // stays resumable (or foldable); the human can replace the agent that
2259        // ran out of quota and pick it up.
2260        if self.state.status == RunStatus::Stalled {
2261            // Persist the stalled marker now — the normal end-of-execute save
2262            // below is below this early return, and without it a resumed run
2263            // would reload a pre-tally status and keep going.
2264            self.state.save()?;
2265            return Ok(());
2266        }
2267        self.finish_after_tally().await?;
2268        Ok(())
2269    }
2270
2271    /// A wave could not start its agents because a shared resource stayed busy
2272    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2273    /// resumable. Unstarted seats have no result; completed peers retain
2274    /// their saved candidate state.
2275    fn stop_for_resource(&mut self) -> Result<()> {
2276        // Resource acquisition observes cancellation inside a node. Complete
2277        // the same durable park as every other node boundary before returning.
2278        if self.pause.parked() || self.interrupt.parked() {
2279            self.park_here()?;
2280            return Ok(());
2281        }
2282        if self.state.resource_block.is_some() {
2283            self.state.status = RunStatus::Blocked;
2284        }
2285        self.state.save()
2286    }
2287
2288    /// Stop walking the graph if a node ended on a resource block.
2289    fn resource_halt(&mut self) -> Result<bool> {
2290        if self.state.resource_block.is_none() {
2291            return Ok(false);
2292        }
2293        self.stop_for_resource()?;
2294        Ok(true)
2295    }
2296
2297    /// Park here if asked to, recording it in the run's own timeline.
2298    ///
2299    /// Returns whether the caller should stop walking the graph. The state is
2300    /// saved either way by the node that just finished; this adds the event so
2301    /// the operator's card says why a run that is neither finished nor moving
2302    /// is sitting where it is.
2303    fn park_here(&mut self) -> Result<bool> {
2304        let parking = self.pause.parked() || self.interrupt.parked();
2305        if !parking && self.state.resource_block.is_some() {
2306            self.stop_for_resource()?;
2307            return Ok(true);
2308        }
2309        // Either handle asking is enough - see `Pause`'s own doc for why
2310        // they are never the same one. `interrupt` is checked second so a
2311        // reason it carries is preferred in the message below over a plain
2312        // shutdown park racing it at the same boundary.
2313        if !self.pause.parked() && !self.interrupt.parked() {
2314            return Ok(false);
2315        }
2316        if self.state.parked {
2317            return Ok(true);
2318        }
2319        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320        if let Some(block) = &mut self.state.resource_block
2321            && matches!(
2322                block.cause,
2323                crate::run::ResourceBlockCause::Cancelled { .. }
2324            )
2325        {
2326            block.cause = crate::run::ResourceBlockCause::Cancelled {
2327                park: cause.clone(),
2328            };
2329        }
2330        let why = match cause.as_ref().map(ParkCause::label) {
2331            Some(reason) => format!(
2332                "parked after `{}` ({reason}) — resume to carry on from here",
2333                self.state.status.as_str()
2334            ),
2335            None => format!(
2336                "parked after `{}` — resume to carry on from here",
2337                self.state.status.as_str()
2338            ),
2339        };
2340        self.state.event("park", why);
2341        self.state
2342            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343        self.state.parked = true;
2344        self.state.save()?;
2345        Ok(true)
2346    }
2347
2348    /// Only a new explicit operator approval may retry failed preparation.
2349    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350        if self.state.prep_failure.is_none() {
2351            return Ok(());
2352        }
2353        let failed_at = self
2354            .state
2355            .events
2356            .iter()
2357            .rev()
2358            .find(|event| event.node == "prep")
2359            .map_or(self.state.updated_at, |event| event.at);
2360        anyhow::ensure!(
2361            approval_at > failed_at,
2362            "preparation retry requires a new explicit release after the failure"
2363        );
2364        self.state.prep_failure = None;
2365        self.state.prep_complete = Some(false);
2366        self.state
2367            .event("prep-retry", "operator approved retry of preparation");
2368        Ok(())
2369    }
2370
2371    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2372    pub fn on_pause(&mut self, pause: Pause) {
2373        self.pause = pause;
2374    }
2375
2376    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2377    /// scheduler asking this one run - and no other - to park so a task
2378    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2379    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2380    /// handle.
2381    pub fn watch_interrupt(&mut self, pause: Pause) {
2382        self.interrupt = pause;
2383    }
2384
2385    /// Abandon this run's own open questions, once `status` has actually
2386    /// settled rather than merely paused.
2387    ///
2388    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2389    /// either back up with the candidates, the review round and the seat
2390    /// sessions already on disk, so a question an implementer asked mid-round
2391    /// may still get a real answer read by a real resume. Only the statuses
2392    /// `resumable` excludes are actually final: the run merged, it reached
2393    /// `Ready` with nothing left to do, it failed outright with no
2394    /// established point to continue from, or every candidate agreed, with
2395    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2396    /// every one of those the seat that asked is gone for good, exactly like
2397    /// the run being deleted under `magi run rm` - so the same cleanup
2398    /// applies, worded for what actually happened instead of "the run was
2399    /// deleted".
2400    ///
2401    /// Best-effort and silent on success: called from every place `status`
2402    /// can land on one of those three, including ones a resumed run revisits,
2403    /// so it must cost nothing when there was nothing open to begin with.
2404    fn settle_questions(&mut self) {
2405        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407        }
2408    }
2409
2410    /// The tail of the graph after a trustworthy tally: fold losers, review,
2411    /// gate, merge, and persist.
2412    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413        Box::pin(self.finish_after_tally_inner())
2414    }
2415
2416    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417        // Before the losers are folded: their branches are what the graft
2418        // reads from.
2419        self.graft().await?;
2420        if self.resource_halt()? {
2421            return Ok(());
2422        }
2423        self.fold_losers().await?;
2424        // Before review starts, and again right before the gate: a run's
2425        // review rounds can themselves take long enough for the base to move
2426        // a second time, and the gate is the one node whose "green" gets
2427        // acted on.
2428        self.sync_to_base().await?;
2429        if self.hold_base_sync().await? || self.resource_halt()? {
2430            return Ok(());
2431        }
2432        if self.state.status == RunStatus::AlreadyInBase {
2433            return Ok(());
2434        }
2435        self.review_loop().await?;
2436        if self.resource_halt()? {
2437            return Ok(());
2438        }
2439        self.sync_to_base().await?;
2440        if self.hold_base_sync().await? || self.resource_halt()? {
2441            return Ok(());
2442        }
2443        if self.state.status == RunStatus::AlreadyInBase {
2444            return Ok(());
2445        }
2446        self.gate().await?;
2447        self.merge().await?;
2448        if self.resource_halt()? {
2449            return Ok(());
2450        }
2451        self.state.save()?;
2452        Ok(())
2453    }
2454
2455    // ---------------------------------------------------------------- prep
2456
2457    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2458    /// run with the step and its output in the event; it is never swallowed.
2459    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2460    /// resume starts over instead of finding candidates and missing seats.
2461    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462        let result = match ensure_setup_config(&mut self.state, repo).await {
2463            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464            Err(e) => Err(e),
2465        };
2466        if let Err(e) = result {
2467            git::worktree_remove_clean(repo, wt).await.ok();
2468            if in_prep {
2469                self.rollback_prep(repo).await;
2470            }
2471            self.state.status = RunStatus::Blocked;
2472            self.state.event(
2473                "setup",
2474                format!("worktree setup failed in {}: {e:#}", wt.display()),
2475            );
2476            self.state.save()?;
2477            return Err(e);
2478        }
2479        Ok(())
2480    }
2481
2482    /// Undo everything `prep` created, so a resume re-runs it whole.
2483    async fn rollback_prep(&mut self, repo: &Path) {
2484        let root = self.state.worktree_root();
2485        for c in std::mem::take(&mut self.state.candidates) {
2486            if matches!(
2487                git::worktree_remove_clean(repo, &c.worktree).await,
2488                Ok(true)
2489            ) {
2490                git::branch_delete(repo, &c.branch).await.ok();
2491            } else {
2492                self.state.candidates.push(c);
2493            }
2494        }
2495        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496        for j in 1..=seats {
2497            remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498                .await
2499                .ok();
2500        }
2501        for k in 1..=self.state.config.graph.advisors {
2502            remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503                .await
2504                .ok();
2505        }
2506    }
2507
2508    async fn prep(&mut self) -> Result<()> {
2509        if let Some(reason) = &self.state.prep_failure {
2510            bail!(
2511                "candidate preparation previously failed: {reason}; correct the source before retrying"
2512            );
2513        }
2514        if self
2515            .state
2516            .prep_complete
2517            .unwrap_or(!self.state.candidates.is_empty())
2518        {
2519            return Ok(());
2520        }
2521        let repo = self.state.repo.clone();
2522        // Includes rollback, config, refs, checkout creation and setup. No
2523        // repository mutation may precede the real shared lock acquisition.
2524        let _coordination = self.coordinate(&repo).await?;
2525        let base = self.state.base_commit.clone();
2526        let plan = self.state.sources.plan(&repo, &base).await?;
2527        self.rollback_prep(&repo).await;
2528        if !self.state.candidates.is_empty() {
2529            bail!(
2530                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531            );
2532        }
2533        self.state.prep_complete = Some(false);
2534        self.state.status = RunStatus::Prep;
2535        self.state.save()?;
2536        let repo = self.state.repo.clone();
2537        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538        let root = self.state.worktree_root();
2539        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541        // The hook is the write-time half of the blindness contract; the
2542        // presentation filter in `blind` is the half that cannot be bypassed.
2543        let hooks_dir = self.state.dir().join("hooks");
2544        if self.state.config.blind.commit_msg_hook {
2545            std::fs::create_dir_all(&hooks_dir)
2546                .with_context(|| format!("create {}", hooks_dir.display()))?;
2547            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548            let path = hooks_dir.join("commit-msg");
2549            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550            make_executable(&path)?;
2551            // Ref-counted rather than a plain idempotent set: with more than
2552            // one run able to be in flight in the same repository at once
2553            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2554            // true?" check cannot tell "another run of mine still needs
2555            // this" from "nobody does", and the run that happens to finish
2556            // first would disable the hook out from under a sibling still
2557            // relying on it.
2558            git::acquire_worktree_config(&repo).await?;
2559            self.state.enabled_worktree_config = true;
2560        }
2561
2562        // `[worktree] setup` hides its products per worktree, which needs the
2563        // same per-worktree config the hook does. Held until fold, like it.
2564        ensure_setup_config(&mut self.state, &repo).await?;
2565
2566        for (index, (spec, label)) in self
2567            .roles
2568            .implementers
2569            .clone()
2570            .into_iter()
2571            .zip(labels)
2572            .enumerate()
2573        {
2574            let branch = self.state.branch_for(label);
2575            let worktree = root.join(format!("cand-{label}"));
2576            // Persist intent before Git can leave a partially created tree.
2577            // Candidate registration follows setup, so it cannot inventory
2578            // crashes or refusals during creation itself.
2579            self.state.event(
2580                "prep-intent",
2581                serde_json::json!({
2582                    "branch": branch,
2583                    "worktree": worktree,
2584                    "start": start,
2585                    "picks": plan.picks,
2586                })
2587                .to_string(),
2588            );
2589            self.state.save()?;
2590            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591            if self.state.config.blind.commit_msg_hook {
2592                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593            }
2594            git::local_exclude(&worktree, "/.magi/").await?;
2595            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596                // Nothing of this prep may survive: a resume must start over,
2597                // not find half the candidates and skip the rest.
2598                git::worktree_remove_clean(&repo, &worktree).await.ok();
2599                git::branch_delete(&repo, &branch).await.ok();
2600                self.rollback_prep(&repo).await;
2601                self.state.status = RunStatus::Blocked;
2602                self.state
2603                    .event("prep", format!("worktree setup failed: {e:#}"));
2604                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605                self.state.save()?;
2606                return Err(e);
2607            }
2608            self.state.candidates.push(Candidate {
2609                index,
2610                label,
2611                agent: spec.id.clone(),
2612                branch,
2613                worktree,
2614                summary: String::new(),
2615                stat: String::new(),
2616                files: 0,
2617                commits: 0,
2618                empty: false,
2619                failed: None,
2620                verified_noop: None,
2621                duration_ms: 0,
2622                folded: false,
2623            });
2624            self.state.save()?;
2625            for pick in &plan.picks {
2626                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627                {
2628                    self.state.status = RunStatus::Blocked;
2629                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630                    self.state
2631                        .event("prep", format!("cannot apply referenced commit: {e}"));
2632                    self.state.save()?;
2633                    return Err(e);
2634                }
2635            }
2636        }
2637
2638        for j in 1..=self.roles.judges.len() {
2639            let wt = root.join(format!("judge-{j}"));
2640            if !wt.exists() {
2641                git::worktree_add_detached(&repo, &wt, &base).await?;
2642                self.setup_seat_worktree(&repo, &wt, true).await?;
2643            }
2644        }
2645
2646        // Disposable, detached checkouts for the design-deliberation stage's
2647        // advisor seats — the same shape as the judges' above, at the same
2648        // base commit, since advisors also only ever read. Sized off the
2649        // configured count directly rather than a resolved roster: unlike
2650        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2651        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2652        // `prep` has no `ResolvedRoles` field to read a count from here.
2653        if self.state.config.graph.advise {
2654            for k in 1..=self.state.config.graph.advisors {
2655                let wt = root.join(format!("advisor-{k}"));
2656                if !wt.exists() {
2657                    git::worktree_add_detached(&repo, &wt, &base).await?;
2658                    self.setup_seat_worktree(&repo, &wt, true).await?;
2659                }
2660            }
2661        }
2662
2663        // A judge cannot tell it is looking at its own patch — the seats keep
2664        // separate conversations — but a panel that shares agents with the
2665        // field is less independent than it looks, and that is worth saying out
2666        // loud once per run rather than leaving it in the config.
2667        let authors: Vec<&str> = self
2668            .roles
2669            .implementers
2670            .iter()
2671            .map(|a| a.id.as_str())
2672            .collect();
2673        let overlap: Vec<String> = self
2674            .roles
2675            .judges
2676            .iter()
2677            .enumerate()
2678            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680            .collect();
2681        if !overlap.is_empty() {
2682            let note = format!(
2683                "{} also authored a candidate; blind, but the panel is less \
2684                 independent than {} distinct agents would be",
2685                overlap.join(", "),
2686                self.roles.judges.len()
2687            );
2688            self.state.event("prep", note);
2689        }
2690
2691        self.state.event(
2692            "prep",
2693            format!(
2694                "{} candidates, {} judges, base {} ({})",
2695                self.state.candidates.len(),
2696                self.roles.judges.len(),
2697                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698                self.state.base_branch
2699            ),
2700        );
2701        self.state.prep_complete = Some(true);
2702        self.state.status = RunStatus::Implementing;
2703        self.state.save()?;
2704        Ok(())
2705    }
2706
2707    // -------------------------------------------------------------- advise
2708
2709    /// The design-deliberation stage: independent, read-only advisor seats
2710    /// each sketch a design before any implementer touches the repository,
2711    /// and (when at least one produced a usable proposal) a synthesis seat
2712    /// blends them into a brief `implement` carries in every candidate's
2713    /// prompt.
2714    ///
2715    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2716    /// advisors` is the proposal count. Everything here is best-effort and
2717    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2718    /// that cannot reach quota, or a synthesis seat that produced nothing
2719    /// usable all leave `implement` exactly as it was before this stage
2720    /// existed — the task instruction alone — rather than failing the whole
2721    /// competition over an enrichment stage. Every outcome is still recorded
2722    /// as an event, so a run that got nothing from this stage says why.
2723    ///
2724    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2725    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2726    /// resumed run whose stage failed would re-run it, and re-spend the
2727    /// agent calls, on every reentry before `implement`.
2728    ///
2729    /// Also skipped once any candidate shows implementation progress — the
2730    /// exact predicate `implement` itself uses to decide a candidate is no
2731    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2732    /// is not enough: a run created by an older binary that predates this
2733    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2734    /// an already-`Implementing`-or-later run under this build would
2735    /// otherwise walk straight back through `prep` (a no-op once candidates
2736    /// exist) into this node and spawn every advisor seat against worktrees
2737    /// `prep` never recreated — after implementation has already started,
2738    /// which is exactly the invariant this stage exists to guarantee.
2739    async fn advise(&mut self) -> Result<()> {
2740        let implement_untouched = self
2741            .state
2742            .candidates
2743            .iter()
2744            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745        if !self.state.config.graph.advise || self.state.advise_attempted {
2746            return Ok(());
2747        }
2748        if !implement_untouched {
2749            self.state.event(
2750                "advise",
2751                "skipping the design-deliberation stage: at least one \
2752                 candidate already shows implementation progress, so this \
2753                 run is past the point the stage exists to run before"
2754                    .to_owned(),
2755            );
2756            self.state.advise_attempted = true;
2757            self.state.save()?;
2758            return Ok(());
2759        }
2760        let run_id = self.state.id.clone();
2761        let prompts = self.state.config.prompts.clone();
2762        let instruction = self.state.instruction.clone();
2763        let language = self.state.config.graph.language.clone();
2764        let root = self.state.worktree_root();
2765        let n = self.state.config.graph.advisors;
2766        let where_recorded = self.state.dir().join("run.json");
2767
2768        let seats = match self.state.config.advisors() {
2769            Ok(seats) if !seats.is_empty() => seats,
2770            Ok(_) => {
2771                self.state.event(
2772                    "advise",
2773                    format!(
2774                        "[graph] advisors is 0; skipping the design-deliberation \
2775                         stage and continuing without a synthesis brief (see {})",
2776                        where_recorded.display()
2777                    ),
2778                );
2779                self.state.advise_attempted = true;
2780                self.state.save()?;
2781                return Ok(());
2782            }
2783            Err(e) => {
2784                self.state.event(
2785                    "advise",
2786                    format!(
2787                        "could not resolve advisor seats ({e:#}); continuing \
2788                         without a design-deliberation brief (see {})",
2789                        where_recorded.display()
2790                    ),
2791                );
2792                self.state.advise_attempted = true;
2793                self.state.save()?;
2794                return Ok(());
2795            }
2796        };
2797
2798        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799        let artifacts = agent::artifacts_dir(&self.state.dir());
2800        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802        let mut jobs = Vec::new();
2803        for (i, spec) in seats.iter().cloned().enumerate() {
2804            let seat_key = format!("advisor-{}", i + 1);
2805            let seat = self.seat(&seat_key, &spec.id);
2806            jobs.push(SeatJob {
2807                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808                spec,
2809                seat,
2810                cwd: worktrees[i % worktrees.len()].clone(),
2811                timeout,
2812                allow_write: false,
2813                sessions: false,
2814                artifacts: artifacts.clone(),
2815                stem: seat_key,
2816                handover: None,
2817            });
2818        }
2819
2820        self.state.event(
2821            "advise",
2822            format!(
2823                "{} advisor seat(s) sketching a design in parallel",
2824                jobs.len()
2825            ),
2826        );
2827        let mut quota_losses = Vec::new();
2828        let cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            carry_seats: false,
2831            pauses: Some((&self.pause, &self.interrupt)),
2832            run: &run_id,
2833            node: "advise",
2834            prompts: &prompts,
2835            cache: cache.as_deref(),
2836            round: None,
2837        };
2838        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839        let results = ask_json_wave::<Proposal>(
2840            jobs,
2841            Arc::clone(&self.sem),
2842            self.state.config.graph.retries,
2843            &advisor_roster,
2844            &ctx,
2845            &mut quota_losses,
2846            &mut self.state,
2847            &|p: &Proposal| p.validate(),
2848        )
2849        .await;
2850        if self.state.resource_block.is_some() {
2851            return self.stop_for_resource();
2852        }
2853        self.state.quota.extend(quota_losses);
2854
2855        let mut records = Vec::with_capacity(results.len());
2856        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857            let agent_id = seat.agent.clone();
2858            self.state.seats.insert(seat.key.clone(), seat);
2859            match res {
2860                Ok((proposal, out)) => {
2861                    self.state
2862                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2863                    records.push(advise::AdvisorRecord::proposed(
2864                        i + 1,
2865                        agent_id,
2866                        proposal,
2867                        out.duration_ms,
2868                    ));
2869                }
2870                Err(e) => {
2871                    self.state.event(
2872                        "advise",
2873                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874                    );
2875                    records.push(advise::AdvisorRecord::failed(
2876                        i + 1,
2877                        agent_id,
2878                        e.to_string(),
2879                    ));
2880                }
2881            }
2882        }
2883
2884        let mut advice = advise::Advice {
2885            records,
2886            synthesis: None,
2887        };
2888        if advice.proposals().is_empty() {
2889            self.state.event(
2890                "advise",
2891                "no advisor produced a usable proposal; continuing without a \
2892                 synthesis brief"
2893                    .to_owned(),
2894            );
2895        } else {
2896            match self
2897                .synthesize_brief(
2898                    &advice,
2899                    &instruction,
2900                    &language,
2901                    &worktrees[0],
2902                    &artifacts,
2903                    &run_id,
2904                    &prompts,
2905                    cache.as_deref(),
2906                )
2907                .await
2908            {
2909                Ok(Some(text)) => {
2910                    self.state.event(
2911                        "advise",
2912                        "synthesized a design brief for the implementer".to_owned(),
2913                    );
2914                    advice.synthesis = Some(text);
2915                }
2916                Ok(None) => {
2917                    self.state.event(
2918                        "advise",
2919                        "the synthesis seat produced nothing usable; continuing \
2920                         without a design brief"
2921                            .to_owned(),
2922                    );
2923                }
2924                Err(e) => {
2925                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2926                        return Err(e);
2927                    }
2928                    self.state.event(
2929                        "advise",
2930                        format!("could not synthesize a design brief: {e:#}"),
2931                    );
2932                }
2933            }
2934        }
2935        advise::apply_reflection(&mut advice);
2936
2937        self.state.advice = Some(advice);
2938        self.state.advise_attempted = true;
2939        self.state.save()?;
2940        Ok(())
2941    }
2942
2943    /// The synthesis seat: reads every advisor's proposal and blends them
2944    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2945    /// out of [`Runner::advise`] only for readability — it is not called
2946    /// anywhere else.
2947    ///
2948    /// Picked the same way [`crate::talk`]'s standing conversation and
2949    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2950    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2951    /// order (a claude seat, else the first runnable agent in roster order)
2952    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2953    /// in [`crate::config`] for why a dedicated field exists here at all.
2954    #[allow(clippy::too_many_arguments)]
2955    async fn synthesize_brief(
2956        &mut self,
2957        advice: &advise::Advice,
2958        instruction: &str,
2959        language: &str,
2960        cwd: &Path,
2961        artifacts: &Path,
2962        run_id: &str,
2963        prompts: &Prompts,
2964        cache: Option<&Path>,
2965    ) -> Result<Option<String>> {
2966        let chain = agent::pick_chain(
2967            &self.state.config.agents,
2968            self.state.config.roles.synthesizer.as_ref(),
2969            &agent::installed,
2970            "synthesizer",
2971        )?;
2972        let proposals = advice.proposals();
2973        let mut prompt = prompt::with_overlay(
2974            prompt::synthesize_brief(instruction, &proposals, language),
2975            prompts.overlay("advise"),
2976        );
2977        if cache.is_some() {
2978            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2979            // below — see `prompt::build_cache_note`'s doc for why telling a
2980            // read-only seat to build through the shared cache is exactly how
2981            // a sandbox's write refusal gets misread as a defect.
2982            prompt.push('\n');
2983            prompt.push_str(&prompt::build_cache_note("advise", false));
2984        }
2985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986        // Each id is tried once, in order; a quota hit, error or unusable
2987        // answer moves to the next. The seat is single-turn (`sessions:
2988        // false`) and the prompt is the whole context, so a fallback agent
2989        // needs nothing carried over.
2990        let mut last = None;
2991        for (n, spec) in chain.iter().enumerate() {
2992            if n > 0 {
2993                self.state
2994                    .event("advise", format!("synthesis falling back to {}", spec.id));
2995            }
2996            let mut seat = self.seat("advise-synthesis", &spec.id);
2997            let wait = self.repository_wait();
2998            let outcome = wait
2999                .scope(agent::invoke(
3000                    spec,
3001                    &mut seat,
3002                    &Invocation {
3003                        coordination: self.state.config.coordination.as_ref(),
3004                        fetched_base: None,
3005                        cwd,
3006                        prompt: &prompt,
3007                        timeout,
3008                        allow_write: false,
3009                        sessions: false,
3010                        artifacts,
3011                        stem: &if n == 0 {
3012                            "advise-synthesis".to_owned()
3013                        } else {
3014                            format!("advise-synthesis-{}", spec.id)
3015                        },
3016                        run: run_id,
3017                        node: "advise",
3018                        cache_dir: None,
3019                        attachments: &[],
3020                        writable: &[],
3021                    },
3022                ))
3023                .await;
3024            if outcome.is_ok() {
3025                self.state.seats.insert(seat.key.clone(), seat);
3026            }
3027            let advance = agent::chain_advances(&outcome);
3028            last = Some(outcome);
3029            if !advance {
3030                break;
3031            }
3032        }
3033        // Exhausted: the last attempt's result is what a single failed seat
3034        // would have produced.
3035        let out = last.expect("a chain holds at least one agent")?;
3036        if !out.usable() {
3037            return Ok(None);
3038        }
3039        let text =
3040            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041        Ok((!text.trim().is_empty()).then_some(text))
3042    }
3043
3044    // ----------------------------------------------------------- implement
3045
3046    async fn implement(&mut self) -> Result<()> {
3047        self.persist_candidates().await?;
3048        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3049        // agent files with `magi task add` name the run that paid for it. The
3050        // prompt overlay is cloned alongside it because the waves borrow it
3051        // while `self` is mutably borrowed by the node's own bookkeeping.
3052        let run_id = self.state.id.clone();
3053        let prompts = self.state.config.prompts.clone();
3054        let todo: Vec<usize> = self
3055            .state
3056            .candidates
3057            .iter()
3058            .enumerate()
3059            .filter(|(_, c)| {
3060                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061                    && c.failed.is_none()
3062                    && !c.empty
3063            })
3064            .map(|(i, _)| i)
3065            .collect();
3066        if todo.is_empty() {
3067            return self.after_implement();
3068        }
3069        self.state.status = RunStatus::Implementing;
3070
3071        let language = self.state.config.graph.language.clone();
3072        let github_language = self.state.config.graph.github_language.clone();
3073        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074        let sessions = self.state.config.graph.sessions;
3075        let artifacts = agent::artifacts_dir(&self.state.dir());
3076        // The design-deliberation stage's blended brief, when `advise` found
3077        // one — carried into every implementer's prompt the same way
3078        // regardless of which candidate it is.
3079        let brief = self
3080            .state
3081            .advice
3082            .as_ref()
3083            .and_then(|a| a.synthesis.as_deref())
3084            .map(str::to_owned);
3085        let attachments = self.state.attachments.clone();
3086
3087        let mut jobs = Vec::new();
3088        for &i in &todo {
3089            let (index, label, worktree) = {
3090                let c = &self.state.candidates[i];
3091                (c.index, c.label, c.worktree.clone())
3092            };
3093            let spec = self.roles.implementers[index].clone();
3094            let seat_key = format!("impl-{label}");
3095            let seat = self.seat(&seat_key, &spec.id);
3096            let instruction = seeded_instruction(&self.state);
3097            jobs.push(SeatJob {
3098                spec,
3099                seat,
3100                prompt: prompt::implement_in(
3101                    &instruction,
3102                    &worktree.to_string_lossy(),
3103                    &language,
3104                    &github_language,
3105                    brief.as_deref(),
3106                    &attachments,
3107                ),
3108                cwd: worktree,
3109                timeout,
3110                allow_write: true,
3111                sessions,
3112                artifacts: artifacts.clone(),
3113                stem: format!("impl-{label}"),
3114                handover: None,
3115            });
3116        }
3117
3118        self.state.event(
3119            "implement",
3120            format!("{} candidates in parallel", jobs.len()),
3121        );
3122        // Kept so a seat whose CLI hung up can be asked again from the same
3123        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3124        // can update a seat's own entry once a fallback agent takes it over —
3125        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3126        // whichever agent actually answered, not the one that quota'd out.
3127        let mut sent = jobs.clone();
3128        let cache = self.state.config.cache_dir();
3129        let ctx = WaveCtx {
3130            carry_seats: false,
3131            pauses: Some((&self.pause, &self.interrupt)),
3132            run: &run_id,
3133            node: "implement",
3134            prompts: &prompts,
3135            cache: cache.as_deref(),
3136            round: None,
3137        };
3138        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139        if !wave_blocked(&results) {
3140            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141                .await;
3142            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143                .await?;
3144            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145                .await;
3146        }
3147        // Do not finalize failed candidates when the entire wave needs machine
3148        // recovery: leaving them pending resumes this node with its work intact.
3149        if !results.is_empty()
3150            && results
3151                .iter()
3152                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153        {
3154            let losses: Vec<_> = results
3155                .iter()
3156                .map(|(_, seat, out)| {
3157                    let AgentOutcome::Quota(o) = out else {
3158                        unreachable!()
3159                    };
3160                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161                })
3162                .collect();
3163            self.state.quota.extend(losses.clone());
3164            for (_, seat, _) in results {
3165                self.state.seats.insert(seat.key.clone(), seat);
3166            }
3167            for &i in &todo {
3168                let label = self.state.candidates[i].label;
3169                self.state.interrupted_implementations.insert(label);
3170                self.state
3171                    .candidate_preservation
3172                    .insert(label, "machine outage awaiting persistence".into());
3173            }
3174            // Preserve partial edits too, while keeping every interrupted seat
3175            // pending even when its tree already contains an inherited patch.
3176            block_machine_outage(&mut self.state, "implement", &losses);
3177            self.state.save()?;
3178            self.persist_candidates().await?;
3179            for &i in &todo {
3180                self.state
3181                    .interrupted_implementations
3182                    .insert(self.state.candidates[i].label);
3183            }
3184            return self.stop_for_resource();
3185        }
3186        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189            && results
3190                .iter()
3191                .any(|(_, _, out)| FailClass::of(out).is_some());
3192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194                // Executed turns interrupted by disk pressure already carry
3195                // a preservation barrier from wave. Admission-only waits do
3196                // not. Keep the interrupted seat's session for recovery too.
3197                if self
3198                    .state
3199                    .candidate_preservation
3200                    .contains_key(&self.state.candidates[i].label)
3201                {
3202                    self.state.seats.insert(seat.key.clone(), seat);
3203                }
3204                continue;
3205            }
3206            let seat_key = seat.key.clone();
3207            // A quota fallback (`resume_seat_handovers`) may have handed this
3208            // seat to a different agent than the one `prep` recorded on the
3209            // candidate; the stats tables and any later fixer-defaults-to-
3210            // winner's-author lookup must credit whoever actually answered —
3211            // unless every fallback also quota'd out, in which case nobody
3212            // actually answered and crediting the last agent tried would
3213            // erase every earlier agent's own quota loss from the stats
3214            // tables instead of just this one seat's.
3215            let agent = seat.agent.clone();
3216            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217            self.state.seats.insert(seat.key.clone(), seat);
3218            let label = self.state.candidates[i].label;
3219            self.state.interrupted_implementations.remove(&label);
3220
3221            let (summary, duration, failed, verified_claim) = match out {
3222                AgentOutcome::Ok(o) => {
3223                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224                    let failed = (!o.usable()).then(|| {
3225                        if o.timed_out {
3226                            "agent timed out".to_owned()
3227                        } else {
3228                            format!(
3229                                "agent exited with {}",
3230                                crate::notices::exit_status(o.exit_code)
3231                            )
3232                        }
3233                    });
3234                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235                    (text, o.duration_ms, failed, verified_claim)
3236                }
3237                // Left un-resumed by `resume_undelivered` (a dirty tree
3238                // already rescues the work, or there was no session left to
3239                // resume into) — reported like the ordinary failure it is,
3240                // never as if `o.text` (the CLI's raw error JSON) were an
3241                // answer.
3242                AgentOutcome::Dropped(o) => {
3243                    let why = o
3244                        .dropped
3245                        .as_ref()
3246                        .map(|d| d.why.as_str())
3247                        .unwrap_or("the CLI ended the stream without delivering its answer");
3248                    (
3249                        String::new(),
3250                        o.duration_ms,
3251                        Some(format!("the CLI dropped the stream ({why})")),
3252                        None,
3253                    )
3254                }
3255                AgentOutcome::Quota(o) => {
3256                    self.state.quota.push(QuotaLoss::from_report(
3257                        seat_key,
3258                        "implement",
3259                        o.quota.as_ref(),
3260                    ));
3261                    (
3262                        String::new(),
3263                        o.duration_ms,
3264                        Some("rate limited (quota); produced no change".to_owned()),
3265                        None,
3266                    )
3267                }
3268                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269                    (String::new(), 0, Some(e), None)
3270                }
3271            };
3272
3273            // Save every completed turn before any fallible Git operation. A failed
3274            // rescue must not erase the reply or prevent collection of other seats.
3275            let c = &mut self.state.candidates[i];
3276            if !exhausted_the_fallback_chain {
3277                c.agent = agent;
3278            }
3279            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280            c.duration_ms = duration;
3281            c.failed = failed;
3282            c.verified_noop = verified_claim;
3283            self.state
3284                .candidate_preservation
3285                .insert(label, "awaiting persistence".to_owned());
3286        }
3287        self.state.save()?;
3288        self.persist_candidates().await?;
3289
3290        if resource_blocked {
3291            return self.stop_for_resource();
3292        }
3293        if selected_failed {
3294            self.state.status = RunStatus::Blocked;
3295            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296            self.state.save()?;
3297            bail!(
3298                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299            );
3300        }
3301        self.after_implement()
3302    }
3303
3304    /// Retry only persistence after a completed implementation turn.
3305    async fn persist_candidates(&mut self) -> Result<()> {
3306        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307        for label in labels {
3308            let i = self
3309                .state
3310                .candidates
3311                .iter()
3312                .position(|c| c.label == label)
3313                .context("pending candidate is missing")?;
3314            let c = self.state.candidates[i].clone();
3315            let interrupted = self.state.interrupted_implementations.contains(&label);
3316            let result = async {
3317                let wait = crate::repo_lock::WaitPolicy::new(
3318                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319                    || false,
3320                );
3321                let _coordination =
3322                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323                // Best effort evidence requires no index writes. A successful snapshot
3324                // never authorizes removal of a tree whose rescue failed.
3325                if let Err(e) = git::snapshot_uncommitted(
3326                    &c.worktree,
3327                    &self
3328                        .state
3329                        .dir()
3330                        .join("artifacts")
3331                        .join(format!("candidate-{label}-recovery")),
3332                )
3333                .await
3334                {
3335                    self.state.event(
3336                        "preserve",
3337                        format!("candidate {label} snapshot incomplete: {e:#}"),
3338                    );
3339                }
3340                let rescued = git::rescue_commit(
3341                    &c.worktree,
3342                    &format!("magi: candidate {label} (uncommitted work)"),
3343                )
3344                .await?;
3345                self.state.note_withheld("implement", &rescued.withheld);
3346                let base = &self.state.base_commit;
3347                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352                if c.empty && commits > 0 && !patch.trim().is_empty()
3353                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355                        || self.state.tally.is_some())
3356                {
3357                    // Keep the old classification and pending marker so every
3358                    // retry stops too. Retain decisions as recovery evidence;
3359                    // only a supported recovery operation may replace them.
3360                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361                }
3362                let c = &mut self.state.candidates[i];
3363                c.commits = commits;
3364                c.stat = stat;
3365                c.files = files;
3366                let no_changes = commits == 0 || patch.trim().is_empty();
3367                c.empty = no_changes && !interrupted;
3368                if interrupted && no_changes {
3369                    c.failed = None;
3370                    c.verified_noop = None;
3371                }
3372                if !c.empty {
3373                    c.failed = None;
3374                    c.verified_noop = None;
3375                }
3376                self.state.event(
3377                    "preserve",
3378                    format!("candidate {label}: {files} files, {commits} commits"),
3379                );
3380                Ok::<_, anyhow::Error>(())
3381            }
3382            .await;
3383            match result {
3384                Ok(()) => {
3385                    self.state.candidate_preservation.remove(&label);
3386                    if !interrupted || self.state.candidates[i].files > 0 {
3387                        self.state.interrupted_implementations.remove(&label);
3388                    }
3389                }
3390                Err(e) => {
3391                    let reason = format!("candidate {label} preservation failed: {e:#}");
3392                    self.state.event("preserve", &reason);
3393                    self.state.candidate_preservation.insert(label, reason);
3394                }
3395            }
3396            if self.state.candidate_preservation.is_empty()
3397                && self.state.status == RunStatus::Blocked
3398                && self.state.candidates.iter().all(|c| c.failed.is_none())
3399            {
3400                // Persist the recovered phase with the cleared pending marker;
3401                // a crash between saves must not turn this into a selected-seat
3402                // failure hold on the next resume.
3403                self.state.status = RunStatus::Implementing;
3404                self.state.event(
3405                    "preserve",
3406                    "candidate persistence recovered; implementation can continue",
3407                );
3408            }
3409            self.state.save()?;
3410        }
3411        if !self.state.candidate_preservation.is_empty() {
3412            self.state.status = RunStatus::Blocked;
3413            self.state.save()?;
3414            bail!(
3415                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416            );
3417        }
3418        Ok(())
3419    }
3420
3421    /// Ask again, once, for work a CLI did and then failed to hand over.
3422    ///
3423    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3424    /// status with an empty response and a usage report showing output tokens,
3425    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3426    /// seven minutes and 14,267 output tokens that arrived as an empty
3427    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3428    ///
3429    /// Two conditions, and both matter:
3430    ///
3431    /// - **Only when the tree is untouched.** Often the agent has already
3432    ///   written its files and only the closing message was lost; the rescue
3433    ///   commit below picks that up and there is nothing to ask for. Re-asking
3434    ///   then would pay for a second implementation of work already on disk.
3435    /// - **Once.** A CLI that drops one stream can drop the next, and this
3436    ///   node is the most expensive in the graph.
3437    ///
3438    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3439    /// dropped reply still carried its `conversation_id`, so the seat is asked
3440    /// to finish what it was doing rather than sent the whole task again. It
3441    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3442    /// node's - for the same reason a re-ranked judge does: restating finished
3443    /// work is not the work.
3444    ///
3445    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3446    /// same way until it resets, while an abandoned conversation is still
3447    /// there to be picked up.
3448    async fn resume_undelivered(
3449        &mut self,
3450        results: &mut [(usize, SeatState, AgentOutcome)],
3451        sent: &[SeatJob],
3452        prompts: &Prompts,
3453        run_id: &str,
3454    ) {
3455        for (wi, seat, out) in results.iter_mut() {
3456            let Some(dropped) = (match &*out {
3457                AgentOutcome::Dropped(o) => o.dropped.clone(),
3458                _ => None,
3459            }) else {
3460                continue;
3461            };
3462            let Some(job) = sent.get(*wi) else { continue };
3463            // Already on disk? Then only the closing message was lost.
3464            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465                self.state.event(
3466                    "implement",
3467                    format!(
3468                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469                         work is in the tree",
3470                        seat.key, dropped.output_tokens, dropped.why
3471                    ),
3472                );
3473                continue;
3474            }
3475            // The re-ask only makes sense as a resume: `resume_after_drop`
3476            // says nothing about the task, trusting the seat to still hold it.
3477            // Without a session to resume — sessions disabled, or this CLI's
3478            // drop shape happened not to carry a session id — that prompt
3479            // would open a brand-new conversation with no context at all,
3480            // which is worse than leaving this as the ordinary failure it
3481            // already is.
3482            if !has_context(&job.spec, seat, job.sessions) {
3483                self.state.event(
3484                    "implement",
3485                    format!(
3486                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487                         is no session left to resume",
3488                        seat.key, dropped.output_tokens, dropped.why
3489                    ),
3490                );
3491                continue;
3492            }
3493            self.state.event(
3494                "implement",
3495                format!(
3496                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497                     conversation",
3498                    seat.key, dropped.output_tokens, dropped.why
3499                ),
3500            );
3501            let mut retry = job.clone();
3502            retry.seat = seat.clone();
3503            retry.prompt = prompt::resume_after_drop(&dropped.why);
3504            retry.timeout = retry_budget(job.timeout, true);
3505            retry.stem = format!("{}-resume", job.stem);
3506            let cache = self.state.config.cache_dir();
3507            let ctx = WaveCtx {
3508                carry_seats: false,
3509                pauses: Some((&self.pause, &self.interrupt)),
3510                run: run_id,
3511                node: "implement",
3512                prompts,
3513                cache: cache.as_deref(),
3514                round: None,
3515            };
3516            let (resumed_seat, resumed) =
3517                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518            *seat = resumed_seat;
3519            *out = resumed;
3520        }
3521    }
3522
3523    /// Fall an implement seat through to the next untried agent in the
3524    /// implementer roster when it lost to quota — or, since the handover was
3525    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3526    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3527    /// quota path itself being unchanged — instead of leaving the
3528    /// seat's loss final the moment one agent's account runs dry.
3529    ///
3530    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3531    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3532    /// truncates to the single slot rotation picked, so a solo task whose one
3533    /// implementer hits quota mid-run used to have nothing else to try. This
3534    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3535    /// unrotated roster — which is the only place the *other* candidates in
3536    /// the machine's roster still exist once `implementers` has been cut down
3537    /// to size.
3538    ///
3539    /// Walks forward from just past the seat's own original position in the
3540    /// roster, never wrapping back to the front: a later candidate slot (say
3541    /// `beta`, the roster's second entry) must fall through to the *next*
3542    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3543    /// almost certainly a different candidate's own agent already — and once
3544    /// the roster's tail is exhausted there is nothing left to fall through
3545    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3546    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3547    /// let this retry that id forever. The loop keeps falling through until
3548    /// an attempt lands something other than `Quota` or the roster's tail
3549    /// runs out of untried ids, at which point the seat is left exactly as
3550    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3551    /// `QuotaLoss` recorded, the candidate failed/empty.
3552    ///
3553    /// `sent` is taken mutably and updated with the fallback agent's spec:
3554    /// `resume_unconfirmed_commands`, which runs after this and also reads
3555    /// `sent`, must see whichever agent actually ended up answering the seat
3556    /// — reading the stale, original spec there would check session
3557    /// eligibility against the wrong CLI and could hand a fallback agent's
3558    /// session id to the agent that just lost the seat to quota.
3559    ///
3560    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3561    /// — `self.seat` only reuses state when the agent id is unchanged, so
3562    /// handing it a different id already gets this for free. Reusing the old
3563    /// seat would resume a different CLI's session as if it were a
3564    /// continuation of this one.
3565    ///
3566    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3567    /// a quota loss cuts an agent off mid-turn, so anything already in the
3568    /// tree is unfinished work, not a completed candidate a re-ask would pay
3569    /// for twice. A dirty tree is rescued into a commit first (the same
3570    /// neutral-identity rescue `implement`'s own outcome loop gives every
3571    /// candidate) so the next agent starts clean.
3572    ///
3573    /// The new agent gets the implementer's full prompt and full
3574    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3575    /// it has no session and no context, and is implementing the task from
3576    /// nothing, unlike a resumed drop which is only restating work already
3577    /// done.
3578    ///
3579    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3580    /// `implement` event, never into `self.state.quota` — that is what
3581    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3582    /// go unspent, and a seat that ultimately recovered on its second or
3583    /// third agent is not the stalled panel that check exists to catch. Only
3584    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3585    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3586    /// outcome loop already has — this helper never pushes to it itself.
3587    async fn resume_seat_handovers(
3588        &mut self,
3589        results: &mut [(usize, SeatState, AgentOutcome)],
3590        sent: &mut [SeatJob],
3591        prompts: &Prompts,
3592        run_id: &str,
3593    ) -> Result<()> {
3594        let instruction = seeded_instruction(&self.state);
3595        let language = self.state.config.graph.language.clone();
3596        let github_language = self.state.config.graph.github_language.clone();
3597        let brief = self
3598            .state
3599            .advice
3600            .as_ref()
3601            .and_then(|a| a.synthesis.as_deref())
3602            .map(str::to_owned);
3603        let attachments = self.state.attachments.clone();
3604        for (wi, seat, out) in results.iter_mut() {
3605            // Who holds the other candidate seats of this wave right now
3606            // (earlier handovers already written back to `sent`).
3607            let others: BTreeSet<String> = sent
3608                .iter()
3609                .enumerate()
3610                .filter(|(j, _)| j != wi)
3611                .map(|(_, j)| j.spec.id.clone())
3612                .collect();
3613            let Some(job) = sent.get_mut(*wi) else {
3614                continue;
3615            };
3616            // Where the seat's own original agent sits in the roster — the
3617            // fallback walk starts just past here, never at the front, so a
3618            // later candidate slot's quota loss does not fall back onto an
3619            // earlier slot's own agent.
3620            let start = self
3621                .roles
3622                .implementer_roster
3623                .iter()
3624                .position(|s| s.id == job.spec.id)
3625                .unwrap_or(0);
3626            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627            let mut fallback_attempt = 0usize;
3628            let mut prev: Option<FailClass> = None;
3629            while let Some(cur) = FailClass::of(&*out) {
3630                if !should_hand_over(prev.as_ref(), &cur) {
3631                    break;
3632                }
3633                let Some(next) =
3634                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635                        .cloned()
3636                else {
3637                    break;
3638                };
3639                tried.insert(next.id.clone());
3640                fallback_attempt += 1;
3641
3642                let coordination = self.coordinate(&job.cwd).await?;
3643                if let Ok(r) = git::rescue_commit(
3644                    &job.cwd,
3645                    &format!(
3646                        "magi: candidate {} (uncommitted work before {} fallback)",
3647                        seat.key,
3648                        if cur == FailClass::Quota {
3649                            "quota"
3650                        } else {
3651                            "handover"
3652                        }
3653                    ),
3654                )
3655                .await
3656                {
3657                    self.state.note_withheld("implement", &r.withheld);
3658                }
3659                drop(coordination);
3660
3661                record_handover(
3662                    &mut self.state,
3663                    "implement",
3664                    &seat.key,
3665                    &seat.agent,
3666                    &next.id,
3667                    &cur,
3668                    &fail_reason(&*out),
3669                );
3670                prev = Some(cur.clone());
3671
3672                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674                // Kept in sync on `sent` itself, not just the local retry: a
3675                // later helper (`resume_unconfirmed_commands`) reads `sent`
3676                // after this one returns and must see whichever agent is now
3677                // occupying the seat, not the one that just quota'd out —
3678                // otherwise it would judge session/continuation eligibility
3679                // by the wrong CLI and could resend a fallback's session id
3680                // to the agent that lost it the seat in the first place.
3681                job.spec = next.clone();
3682                let mut retry = job.clone();
3683                retry.seat = new_seat;
3684                retry.prompt = prompt::implement_in(
3685                    &instruction,
3686                    &job.cwd.to_string_lossy(),
3687                    &language,
3688                    &github_language,
3689                    brief.as_deref(),
3690                    &attachments,
3691                );
3692                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693                let cache = self.state.config.cache_dir();
3694                let ctx = WaveCtx {
3695                    carry_seats: false,
3696                    pauses: Some((&self.pause, &self.interrupt)),
3697                    run: run_id,
3698                    node: "implement",
3699                    prompts,
3700                    cache: cache.as_deref(),
3701                    round: None,
3702                };
3703                let (fallback_seat, fallback_out) = run_one(
3704                    retry,
3705                    Arc::clone(&self.sem),
3706                    &ctx,
3707                    &mut self.state,
3708                    fallback_attempt,
3709                )
3710                .await;
3711                *seat = fallback_seat;
3712                *out = fallback_out;
3713            }
3714        }
3715        Ok(())
3716    }
3717
3718    /// Ask an implement seat's own CLI to confirm what it started, once, when
3719    /// its reply reported a command whose completion status it never
3720    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3721    /// that does and does not mean.
3722    ///
3723    /// The completion contract this task asks for, extended to `implement`
3724    /// with the same signal `continue_fix_report` reads for the fixer,
3725    /// rather than a keyword search over the reply or a hard requirement on
3726    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3727    /// a candidate's CLI turn ended cleanly while a test run it had started
3728    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3729    /// named in it at all is untouched by this: `commands` is empty, so
3730    /// there is nothing to be unconfirmed.
3731    ///
3732    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3733    /// this is not about recovering edits that might already be on disk, it
3734    /// is about a result the seat itself never vouched for, which resuming
3735    /// asks for regardless of what the tree already holds. Bounded to one
3736    /// attempt for the same reason `resume_undelivered` is — this is the
3737    /// most expensive node in the graph — and a seat that still cannot
3738    /// confirm on that attempt is left as whatever its (possibly still
3739    /// unconfirmed) reply says; this does not invent a new "failed" reason
3740    /// for a candidate that otherwise produced a real, committed change.
3741    async fn resume_unconfirmed_commands(
3742        &mut self,
3743        results: &mut [(usize, SeatState, AgentOutcome)],
3744        sent: &[SeatJob],
3745        prompts: &Prompts,
3746        run_id: &str,
3747    ) {
3748        for (wi, seat, out) in results.iter_mut() {
3749            let AgentOutcome::Ok(o) = &*out else {
3750                continue;
3751            };
3752            if !has_unconfirmed_command(&o.commands) {
3753                continue;
3754            }
3755            let Some(job) = sent.get(*wi) else { continue };
3756            if !has_context(&job.spec, seat, job.sessions) {
3757                self.state.event(
3758                    "implement",
3759                    format!(
3760                        "{}: the reply named a command whose own CLI never confirmed the exit \
3761                         status of, but there is no session left to resume",
3762                        seat.key
3763                    ),
3764                );
3765                continue;
3766            }
3767            self.state.event(
3768                "implement",
3769                format!(
3770                    "{}: the reply named a command whose own CLI never confirmed the exit \
3771                     status of; resuming the conversation",
3772                    seat.key
3773                ),
3774            );
3775            let mut retry = job.clone();
3776            retry.seat = seat.clone();
3777            retry.prompt = prompt::resume_incomplete(
3778                "a command in your last reply had no confirmed exit status",
3779            );
3780            retry.timeout = retry_budget(job.timeout, true);
3781            retry.stem = format!("{}-confirm", job.stem);
3782            let cache = self.state.config.cache_dir();
3783            let ctx = WaveCtx {
3784                carry_seats: false,
3785                pauses: Some((&self.pause, &self.interrupt)),
3786                run: run_id,
3787                node: "implement",
3788                prompts,
3789                cache: cache.as_deref(),
3790                round: None,
3791            };
3792            let (resumed_seat, resumed) =
3793                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794            *seat = resumed_seat;
3795            *out = resumed;
3796        }
3797    }
3798
3799    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3800    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3801    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3802    /// that motivated this.
3803    ///
3804    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3805    /// own nudge loop already covers for judge/review/vote seats, and not a
3806    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3807    /// implement seats: here the CLI turn genuinely finished while the node's
3808    /// own work — the fixer's account of what it did — had not. Gated purely
3809    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3810    /// reply, never on any wording in it, so a fixer whose valid, first-try
3811    /// `FixReport` happens to mention having waited on a background test is
3812    /// never resumed — the `Ok(report)` branch at the call site returns
3813    /// before this is ever invoked.
3814    ///
3815    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3816    /// attempt ([`retry_budget`]), nothing attempted once the session is
3817    /// gone, and a quota hit ends the loop immediately rather than retrying a
3818    /// rate limit that fails the same way again.
3819    async fn continue_fix_report(
3820        &mut self,
3821        mut seat: SeatState,
3822        parse_err: String,
3823        job: &SeatJob,
3824        prompts: &Prompts,
3825        run_id: &str,
3826        round: usize,
3827    ) -> (
3828        SeatState,
3829        Option<FixReport>,
3830        Option<String>,
3831        ContinuationRecord,
3832    ) {
3833        let mut last_err = parse_err;
3834        let mut cumulative_wait_ms = 0u64;
3835        let mut attempts = 0usize;
3836        loop {
3837            if !has_context(&job.spec, &seat, job.sessions) {
3838                self.state.event(
3839                    "fix",
3840                    format!(
3841                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842                         session left to resume into"
3843                    ),
3844                );
3845                let outcome = if attempts == 0 {
3846                    ContinuationOutcome::NoSession
3847                } else {
3848                    ContinuationOutcome::Exhausted
3849                };
3850                return (
3851                    seat,
3852                    None,
3853                    Some(format!("unparsable fix report: {last_err}")),
3854                    ContinuationRecord {
3855                        attempts,
3856                        cumulative_wait_ms,
3857                        outcome,
3858                    },
3859                );
3860            }
3861            if attempts >= MAX_FIX_CONTINUATIONS {
3862                self.state.event(
3863                    "fix",
3864                    format!(
3865                        "round {round}: fixer's reply still had no adoption report after \
3866                         {attempts} continuation(s) ({last_err}); giving up"
3867                    ),
3868                );
3869                return (
3870                    seat,
3871                    None,
3872                    Some(format!(
3873                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3874                    )),
3875                    ContinuationRecord {
3876                        attempts,
3877                        cumulative_wait_ms,
3878                        outcome: ContinuationOutcome::Exhausted,
3879                    },
3880                );
3881            }
3882            attempts += 1;
3883            self.state.event(
3884                "fix",
3885                format!(
3886                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888                ),
3889            );
3890            let mut retry = job.clone();
3891            retry.seat = seat.clone();
3892            retry.prompt = prompt::resume_incomplete(&last_err);
3893            retry.timeout = retry_budget(job.timeout, true);
3894            retry.stem = format!("{}-continue{attempts}", job.stem);
3895            let cache = self.state.config.cache_dir();
3896            let ctx = WaveCtx {
3897                carry_seats: false,
3898                pauses: Some((&self.pause, &self.interrupt)),
3899                run: run_id,
3900                node: "fix",
3901                prompts,
3902                cache: cache.as_deref(),
3903                round: Some(round),
3904            };
3905            let (resumed_seat, resumed_out) = run_one(
3906                retry,
3907                Arc::clone(&self.sem),
3908                &ctx,
3909                &mut self.state,
3910                attempts,
3911            )
3912            .await;
3913            seat = resumed_seat;
3914            match resumed_out {
3915                AgentOutcome::Ok(o) => {
3916                    cumulative_wait_ms += o.duration_ms;
3917                    match verdict::extract_json::<FixReport>(&o.text) {
3918                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919                            self.state.event(
3920                                "fix",
3921                                format!(
3922                                    "round {round}: fixer's adoption report recovered after \
3923                                     {attempts} continuation(s)"
3924                                ),
3925                            );
3926                            return (
3927                                seat,
3928                                Some(report),
3929                                None,
3930                                ContinuationRecord {
3931                                    attempts,
3932                                    cumulative_wait_ms,
3933                                    outcome: ContinuationOutcome::Resumed,
3934                                },
3935                            );
3936                        }
3937                        // The report parsed, but this same reply's own
3938                        // CommandEvidence — the identical record `state.jobs`
3939                        // renders — names a command whose CLI never
3940                        // confirmed an exit status. Read together, that is
3941                        // not a resolved answer: keep nudging rather than
3942                        // accept a report standing next to a command the
3943                        // seat's own CLI cannot vouch for.
3944                        Ok(_) => {
3945                            last_err = "the reply parsed, but it reported a command whose own CLI \
3946                                 never confirmed an exit status"
3947                                .to_owned();
3948                        }
3949                        Err(e) => last_err = e.to_string(),
3950                    }
3951                }
3952                AgentOutcome::Quota(o) => {
3953                    cumulative_wait_ms += o.duration_ms;
3954                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956                    self.state.quota.push(loss);
3957                    self.state.event(
3958                        "fix",
3959                        format!(
3960                            "round {round}: continuation rate limited (quota); not retrying now"
3961                        ),
3962                    );
3963                    return (
3964                        seat,
3965                        None,
3966                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3967                        ContinuationRecord {
3968                            attempts,
3969                            cumulative_wait_ms,
3970                            outcome: ContinuationOutcome::QuotaLost,
3971                        },
3972                    );
3973                }
3974                AgentOutcome::Dropped(o) => {
3975                    cumulative_wait_ms += o.duration_ms;
3976                    let why = o
3977                        .dropped
3978                        .as_ref()
3979                        .map(|d| d.why.as_str())
3980                        .unwrap_or("the CLI ended the stream without delivering its answer");
3981                    last_err = format!("the CLI dropped the stream ({why})");
3982                }
3983                AgentOutcome::Failed(e) => last_err = e,
3984                // The continuation was never started; the report stays lost
3985                // and the caller records that, without any handover.
3986                AgentOutcome::ResourceBlocked(e) => {
3987                    return (
3988                        seat,
3989                        None,
3990                        Some(format!("the continuation was not started: {e}")),
3991                        ContinuationRecord {
3992                            attempts: attempts.saturating_sub(1),
3993                            cumulative_wait_ms,
3994                            outcome: ContinuationOutcome::NoSession,
3995                        },
3996                    );
3997                }
3998            }
3999        }
4000    }
4001
4002    fn after_implement(&mut self) -> Result<()> {
4003        if !self.state.candidate_preservation.is_empty() {
4004            bail!("candidate preservation incomplete; judging is unsafe");
4005        }
4006        // Scan every candidate patch once the set is complete.
4007        if self.state.leaks.is_empty() {
4008            let cfg = self.state.config.blind.clone();
4009            let mut leaks = Vec::new();
4010            for c in &self.state.candidates {
4011                let Some(patch) =
4012                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013                else {
4014                    continue;
4015                };
4016                leaks.extend(blind::scan(
4017                    &format!("candidate {} patch", c.label),
4018                    &patch,
4019                    &cfg.vendor_tokens,
4020                ));
4021            }
4022            if !leaks.is_empty() {
4023                let summary = leaks
4024                    .iter()
4025                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026                    .collect::<Vec<_>>()
4027                    .join(", ");
4028                match cfg.on_leak {
4029                    LeakPolicy::Fail => {
4030                        self.state.status = RunStatus::Failed;
4031                        self.state
4032                            .event("blind", format!("vendor text in a patch: {summary}"));
4033                        self.state.leaks = leaks;
4034                        self.state.save()?;
4035                        self.settle_questions();
4036                        bail!(
4037                            "blind.on_leak = \"fail\" and vendor text reached a \
4038                             judged patch: {summary}"
4039                        );
4040                    }
4041                    LeakPolicy::Redact => self.state.event(
4042                        "blind",
4043                        format!("redacting vendor text for judging: {summary}"),
4044                    ),
4045                    LeakPolicy::Warn => self.state.event(
4046                        "blind",
4047                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048                    ),
4049                }
4050                self.state.leaks = leaks;
4051            }
4052        }
4053
4054        if self.state.viable().is_empty() {
4055            if self.state.all_candidates_verified_noop() {
4056                // Every candidate agreed, with evidence the adoption guard
4057                // accepted, that nothing belongs in this worktree. That is
4058                // not the same fact as a candidate that simply failed to
4059                // write anything, and settling it as an ordinary `Failed`
4060                // (see `SCHEMA`'s doc for schema 10) is what let two of
4061                // task 391f's attempts burn a retry each re-discovering the
4062                // same already-landed fix. Terminal either way, so `judge`
4063                // must never run over an empty candidate set — unlike the
4064                // `Failed` branch below this returns `Ok`, not an error:
4065                // nothing here failed.
4066                self.state.status = RunStatus::VerifiedNoop;
4067                self.state.save()?;
4068                self.settle_questions();
4069                return Ok(());
4070            }
4071            self.state.status = RunStatus::Failed;
4072            self.state.save()?;
4073            self.settle_questions();
4074            bail!("no candidate produced a change; nothing to judge");
4075        }
4076        self.state.status = RunStatus::Judging;
4077        self.state.save()?;
4078        Ok(())
4079    }
4080
4081    // --------------------------------------------------------------- judge
4082
4083    async fn judge(&mut self) -> Result<()> {
4084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4085        // agent files with `magi task add` name the run that paid for it. The
4086        // prompt overlay is cloned alongside it because the waves borrow it
4087        // while `self` is mutably borrowed by the node's own bookkeeping.
4088        let run_id = self.state.id.clone();
4089        let prompts = self.state.config.prompts.clone();
4090        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091            return Ok(());
4092        }
4093        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094        if viable.len() == 1 {
4095            // Recorded so this is a one-time event: `judgements` stays empty
4096            // either way, which without this flag is indistinguishable from
4097            // "not yet judged" on the next reentry — and status is left
4098            // untouched, so a later node's conclusion (e.g. `Blocked` after
4099            // the review budget ran out) survives a resume instead of being
4100            // clobbered back to `Judging` by this node running again.
4101            self.state.judge_skipped = true;
4102            self.state.event(
4103                "judge",
4104                format!(
4105                    "only candidate {} produced a change; judging skipped",
4106                    viable[0].label
4107                ),
4108            );
4109            self.state.save()?;
4110            return Ok(());
4111        }
4112        self.state.status = RunStatus::Judging;
4113
4114        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115        let language = self.state.config.graph.language.clone();
4116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117        let sessions = self.state.config.graph.sessions;
4118        let artifacts = agent::artifacts_dir(&self.state.dir());
4119        let root = self.state.worktree_root();
4120        let base_short = short(&self.state.base_commit);
4121
4122        let mut jobs = Vec::new();
4123        let mut orders = Vec::new();
4124        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128            let seat_key = format!("judge-{}", j + 1);
4129            let seat = self.seat(&seat_key, &spec.id);
4130            jobs.push(SeatJob {
4131                prompt: prompt::judge(
4132                    &self.state.instruction,
4133                    &views,
4134                    self.roles.judges.len(),
4135                    &base_short,
4136                    &language,
4137                ),
4138                spec,
4139                seat,
4140                cwd: root.join(format!("judge-{}", j + 1)),
4141                timeout,
4142                allow_write: false,
4143                sessions,
4144                artifacts: artifacts.clone(),
4145                stem: format!("judge-{}", j + 1),
4146                handover: None,
4147            });
4148        }
4149
4150        self.state.event(
4151            "judge",
4152            format!(
4153                "{} judges ranking {} candidates blind",
4154                jobs.len(),
4155                viable.len()
4156            ),
4157        );
4158        let labels_for_check = labels.clone();
4159        let mut quota_losses = Vec::new();
4160        let cache = self.state.config.cache_dir();
4161        let ctx = WaveCtx {
4162            carry_seats: false,
4163            pauses: Some((&self.pause, &self.interrupt)),
4164            run: &run_id,
4165            node: "judge",
4166            prompts: &prompts,
4167            cache: cache.as_deref(),
4168            round: None,
4169        };
4170        let results = ask_json_wave::<Ranking>(
4171            jobs,
4172            Arc::clone(&self.sem),
4173            self.state.config.graph.retries,
4174            &self.roles.judge_roster,
4175            &ctx,
4176            &mut quota_losses,
4177            &mut self.state,
4178            &move |r: &Ranking| r.validate(&labels_for_check),
4179        )
4180        .await;
4181        if self.state.resource_block.is_some() {
4182            return self.stop_for_resource();
4183        }
4184        self.state.quota.extend(quota_losses);
4185
4186        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187            let agent_id = seat.agent.clone();
4188            self.state.seats.insert(seat.key.clone(), seat);
4189            let mut record = Judgement {
4190                judge: j + 1,
4191                seat: format!("judge-{}", j + 1),
4192                agent: agent_id,
4193                ranking: Vec::new(),
4194                reasons: BTreeMap::new(),
4195                confidence: None,
4196                strengths: BTreeMap::new(),
4197                order: orders[j].clone(),
4198                failed: None,
4199                duration_ms: 0,
4200            };
4201            match res {
4202                Ok((ranking, out)) => {
4203                    record.ranking = ranking.normalized();
4204                    record.reasons = ranking.reasons;
4205                    record.strengths = ranking.strengths;
4206                    record.confidence = ranking.confidence;
4207                    record.duration_ms = out.duration_ms;
4208                    self.state.event(
4209                        "judge",
4210                        format!(
4211                            "judge {} ranked {}",
4212                            j + 1,
4213                            record.ranking.iter().collect::<String>()
4214                        ),
4215                    );
4216                }
4217                Err(e) => {
4218                    record.failed = Some(e.to_string());
4219                    self.state
4220                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221                }
4222            }
4223            self.state.judgements.push(record);
4224            self.state.save()?;
4225        }
4226        Ok(())
4227    }
4228
4229    // ---------------------------------------------------------- deliberate
4230
4231    async fn deliberate(&mut self) -> Result<()> {
4232        if self.hold_base_sync().await? {
4233            return Ok(());
4234        }
4235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4236        // agent files with `magi task add` name the run that paid for it. The
4237        // prompt overlay is cloned alongside it because the waves borrow it
4238        // while `self` is mutably borrowed by the node's own bookkeeping.
4239        let run_id = self.state.id.clone();
4240        let prompts = self.state.config.prompts.clone();
4241        let rounds = self.state.config.graph.deliberate_rounds;
4242        // Resume unfinished rounds after an outage, but never reopen a tally.
4243        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244            return Ok(());
4245        }
4246        let tops: Vec<char> = self
4247            .state
4248            .judgements
4249            .iter()
4250            .filter_map(|j| j.ranking.first().copied())
4251            .collect();
4252        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254                self.state.event(
4255                    "deliberate",
4256                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4257                );
4258            }
4259            self.state.status = RunStatus::Voting;
4260            self.state.save()?;
4261            return Ok(());
4262        }
4263
4264        self.state.status = RunStatus::Deliberating;
4265        self.state.event(
4266            "deliberate",
4267            format!(
4268                "split: first choices were {} — opening {rounds} round(s)",
4269                tops.iter().collect::<String>()
4270            ),
4271        );
4272
4273        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274        let language = self.state.config.graph.language.clone();
4275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276        let sessions = self.state.config.graph.sessions;
4277        let artifacts = agent::artifacts_dir(&self.state.dir());
4278        let root = self.state.worktree_root();
4279        let base_short = short(&self.state.base_commit);
4280
4281        // Judges argue in sequence so that a turn can answer the one before it;
4282        // that is the difference between deliberation and three parallel
4283        // monologues.
4284        for round in self.state.deliberation.len() + 1..=rounds {
4285            let mut turns: Vec<DeliberationTurn> = Vec::new();
4286            let mut round_losses = Vec::new();
4287            let mut asked = 0;
4288            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289                if self.state.judgements[j].failed.is_some() {
4290                    continue;
4291                }
4292                let seat_key = format!("judge-{}", j + 1);
4293                let spec = self.occupant(&seat_key, spec);
4294                let mut seat = self.seat(&seat_key, &spec.id);
4295                let transcript = self.transcript(&turns, j);
4296                let build = |context: Option<&str>| {
4297                    prompt::deliberate(
4298                        &self.state.instruction,
4299                        context,
4300                        &transcript,
4301                        round,
4302                        rounds,
4303                        &language,
4304                    )
4305                };
4306                let block = self.candidate_block(&viable, &base_short);
4307                let full = build(Some(&block));
4308                let text = if has_context(&spec, &seat, sessions) {
4309                    build(None)
4310                } else {
4311                    full.clone()
4312                };
4313                let job = SeatJob {
4314                    spec,
4315                    seat: seat.clone(),
4316                    prompt: text,
4317                    cwd: root.join(format!("judge-{}", j + 1)),
4318                    timeout,
4319                    allow_write: false,
4320                    sessions,
4321                    artifacts: artifacts.clone(),
4322                    stem: format!("delib-{round}-judge-{}", j + 1),
4323                    handover: Some(full),
4324                };
4325                let cache = self.state.config.cache_dir();
4326                let ctx = WaveCtx {
4327                    carry_seats: false,
4328                    pauses: Some((&self.pause, &self.interrupt)),
4329                    run: &run_id,
4330                    node: "deliberate",
4331                    prompts: &prompts,
4332                    cache: cache.as_deref(),
4333                    round: None,
4334                };
4335                // A turn is never nudged (`retries` 0): a failed seat is
4336                // handed to the next roster agent, which gets the full
4337                // context. An empty answer is a turn, not a failure.
4338                let mut losses = Vec::new();
4339                asked += 1;
4340                let mut results = ask_wave_with::<String>(
4341                    vec![job],
4342                    Arc::clone(&self.sem),
4343                    0,
4344                    &self.roles.judge_roster,
4345                    &ctx,
4346                    &mut losses,
4347                    &mut self.state,
4348                    &|text: &str| {
4349                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350                    },
4351                )
4352                .await;
4353                if self.state.resource_block.is_some() {
4354                    return self.stop_for_resource();
4355                }
4356                round_losses.extend(losses.iter().cloned());
4357                self.state.quota.extend(losses);
4358                let (updated, res, _) = results.pop().expect("one job in, one result out");
4359                seat = updated;
4360                let agent_id = seat.agent.clone();
4361                self.state.seats.insert(seat.key.clone(), seat);
4362                let body = match res {
4363                    Ok((body, _)) => body,
4364                    // Skip the seat; a CLI's raw error JSON is never read as
4365                    // this judge's position.
4366                    Err(e) => {
4367                        self.state
4368                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369                        continue;
4370                    }
4371                };
4372                let tentative = verdict::extract_json::<Position>(&body)
4373                    .ok()
4374                    .and_then(|p| p.tentative)
4375                    .and_then(|s| s.trim().chars().next())
4376                    .map(|c| c.to_ascii_uppercase());
4377                self.state.event(
4378                    "deliberate",
4379                    format!(
4380                        "round {round}: judge {} now favours {}",
4381                        j + 1,
4382                        tentative.map_or("—".to_owned(), |c| c.to_string())
4383                    ),
4384                );
4385                turns.push(DeliberationTurn {
4386                    judge: j + 1,
4387                    agent: agent_id,
4388                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4389                    tentative,
4390                });
4391            }
4392            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394                return self.stop_for_resource();
4395            }
4396            self.state
4397                .deliberation
4398                .push(DeliberationRound { round, turns });
4399            self.state.save()?;
4400        }
4401
4402        self.state.status = RunStatus::Voting;
4403        self.state.save()?;
4404        Ok(())
4405    }
4406
4407    // ---------------------------------------------------------------- vote
4408
4409    async fn vote(&mut self) -> Result<()> {
4410        if self.hold_base_sync().await? {
4411            return Ok(());
4412        }
4413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4414        // agent files with `magi task add` name the run that paid for it. The
4415        // prompt overlay is cloned alongside it because the waves borrow it
4416        // while `self` is mutably borrowed by the node's own bookkeeping.
4417        let run_id = self.state.id.clone();
4418        let prompts = self.state.config.prompts.clone();
4419        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420            return Ok(());
4421        }
4422        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423        if viable.len() == 1 {
4424            return Ok(());
4425        }
4426        self.state.status = RunStatus::Voting;
4427
4428        let language = self.state.config.graph.language.clone();
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let root = self.state.worktree_root();
4433        let base_short = short(&self.state.base_commit);
4434        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436        let mut jobs = Vec::new();
4437        let mut seats_at = Vec::new();
4438        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439            if self
4440                .state
4441                .judgements
4442                .get(j)
4443                .is_some_and(|r| r.failed.is_some())
4444            {
4445                continue;
4446            }
4447            let seat_key = format!("judge-{}", j + 1);
4448            let spec = self.occupant(&seat_key, spec);
4449            let seat = self.seat(&seat_key, &spec.id);
4450            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451            let text = if has_context(&spec, &seat, sessions) {
4452                prompt::final_vote(&viable, &language)
4453            } else {
4454                full.clone()
4455            };
4456            jobs.push(SeatJob {
4457                spec,
4458                seat,
4459                prompt: text,
4460                cwd: root.join(format!("judge-{}", j + 1)),
4461                timeout,
4462                allow_write: false,
4463                sessions,
4464                artifacts: artifacts.clone(),
4465                stem: format!("vote-judge-{}", j + 1),
4466                handover: Some(full),
4467            });
4468            seats_at.push(j);
4469        }
4470
4471        self.state.event(
4472            "vote",
4473            format!(
4474                "collecting {} final votes one by one, privately",
4475                jobs.len()
4476            ),
4477        );
4478        let allowed = viable.clone();
4479        let mut quota_losses = Vec::new();
4480        let cache = self.state.config.cache_dir();
4481        let ctx = WaveCtx {
4482            carry_seats: false,
4483            pauses: Some((&self.pause, &self.interrupt)),
4484            run: &run_id,
4485            node: "vote",
4486            prompts: &prompts,
4487            cache: cache.as_deref(),
4488            round: None,
4489        };
4490        let results = ask_json_wave::<FinalVote>(
4491            jobs,
4492            Arc::clone(&self.sem),
4493            self.state.config.graph.retries,
4494            &self.roles.judge_roster,
4495            &ctx,
4496            &mut quota_losses,
4497            &mut self.state,
4498            &move |v: &FinalVote| match v.label() {
4499                Some(c) if allowed.contains(&c) => Ok(()),
4500                other => bail!("vote {other:?} is not one of {allowed:?}"),
4501            },
4502        )
4503        .await;
4504        if self.state.resource_block.is_some() {
4505            return self.stop_for_resource();
4506        }
4507        self.state.quota.extend(quota_losses);
4508
4509        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510            let agent_id = seat.agent.clone();
4511            self.state.seats.insert(seat.key.clone(), seat);
4512            let initial = self
4513                .state
4514                .judgements
4515                .get(j)
4516                .and_then(|r| r.ranking.first().copied());
4517            let mut record = VoteRecord {
4518                judge: j + 1,
4519                agent: agent_id,
4520                vote: None,
4521                reason: String::new(),
4522                changed: false,
4523            };
4524            match res {
4525                Ok((v, _)) => {
4526                    record.vote = v.label();
4527                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529                    self.state.event(
4530                        "vote",
4531                        format!(
4532                            "judge {} voted {}{}",
4533                            j + 1,
4534                            record.vote.unwrap_or('?'),
4535                            if record.changed { " (changed)" } else { "" }
4536                        ),
4537                    );
4538                }
4539                Err(e) => {
4540                    self.state
4541                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542                }
4543            }
4544            self.state.votes.push(record);
4545            self.state.save()?;
4546        }
4547        Ok(())
4548    }
4549
4550    // --------------------------------------------------------------- tally
4551
4552    fn tally(&mut self) -> Result<()> {
4553        if self.state.tally.is_some() {
4554            return Ok(());
4555        }
4556        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557        let tops: Vec<char> = self
4558            .state
4559            .judgements
4560            .iter()
4561            .filter_map(|j| j.ranking.first().copied())
4562            .collect();
4563        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565        // A judge whose private vote failed still counted once, in the initial
4566        // ranking; using it beats discarding a whole seat.
4567        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568        let mut cast: Vec<char> = Vec::new();
4569        for (i, j) in self.state.judgements.iter().enumerate() {
4570            let vote = self
4571                .state
4572                .votes
4573                .iter()
4574                .find(|v| v.judge == i + 1)
4575                .and_then(|v| v.vote)
4576                .or_else(|| j.ranking.first().copied());
4577            if let Some(v) = vote {
4578                *first_choice.entry(v).or_insert(0) += 1;
4579                cast.push(v);
4580            }
4581        }
4582
4583        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584        for j in &self.state.judgements {
4585            let n = j.ranking.len();
4586            for (pos, label) in j.ranking.iter().enumerate() {
4587                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588            }
4589        }
4590
4591        let best = first_choice.values().copied().max().unwrap_or(0);
4592        let mut leaders: Vec<char> = first_choice
4593            .iter()
4594            .filter(|(_, v)| **v == best)
4595            .map(|(k, _)| *k)
4596            .collect();
4597        let mut tie_break = None;
4598        if leaders.len() > 1 {
4599            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600            let borda_leaders: Vec<char> = leaders
4601                .iter()
4602                .copied()
4603                .filter(|l| borda[l] == top_borda)
4604                .collect();
4605            tie_break = Some(if borda_leaders.len() == 1 {
4606                format!(
4607                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608                    leaders.len()
4609                )
4610            } else {
4611                format!(
4612                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4613                    leaders.len()
4614                )
4615            });
4616            leaders = borda_leaders;
4617            leaders.sort_unstable();
4618        }
4619        let winner = *leaders
4620            .first()
4621            .or(viable.first())
4622            .context("no candidate to declare a winner from")?;
4623
4624        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626        let deliberated = !self.state.deliberation.is_empty();
4627
4628        // Whose verdict is this? A rate-limited seat is absent even if it
4629        // ranked before the limit hit, so presence is measured against the
4630        // recorded losses, not just "did a ranking ever appear".
4631        let quota_seats: std::collections::BTreeSet<&str> =
4632            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633        let mut present = 0usize;
4634        for (i, j) in self.state.judgements.iter().enumerate() {
4635            if quota_seats.contains(j.seat.as_str()) {
4636                continue;
4637            }
4638            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639            let voted = self
4640                .state
4641                .votes
4642                .iter()
4643                .any(|v| v.judge == i + 1 && v.vote.is_some());
4644            if ranked || voted {
4645                present += 1;
4646            }
4647        }
4648        // Strict majority of the configured panel. A bare majority is real
4649        // signal we can act on, while a minority verdict must never stand in
4650        // for a healthy one. A one-candidate run needs no panel at all, and
4651        // `judges` stays `0` rather than the roster size a panel that never
4652        // sat would otherwise be credited with.
4653        let needs_quorum = viable.len() > 1;
4654        let judges_total = if needs_quorum {
4655            self.roles.judges.len()
4656        } else {
4657            0
4658        };
4659        let quorum = if needs_quorum {
4660            judges_total / 2 + 1
4661        } else {
4662            0
4663        };
4664        let met_quorum = !needs_quorum || present >= quorum;
4665        let uncontested = (!needs_quorum).then(|| {
4666            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667        });
4668
4669        self.state.event(
4670            "tally",
4671            match &uncontested {
4672                Some(reason) => format!("winner {winner} — {reason}"),
4673                None => format!(
4674                    "winner {winner} — votes {} | initial {} | {} changed | \
4675                     {present}/{judges_total} judges{}",
4676                    first_choice
4677                        .iter()
4678                        .map(|(k, v)| format!("{k}:{v}"))
4679                        .collect::<Vec<_>>()
4680                        .join(" "),
4681                    if unanimous_initial {
4682                        "unanimous"
4683                    } else {
4684                        "split"
4685                    },
4686                    changed_votes,
4687                    if met_quorum {
4688                        String::new()
4689                    } else {
4690                        format!(" — below quorum ({quorum} required)")
4691                    },
4692                ),
4693            },
4694        );
4695        if !met_quorum {
4696            self.state.event(
4697                "stall",
4698                format!(
4699                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700                     the run stops here, resumable"
4701                ),
4702            );
4703        }
4704        self.state.tally = Some(Tally {
4705            first_choice,
4706            borda,
4707            winner,
4708            rankings: tops.len(),
4709            unanimous_initial,
4710            deliberated,
4711            changed_votes,
4712            unanimous_final,
4713            tie_break,
4714            judges: judges_total,
4715            present,
4716            quorum,
4717            met_quorum,
4718            uncontested,
4719        });
4720        self.state.status = if met_quorum {
4721            RunStatus::Reviewing
4722        } else {
4723            RunStatus::Stalled
4724        };
4725        self.state.save()?;
4726        Ok(())
4727    }
4728
4729    // ------------------------------------------------------------- recover
4730
4731    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4732    /// resumed toward completion once the transient cause clears.
4733    ///
4734    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4735    /// it toward the quorum, which is exactly the set of seats whose absence
4736    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4737    /// not depend on which node happened to hit the limit), or an ordinary
4738    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4739    /// seat is never disturbed.
4740    ///
4741    /// A seat that now answers with a usable ranking is "recovered": its
4742    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4743    /// `tally` counts it present again), and its vote re-collected. A seat that
4744    /// still fails keeps its loss and stays absent.
4745    ///
4746    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4747    /// to review/gate/merge), `false` when it is still below quorum (the run
4748    /// stays `Stalled`, still resumable for a later retry).
4749    #[allow(clippy::too_many_lines)]
4750    async fn recover_stall(&mut self) -> Result<bool> {
4751        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4752        // agent files with `magi task add` name the run that paid for it. The
4753        // prompt overlay is cloned alongside it because the waves borrow it
4754        // while `self` is mutably borrowed by the node's own bookkeeping.
4755        let run_id = self.state.id.clone();
4756        let prompts = self.state.config.prompts.clone();
4757        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4758        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4759        // is what keeps a non-quota collapse — or a quota loss recorded at the
4760        // deliberate node — from being a permanent dead-end on `--resume`.
4761        let quota_seats: BTreeSet<&str> =
4762            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763        let absent: Vec<String> = self
4764            .state
4765            .judgements
4766            .iter()
4767            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768            .map(|j| j.seat.clone())
4769            .collect();
4770        if absent.is_empty() {
4771            return Ok(false);
4772        }
4773        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774        if viable.len() <= 1 {
4775            return Ok(false);
4776        }
4777        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778        let language = self.state.config.graph.language.clone();
4779        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780        let sessions = self.state.config.graph.sessions;
4781        let artifacts = agent::artifacts_dir(&self.state.dir());
4782        let root = self.state.worktree_root();
4783        let base_short = short(&self.state.base_commit);
4784        let candidates: Vec<Candidate> = viable.clone();
4785
4786        // Map each absent seat key to its 0-based position in `roles.judges`.
4787        let mut positions: Vec<usize> = absent
4788            .iter()
4789            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790            .collect();
4791        if positions.is_empty() {
4792            return Ok(false);
4793        }
4794        positions.sort_unstable();
4795        positions.dedup();
4796
4797        // Re-rank the lost seats, one blind prompt each.
4798        let mut judge_jobs = Vec::new();
4799        for &j in &positions {
4800            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802            let seat_key = format!("judge-{}", j + 1);
4803            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804            let seat = self.seat(&seat_key, &spec.id);
4805            judge_jobs.push(SeatJob {
4806                spec,
4807                seat,
4808                prompt: prompt::judge(
4809                    &self.state.instruction,
4810                    &views,
4811                    self.roles.judges.len(),
4812                    &base_short,
4813                    &language,
4814                ),
4815                cwd: root.join(seat_key),
4816                timeout,
4817                allow_write: false,
4818                sessions,
4819                artifacts: artifacts.clone(),
4820                stem: format!("judge-{}-recover", j + 1),
4821                handover: None,
4822            });
4823        }
4824
4825        let labels_for_check = labels.clone();
4826        let mut judge_losses = Vec::new();
4827        let retries = self.state.config.graph.retries;
4828        let cache = self.state.config.cache_dir();
4829        let ctx = WaveCtx {
4830            carry_seats: false,
4831            pauses: Some((&self.pause, &self.interrupt)),
4832            run: &run_id,
4833            node: "judge",
4834            prompts: &prompts,
4835            cache: cache.as_deref(),
4836            round: None,
4837        };
4838        let results = ask_json_wave::<Ranking>(
4839            judge_jobs,
4840            Arc::clone(&self.sem),
4841            retries,
4842            &self.roles.judge_roster,
4843            &ctx,
4844            &mut judge_losses,
4845            &mut self.state,
4846            &move |r: &Ranking| r.validate(&labels_for_check),
4847        )
4848        .await;
4849        if self.state.resource_block.is_some() {
4850            self.stop_for_resource()?;
4851            return Ok(false);
4852        }
4853
4854        // Refresh the judgement of every seat that ranked again.
4855        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857            let agent_id = seat.agent.clone();
4858            self.state.seats.insert(seat.key.clone(), seat);
4859            let record = &mut self.state.judgements[j];
4860            match res {
4861                Ok((ranking, out)) => {
4862                    record.agent = agent_id;
4863                    record.ranking = ranking.normalized();
4864                    record.reasons = ranking.reasons;
4865                    record.strengths = ranking.strengths;
4866                    record.confidence = ranking.confidence;
4867                    record.failed = None;
4868                    record.duration_ms = out.duration_ms;
4869                    recovered.insert(j);
4870                    self.state.event(
4871                        "recover",
4872                        format!("judge {} ranked again after the limit", j + 1),
4873                    );
4874                }
4875                Err(e) => {
4876                    self.state
4877                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878                }
4879            }
4880        }
4881
4882        // Re-ask the votes of the seats that recovered a ranking.
4883        let mut vote_jobs = Vec::new();
4884        let mut vote_pos: Vec<usize> = Vec::new();
4885        for &j in &recovered {
4886            let seat_key = format!("judge-{}", j + 1);
4887            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888            let seat = self.seat(&seat_key, &spec.id);
4889            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890            let text = if has_context(&spec, &seat, sessions) {
4891                prompt::final_vote(&labels, &language)
4892            } else {
4893                full.clone()
4894            };
4895            vote_jobs.push(SeatJob {
4896                spec,
4897                seat,
4898                prompt: text,
4899                cwd: root.join(seat_key),
4900                timeout,
4901                allow_write: false,
4902                sessions,
4903                artifacts: artifacts.clone(),
4904                stem: format!("vote-judge-{}-recover", j + 1),
4905                handover: Some(full),
4906            });
4907            vote_pos.push(j);
4908        }
4909        let allowed = labels.clone();
4910        let mut vote_losses = Vec::new();
4911        let vote_retries = self.state.config.graph.retries;
4912        let vote_cache = self.state.config.cache_dir();
4913        let ctx = WaveCtx {
4914            carry_seats: false,
4915            pauses: Some((&self.pause, &self.interrupt)),
4916            run: &run_id,
4917            node: "vote",
4918            prompts: &prompts,
4919            cache: vote_cache.as_deref(),
4920            round: None,
4921        };
4922        let votes = ask_json_wave::<FinalVote>(
4923            vote_jobs,
4924            Arc::clone(&self.sem),
4925            vote_retries,
4926            &self.roles.judge_roster,
4927            &ctx,
4928            &mut vote_losses,
4929            &mut self.state,
4930            &move |v: &FinalVote| match v.label() {
4931                Some(c) if allowed.contains(&c) => Ok(()),
4932                other => bail!("vote {other:?} is not one of {allowed:?}"),
4933            },
4934        )
4935        .await;
4936        if self.state.resource_block.is_some() {
4937            self.stop_for_resource()?;
4938            return Ok(false);
4939        }
4940        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941            let agent_id = seat.agent.clone();
4942            self.state.seats.insert(seat.key.clone(), seat);
4943            match res {
4944                Ok((v, _)) => {
4945                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946                        rec.vote = v.label();
4947                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948                    } else {
4949                        self.state.votes.push(VoteRecord {
4950                            judge: j + 1,
4951                            agent: agent_id,
4952                            vote: v.label(),
4953                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954                            changed: false,
4955                        });
4956                    }
4957                    self.state.event(
4958                        "recover",
4959                        format!("judge {} voted again after the limit", j + 1),
4960                    );
4961                }
4962                Err(e) => {
4963                    self.state
4964                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965                }
4966            }
4967        }
4968
4969        // A seat that ranked again is present even if its re-vote failed —
4970        // `tally` falls back to the initial ranking's first choice — so clear
4971        // its quota loss. Seats that still fail keep theirs and stay absent.
4972        let recovered_keys: BTreeSet<String> = recovered
4973            .iter()
4974            .map(|&j| format!("judge-{}", j + 1))
4975            .collect();
4976        self.state
4977            .quota
4978            .retain(|q| !recovered_keys.contains(&q.seat));
4979        // A seat that hit the limit again is a fresh loss, not the old one:
4980        // replace the stale entry so the history stays one-per-seat and the
4981        // daemon can tell this attempt's loss from a previous session's.
4982        for loss in judge_losses.into_iter().chain(vote_losses) {
4983            if recovered_keys.contains(&loss.seat) {
4984                continue;
4985            }
4986            self.state.quota.retain(|q| q.seat != loss.seat);
4987            self.state.quota.push(loss);
4988        }
4989
4990        // Recompute the verdict from the refreshed panel.
4991        self.state.tally = None;
4992        self.tally()?;
4993        Ok(self
4994            .state
4995            .tally
4996            .as_ref()
4997            .map(|t| t.met_quorum)
4998            .unwrap_or(false))
4999    }
5000
5001    // ---------------------------------------------------------------- graft
5002
5003    /// Port the strengths the judges attributed to the losing candidates onto
5004    /// the winner, once, between the tally and the review loop.
5005    ///
5006    /// Best-effort by construction: it returns nothing and fails nothing. A
5007    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5008    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5009    /// the run carries on as if the phase were off. See [`graft`] for the
5010    /// decisions and `AGENTS.md` for the invariants.
5011    ///
5012    /// The one exception is a rollback that could not be confirmed: the
5013    /// rejected commit may still be the winner's tip, so the run is blocked
5014    /// (resumable; the record stays `Running`, which makes a resume retry the
5015    /// restore) instead of reviewing, gating and merging it.
5016    async fn graft(&mut self) -> Result<()> {
5017        let Err(e) = self.graft_inner().await else {
5018            return Ok(());
5019        };
5020        if e.is::<crate::repo_lock::ResourceBlocked>() {
5021            return Err(e);
5022        }
5023        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024            self.state.status = RunStatus::Blocked;
5025            self.state.event("graft", format!("blocked: {e:#}"));
5026            self.state.save()?;
5027            return Err(e);
5028        }
5029        // An error after the `Running` record was saved may have left the
5030        // fixer's work in the tree: the record must not stay unsettled
5031        // (a later resume would reset past the review fixes), so confirm the
5032        // restore now, or block.
5033        if let Some(rec) = self.state.graft.clone()
5034            && !rec.outcome.settled()
5035        {
5036            if let Some(winner) = self.state.winner().cloned() {
5037                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5039                        return Err(re);
5040                    }
5041                    self.state.status = RunStatus::Blocked;
5042                    self.state.event("graft", format!("blocked: {re:#}"));
5043                    self.state.save()?;
5044                    return Err(re);
5045                }
5046            }
5047            if let Some(g) = self.state.graft.as_mut() {
5048                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049            }
5050        }
5051        self.state.event("graft", format!("skipped: {e:#}"));
5052        if let Err(e) = self.state.save() {
5053            tracing::warn!("save after graft: {e:#}");
5054        }
5055        Ok(())
5056    }
5057
5058    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5059    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5060    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061        let _coordination = self.coordinate(wt).await?;
5062        crate::recovery::restore_repair(wt, pre)
5063            .await
5064            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065    }
5066
5067    /// A skip is a run event, recorded once however often the node is re-entered.
5068    fn graft_skipped(&mut self, why: &str) {
5069        let message = format!("skipped: {why}");
5070        if !self
5071            .state
5072            .events
5073            .iter()
5074            .any(|e| e.node == "graft" && e.message == message)
5075        {
5076            self.state.event("graft", message);
5077        }
5078    }
5079
5080    async fn graft_inner(&mut self) -> Result<()> {
5081        // A round that was started and never finished (crash, kill): the
5082        // fixer may have left anything in the tree, so put the winner back.
5083        if let Some(rec) = self.state.graft.clone()
5084            && !rec.outcome.settled()
5085        {
5086            // Reviews exist only if the graft ended without its record being
5087            // settled (a swallowed error): the tree has moved on since, and
5088            // a reset to `pre` would discard the review fixes.
5089            if self.state.reviews.is_empty()
5090                && let Some(winner) = self.state.winner().cloned()
5091            {
5092                self.graft_restore(&winner.worktree, &rec.pre).await?;
5093            }
5094            if rec.outcome == GraftOutcome::WaitingForRestore {
5095                self.state.graft = None;
5096                self.state.save()?;
5097            } else {
5098                if let Some(g) = self.state.graft.as_mut() {
5099                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100                }
5101                self.state.event(
5102                    "graft",
5103                    "rolled back: the previous round was interrupted; the winner is untouched",
5104                );
5105                self.state.save()?;
5106                return Ok(());
5107            }
5108        }
5109        let plan = match graft::plan(&self.state) {
5110            Ok(plan) => plan,
5111            Err(skip) => {
5112                if skip != graft::Skip::AlreadyDone {
5113                    self.graft_skipped(skip.reason());
5114                    self.state.save()?;
5115                }
5116                return Ok(());
5117            }
5118        };
5119        let Some(winner) = self.state.winner().cloned() else {
5120            return Ok(());
5121        };
5122        let wt = winner.worktree.clone();
5123        if !git::is_clean(&wt).await? {
5124            self.graft_skipped("the winner's worktree has uncommitted changes");
5125            self.state.save()?;
5126            return Ok(());
5127        }
5128        let pre = git::rev_parse(&wt, "HEAD").await?;
5129        // Saved before the fixer is called: a stop from here on is repaired
5130        // by the branch above on the next entry.
5131        self.state.graft = Some(GraftRecord {
5132            pre: pre.clone(),
5133            post: None,
5134            lines: None,
5135            limit: None,
5136            agent: None,
5137            outcome: GraftOutcome::Running,
5138            requested: graft::requested(&plan),
5139            adoption: None,
5140            adoption_error: None,
5141        });
5142        self.state.event(
5143            "graft",
5144            format!(
5145                "porting strengths of candidate(s) {} onto {}",
5146                plan.strengths
5147                    .iter()
5148                    .map(|(l, _, _)| l.to_string())
5149                    .collect::<Vec<_>>()
5150                    .join(", "),
5151                plan.winner
5152            ),
5153        );
5154        self.state.save()?;
5155
5156        let fix_prompt = prompt::graft(
5157            &self.state.instruction,
5158            plan.winner,
5159            &plan.strengths,
5160            &self.state.config.graph.language,
5161        );
5162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163        let sessions = self.state.config.graph.sessions;
5164        let artifacts = agent::artifacts_dir(&self.state.dir());
5165        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166        let (_, seat, out) = self
5167            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168                SeatJob {
5169                    prompt: fix_prompt.clone(),
5170                    spec,
5171                    seat,
5172                    cwd: wt.clone(),
5173                    timeout,
5174                    allow_write: true,
5175                    sessions,
5176                    artifacts: artifacts.clone(),
5177                    stem: "graft".to_owned(),
5178                    handover: None,
5179                }
5180            })
5181            .await;
5182        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183            // Preserve the need to restore earlier fallback edits without
5184            // confusing an unstarted seat with an interrupted invocation.
5185            self.state.graft.as_mut().expect("graft record").outcome =
5186                GraftOutcome::WaitingForRestore;
5187            return self.stop_for_resource();
5188        }
5189        let mut why: Option<String> = None;
5190        match &out {
5191            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192                why = Some("the fixer produced nothing usable".to_owned());
5193            }
5194            AgentOutcome::Ok(_) => {}
5195            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196            AgentOutcome::Quota(o) => {
5197                self.state.quota.push(QuotaLoss::from_report(
5198                    seat.key.clone(),
5199                    "graft",
5200                    o.quota.as_ref(),
5201                ));
5202                why = Some("rate limited (quota)".to_owned());
5203            }
5204            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205                why = Some(format!(
5206                    "the fixer failed: {}",
5207                    e.lines()
5208                        .next()
5209                        .unwrap_or("")
5210                        .chars()
5211                        .take(160)
5212                        .collect::<String>()
5213                ));
5214            }
5215        }
5216        let agent_id = seat.agent.clone();
5217        self.state.seats.insert(seat.key.clone(), seat);
5218
5219        let mut post = None;
5220        let mut lines = None;
5221        let _coordination = self.coordinate(&wt).await?;
5222        if why.is_none() {
5223            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224            if let Ok(r) = &rescued {
5225                self.state.note_withheld("graft", &r.withheld);
5226            }
5227            let head = git::rev_parse(&wt, "HEAD").await?;
5228            // What is measured must be what gets reviewed and gated: a failed
5229            // rescue, or anything still uncommitted (withheld lockfiles
5230            // aside, which are never part of the tree), rejects the graft.
5231            let dirty = match &rescued {
5232                Ok(r) => git::status_porcelain(&wt)
5233                    .await
5234                    .map(|st| {
5235                        st.lines().any(|l| {
5236                            !r.withheld
5237                                .iter()
5238                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239                        })
5240                    })
5241                    .unwrap_or(true),
5242                Err(_) => true,
5243            };
5244            if dirty {
5245                why = Some("the fixer left changes that could not be committed".to_owned());
5246            } else if head == pre {
5247                why = Some("the fixer changed nothing".to_owned());
5248            } else {
5249                lines = git::changed_lines(&wt, &pre, &head).await;
5250                post = Some(head);
5251            }
5252        }
5253        let (adoption, adoption_error) = match &out {
5254            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255            _ => (None, None),
5256        };
5257        let requested = graft::requested(&plan);
5258        let outcome = match why {
5259            None => {
5260                self.state.event(
5261                    "graft",
5262                    format!(
5263                        "applied: {}",
5264                        lines.map_or_else(
5265                            || "changed lines unavailable".to_owned(),
5266                            |n| format!("{n} changed lines")
5267                        )
5268                    ),
5269                );
5270                GraftOutcome::Applied
5271            }
5272            Some(why) => {
5273                // Not settled until the restore is confirmed: the record
5274                // stays `Running`, so a resume retries it.
5275                crate::recovery::restore_repair(&wt, &pre)
5276                    .await
5277                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278                self.state.event(
5279                    "graft",
5280                    format!("rolled back: {why}; the winner is untouched"),
5281                );
5282                GraftOutcome::RolledBack(why)
5283            }
5284        };
5285        self.state.graft = Some(GraftRecord {
5286            pre,
5287            post,
5288            lines,
5289            limit: None,
5290            agent: Some(agent_id),
5291            outcome,
5292            requested,
5293            adoption,
5294            adoption_error,
5295        });
5296        self.state.save()?;
5297        Ok(())
5298    }
5299
5300    // ----------------------------------------------------------------- fold
5301
5302    async fn fold_losers(&mut self) -> Result<()> {
5303        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304            return Ok(());
5305        };
5306        if !self.state.candidate_preservation.is_empty() {
5307            bail!("candidate preservation incomplete; refusing cleanup");
5308        }
5309        let repo = self.state.repo.clone();
5310        let _coordination = self.coordinate(&repo).await?;
5311        let mut folded = Vec::new();
5312        for i in 0..self.state.candidates.len() {
5313            let c = &self.state.candidates[i];
5314            if c.label == winner || c.folded {
5315                continue;
5316            }
5317            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318            remove_preserved_worktree(&repo, &wt).await?;
5319            if git::branch_exists(&repo, &branch).await?
5320                && !git::branch_delete(&repo, &branch).await?
5321            {
5322                bail!("candidate branch removal refused");
5323            }
5324            self.state.candidates[i].folded = true;
5325            self.state.save()?;
5326            folded.push(label.to_string());
5327        }
5328        // The judges are finished; their checkouts are pure cost from here.
5329        let root = self.state.worktree_root();
5330        for j in 1..=self.roles.judges.len() {
5331            let wt = root.join(format!("judge-{j}"));
5332            if wt.exists() {
5333                remove_read_seat_worktree(&mut self.state, &wt).await?;
5334                self.state.save()?;
5335            }
5336        }
5337        // The design-deliberation stage is finished by the time a tally
5338        // exists — same reasoning as the judges above.
5339        if self.state.config.graph.advise {
5340            for k in 1..=self.state.config.graph.advisors {
5341                let wt = root.join(format!("advisor-{k}"));
5342                if wt.exists() {
5343                    remove_read_seat_worktree(&mut self.state, &wt).await?;
5344                    self.state.save()?;
5345                }
5346            }
5347        }
5348        if !folded.is_empty() {
5349            self.state
5350                .event("fold", format!("folded candidates {}", folded.join(", ")));
5351            self.state.save()?;
5352        }
5353        Ok(())
5354    }
5355
5356    // ------------------------------------------------------------ base sync
5357
5358    async fn hold_base_sync(&mut self) -> Result<bool> {
5359        self.hold_base_sync_as(None).await
5360    }
5361
5362    async fn hold_base_sync_kind(
5363        &mut self,
5364        kind: crate::base_sync_recovery::StopKind,
5365    ) -> Result<()> {
5366        self.hold_base_sync_as(Some(kind)).await?;
5367        Ok(())
5368    }
5369
5370    async fn hold_base_sync_as(
5371        &mut self,
5372        kind: Option<crate::base_sync_recovery::StopKind>,
5373    ) -> Result<bool> {
5374        if !self.state.base_sync_blocked() {
5375            self.state.base_sync_recovery.stop = None;
5376            return Ok(false);
5377        }
5378        let head = match self.state.winner() {
5379            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380                .await
5381                .unwrap_or_default(),
5382            None => String::new(),
5383        };
5384        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385            && !authorization.consumed
5386            && authorization.stop.head != head
5387        {
5388            authorization.refuse(format!(
5389                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390                authorization.stop.head
5391            ));
5392        }
5393        self.state.record_base_sync_stop(&head);
5394        if let Some(kind) = kind {
5395            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397                stop.kind = kind;
5398            }
5399        }
5400        self.state.save()?;
5401        Ok(true)
5402    }
5403
5404    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5405    /// anything verifies it.
5406    ///
5407    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5408    /// read whatever is checked out in the winner's worktree. Left alone that
5409    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5410    /// it when the run *branched* - and a run takes long enough that the base
5411    /// has usually moved by the time it gets here. A gate that ran there
5412    /// answers "green on the commit this run started from", not "green on
5413    /// what is about to land", and the difference showed up three times in
5414    /// one day as a green run whose merge would have reverted a file another
5415    /// pull request had already landed.
5416    ///
5417    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5418    /// `land::Step::Rebase` calls, rather than a second implementation of the
5419    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5420    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5421    /// doc warns about ("two answers to a question nobody notices until a
5422    /// diff is wrong").
5423    ///
5424    /// Foreign-checkout reviews instead merge the base into the owned shadow
5425    /// worktree, preserving the ancestry required for fast-forward delivery.
5426    /// A conflicting merge is aborted and held for operator resolution.
5427    ///
5428    /// A conflict is not the end of the road: the standing rebase is handed
5429    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5430    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5431    /// land). Once it finishes, review and the gate run as usual on the
5432    /// rebased tree, which is where a breakage the new base caused is caught
5433    /// by the ordinary gate-fix round. magi resolves nothing itself.
5434    ///
5435    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5436    /// the current durable recovery interval, bounds how often the base is
5437    /// synchronized during active recovery (lifetime attempts remain evidence).
5438    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5439    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5440    /// what was tried (rounds spent, paths still conflicted) and the branch
5441    /// and worktree stay exactly as they were - untouched, for a person to
5442    /// look at - which is also what makes re-entering this function
5443    /// afterwards a no-op instead of a second attempt at the same wall. A
5444    /// push failure ends the same way.
5445    async fn sync_to_base(&mut self) -> Result<()> {
5446        if self.state.status == RunStatus::AlreadyInBase {
5447            return Ok(());
5448        }
5449        self.hold_base_sync().await?;
5450        let conflicted = self
5451            .state
5452            .base_sync
5453            .as_ref()
5454            .is_some_and(|s| s.conflict.is_some());
5455        let Some(winner) = self.state.winner().cloned() else {
5456            return Ok(());
5457        };
5458
5459        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461        let repo = self.state.repo.clone();
5462        let remote = self.state.config.merge.remote.clone();
5463        let base_branch = self.state.base_branch.clone();
5464        let tracking = format!("{remote}/{base_branch}");
5465
5466        let fetched_base =
5467            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468        // No network, or the remote never had this branch: the run already
5469        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5470        // otherwise), and one that got this far is not blocked by a fetch
5471        // that fails now. It just has no newer tip to compare against.
5472        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473            return Ok(());
5474        };
5475
5476        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477        self.state.ensure_base_episode();
5478        let mut pending = self
5479            .state
5480            .base_sync_recovery
5481            .operations
5482            .last()
5483            .filter(|o| !o.finished)
5484            .cloned();
5485        let scratch = self.state.dir().join("base-sync");
5486        if git::rebase_in_progress(&scratch).await {
5487            // Older resource stops predate operation reservations. Import the
5488            // standing Git operation, never retarget it to today's main.
5489            if pending.is_none()
5490                && self.state.base_sync_recovery.operations.is_empty()
5491                && !conflicted
5492                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493                    .await
5494                    .ok()
5495                    .as_deref()
5496                    == Some(head.as_str())
5497                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498                && git::is_clean(&winner.worktree).await?
5499            {
5500                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501                if self.state.base_sync.is_none() {
5502                    self.state.base_sync = Some(BaseSync {
5503                        tip: saved_tip.clone(),
5504                        behind: 0,
5505                        attempts: 0,
5506                        conflict: None,
5507                        already_in: None,
5508                    });
5509                }
5510                self.state.reserve_base_operation(&head, &saved_tip);
5511                self.state.event(
5512                    "base-recovery",
5513                    "imported standing legacy rebase as a spent synchronization reservation",
5514                );
5515                self.state.save()?;
5516                pending = self.state.base_sync_recovery.operations.last().cloned();
5517            }
5518            let expected = pending.as_ref().map(|op| op.tip.as_str());
5519            if expected.is_none()
5520                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521            {
5522                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523                if self.state.base_sync.is_none() {
5524                    self.state.base_sync = Some(BaseSync {
5525                        tip: tip.clone(),
5526                        behind: 0,
5527                        attempts: 0,
5528                        conflict: None,
5529                        already_in: None,
5530                    });
5531                }
5532                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533                self.state.event("land", why);
5534                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535                    .await?;
5536                return Ok(());
5537            }
5538        }
5539        if let Some(op) = &pending {
5540            // A replay may add the pinned base's commits, never foreign work.
5541            let mut replay_proven =
5542                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544                let wait = repository_wait(&self.state, None);
5545                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547                    Ok(true) => {
5548                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549                        self.state.save()?;
5550                        git::worktree_remove(&repo, &scratch).await.ok();
5551                        replay_proven = true;
5552                    }
5553                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555                }
5556            }
5557            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558                if self.state.foreign_checkout.is_some() {
5559                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560                        .await
5561                        .ok()
5562                        .as_deref()
5563                        == Some(op.head.as_str())
5564                        && git::rev_parse(&repo, &format!("{head}^2"))
5565                            .await
5566                            .ok()
5567                            .as_deref()
5568                            == Some(op.tip.as_str());
5569                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570                    .await
5571                    .is_ok_and(|m| m.is_empty())
5572                    && let Ok(added) =
5573                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574                {
5575                    replay_proven = true;
5576                    for commit in added {
5577                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578                            replay_proven = false;
5579                            break;
5580                        }
5581                    }
5582                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584                }
5585            }
5586            // Never retarget an operation across a crash, even if main moved.
5587            // sync_to_head resets and cleans with -x, so even ignored untracked
5588            // files must stop recovery independently of the old tracked tree.
5589            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590                || (head != op.head
5591                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592                        .await?
5593                        .is_empty())
5594                || (!git::is_clean(&winner.worktree).await?
5595                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596                        .await
5597                        .is_ok_and(|o| o.ok()))
5598                || !replay_proven
5599            {
5600                let why = format!(
5601                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602                    op.id
5603                );
5604                let sync = self
5605                    .state
5606                    .base_sync
5607                    .as_mut()
5608                    .context("reserved synchronization has no lifetime accounting")?;
5609                sync.conflict = Some(why.clone());
5610                self.state.event("land", why);
5611                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612                    .await?;
5613                return Ok(());
5614            }
5615            if head != op.head {
5616                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617            }
5618            tip = op.tip.clone();
5619        }
5620        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622        if pending.is_none()
5623            && !conflicted
5624            && fetched_base
5625            && self
5626                .state
5627                .base_sync_recovery
5628                .episodes
5629                .last()
5630                .is_some_and(|e| e.boundary.is_some())
5631            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633            && owner.execution_runs().last() == Some(&self.state.id)
5634            && owner.status != crate::queue::TaskStatus::Done
5635            && !owner.fresh_start
5636            && owner.review_branch.is_none()
5637            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638            && let Some(approval) = owner.resumption.as_ref()
5639            && approval.instruction == owner.instruction
5640            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641            && git::is_clean(&winner.worktree).await?
5642            && self.state.renew_base_episode(&task, approval, &head, &tip)
5643        {
5644            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645            self.state.save()?;
5646        }
5647
5648        // Before any rebase, and before a recorded conflict is honoured: a
5649        // branch whose change reached the base under other commit ids has
5650        // nothing to rebase and nothing to conflict with, and a run that
5651        // already stopped on that phantom conflict recovers here on resume.
5652        // `behind == 0` with head == tip is a branch the base has since taken
5653        // in whole, whether or not a conflict was ever recorded: the ancestry
5654        // proof must still run (`classify` ignores a head still on the start
5655        // commit).
5656        if (behind > 0 || conflicted || head == tip)
5657            && self
5658                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659                .await?
5660        {
5661            return Ok(());
5662        }
5663        if conflicted && self.state.base_recovery_pending() {
5664            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665            let owner =
5666                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667            let owner_valid = owner.as_ref().is_ok_and(|t| {
5668                t.execution_runs().last() == Some(&self.state.id)
5669                    && t.status != crate::queue::TaskStatus::Done
5670                    && !t.fresh_start
5671                    && t.review_branch.is_none()
5672            });
5673            let invalidation = if !owner_valid {
5674                Some("latest task ownership changed")
5675            } else if !owner.as_ref().is_ok_and(|t| {
5676                authorization
5677                    .generation
5678                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679            }) {
5680                Some("owner approval generation changed")
5681            } else if authorization.question.as_ref().is_some_and(|question| {
5682                !crate::base_recovery_control::decision_current(
5683                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5684                    &self.state,
5685                    &authorization.task,
5686                    question,
5687                )
5688            }) {
5689                Some("recovery decision ownership, routing or stop evidence changed")
5690            } else if authorization.request.is_some()
5691                && !owner
5692                    .as_ref()
5693                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694            {
5695                Some("owner decision changed")
5696            } else if let Some(scope) = &authorization.scope {
5697                if scope.tip != tip {
5698                    Some("fetched base changed")
5699                } else if !self
5700                    .state
5701                    .base_sync_recovery
5702                    .episodes
5703                    .last()
5704                    .is_some_and(|e| e.id == scope.episode)
5705                {
5706                    Some("active synchronization interval changed")
5707                } else if !owner
5708                    .as_ref()
5709                    .is_ok_and(|t| t.instruction == scope.instruction)
5710                {
5711                    Some("task instruction changed")
5712                } else {
5713                    None
5714                }
5715            } else {
5716                None
5717            };
5718            if let Some(reason) = invalidation {
5719                let authorization = self
5720                    .state
5721                    .base_sync_recovery
5722                    .authorizations
5723                    .last_mut()
5724                    .unwrap();
5725                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726                self.state.save()?;
5727                return Ok(());
5728            }
5729            if !fetched_base
5730                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731                || !git::is_clean(&winner.worktree).await?
5732                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733            {
5734                return Ok(());
5735            }
5736        }
5737        let recovery = conflicted && self.state.consume_base_recovery(&head);
5738        if conflicted {
5739            // Consumption (including a refused changed HEAD) is durable before Git.
5740            self.state.save()?;
5741            if !recovery {
5742                return Ok(());
5743            }
5744        }
5745
5746        if behind == 0 {
5747            // A fixer-finished rebase moves the branch ref before the
5748            // winner's worktree is told (`sync_to_head` below). A run that
5749            // died in between resumes here with `behind == 0` and a tree still
5750            // holding the pre-rebase files, which review and the gate would
5751            // then read. That state is exactly: HEAD moved off the tip the
5752            // rebase started from, yet the tree is still identical to that
5753            // tip. A tree with edits of its own differs from it, so nothing
5754            // is thrown away.
5755            if let Some(from) = self
5756                .state
5757                .rebase_fixes
5758                .iter()
5759                .rev()
5760                .find_map(|r| r.from.clone())
5761                && from != head
5762                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763                    .await
5764                    .is_ok_and(|o| o.ok())
5765            {
5766                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767            }
5768            // An earlier attempt may have rebased the branch locally and died
5769            // before pushing it (only the fresh-rebase arm below pushes).
5770            // Publish it now, so the plain push at PR time is not refused as
5771            // a non-fast-forward. A run already holding a recorded conflict
5772            // never reaches here; that case is out of scope.
5773            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774            if let Some(why) = &conflict {
5775                self.state.status = RunStatus::Blocked;
5776                self.state.event("land", why.clone());
5777            }
5778            self.state.base_sync = Some(BaseSync {
5779                tip,
5780                behind: 0,
5781                attempts,
5782                conflict,
5783                already_in: None,
5784            });
5785            if pending.is_some() {
5786                self.state.finish_base_operation(Some(&head));
5787            }
5788            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789                .await?;
5790            self.state.save()?;
5791            return Ok(());
5792        }
5793
5794        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795            let why = format!(
5796                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797                 synchronization(s); synchronizing again would only race it",
5798                winner.branch
5799            );
5800            self.state.status = RunStatus::Blocked;
5801            self.state.base_sync = Some(BaseSync {
5802                tip,
5803                behind,
5804                attempts,
5805                conflict: Some(why.clone()),
5806                already_in: None,
5807            });
5808            self.state.event("land", why);
5809            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810                .await?;
5811            self.state.save()?;
5812            return Ok(());
5813        }
5814
5815        if self.state.foreign_checkout.is_some() {
5816            // Delivery to the operator's checkout is fast-forward only. A
5817            // rebase here would discard its saved head from our ancestry.
5818            let wait = repository_wait(&self.state, None);
5819            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820            if self.state.base_sync.is_none() {
5821                self.state.base_sync = Some(BaseSync {
5822                    tip: tip.clone(),
5823                    behind,
5824                    attempts: 0,
5825                    conflict: None,
5826                    already_in: None,
5827                });
5828            }
5829            self.state.reserve_base_operation(&head, &tip);
5830            self.state.save()?;
5831            let merged = git::git_raw(
5832                &winner.worktree,
5833                &[
5834                    "-c",
5835                    "core.hooksPath=/dev/null",
5836                    "merge",
5837                    "--no-ff",
5838                    "--no-autostash",
5839                    "--no-edit",
5840                    "-m",
5841                    &format!("Merge {tracking} before verifying"),
5842                    &tip,
5843                ],
5844            )
5845            .await?;
5846            let conflict = if merged.ok() {
5847                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848                self.state.event(
5849                    "land",
5850                    format!("merged {tracking} into {} before verifying", winner.branch),
5851                );
5852                None
5853            } else {
5854                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856                }
5857                let why = format!(
5858                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859                    winner.branch,
5860                    merged.stderr.chars().take(600).collect::<String>()
5861                );
5862                self.state.status = RunStatus::Blocked;
5863                self.state.event("land", &why);
5864                Some(why)
5865            };
5866            let synced = conflict.is_none();
5867            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868            self.state
5869                .finish_base_operation(synced.then_some(fresh.as_str()));
5870            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871            self.state.base_sync = Some(BaseSync {
5872                tip,
5873                behind: if synced { 0 } else { behind },
5874                attempts,
5875                conflict,
5876                already_in: None,
5877            });
5878            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879                .await?;
5880            self.state.save()?;
5881            return Ok(());
5882        }
5883
5884        self.state.event(
5885            "land",
5886            format!(
5887                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888                winner.branch
5889            ),
5890        );
5891        self.state.save()?;
5892
5893        // The remote's copy of the branch, read now and only if the fetch
5894        // really succeeded (a stale tracking ref must never pin a lease). It is
5895        // pushed over after a rebase only when it is a commit this branch
5896        // already contains, by ancestry or by patch (an earlier rebase of ours
5897        // that never reached the remote): anything else is somebody else's work.
5898        let branch_tracking = format!("{remote}/{}", winner.branch);
5899        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900        let remote_tip = if self.state.task_pr_branch().is_none()
5901            && matches!(&fetched_branch, Ok(o) if o.ok())
5902        {
5903            git::rev_parse(&repo, &branch_tracking).await.ok()
5904        } else {
5905            None
5906        };
5907        // A remote tip this branch does not contain is somebody else's work:
5908        // rebasing would leave a local tip that can never be pushed. Stop
5909        // before touching anything and say so.
5910        if let Some(theirs) = &remote_tip
5911            && !git::is_ancestor(&repo, theirs, &head).await
5912            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913                .await
5914                .is_ok_and(|missing| missing.is_empty())
5915        {
5916            let why = format!(
5917                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918                 them",
5919                short(theirs),
5920                winner.branch
5921            );
5922            self.state.status = RunStatus::Blocked;
5923            self.state.base_sync = Some(BaseSync {
5924                tip,
5925                behind,
5926                attempts,
5927                conflict: Some(why.clone()),
5928                already_in: None,
5929            });
5930            self.state.event("land", why);
5931            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932                .await?;
5933            self.state.save()?;
5934            return Ok(());
5935        }
5936
5937        if self.state.base_sync.is_none() {
5938            self.state.base_sync = Some(BaseSync {
5939                tip: tip.clone(),
5940                behind,
5941                attempts: 0,
5942                conflict: None,
5943                already_in: None,
5944            });
5945        }
5946        let scratch = self.state.dir().join("base-sync");
5947        if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948        {
5949            if error.is::<crate::repo_lock::ResourceBlocked>() {
5950                return Ok(());
5951            }
5952            return Err(error);
5953        }
5954        self.state.reserve_base_operation(&head, &tip);
5955        self.state.save()?;
5956        let rebased =
5957            match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958                .await
5959            {
5960                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962                Err(e) => Err(e),
5963            };
5964        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965        let stop_kind = match &rebased {
5966            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969        };
5970        match rebased {
5971            Ok(None) => {
5972                // The branch ref moved, but a worktree that already had it
5973                // checked out (the winner's) was not told; sync its index and
5974                // files before anything reads them.
5975                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977                self.state.finish_base_operation(Some(&fresh));
5978                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979                let mut conflict = None;
5980                if let Some(pinned) = &remote_tip {
5981                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982                    match pushed {
5983                        Ok(o) if o.ok() => self.state.event(
5984                            "land",
5985                            format!("pushed rebased {} to {remote}", winner.branch),
5986                        ),
5987                        Ok(o) => {
5988                            conflict = Some(format!(
5989                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5990                                winner.branch,
5991                                short(pinned),
5992                                o.stderr.chars().take(600).collect::<String>()
5993                            ));
5994                        }
5995                        Err(e) => {
5996                            conflict = Some(format!(
5997                                "rebased {} locally but could not push it: {e:#}",
5998                                winner.branch
5999                            ));
6000                        }
6001                    }
6002                }
6003                if let Some(why) = &conflict {
6004                    self.state.status = RunStatus::Blocked;
6005                    self.state.event("land", why.clone());
6006                }
6007                self.state.base_sync = Some(BaseSync {
6008                    tip: tip.clone(),
6009                    behind: 0,
6010                    attempts,
6011                    conflict,
6012                    already_in: None,
6013                });
6014                self.state
6015                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016            }
6017            Ok(Some(conflict)) => {
6018                self.state.finish_base_operation(None);
6019                let why = format!(
6020                    "{} conflicts with {tracking} and did not rebase: {}",
6021                    winner.branch,
6022                    conflict.chars().take(600).collect::<String>()
6023                );
6024                self.state.status = RunStatus::Blocked;
6025                self.state.base_sync = Some(BaseSync {
6026                    tip,
6027                    behind,
6028                    attempts,
6029                    conflict: Some(why.clone()),
6030                    already_in: None,
6031                });
6032                self.state.event("land", why);
6033            }
6034            // A refused Git write or a busy repository: nothing is wrong with
6035            // the branch. The scratch rebase stays standing and the run
6036            // resumes into it (`base_sync` must not claim a conflict).
6037            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038                self.state.save()?;
6039                return Ok(());
6040            }
6041            Err(e) => {
6042                self.state.finish_base_operation(None);
6043                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044                self.state.status = RunStatus::Blocked;
6045                self.state.base_sync = Some(BaseSync {
6046                    tip,
6047                    behind,
6048                    attempts,
6049                    conflict: Some(why.clone()),
6050                    already_in: None,
6051                });
6052                self.state.event("land", why);
6053            }
6054        }
6055        if !self.state.base_sync_blocked() {
6056            self.state.base_sync_recovery.stop = None;
6057        }
6058        self.hold_base_sync_kind(stop_kind).await?;
6059        self.state.save()?;
6060        Ok(())
6061    }
6062
6063    /// Push a branch an earlier attempt rebased locally but never published,
6064    /// pinned to the remote tip read right after a successful fetch. Returns
6065    /// the reason when the run must stop; `None` when there was nothing to do
6066    /// (no remote copy, the same tip, or a remote copy this branch already
6067    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6068    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069        // Task competitions publish only after the PR text gate, to their
6070        // public head; never resurrect a candidate branch on the remote.
6071        if self.state.task_pr_branch().is_some() {
6072            return None;
6073        }
6074        let repo = self.state.repo.clone();
6075        let remote = self.state.config.merge.remote.clone();
6076        let fetched = git::fetch(&repo, &remote, branch).await;
6077        if !matches!(&fetched, Ok(o) if o.ok()) {
6078            return None;
6079        }
6080        let branch_tracking = format!("{remote}/{branch}");
6081        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083            return None;
6084        }
6085        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086            .await
6087            .is_ok_and(|missing| missing.is_empty())
6088        {
6089            return Some(format!(
6090                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091                 them",
6092                short(&theirs)
6093            ));
6094        }
6095        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096            Ok(o) if o.ok() => {
6097                self.state
6098                    .event("land", format!("pushed rebased {branch} to {remote}"));
6099                None
6100            }
6101            Ok(o) => Some(format!(
6102                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103                 someone may have pushed): {}",
6104                short(&theirs),
6105                o.stderr.chars().take(600).collect::<String>()
6106            )),
6107            Err(e) => Some(format!(
6108                "{branch} is rebased locally but could not be pushed: {e:#}"
6109            )),
6110        }
6111    }
6112
6113    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6114    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6115    /// whether it did.
6116    ///
6117    /// Checked only when the base is ahead of the branch. A failing check is
6118    /// "not proven" - the ordinary rebase path then decides - never a reason to
6119    /// stop the run.
6120    ///
6121    /// The remote copy of the branch is held to the same standard as the local
6122    /// one: if it carries a tip this worktree does not, that tip must itself be
6123    /// proven in the base, or nothing is settled (a pull request would
6124    /// otherwise be closed over commits nobody checked). The pull request is
6125    /// closed *before* the terminal status is saved; if that fails for a
6126    /// reason other than a refusal (no network, a `gh` error) the run is left
6127    /// `Blocked` with the reason as its conflict, which a resume retries -
6128    /// the same recovery a phantom conflict gets.
6129    async fn settle_already_in(
6130        &mut self,
6131        branch: &str,
6132        tip: &str,
6133        head: &str,
6134        attempts: usize,
6135        behind: usize,
6136    ) -> Result<bool> {
6137        let repo = self.state.repo.clone();
6138        let remote = self.state.config.merge.remote.clone();
6139        let start = self.state.base_commit.clone();
6140        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141            Ok(Some(e)) => e,
6142            Ok(None) => return Ok(false),
6143            Err(e) => {
6144                tracing::warn!("already-in-base check for {branch}: {e:#}");
6145                return Ok(false);
6146            }
6147        };
6148        let mut verified = vec![head.to_owned()];
6149        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151        if matches!(&fetched, Ok(o) if o.ok())
6152            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153            && theirs != head
6154        {
6155            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156                Ok(Some(_)) => verified.push(theirs),
6157                _ => return Ok(false),
6158            }
6159        }
6160        let base_branch = self.state.base_branch.clone();
6161        let message = format!(
6162            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163             land",
6164            evidence.names(),
6165            evidence.proof.as_str()
6166        );
6167        let closed =
6168            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169                .await;
6170        match closed {
6171            Ok(Ok(url)) => self
6172                .state
6173                .event("land", format!("closed {url}: superseded on {base_branch}")),
6174            Ok(Err(why)) => self
6175                .state
6176                .event("land", format!("did not close a pull request: {why}")),
6177            Err(e) => {
6178                let why = format!(
6179                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180                     not be closed ({e:#}); resume to retry"
6181                );
6182                self.state.status = RunStatus::Blocked;
6183                self.state.base_sync = Some(BaseSync {
6184                    tip: tip.to_owned(),
6185                    behind,
6186                    attempts,
6187                    conflict: Some(why.clone()),
6188                    already_in: None,
6189                });
6190                self.state.event("land", why);
6191                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192                    .await?;
6193                self.state.save()?;
6194                return Ok(true);
6195            }
6196        }
6197        self.state.status = RunStatus::AlreadyInBase;
6198        self.state.base_sync_recovery.stop = None;
6199        self.state.base_sync = Some(BaseSync {
6200            tip: tip.to_owned(),
6201            behind,
6202            attempts,
6203            conflict: None,
6204            already_in: Some(evidence),
6205        });
6206        self.state.event("land", message);
6207        self.state.save()?;
6208        self.settle_questions();
6209        Ok(true)
6210    }
6211
6212    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6213    /// last landed the winner on, once it has run, else the commit the run
6214    /// branched from.
6215    ///
6216    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6217    /// and `vote` all happen before there is a winner to rebase, so they
6218    /// compare every candidate against the branch point on purpose, and a
6219    /// base that moves after they are already done cannot change an answer
6220    /// they already gave.
6221    fn landing_base(&self) -> String {
6222        self.state
6223            .base_sync
6224            .as_ref()
6225            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226    }
6227
6228    // ------------------------------------------------------- operator fix
6229
6230    /// Route specific, already-recorded review findings to a fixer for a
6231    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6232    /// entry point.
6233    ///
6234    /// Distinct from `review_loop`'s own fix step in three ways: it never
6235    /// runs a reviewer wave, it never spends review-round budget, and what
6236    /// happened is recorded as an [`OperatorFixRequest`] appended to
6237    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6238    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6239    /// and vote must never be rewritten to look like a manufactured blocking
6240    /// verdict.
6241    ///
6242    /// Only meaningful once review has actually concluded: `Ready` (handed
6243    /// off with findings still open, or simply concluded clean while minor
6244    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6245    /// gate failed). Everything else is refused: a run still in progress
6246    /// should simply be resumed, and a `Merged` run's branch has already
6247    /// landed — reopening *this* run's own record cannot change that, so the
6248    /// answer there is a fresh `magi review <branch>`.
6249    ///
6250    /// A real commit here re-verifies through a fresh, ordinary review-only
6251    /// run on the same branch ([`Self::review`]) rather than reopening this
6252    /// run's own `review_loop`: once any round in this run's history went
6253    /// clean, `review_conclusion` treats that as permanent by design (the
6254    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6255    /// way to force one more genuine reviewer wave out of *this* run without
6256    /// either rewriting history or weakening that guarantee for every other
6257    /// caller. A review-only run costs nothing extra — no implementation, no
6258    /// judging, no vote — and exercises the exact same review → verify →
6259    /// gate → (human) merge path, unmodified.
6260    pub async fn fix_selected(
6261        &mut self,
6262        ids: &[String],
6263        reason: &str,
6264        allow_stale: bool,
6265    ) -> Result<()> {
6266        let reason = reason.trim();
6267        if reason.is_empty() {
6268            bail!("a fix request needs a reason — that is the operator's own record of why");
6269        }
6270        if ids.is_empty() {
6271            bail!("no finding id given");
6272        }
6273        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274            bail!(
6275                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276                 has already concluded — can be given a targeted fix. A run still \
6277                 in progress should simply be resumed; a `merged` run's branch has \
6278                 already landed, so its answer is a fresh `magi review <branch>`, \
6279                 not reopening this run's own record",
6280                self.state.id,
6281                self.state.status.as_str()
6282            );
6283        }
6284        // Reassignment shares this claim through its atomic task save. Check
6285        // the canonical run identity inside it, before preflight or any await.
6286        let _claim = FixClaim::acquire(&self.state.dir())?;
6287        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288        let Some(winner) = self.state.winner().cloned() else {
6289            bail!("run {} has no winning candidate to fix", self.state.id);
6290        };
6291        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292            bail!(
6293                "branch `{}` no longer exists; this run cannot be extended",
6294                winner.branch
6295            );
6296        }
6297        let home = crate::run::home();
6298        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299            bail!(
6300                "run {} is currently being worked on by another magi process",
6301                self.state.id
6302            );
6303        }
6304        // Resolve every id before spending anything — an unknown id refuses
6305        // the whole request rather than silently dropping it — and dedup
6306        // while keeping the operator's own order.
6307        let mut seen = BTreeSet::new();
6308        let mut findings = Vec::new();
6309        let mut missing = Vec::new();
6310        for id in ids {
6311            if !seen.insert(id.clone()) {
6312                continue;
6313            }
6314            match self.state.finding(id) {
6315                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316                    id: f.id.clone(),
6317                    severity: f.severity,
6318                    reviewer_vote: rec.vote,
6319                    round: round.round,
6320                    round_head: round.head.clone(),
6321                    reviewer: rec.reviewer,
6322                    agent: rec.agent.clone(),
6323                    file: f.file.clone(),
6324                    line: f.line,
6325                    title: f.title.clone(),
6326                    detail: f.detail.clone(),
6327                    outcome: OperatorFixOutcome::Pending,
6328                }),
6329                None => missing.push(id.clone()),
6330            }
6331        }
6332        if !missing.is_empty() {
6333            bail!(
6334                "unknown finding id(s): {}; nothing was changed",
6335                missing.join(", ")
6336            );
6337        }
6338
6339        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340        let stale_details: Vec<(String, String)> = findings
6341            .iter()
6342            .filter(|f| f.round_head != head_at_request)
6343            .map(|f| (f.id.clone(), f.round_head.clone()))
6344            .collect();
6345        let stale = !stale_details.is_empty();
6346        if stale && !allow_stale {
6347            bail!(
6348                "the branch has moved since some finding(s) were raised — {} — now \
6349                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350                stale_details
6351                    .iter()
6352                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353                    .collect::<Vec<_>>()
6354                    .join(", "),
6355                short(&head_at_request)
6356            );
6357        }
6358
6359        let request = OperatorFixRequest {
6360            requested_at: Timestamp::now(),
6361            reason: reason.to_owned(),
6362            findings,
6363            head_at_request: head_at_request.clone(),
6364            allow_stale,
6365            stale,
6366            fix: None,
6367            result_head: None,
6368            follow_up_review_run: None,
6369        };
6370        self.state.event(
6371            "fix",
6372            format!(
6373                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374                request.findings.len(),
6375                request
6376                    .findings
6377                    .iter()
6378                    .map(|f| f.id.as_str())
6379                    .collect::<Vec<_>>()
6380                    .join(", "),
6381            ),
6382        );
6383        // Recorded now, before any worktree work or the fixer call itself —
6384        // and re-saved at each checkpoint below: a crash at any point after
6385        // this (mid fixer call, mid follow-up review) must not lose the fact
6386        // that this was requested, for which findings, and why. Everything
6387        // past this point reads and writes through `request_index` rather
6388        // than a local variable, since `request` itself is moved here.
6389        self.state.operator_fixes.push(request);
6390        self.state.save()?;
6391        let request_index = self.state.operator_fixes.len() - 1;
6392
6393        // A fresh, dedicated worktree for this one call, never the winner's
6394        // own worktree in place: that one may already be gone (folded away),
6395        // and reusing it in place would leave the branch checked out there
6396        // when the follow-up review below tries to check it out again. Freed
6397        // immediately after, either way — but only once confirmed clean:
6398        // `worktree_remove` is a `git worktree remove --force`, which would
6399        // otherwise discard uncommitted work left there by the operator or
6400        // another process before this had a chance to even look at it.
6401        let repo = self.state.repo.clone();
6402        let coordination = self.coordinate(&repo).await?;
6403        if winner.worktree.exists() {
6404            // Lockfiles a rescue commit withheld stay untracked on purpose and
6405            // are already recorded; they are not the operator's work to protect.
6406            let dirty = git::git(
6407                &winner.worktree,
6408                &["status", "--porcelain", "--untracked-files=all"],
6409            )
6410            .await?;
6411            let only_withheld = dirty.lines().all(|l| {
6412                l.strip_prefix("?? ")
6413                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414            });
6415            if !only_withheld {
6416                bail!(
6417                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6418                     discard them first",
6419                    winner.worktree.display()
6420                );
6421            }
6422            git::worktree_remove(&self.state.repo, &winner.worktree)
6423                .await
6424                .ok();
6425        }
6426        let fix_worktree = self.state.worktree_root().join("operator-fix");
6427        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428        git::git(
6429            &self.state.repo,
6430            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431        )
6432        .await
6433        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434        if !git::is_clean(&fix_worktree).await? {
6435            git::worktree_remove(&self.state.repo, &fix_worktree)
6436                .await
6437                .ok();
6438            bail!(
6439                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440                winner.branch
6441            );
6442        }
6443        let repo_for_setup = self.state.repo.clone();
6444        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445            Ok(()) => {
6446                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447            }
6448            Err(e) => Err(e),
6449        };
6450        if let Err(e) = setup {
6451            git::worktree_remove(&self.state.repo, &fix_worktree)
6452                .await
6453                .ok();
6454            return Err(e);
6455        }
6456
6457        // This explicit operator request retries availability independently
6458        // of execute(); the previous call has already saved and cleaned up.
6459        self.state.resource_block = None;
6460
6461        let run_id = self.state.id.clone();
6462        let prompts = self.state.config.prompts.clone();
6463        let language = self.state.config.graph.language.clone();
6464        let sessions = self.state.config.graph.sessions;
6465        let artifacts = agent::artifacts_dir(&self.state.dir());
6466        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467            .findings
6468            .iter()
6469            .map(|f| Finding {
6470                id: f.id.clone(),
6471                severity: f.severity,
6472                file: f.file.clone(),
6473                line: f.line,
6474                title: f.title.clone(),
6475                detail: f.detail.clone(),
6476            })
6477            .collect();
6478        let fix_prompt = prompt::operator_fix_in(
6479            &self.state.instruction,
6480            &finding_list,
6481            reason,
6482            &stale_details,
6483            &head_at_request,
6484            &language,
6485            &self.state.config.graph.github_language,
6486        );
6487        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488        drop(coordination);
6489        let (job, seat, out) = self
6490            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491                prompt: fix_prompt.clone(),
6492                spec,
6493                seat,
6494                cwd: fix_worktree.clone(),
6495                timeout,
6496                allow_write: true,
6497                sessions,
6498                artifacts: artifacts.clone(),
6499                stem: "operator-fix".to_owned(),
6500                handover: None,
6501            })
6502            .await;
6503        let agent_id = seat.agent.clone();
6504
6505        let mut fix = FixRecord {
6506            agent: agent_id,
6507            addressed: Vec::new(),
6508            rejected: Vec::new(),
6509            notes: String::new(),
6510            committed: false,
6511            failed: None,
6512            duration_ms: 0,
6513            continuation: None,
6514        };
6515        let mut final_seat = seat.clone();
6516        match out {
6517            AgentOutcome::Ok(o) => {
6518                fix.duration_ms = o.duration_ms;
6519                let parsed = verdict::extract_json::<FixReport>(&o.text);
6520                let incomplete_reason = match &parsed {
6521                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522                        "the reply parsed, but it reported a command whose own CLI \
6523                         never confirmed an exit status"
6524                            .to_owned(),
6525                    ),
6526                    Ok(_) => None,
6527                    Err(e) => Some(e.to_string()),
6528                };
6529                match incomplete_reason {
6530                    None => {
6531                        let report = parsed.expect("checked Ok above");
6532                        fix.addressed = report.addressed;
6533                        fix.rejected = report.rejected;
6534                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535                    }
6536                    Some(reason) => {
6537                        let (resumed_seat, resolved, failure, cont) = self
6538                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539                            .await;
6540                        fix.duration_ms += cont.cumulative_wait_ms;
6541                        fix.continuation = Some(cont);
6542                        final_seat = resumed_seat;
6543                        match resolved {
6544                            Some(report) => {
6545                                fix.addressed = report.addressed;
6546                                fix.rejected = report.rejected;
6547                                fix.notes =
6548                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549                            }
6550                            None => fix.failed = failure,
6551                        }
6552                    }
6553                }
6554            }
6555            AgentOutcome::Dropped(o) => {
6556                fix.duration_ms = o.duration_ms;
6557                let why = o
6558                    .dropped
6559                    .as_ref()
6560                    .map(|d| d.why.as_str())
6561                    .unwrap_or("the CLI ended the stream without delivering its answer");
6562                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563            }
6564            AgentOutcome::Quota(o) => {
6565                self.state.quota.push(QuotaLoss::from_report(
6566                    final_seat.key.clone(),
6567                    "fix",
6568                    o.quota.as_ref(),
6569                ));
6570                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571            }
6572            AgentOutcome::Failed(e) => fix.failed = Some(e),
6573            AgentOutcome::ResourceBlocked(e) => {
6574                fix.failed = Some(format!("the fixer was not started: {e}"));
6575            }
6576        }
6577        // Even a deferred call must rescue edits, release the worktree and
6578        // finalize its record below so an explicit `magi fix` can retry.
6579        if fix.continuation.is_none() {
6580            fix.continuation = Some(ContinuationRecord::not_needed());
6581        }
6582        self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584        let rescue_message = format!(
6585            "magi: operator-selected fix ({}) (uncommitted work)",
6586            self.state.operator_fixes[request_index]
6587                .findings
6588                .iter()
6589                .map(|f| f.id.as_str())
6590                .collect::<Vec<_>>()
6591                .join(", ")
6592        );
6593        let repo = self.state.repo.clone();
6594        let coordination = self.coordinate(&repo).await?;
6595        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596            self.state.note_withheld("fix", &r.withheld);
6597        }
6598        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599        fix.committed = after != head_at_request;
6600        git::worktree_remove(&self.state.repo, &fix_worktree)
6601            .await
6602            .ok();
6603        drop(coordination);
6604
6605        self.state.event(
6606            "fix",
6607            match &fix.failed {
6608                Some(reason) => format!(
6609                    "operator fix: adoption report was lost ({reason}); {}",
6610                    if fix.committed {
6611                        "committed"
6612                    } else {
6613                        "NO new commit"
6614                    }
6615                ),
6616                None => format!(
6617                    "operator fix: {} addressed, {} rejected, {}",
6618                    fix.addressed.len(),
6619                    fix.rejected.len(),
6620                    if fix.committed {
6621                        "committed"
6622                    } else {
6623                        "NO new commit"
6624                    }
6625                ),
6626            },
6627        );
6628
6629        // Every selected finding gets an outcome — never left `Pending` once
6630        // the fixer's own turn is over. A report that never came back at all
6631        // marks every one of them `Unreported`, not silently "not addressed":
6632        // quota, a dropped stream, or an exhausted continuation are gaps in
6633        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6634        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6635        for f in &mut self.state.operator_fixes[request_index].findings {
6636            f.outcome = if fix.failed.is_some() {
6637                OperatorFixOutcome::Unreported
6638            } else if fix.addressed.contains(&f.id) {
6639                OperatorFixOutcome::Addressed
6640            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641                OperatorFixOutcome::Rejected { why: r.why.clone() }
6642            } else {
6643                OperatorFixOutcome::Unreported
6644            };
6645        }
6646
6647        let committed = fix.committed;
6648        if committed {
6649            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650        }
6651        self.state.operator_fixes[request_index].fix = Some(fix);
6652        // Saved again now that the fixer's own outcome is final, on top of
6653        // the save right after the request was first pushed above.
6654        self.state.save()?;
6655        if self.state.resource_block.is_some() {
6656            return self.stop_for_resource();
6657        }
6658
6659        if committed {
6660            self.state.event(
6661                "fix",
6662                format!(
6663                    "operator fix committed {}; opening a follow-up review-only run",
6664                    short(&after)
6665                ),
6666            );
6667            // The operator asked for the fix, and the follow-up serves whatever
6668            // task the run it follows served.
6669            let origin =
6670                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671            match Self::review(
6672                &self.state.repo,
6673                &winner.branch,
6674                self.state.config.clone(),
6675                origin,
6676            )
6677            .await
6678            {
6679                Ok(mut follow_up) => {
6680                    follow_up.state.event(
6681                        "start",
6682                        format!(
6683                            "requested by an operator fix on run {} for finding(s) {}",
6684                            self.state.id,
6685                            self.state.operator_fixes[request_index]
6686                                .findings
6687                                .iter()
6688                                .map(|f| f.id.as_str())
6689                                .collect::<Vec<_>>()
6690                                .join(", "),
6691                        ),
6692                    );
6693                    follow_up.state.save()?;
6694                    let follow_up_id = follow_up.state.id.clone();
6695                    // The follow-up is a run like any other: it belongs to the
6696                    // task of the run it follows, or to one filed for it.
6697                    let adopted = match crate::direct::adopt(
6698                        &follow_up.state,
6699                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700                    ) {
6701                        Ok(a) => a,
6702                        Err(e) => {
6703                            // An ownerless run must not spend agent calls; it
6704                            // stays saved, and `magi run --resume` adopts it.
6705                            self.state.event(
6706                                "fix",
6707                                format!(
6708                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709                                ),
6710                            );
6711                            self.state.save()?;
6712                            return Ok(());
6713                        }
6714                    };
6715                    let executed = follow_up.execute().await;
6716                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717                    if let Some(a) = adopted {
6718                        a.finish(&follow_up.state, executed);
6719                    }
6720                    if let Some(e) = failure {
6721                        self.state.event(
6722                            "fix",
6723                            format!(
6724                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725                            ),
6726                        );
6727                    }
6728                    self.state.operator_fixes[request_index].follow_up_review_run =
6729                        Some(follow_up_id);
6730                }
6731                Err(e) => {
6732                    self.state.event(
6733                        "fix",
6734                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6735                    );
6736                }
6737            }
6738            self.state.save()?;
6739        }
6740
6741        Ok(())
6742    }
6743
6744    // --------------------------------------------------------------- review
6745
6746    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6747    /// names, in order, each at most once, moving on only when the call
6748    /// advances (an error, a quota hit or nothing usable - the same decision
6749    /// point `agent::chain_advances` is for the other chained roles).
6750    ///
6751    /// Each agent gets its own seat from `Runner::seat`: the same agent
6752    /// continues its conversation (the winner's own implementer seat when it
6753    /// is the winner's author, now that the competition is over), another
6754    /// takes a fresh one so the full prompt is sent again. A handover is
6755    /// recorded under `node`, which is what makes the fallback stick for the
6756    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6757    /// is returned, so an exhausted chain reads exactly like a single failed
6758    /// fixer: one quota loss, the same wording, the same refund. An earlier
6759    /// attempt's edits are left in the tree and judged, with the final
6760    /// attempt's, by what git says afterwards.
6761    ///
6762    /// Returns the job that produced the outcome, for `continue_fix_report`.
6763    async fn ask_fixer(
6764        &mut self,
6765        winner: &Candidate,
6766        node: &'static str,
6767        round: Option<usize>,
6768        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769    ) -> (SeatJob, SeatState, AgentOutcome) {
6770        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771        self.ask_fixer_among(attempts, node, round, None, build)
6772            .await
6773    }
6774
6775    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6776    /// worktree and a commit: before the next agent is asked after one that
6777    /// did not satisfy the call, the worktree is put back on that commit, so
6778    /// a failed agent's partial edits are not carried into the next one.
6779    async fn ask_fixer_among(
6780        &mut self,
6781        attempts: Vec<(AgentSpec, String)>,
6782        node: &'static str,
6783        round: Option<usize>,
6784        restore: Option<(&Path, &str)>,
6785        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786    ) -> (SeatJob, SeatState, AgentOutcome) {
6787        let run_id = self.state.id.clone();
6788        let prompts = self.state.config.prompts.clone();
6789        let cache = self.state.config.cache_dir();
6790        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791        let mut last = None;
6792        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793            let seat = self.seat(&key, &spec.id);
6794            let mut job = build(spec.clone(), seat);
6795            if i > 0 {
6796                job.stem = format!("{}-{}", job.stem, spec.id);
6797            }
6798            let ctx = WaveCtx {
6799                carry_seats: false,
6800                pauses: Some((&self.pause, &self.interrupt)),
6801                run: &run_id,
6802                node,
6803                prompts: &prompts,
6804                cache: cache.as_deref(),
6805                round,
6806            };
6807            let (seat, out) =
6808                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809            // A resource block never advances the chain: the chosen agent
6810            // was not asked, so it must be the one asked again on resume.
6811            let advances = match &out {
6812                AgentOutcome::Ok(o) => agent::output_advances(o),
6813                AgentOutcome::ResourceBlocked(_) => false,
6814                _ => true,
6815            };
6816            if let Some(next) = ids.get(i + 1)
6817                && advances
6818            {
6819                self.state.seats.insert(seat.key.clone(), seat.clone());
6820                let class =
6821                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822                let reason = match &out {
6823                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824                    other => fail_reason(other),
6825                };
6826                // A failed agent's edits must not reach the next one, and a
6827                // restore that did not work ends the chain: the call fails
6828                // and the caller's own (confirmed) restore decides.
6829                if let Some((wt, rev)) = restore
6830                    && let Err(e) = self.graft_restore(wt, rev).await
6831                {
6832                    let why =
6833                        format!("could not restore the worktree before the next fixer: {e:#}");
6834                    tracing::warn!("{why}");
6835                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836                        AgentOutcome::ResourceBlocked(why)
6837                    } else {
6838                        AgentOutcome::Failed(why)
6839                    };
6840                    last = Some((job, seat, out));
6841                    break;
6842                }
6843                record_handover(
6844                    &mut self.state,
6845                    node,
6846                    &seat.key,
6847                    &spec.id,
6848                    next,
6849                    &class,
6850                    &reason,
6851                );
6852                continue;
6853            }
6854            last = Some((job, seat, out));
6855            break;
6856        }
6857        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858        if let AgentOutcome::Quota(output) = &out
6859            && node != "graft"
6860        {
6861            // Graft is best effort: its caller records the loss and restores
6862            // the pre-graft tree, then proceeds to review and gate.
6863            // Every fallback was tried. Machine outages cannot be repaired by
6864            // spending a fix round; use each caller's resumable resource path.
6865            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867            self.state.quota.push(loss);
6868            self.state.seats.insert(seat.key.clone(), seat.clone());
6869            self.state.status = RunStatus::Blocked;
6870            return (
6871                job,
6872                seat,
6873                AgentOutcome::ResourceBlocked(
6874                    "fixer unavailable; restore agent availability and resume".into(),
6875                ),
6876            );
6877        }
6878        (job, seat, out)
6879    }
6880
6881    /// The one repair a verification-only failure earns (see
6882    /// [`crate::recovery`]), before the first review round of a run that took
6883    /// a branch over.
6884    ///
6885    /// Saved as started *before* the fixer is called, so a resume never
6886    /// repairs twice; a start found on re-entry is rolled back to the commit
6887    /// it began from. A failed or unusable fixer leaves the tree as it was and
6888    /// the review goes on over it, so the first round's own verification
6889    /// reports the failure again. The failure is never carried into a round as
6890    /// a result: verification is re-run on whatever commit this leaves.
6891    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892        use crate::recovery::{Mode, Repair};
6893        if !self.state.reviews.is_empty() {
6894            return Ok(());
6895        }
6896        let Some(mut rec) = self.state.recovery.clone() else {
6897            return Ok(());
6898        };
6899        if rec.mode != Mode::VerifyRepair {
6900            return Ok(());
6901        }
6902        let wt = winner.worktree.clone();
6903        let mut head = git::rev_parse(&wt, "HEAD").await?;
6904        match rec.repair.clone() {
6905            Repair::Done { .. } => return Ok(()),
6906            Repair::WaitingForRestore { before } => {
6907                self.graft_restore(&wt, &before).await?;
6908                head = before;
6909                rec.repair = Repair::NotStarted;
6910                self.state.recovery = Some(rec.clone());
6911                self.state.save()?;
6912            }
6913            Repair::Started { before } => {
6914                // Interrupted: the unfinished edits are not trusted.
6915                // An unconfirmed rollback blocks the run with `Started` kept,
6916                // so a resume retries it; the tree is never reviewed as is.
6917                if let Err(e) = self.graft_restore(&wt, &before).await {
6918                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6919                        return Err(e);
6920                    }
6921                    return self
6922                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923                        .await;
6924                }
6925                rec.repair = Repair::Done {
6926                    before: before.clone(),
6927                    after: before,
6928                    committed: false,
6929                    failed: Some("interrupted; rolled back".to_owned()),
6930                };
6931                self.state
6932                    .event("recovery", "the repair was interrupted and rolled back");
6933                self.state.recovery = Some(rec);
6934                self.state.save()?;
6935                return Ok(());
6936            }
6937            Repair::NotStarted => {}
6938        }
6939        if crate::recovery::recheck(&mut rec, &head) {
6940            self.state.event("recovery", rec.summary());
6941            self.state.recovery = Some(rec);
6942            self.state.save()?;
6943            return Ok(());
6944        }
6945        rec.repair = Repair::Started {
6946            before: head.clone(),
6947        };
6948        self.state.recovery = Some(rec.clone());
6949        self.state.save()?;
6950
6951        let language = self.state.config.graph.language.clone();
6952        let sessions = self.state.config.graph.sessions;
6953        let artifacts = agent::artifacts_dir(&self.state.dir());
6954        let fix_prompt = prompt::recovery_fix(
6955            &self.state.instruction,
6956            &rec,
6957            self.state.config.graph.review_rounds,
6958            &language,
6959        );
6960        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961        self.state.event(
6962            "recovery",
6963            format!(
6964                "repair: the fixer starts from the failed verification on {}",
6965                short(&head)
6966            ),
6967        );
6968        let (_job, seat, out) = self
6969            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970                prompt: fix_prompt.clone(),
6971                spec,
6972                seat,
6973                cwd: wt.clone(),
6974                timeout,
6975                allow_write: true,
6976                sessions,
6977                artifacts: artifacts.clone(),
6978                stem: "fix-recovery".to_owned(),
6979                handover: None,
6980            })
6981            .await;
6982        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983            rec.repair = Repair::WaitingForRestore { before: head };
6984            self.state.recovery = Some(rec);
6985            return self.stop_for_resource();
6986        }
6987        let failed = match &out {
6988            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989            AgentOutcome::Quota(o) => {
6990                self.state.quota.push(QuotaLoss::from_report(
6991                    seat.key.clone(),
6992                    "fix",
6993                    o.quota.as_ref(),
6994                ));
6995                Some("rate limited (quota); the fixer could not run".to_owned())
6996            }
6997            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998            AgentOutcome::Failed(e) => Some(e.clone()),
6999        };
7000        self.state.seats.insert(seat.key.clone(), seat);
7001        let coordination = self.coordinate(&wt).await?;
7002        let failed = match failed {
7003            // The agent's partial edits are not kept.
7004            Some(why) => {
7005                if let Err(e) = git::restore_to(&wt, &head).await {
7006                    return self
7007                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008                        .await;
7009                }
7010                Some(why)
7011            }
7012            None => {
7013                if let Ok(r) = git::rescue_commit(
7014                    &wt,
7015                    "magi: recovery repair of the failed verification (uncommitted work)",
7016                )
7017                .await
7018                {
7019                    self.state.note_withheld("fix", &r.withheld);
7020                }
7021                None
7022            }
7023        };
7024        let after = git::rev_parse(&wt, "HEAD").await?;
7025        drop(coordination);
7026        self.state.event(
7027            "recovery",
7028            match &failed {
7029                Some(why) => format!("repair: nothing kept ({why})"),
7030                None if after != head => {
7031                    format!("repair: committed {}", short(&after))
7032                }
7033                None => "repair: the fixer made no commit".to_owned(),
7034            },
7035        );
7036        rec.repair = Repair::Done {
7037            before: head.clone(),
7038            committed: after != head,
7039            after,
7040            failed,
7041        };
7042        self.state.recovery = Some(rec);
7043        self.state.save()?;
7044        Ok(())
7045    }
7046
7047    /// A repair whose rollback could not be confirmed: the run is blocked and
7048    /// resumable, and the unconfirmed state is never reviewed.
7049    async fn stop_recovery(
7050        &mut self,
7051        mut rec: crate::recovery::RecoveryRecord,
7052        head: &str,
7053        why: String,
7054    ) -> Result<()> {
7055        // `Started` stays recorded: a resume retries the restore.
7056        rec.repair = crate::recovery::Repair::Started {
7057            before: head.to_owned(),
7058        };
7059        self.state.recovery = Some(rec);
7060        self.state
7061            .event("recovery", format!("repair could not be undone: {why}"));
7062        self.state.status = RunStatus::Blocked;
7063        self.state.save()?;
7064        bail!("recovery repair could not be rolled back: {why}")
7065    }
7066
7067    async fn review_loop(&mut self) -> Result<()> {
7068        if self.hold_base_sync().await? {
7069            return Ok(());
7070        }
7071        loop {
7072            if self
7073                .state
7074                .base_sync
7075                .as_ref()
7076                .is_some_and(|s| s.conflict.is_some())
7077            {
7078                return Ok(());
7079            }
7080            if self.catchup_repairable() {
7081                let record = self.state.reviews.last().expect("repairable round").clone();
7082                let winner = self
7083                    .state
7084                    .winner()
7085                    .cloned()
7086                    .context("catch-up repair has no winner")?;
7087                if !self.fix_review_round(&winner, record).await? {
7088                    return Ok(());
7089                }
7090            }
7091            self.review_loop_pass().await?;
7092            if !self.catchup_repairable() {
7093                return Ok(());
7094            }
7095        }
7096    }
7097
7098    /// Derive an owed catch-up repair from durable round evidence, including
7099    /// reentry after a crash before the fixer was dispatched.
7100    fn catchup_repairable(&self) -> bool {
7101        self.state.reviews.len() < self.state.config.graph.review_rounds
7102            && self.state.reviews.last().is_some_and(|r| {
7103                !r.clean
7104                    && r.blocking == 0
7105                    && r.fix.is_none()
7106                    && r.e2e_status() == E2eStatus::Failed
7107            })
7108    }
7109
7110    async fn review_loop_pass(&mut self) -> Result<()> {
7111        // A base that would not rebase is a person's decision, not a review
7112        // round: nothing here would change the answer, and reviewers and a
7113        // fixer would be spending real budget on a tree that cannot land
7114        // regardless of what they find.
7115        if self
7116            .state
7117            .base_sync
7118            .as_ref()
7119            .is_some_and(|s| s.conflict.is_some())
7120        {
7121            return Ok(());
7122        }
7123        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7124        // agent files with `magi task add` name the run that paid for it. The
7125        // prompt overlay is cloned alongside it because the waves borrow it
7126        // while `self` is mutably borrowed by the node's own bookkeeping.
7127        let run_id = self.state.id.clone();
7128        let prompts = self.state.config.prompts.clone();
7129        let Some(winner) = self.state.winner().cloned() else {
7130            return Ok(());
7131        };
7132        let granted_review = self
7133            .state
7134            .repair_grants
7135            .last()
7136            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137            .cloned();
7138        let fresh_review = granted_review
7139            .as_ref()
7140            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141        let max_rounds = granted_review
7142            .as_ref()
7143            .map_or(self.state.config.graph.review_rounds, |g| {
7144                g.review_start.saturating_add(1)
7145            });
7146        // A clean round, an exhausted round budget, or a stalled tree (see
7147        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7148        // they are recorded — recomputed here, not read off `status`, so a
7149        // reentry into a run that already stopped restates the identical
7150        // verdict instead of silently handing back whatever an earlier node
7151        // in this same walk clobbered `status` to (a solo-candidate
7152        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7153        // below runs an empty range once the budget is spent, and would
7154        // otherwise fall through without touching `status` at all.
7155        if let Some(status) =
7156            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157        {
7158            // A reentry after a crash between the last round's save and
7159            // `stop_reviewing` reaches the hand-off here, not there.
7160            if status == RunStatus::Gating {
7161                self.record_contested_handoff();
7162            }
7163            self.state.status = status;
7164            self.state.save()?;
7165            return Ok(());
7166        }
7167        self.state.status = RunStatus::Reviewing;
7168        // A last recorded round whose own verification never resolved
7169        // (`ResourceBlocked` — the shared build cache, not the patch) is
7170        // never a concluded round, whatever the round budget says: starting
7171        // a fresh round on top of it would spend a whole new reviewer wave
7172        // re-reading an unchanged patch instead of just retrying the one
7173        // check that actually needs it, and once the budget is spent the
7174        // loop below has nothing left to do at all (its range is empty).
7175        // Retry that check directly instead, exactly the same retry
7176        // `stop_reviewing` already does for its own catch-up case.
7177        if !fresh_review
7178            && self.state.reviews.last().is_some_and(|r| {
7179                r.e2e_status() == E2eStatus::ResourceBlocked
7180                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181            })
7182        {
7183            let shell = self.state.config.shell();
7184            return self
7185                .stop_unresolved_review(
7186                    "the last round's own verification never resolved",
7187                    &shell,
7188                    &winner.worktree,
7189                )
7190                .await;
7191        }
7192
7193        if granted_review.is_none() {
7194            self.recovery_repair(&winner).await?;
7195        }
7196        if self.resource_halt()? {
7197            return Ok(());
7198        }
7199
7200        let repo = self.state.repo.clone();
7201        let root = self.state.worktree_root();
7202        let language = self.state.config.graph.language.clone();
7203        let sessions = self.state.config.graph.sessions;
7204        let artifacts = self.merge_repair_artifacts();
7205        let base = self.landing_base();
7206        let base_short = short(&base);
7207        let reviewers = self.roles.reviewers.clone();
7208        let shell = self.state.config.shell();
7209
7210        for round in (self.state.reviews.len() + 1)..=max_rounds {
7211            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214            // The prior round's own record, already persisted — never a
7215            // hand-carried variable of just its failing output: that is
7216            // exactly what let a round's e2e result drift out of sync with
7217            // which commit it was actually about (see `SCHEMA`'s doc for
7218            // schema 8). Judged against `head`, the commit reviewers are
7219            // about to look at now, so the summary always reads as "an
7220            // earlier head" here — this round's own patch has not been
7221            // checked yet.
7222            let prev_verification = self
7223                .state
7224                .reviews
7225                .last()
7226                .and_then(|r| r.verification_summary(&head));
7227            // Only the first round of a run that took a branch over: later
7228            // rounds have this run's own history.
7229            let prior_section = match (&self.state.recovery, round) {
7230                (Some(rec), 1) if self.state.reviews.is_empty() => {
7231                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233                            Some(crate::recovery::head_of(
7234                                &git::diff(&winner.worktree, reviewed, "HEAD")
7235                                    .await
7236                                    .unwrap_or_default(),
7237                                crate::recovery::SINCE_DIFF_MAX,
7238                            ))
7239                        }
7240                        _ => None,
7241                    };
7242                    Some(prompt::recovery_review(rec, since.as_deref()))
7243                }
7244                _ => None,
7245            };
7246
7247            // Each reviewer gets its own detached checkout of exactly this
7248            // commit: nobody can perturb the winner's tree, and the fixer can
7249            // keep working without racing a reviewer.
7250            let mut jobs = Vec::new();
7251            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252                self.coordinate(&repo).await?
7253            } else {
7254                None
7255            };
7256            for (r, spec) in reviewers.iter().cloned().enumerate() {
7257                let wt = root.join(format!("review-{}", r + 1));
7258                // `clean -fdx` in a reset wipes setup's products, so every
7259                // round sets the seat up again.
7260                if wt.exists() {
7261                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262                } else {
7263                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264                        None
7265                    } else {
7266                        self.coordinate(&repo).await?
7267                    };
7268                    git::worktree_add_detached(&repo, &wt, &head).await?;
7269                }
7270                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271                    None
7272                } else {
7273                    self.coordinate(&repo).await?
7274                };
7275                self.setup_seat_worktree(&repo, &wt, false).await?;
7276                let seat_key = format!("review-{}", r + 1);
7277                // The seat starts the round on whoever answered it last, not
7278                // on the agent the spec names, so a failure is not re-paid.
7279                let spec = pick_start_spec(
7280                    &self.roles.reviewer_roster,
7281                    spec,
7282                    self.state.seat_history.get(&seat_key),
7283                );
7284                let seat = self.seat(&seat_key, &spec.id);
7285                jobs.push(SeatJob {
7286                    prompt: prompt::review(&prompt::ReviewCtx {
7287                        instruction: &self.state.instruction,
7288                        branch: &winner.branch,
7289                        base_short: &base_short,
7290                        stat: &stat,
7291                        patch: &patch,
7292                        verification: prev_verification.as_ref(),
7293                        prior: prior_section.as_deref(),
7294                        reviewers: reviewers.len(),
7295                        round,
7296                        rounds: max_rounds,
7297                        // A review-only run has no rankings, so nothing
7298                        // competed for this patch and the reviewer is told so.
7299                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300                        lens: Lens::for_seat(r),
7301                        language: &language,
7302                        github_language: &self.state.config.graph.github_language,
7303                    }),
7304                    spec,
7305                    seat,
7306                    cwd: wt,
7307                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308                    allow_write: false,
7309                    sessions,
7310                    artifacts: artifacts.clone(),
7311                    stem: format!("review-{round}-{}", r + 1),
7312                    handover: None,
7313                });
7314            }
7315
7316            drop(coordination);
7317            self.state.event(
7318                "review",
7319                format!(
7320                    "round {round}: {} reviewers on {}",
7321                    jobs.len(),
7322                    short(&head)
7323                ),
7324            );
7325            let mut quota_losses = Vec::new();
7326            let review_retries = self.state.config.graph.retries;
7327            let review_cache = self.state.config.cache_dir();
7328            let ctx = WaveCtx {
7329                carry_seats: true,
7330                pauses: Some((&self.pause, &self.interrupt)),
7331                run: &run_id,
7332                node: "review",
7333                prompts: &prompts,
7334                cache: review_cache.as_deref(),
7335                round: Some(round),
7336            };
7337            let results = ask_json_wave::<Review>(
7338                jobs,
7339                Arc::clone(&self.sem),
7340                review_retries,
7341                &self.roles.reviewer_roster,
7342                &ctx,
7343                &mut quota_losses,
7344                &mut self.state,
7345                &|_: &Review| Ok(()),
7346            )
7347            .await;
7348            if self.state.resource_block.is_some() {
7349                return self.stop_for_resource();
7350            }
7351            // Counted before the move below: how many of *this* round's
7352            // reviewer seats were lost to their own rate limit, as opposed to
7353            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7354            let round_quota_missing = quota_losses.len();
7355            self.state.quota.extend(quota_losses);
7356
7357            let mut records = Vec::new();
7358            let mut all_findings = Vec::new();
7359            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360                let agent_id = seat.agent.clone();
7361                self.state.seats.insert(seat.key.clone(), seat);
7362                let mut record = ReviewRecord {
7363                    reviewer: r + 1,
7364                    agent: agent_id,
7365                    summary: String::new(),
7366                    findings: Vec::new(),
7367                    vote: None,
7368                    failed: None,
7369                    duration_ms: 0,
7370                    // Set for both outcomes: `failed: Some(_)` with
7371                    // `attempts > 0` is a seat every retry still lost, not a
7372                    // recovered one — only `failed: None` with `attempts > 0`
7373                    // reads as "answered after a nudge" (see this field's own
7374                    // doc).
7375                    attempts,
7376                };
7377                match res {
7378                    Ok((review, out)) => {
7379                        // Sanitized here, at the point every other piece of
7380                        // agent prose in this file is (candidate summaries,
7381                        // deliberation turns, vote reasons): a reviewer's own
7382                        // words are the one thing about it that could name
7383                        // it, and reconsideration below broadcasts this same
7384                        // summary and these same findings to every other
7385                        // seat on the panel.
7386                        record.summary =
7387                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388                        record.vote = Some(review.vote);
7389                        record.duration_ms = out.duration_ms;
7390                        for (n, mut f) in review.findings.into_iter().enumerate() {
7391                            // ids are magi's, never the agent's: the fixer's
7392                            // adoption report is keyed by them.
7393                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396                            // `file` is agent-supplied prose too, never
7397                            // checked against the real tree — the same
7398                            // exposure `title`/`detail` above have, just in
7399                            // a field easy to forget because it looks like a
7400                            // path rather than free text.
7401                            f.file = f
7402                                .file
7403                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404                            all_findings.push(f.clone());
7405                            record.findings.push(f);
7406                        }
7407                        self.state.event(
7408                            "review",
7409                            format!(
7410                                "round {round}: reviewer {} voted {} with {} finding(s)",
7411                                r + 1,
7412                                review.vote.label(),
7413                                record.findings.len()
7414                            ),
7415                        );
7416                    }
7417                    Err(e) => {
7418                        record.failed = Some(e.to_string());
7419                        self.state.event(
7420                            "review",
7421                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422                        );
7423                    }
7424                }
7425                records.push(record);
7426            }
7427
7428            // Tally the round's votes and, if they split, spend the one
7429            // round of reconsideration the split -> deliberate -> revote
7430            // shape `judge`/`vote` use for the panel, sized down to what a
7431            // read-only review round can afford: one round, and a revote
7432            // rather than an argument, because the panel already wrote its
7433            // reasoning down as findings the first time around.
7434            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435            let vote_split =
7436                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438            if vote_split {
7439                self.state.event(
7440                    "review",
7441                    format!(
7442                        "round {round}: votes split ({}) — one round of reconsideration",
7443                        initial_votes
7444                            .iter()
7445                            .map(|v| v.label())
7446                            .collect::<Vec<_>>()
7447                            .join(", ")
7448                    ),
7449                );
7450                // Seats read every seat's findings and votes, still numbered
7451                // and never named — the same anonymity `review` itself keeps.
7452                let panel: Vec<ReviewSeatReport<'_>> = records
7453                    .iter()
7454                    .filter_map(|r| {
7455                        r.vote.map(|vote| ReviewSeatReport {
7456                            reviewer: r.reviewer,
7457                            vote,
7458                            summary: &r.summary,
7459                            findings: &r.findings,
7460                        })
7461                    })
7462                    .collect();
7463
7464                let mut jobs = Vec::new();
7465                let mut seats_at = Vec::new();
7466                for (r, spec) in reviewers.iter().cloned().enumerate() {
7467                    // A seat with no initial vote has nothing to reconsider
7468                    // from and stays absent, the same as it stayed absent
7469                    // from `panel` above.
7470                    if records[r].vote.is_none() {
7471                        continue;
7472                    }
7473                    let wt = root.join(format!("review-{}", r + 1));
7474                    let seat_key = format!("review-{}", r + 1);
7475                    let spec = self.occupant(&seat_key, spec);
7476                    let seat = self.seat(&seat_key, &spec.id);
7477                    // A seat with no live session has already forgotten the
7478                    // initial review's prompt — restate the patch it is
7479                    // voting on, the same as `deliberate`/`vote` do for a
7480                    // judge in the same position.
7481                    // The panel already carries this seat's own review and
7482                    // vote, so restating the patch makes the prompt whole for
7483                    // a seat handed to another agent.
7484                    let build = |with_patch: bool| {
7485                        prompt::review_reconsider(&ReviewReconsiderCtx {
7486                            instruction: &self.state.instruction,
7487                            reviewer: r + 1,
7488                            lens: Lens::for_seat(r),
7489                            panel: &panel,
7490                            patch: with_patch.then_some(ReviewPatch {
7491                                branch: &winner.branch,
7492                                base_short: &base_short,
7493                                stat: &stat,
7494                                patch: &patch,
7495                            }),
7496                            round,
7497                            rounds: max_rounds,
7498                            language: &language,
7499                        })
7500                    };
7501                    let full = build(true);
7502                    let prompt = if has_context(&spec, &seat, sessions) {
7503                        build(false)
7504                    } else {
7505                        full.clone()
7506                    };
7507                    jobs.push(SeatJob {
7508                        prompt,
7509                        spec,
7510                        seat,
7511                        cwd: wt,
7512                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513                        allow_write: false,
7514                        sessions,
7515                        artifacts: artifacts.clone(),
7516                        stem: format!("review-{round}-reconsider-{}", r + 1),
7517                        handover: Some(full),
7518                    });
7519                    seats_at.push(r);
7520                }
7521
7522                let mut recon_quota_losses = Vec::new();
7523                let recon_cache = self.state.config.cache_dir();
7524                let recon_ctx = WaveCtx {
7525                    carry_seats: true,
7526                    pauses: Some((&self.pause, &self.interrupt)),
7527                    run: &run_id,
7528                    node: "review",
7529                    prompts: &prompts,
7530                    cache: recon_cache.as_deref(),
7531                    round: Some(round),
7532                };
7533                let recon_results = ask_json_wave::<ReviewRevote>(
7534                    jobs,
7535                    Arc::clone(&self.sem),
7536                    review_retries,
7537                    &self.roles.reviewer_roster,
7538                    &recon_ctx,
7539                    &mut recon_quota_losses,
7540                    &mut self.state,
7541                    &|_: &ReviewRevote| Ok(()),
7542                )
7543                .await;
7544                if self.state.resource_block.is_some() {
7545                    return self.stop_for_resource();
7546                }
7547                self.state.quota.extend(recon_quota_losses);
7548
7549                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550                    let agent_id = seat.agent.clone();
7551                    self.state.seats.insert(seat.key.clone(), seat);
7552                    let mut rec = ReviewRevoteRecord {
7553                        reviewer: r + 1,
7554                        agent: agent_id,
7555                        vote: None,
7556                        reason: String::new(),
7557                        failed: None,
7558                    };
7559                    match res {
7560                        Ok((rv, _)) => {
7561                            rec.vote = Some(rv.vote);
7562                            rec.reason =
7563                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564                            self.state.event(
7565                                "review",
7566                                format!(
7567                                    "round {round}: reviewer {} revoted {}",
7568                                    r + 1,
7569                                    rv.vote.label()
7570                                ),
7571                            );
7572                        }
7573                        Err(e) => {
7574                            rec.failed = Some(e.to_string());
7575                            self.state.event(
7576                                "review",
7577                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578                            );
7579                        }
7580                    }
7581                    reconsideration.push(rec);
7582                }
7583            } else if initial_votes.len() > 1 {
7584                self.state.event(
7585                    "review",
7586                    format!(
7587                        "round {round}: votes agreed ({}) — no reconsideration",
7588                        initial_votes[0].label()
7589                    ),
7590                );
7591            }
7592
7593            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595            // A round that already has a blocking finding and a round left to
7596            // try is going back to the fixer no matter what `verify.e2e`
7597            // says, so running it first only spends the loop's slowest step
7598            // (minutes, for a Rust repo's full test suite) on a head about
7599            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7600            // runs once a round has no blocking findings left (see
7601            // `round_is_clean`, which a deferred — empty — `e2e` can never
7602            // satisfy since `blocking` is nonzero whenever this branch is
7603            // taken), and `stop_reviewing` forces a real run before it will
7604            // ever read a deferred round as green.
7605            let defer_e2e =
7606                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608                let reason =
7609                    format!("{blocking} blocking finding(s) already required a fix this round");
7610                self.state.event(
7611                    "verify",
7612                    format!(
7613                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614                         {}); it will run once a round has none left",
7615                        short(&head)
7616                    ),
7617                );
7618                (Vec::new(), false, true, Some(reason))
7619            } else {
7620                let e2e_commands = self.state.config.verify.e2e.clone();
7621                let cache_dir = self.state.config.cache_dir();
7622                let context = format!("round {round}");
7623                let wait = self.repository_wait();
7624                let (e2e, verify_retried) = with_cache_lease(
7625                    &mut self.state,
7626                    cache_dir.as_deref(),
7627                    "e2e",
7628                    "e2e",
7629                    &winner.worktree,
7630                    &head,
7631                    verify_timeout,
7632                    &context,
7633                    &wait,
7634                    |state, budget| {
7635                        let shell = shell.clone();
7636                        let e2e_commands = e2e_commands.clone();
7637                        let worktree = winner.worktree.clone();
7638                        let context = context.clone();
7639                        let winner = winner.clone();
7640                        async move {
7641                            if !e2e_commands.is_empty() {
7642                                // The suite certifies one SHA. Format before it
7643                                // runs, or the commit `pre_gate` makes at the
7644                                // gate owes the whole suite again.
7645                                Self::run_pre_gate(state, &winner).await;
7646                            }
7647                            run_e2e_with_retry(
7648                                state,
7649                                &shell,
7650                                &e2e_commands,
7651                                &worktree,
7652                                budget,
7653                                &context,
7654                                Some(round),
7655                            )
7656                            .await
7657                        }
7658                    },
7659                )
7660                .await?;
7661                (e2e, verify_retried, false, None)
7662            };
7663
7664            let expected = records.len();
7665            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7666            let incomplete = answered < expected;
7667            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7668            let policy = self.state.config.graph.incomplete_review;
7669            let clean = round_is_clean(
7670                blocking,
7671                e2e_ok,
7672                answered,
7673                expected,
7674                round_quota_missing,
7675                policy,
7676            );
7677
7678            let mut round_record = ReviewRound {
7679                round,
7680                head: head.clone(),
7681                verified_head: None,
7682                verified_at: None,
7683                reviews: records,
7684                e2e,
7685                verify_retried,
7686                e2e_deferred,
7687                e2e_defer_reason,
7688                fix: None,
7689                blocking,
7690                answered,
7691                expected,
7692                clean,
7693                progressed: false,
7694                vote_split,
7695                reconsideration,
7696                verdict: None,
7697            };
7698            // The final vote per seat is its revote where reconsideration
7699            // ran and answered, its initial vote otherwise — the same
7700            // fallback `tally` uses for a judge whose private vote failed.
7701            round_record.verdict = ReviewVote::worst(
7702                round_record
7703                    .final_votes()
7704                    .into_iter()
7705                    .map(|(_, _, vote)| vote),
7706            );
7707            // Which commit and when magi actually attempted to check —
7708            // known the moment a command was dispatched against `head`,
7709            // whether or not it finished: a resource-blocked attempt still
7710            // targeted a specific commit at a specific time, and leaving
7711            // that unrecorded is exactly what made `verification_summary`
7712            // report a fresh attempt as "commit unknown ... recorded before
7713            // this was tracked", indistinguishable from a genuinely old,
7714            // untracked record. Only a deferred or unconfigured round never
7715            // ran at all and has nothing to record — see
7716            // `ReviewRound::verified_head`'s own doc.
7717            if !matches!(
7718                round_record.e2e_status(),
7719                E2eStatus::Deferred | E2eStatus::NotConfigured
7720            ) {
7721                // HEAD, not the reviewed `head`: `pre_gate` may have committed
7722                // mechanical fixes before the commands ran.
7723                round_record.verified_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
7724                round_record.verified_at = Some(Timestamp::now());
7725            }
7726
7727            if incomplete {
7728                let missing: Vec<String> = round_record
7729                    .reviews
7730                    .iter()
7731                    .filter(|r| r.failed.is_some())
7732                    .map(|r| format!("review-{}", r.reviewer))
7733                    .collect();
7734                self.state.event(
7735                    "review",
7736                    format!(
7737                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7738                        missing.join(", ")
7739                    ),
7740                );
7741            }
7742
7743            if clean {
7744                self.state.event(
7745                    "review",
7746                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7747                        format!(
7748                            "round {round}: clean (warn policy, incomplete panel) — no \
7749                             blocking findings from the seats that answered, verification green"
7750                        )
7751                    } else if incomplete {
7752                        format!(
7753                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7754                             quorum) — no blocking findings from the seats that answered, \
7755                             verification green",
7756                            expected - answered
7757                        )
7758                    } else {
7759                        format!("round {round}: clean — no blocking findings, verification green")
7760                    },
7761                );
7762                self.state.reviews.push(round_record);
7763                self.state.status = RunStatus::Gating;
7764                self.state.save()?;
7765                return Ok(());
7766            }
7767
7768            // Nothing was raised and verification passed, but not every seat
7769            // answered and `round_is_clean` still refused to call it clean —
7770            // either a seat is missing for a reason other than its own quota
7771            // (a crash, a timeout, unparsable output — worth another try), or
7772            // every seat that could have answered lost its quota and nobody
7773            // is left to decide on: re-review rather than send the fixer
7774            // after a round with nothing to fix.
7775            if incomplete && blocking == 0 && e2e_ok {
7776                self.state.reviews.push(round_record);
7777                self.state.save()?;
7778                if round == max_rounds {
7779                    self.state.status = RunStatus::Blocked;
7780                    self.state.event(
7781                        "review",
7782                        format!(
7783                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7784                             refusing to call it clean",
7785                            expected - answered
7786                        ),
7787                    );
7788                    return Ok(());
7789                }
7790                continue;
7791            }
7792
7793            // Nothing for the fixer to act on (`blocking == 0`) and the only
7794            // reason this round is not clean is that magi itself never got
7795            // a command to run — the shared build cache, not the patch (see
7796            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7797            // the fixer would invite a change to appease contention that has
7798            // nothing to do with the diff, and would leave this attempt
7799            // sitting in the next round's prompt as if it were about an
7800            // earlier, superseded commit rather than what it actually is:
7801            // the same head, still waiting to be checked. Wait for it the
7802            // same way the final round's own contention is already handled,
7803            // whatever round this happens to be.
7804            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7805                self.state.reviews.push(round_record);
7806                return self
7807                    .stop_unresolved_review(
7808                        "the round's own verification could not run",
7809                        &shell,
7810                        &winner.worktree,
7811                    )
7812                    .await;
7813            }
7814
7815            if round == max_rounds {
7816                self.state.reviews.push(round_record);
7817                return self
7818                    .stop_reviewing(
7819                        &format!(
7820                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7821                        ),
7822                        &shell,
7823                        &winner.worktree,
7824                    )
7825                    .await;
7826            }
7827
7828            if !self.fix_review_round(&winner, round_record).await? {
7829                return Ok(());
7830            }
7831        }
7832        Ok(())
7833    }
7834
7835    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7836    /// False means admission or continuation policy stopped the review loop.
7837    async fn fix_review_round(
7838        &mut self,
7839        winner: &Candidate,
7840        mut round_record: ReviewRound,
7841    ) -> Result<bool> {
7842        let round = round_record.round;
7843        let shell = self.state.config.shell();
7844        let step = self.run_fix_round(winner, &mut round_record).await?;
7845        if matches!(step, FixStep::Resource) {
7846            self.stop_for_resource()?;
7847            return Ok(false);
7848        }
7849        if self.state.reviews.len() < round {
7850            self.state.reviews.push(round_record);
7851        } else {
7852            self.state.reviews[round - 1] = round_record;
7853        }
7854        self.state.save()?;
7855
7856        // The fixer's own report never came back this round, even after
7857        // `continue_fix_report`'s own budget was spent on it — not an
7858        // ordinary "no report" (dropped stream, quota, plain failure),
7859        // which already reads that way and is left to the existing round
7860        // budget. Stopping here, rather than opening another round, is
7861        // what keeps a next reviewer/fixer wave from ever being
7862        // dispatched onto `winner.worktree` while whatever the seat's
7863        // last call may still have running there is unaccounted for: no
7864        // process liveness check exists (and none is being added — see
7865        // AGENTS.md/this task's own scope), so the only way to honour
7866        // "nothing starts before a valid report returns" is to not start
7867        // anything further on this worktree from this run at all.
7868        if matches!(step, FixStep::Recorded { report_lost: true }) {
7869            self.stop_reviewing(
7870                "the fixer's adoption report never came back, even after resuming its \
7871                         own seat; refusing to start another round against the same worktree \
7872                         while that is unresolved",
7873                &shell,
7874                &winner.worktree,
7875            )
7876            .await?;
7877            return Ok(false);
7878        }
7879
7880        let streak = self
7881            .state
7882            .reviews
7883            .iter()
7884            .rev()
7885            .take_while(|r| !r.progressed)
7886            .count();
7887        if streak >= STAGNANT_LIMIT {
7888            self.stop_reviewing(
7889                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7890                &shell,
7891                &winner.worktree,
7892            )
7893            .await?;
7894            return Ok(false);
7895        }
7896        Ok(true)
7897    }
7898
7899    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7900    async fn run_fix_round(
7901        &mut self,
7902        winner: &Candidate,
7903        round_record: &mut ReviewRound,
7904    ) -> Result<FixStep> {
7905        let round = round_record.round;
7906        let max_rounds = self.state.config.graph.review_rounds;
7907        let run_id = self.state.id.clone();
7908        let prompts = self.state.config.prompts.clone();
7909        let language = self.state.config.graph.language.clone();
7910        let sessions = self.state.config.graph.sessions;
7911        let artifacts = self.merge_repair_artifacts();
7912        let base = self.landing_base();
7913        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7914        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7915        let this_round_verification = round_record.verification_summary(&head);
7916        let all_findings: Vec<_> = round_record
7917            .reviews
7918            .iter()
7919            .flat_map(|r| r.findings.iter().cloned())
7920            .collect();
7921        // Fix. The winner's own implementer seat continues its conversation:
7922        // the competition is over, so context is pure benefit now.
7923        let blocking_findings: Vec<_> = all_findings
7924            .iter()
7925            .filter(|f| f.severity.blocks())
7926            .cloned()
7927            .collect();
7928        let fix_prompt = prompt::fix_in(
7929            &self.state.instruction,
7930            &blocking_findings,
7931            this_round_verification.as_ref(),
7932            round,
7933            max_rounds,
7934            &language,
7935            &self.state.config.graph.github_language,
7936        );
7937        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7938        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7939        let (job, seat, out) = self
7940            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7941                prompt: fix_prompt.clone(),
7942                spec,
7943                seat,
7944                cwd: winner.worktree.clone(),
7945                timeout,
7946                allow_write: true,
7947                sessions,
7948                artifacts: artifacts.clone(),
7949                stem: format!("fix-{round}"),
7950                handover: None,
7951            })
7952            .await;
7953        // Not asked: the round is not recorded (so a resume repeats it),
7954        // no fixer was handed over to, and the seat is left as it was.
7955        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7956            return Ok(FixStep::Resource);
7957        }
7958        let agent_id = seat.agent.clone();
7959
7960        let mut fix = FixRecord {
7961            agent: agent_id,
7962            addressed: Vec::new(),
7963            rejected: Vec::new(),
7964            notes: String::new(),
7965            committed: false,
7966            failed: None,
7967            duration_ms: 0,
7968            continuation: None,
7969        };
7970        let mut continuation = ContinuationRecord::not_needed();
7971        let mut final_seat = seat.clone();
7972        match out {
7973            AgentOutcome::Ok(o) => {
7974                fix.duration_ms = o.duration_ms;
7975                let parsed = verdict::extract_json::<FixReport>(&o.text);
7976                // A parsed report standing next to a command this same
7977                // reply's own CLI never confirmed the exit status of is
7978                // not a resolved answer — the identical `CommandEvidence`
7979                // `state.jobs` renders, read here instead of only on
7980                // display, per the completion judgment and the shown
7981                // record needing to agree.
7982                let incomplete_reason = match &parsed {
7983                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7984                        "the reply parsed, but it reported a command whose own CLI never \
7985                             confirmed an exit status"
7986                            .to_owned(),
7987                    ),
7988                    Ok(_) => None,
7989                    Err(e) => Some(e.to_string()),
7990                };
7991                match incomplete_reason {
7992                    None => {
7993                        let report = parsed.expect("checked Ok above");
7994                        fix.addressed = report.addressed;
7995                        fix.rejected = report.rejected;
7996                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7997                    }
7998                    Some(reason) => {
7999                        let (resumed_seat, resolved, failure, cont) = self
8000                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
8001                            .await;
8002                        fix.duration_ms += cont.cumulative_wait_ms;
8003                        continuation = cont;
8004                        final_seat = resumed_seat;
8005                        match resolved {
8006                            Some(report) => {
8007                                fix.addressed = report.addressed;
8008                                fix.rejected = report.rejected;
8009                                fix.notes =
8010                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
8011                            }
8012                            None => fix.failed = failure,
8013                        }
8014                    }
8015                }
8016            }
8017            // The CLI's raw error JSON is not a fix report to parse.
8018            AgentOutcome::Dropped(o) => {
8019                fix.duration_ms = o.duration_ms;
8020                let why = o
8021                    .dropped
8022                    .as_ref()
8023                    .map(|d| d.why.as_str())
8024                    .unwrap_or("the CLI ended the stream without delivering its answer");
8025                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8026            }
8027            AgentOutcome::Quota(o) => {
8028                self.state.quota.push(QuotaLoss::from_report(
8029                    final_seat.key.clone(),
8030                    "fix",
8031                    o.quota.as_ref(),
8032                ));
8033                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8034            }
8035            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8036        }
8037        if self.state.resource_block.is_some() {
8038            self.state.seats.insert(final_seat.key.clone(), final_seat);
8039            return Ok(FixStep::Resource);
8040        }
8041        fix.continuation = Some(continuation);
8042        self.state.seats.insert(final_seat.key.clone(), final_seat);
8043        let coordination = self.coordinate(&winner.worktree).await?;
8044        if let Ok(r) = git::rescue_commit(
8045            &winner.worktree,
8046            &format!("magi: review round {round} fixes (uncommitted work)"),
8047        )
8048        .await
8049        {
8050            self.state.note_withheld("fix", &r.withheld);
8051        }
8052        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8053        fix.committed = after != before;
8054        // Judged by what `git` says moved against base, never by the
8055        // fixer's own `addressed`/`rejected` count — see
8056        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8057        // `patch` snapshot above: swallowing this error would default
8058        // `diff_after` to empty, which almost always differs from a
8059        // non-empty `patch` and reads as "progressed" — exactly backwards
8060        // for a `git` failure the stagnation check cannot see through.
8061        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8062        drop(coordination);
8063        let progressed = diff_after != patch;
8064        let commit_note = if fix.committed {
8065            "committed"
8066        } else {
8067            "NO new commit"
8068        };
8069        let tree_note = if progressed {
8070            "changed vs base"
8071        } else {
8072            "unchanged vs base"
8073        };
8074        self.state.event(
8075            "fix",
8076            match &fix.failed {
8077                // Distinct on purpose from "0 addressed, 0 rejected": the
8078                // fixer's own diff still landed (blocking counts do keep
8079                // falling round over round), only its adoption report did
8080                // not come back, so this must never read like every
8081                // finding was reviewed and declined.
8082                Some(reason) => {
8083                    format!(
8084                        "round {round}: fixer's adoption report was lost ({reason}); \
8085                             {commit_note}, tree {tree_note}"
8086                    )
8087                }
8088                None => format!(
8089                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8090                         {tree_note}{}",
8091                    fix.addressed.len(),
8092                    fix.rejected.len(),
8093                    if continuation.outcome == ContinuationOutcome::Resumed {
8094                        format!(
8095                            " (adoption report recovered after {} continuation(s))",
8096                            continuation.attempts
8097                        )
8098                    } else {
8099                        String::new()
8100                    },
8101                ),
8102            },
8103        );
8104        round_record.fix = Some(fix);
8105        round_record.progressed = progressed;
8106        Ok(FixStep::Recorded {
8107            report_lost: matches!(
8108                continuation.outcome,
8109                ContinuationOutcome::Exhausted
8110                    | ContinuationOutcome::QuotaLost
8111                    | ContinuationOutcome::NoSession
8112            ),
8113        })
8114    }
8115
8116    /// Decide, from the last recorded round's own verification, whether
8117    /// stopping the review loop is a hand-off or a genuine block.
8118    ///
8119    /// Called once the loop has given up trying — the round budget is spent,
8120    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8121    /// findings still open, never while a round is still clean or the
8122    /// incomplete-panel case handled inline above. Gate and e2e are facts
8123    /// about the tree; a lingering review finding is an opinion, and this
8124    /// workload's own `magi stats` puts reviewer precision low enough
8125    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8126    /// must not by itself stand between a green, verified change and the
8127    /// human who decides what to do with it. A red e2e is not an opinion, so
8128    /// that case still blocks, with the failing command and a tail of its
8129    /// output recorded here rather than left in `run.json` for someone to go
8130    /// find.
8131    ///
8132    /// A round that deferred its own e2e (see [`Config::graph`]'s
8133    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8134    /// only because nothing ran, and treating an empty list as a passing one
8135    /// here is exactly the "deferred painted green" bug this function exists
8136    /// to not have. When the last round's own verification never resolved —
8137    /// deferred on purpose, or a real attempt the shared build cache blocked
8138    /// — this makes (or retries) the real run, on the actual worktree this
8139    /// loop is about to stop touching, before deciding anything. A
8140    /// resource-blocked attempt is likewise never read as either green or
8141    /// red: it is evidence about the machine, not the patch (see
8142    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8143    /// blocked cache leaves this call without deciding rather than guessing
8144    /// — the caller retries on a later reentry.
8145    /// Record, once, that the review loop handed off over a blocking finding
8146    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8147    /// `land` asks the owner even with `land_approval` off. Called from every
8148    /// path that concludes `Gating`; a reentry keeps the first record.
8149    fn record_contested_handoff(&mut self) {
8150        if self.state.contested_handoff.is_some() {
8151            return;
8152        }
8153        let Some(contested) = self
8154            .state
8155            .reviews
8156            .last()
8157            .and_then(ReviewRound::contested_handoff)
8158        else {
8159            return;
8160        };
8161        self.state.event(
8162            "review",
8163            format!(
8164                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8165                 wait for the owner's approval",
8166                contested.findings.len(),
8167                contested.rejecters.len()
8168            ),
8169        );
8170        self.state.contested_handoff = Some(contested);
8171    }
8172
8173    async fn stop_unresolved_review(
8174        &mut self,
8175        why: &str,
8176        shell: &[String],
8177        worktree: &Path,
8178    ) -> Result<()> {
8179        self.stop_reviewing_inner(why, shell, worktree, true).await
8180    }
8181
8182    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8183        self.stop_reviewing_inner(why, shell, worktree, false).await
8184    }
8185
8186    async fn stop_reviewing_inner(
8187        &mut self,
8188        why: &str,
8189        shell: &[String],
8190        worktree: &Path,
8191        repairable: bool,
8192    ) -> Result<()> {
8193        let round_idx = self.state.reviews.len() - 1;
8194        // A deferred round and a resource-blocked one are the same shape
8195        // here: neither has a real result yet, and both get one more
8196        // attempt. Read off `e2e_status` — the single source for this —
8197        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8198        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8199        // still retries instead of being read as a settled result the
8200        // instant it stops being empty.
8201        let needs_catchup_run = matches!(
8202            self.state.reviews[round_idx].e2e_status(),
8203            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8204        );
8205        if needs_catchup_run {
8206            let round = self.state.reviews[round_idx].round;
8207            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8208            let commands = self.state.config.verify.e2e.clone();
8209            let before_head = git::rev_parse(worktree, "HEAD").await?;
8210            let winner = self.state.winner().cloned();
8211            let cache_dir = self.state.config.cache_dir();
8212            let context = format!(
8213                "round {round}: verification unresolved, catching up before the final decision"
8214            );
8215            let wait = self.repository_wait();
8216            let (outcomes, verify_retried) = with_cache_lease(
8217                &mut self.state,
8218                cache_dir.as_deref(),
8219                "e2e",
8220                "e2e",
8221                worktree,
8222                &before_head,
8223                timeout,
8224                &context,
8225                &wait,
8226                |state, budget| {
8227                    let shell = shell.to_vec();
8228                    let commands = commands.clone();
8229                    let context = context.clone();
8230                    async move {
8231                        if let Some(winner) = winner.filter(|_| !commands.is_empty()) {
8232                            // Same reason as in the round itself: format before
8233                            // the suite, not after it.
8234                            Self::run_pre_gate(state, &winner).await;
8235                        }
8236                        run_e2e_with_retry(
8237                            state,
8238                            &shell,
8239                            &commands,
8240                            worktree,
8241                            budget,
8242                            &context,
8243                            Some(round),
8244                        )
8245                        .await
8246                    }
8247                },
8248            )
8249            .await?;
8250            // After `pre_gate`: the commit the commands actually ran against.
8251            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8252            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8253                self.state.record_base_progress(&attempted_head);
8254            }
8255            let last = &mut self.state.reviews[round_idx];
8256            last.e2e = outcomes;
8257            last.verify_retried = verify_retried;
8258            // Always the commit and time this attempt actually targeted,
8259            // whether or not it happens to equal the reviewed `head` and
8260            // whether or not a command finished — see
8261            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8262            // attempt is recorded too, so a later reader sees "attempted
8263            // again at T2" rather than silence.
8264            last.verified_head = Some(attempted_head);
8265            last.verified_at = Some(Timestamp::now());
8266            if verify_inconclusive(&last.e2e) {
8267                // Still not a real result: `e2e_deferred` is left exactly
8268                // as it was, so `needs_catchup_run` above reads
8269                // `ResourceBlocked` (via `e2e_status`, which checks
8270                // `resource_blocked` before `e2e_deferred`) and retries
8271                // again on the next reentry, rather than recording
8272                // contention as a red e2e and blocking the run on it.
8273                self.state.save()?;
8274                return Ok(());
8275            }
8276            last.e2e_deferred = false;
8277            // A complete panel may have been saved as unclean solely because
8278            // verification could not run. Refresh that verdict with the new
8279            // outcomes before a granted repair consumes the round's result.
8280            // Incomplete panels retain their original policy/quota decision.
8281            if last.answered == last.expected {
8282                last.clean = round_is_clean(
8283                    last.blocking,
8284                    last.e2e.iter().all(CommandOutcome::ok),
8285                    last.answered,
8286                    last.expected,
8287                    0,
8288                    self.state.config.graph.incomplete_review,
8289                );
8290            }
8291        }
8292        let last = &self.state.reviews[round_idx];
8293        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8294
8295        match last.e2e_status() {
8296            E2eStatus::Failed => {
8297                let red: Vec<String> = last
8298                    .e2e
8299                    .iter()
8300                    .filter(|o| !o.ok())
8301                    .map(|o| {
8302                        format!(
8303                            "`{}` -> {:?}\n{}",
8304                            o.command,
8305                            o.code,
8306                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8307                        )
8308                    })
8309                    .collect();
8310                let can_repair = last.blocking == 0 && last.fix.is_none();
8311                self.state.reviews[round_idx].clean = false;
8312                self.state
8313                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8314                self.state.status = RunStatus::Blocked;
8315                if repairable
8316                    && can_repair
8317                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8318                {
8319                    self.state.status = RunStatus::Reviewing;
8320                }
8321            }
8322            // `needs_catchup_run` above already retried once this call; if
8323            // it is still blocked, this is magi's own admission it could
8324            // not get a command to run, never a verdict on the patch — the
8325            // run is left exactly where a later reentry can retry again.
8326            E2eStatus::ResourceBlocked => {
8327                self.state.event(
8328                    "review",
8329                    format!(
8330                        "{why}; e2e could not run (resource unavailable); not \
8331                         deciding yet"
8332                    ),
8333                );
8334            }
8335            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8336                self.state.event(
8337                    "review",
8338                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8339                );
8340                self.record_contested_handoff();
8341                self.state.status = RunStatus::Gating;
8342            }
8343        }
8344        self.state.save()?;
8345        Ok(())
8346    }
8347
8348    // ----------------------------------------------------------------- gate
8349
8350    async fn gate(&mut self) -> Result<()> {
8351        if self.hold_base_sync().await? {
8352            return Ok(());
8353        }
8354        // Judged by the review record itself, not by `status`: a solo
8355        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8356        // reentry (see `judge`), and trusting it here is exactly how a run
8357        // that exhausted its review budget got gated and merged a second
8358        // time around. `review_conclusion` recomputes the review loop's own
8359        // verdict from the round records themselves — `Gating` for a clean
8360        // round or a hand-off (see `stop_reviewing`), anything else means the
8361        // loop is still going or genuinely blocked.
8362        // A base the winner could not be replayed onto is a decision, not a
8363        // round: there is no landing tree to gate. Read as its own record for
8364        // the same reason the review verdict is.
8365        if self.state.status == RunStatus::Failed
8366            || self
8367                .state
8368                .base_sync
8369                .as_ref()
8370                .is_some_and(|s| s.conflict.is_some())
8371            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8372                != Some(RunStatus::Gating)
8373        {
8374            return Ok(());
8375        }
8376        let Some(winner) = self.state.winner().cloned() else {
8377            return Ok(());
8378        };
8379        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8380        self.state.reconcile_verification_head(&head);
8381        self.state.save()?;
8382        let wait = self.repository_wait();
8383        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8384            return Ok(());
8385        }
8386        if self.state.gate_ran {
8387            // `review_loop` derives its conclusion from the clean review
8388            // record on every reentry and therefore puts a completed run back
8389            // in `Gating`. A recorded gate is a stronger, terminal fact:
8390            // retain its original command output (or lack of any, for a repo
8391            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8392            // doc) and restore `Blocked` on a real failure rather than
8393            // pretending the command is still running or running it a second
8394            // time. `gate_ran == false` remains the only shape — unattempted,
8395            // or a resource-blocked retry — that may still need to execute a
8396            // command.
8397            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8398                self.state.status = RunStatus::Blocked;
8399                self.state.save()?;
8400            }
8401            return Ok(());
8402        }
8403        self.state.status = RunStatus::Gating;
8404        let mut outcomes = self.run_gate(&winner).await?;
8405        let mut attempted_head = self.state.gate_head.clone();
8406        loop {
8407            // A resource-blocked outcome means the gate command never actually
8408            // ran - the shared build cache could not be acquired or confirmed
8409            // fresh in time - which is evidence about the machine, not about
8410            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8411            // Recording it as a red gate would mark a run `Blocked` on nothing
8412            // but contention magi has already logged; leaving `self.state.gate`
8413            // empty and `self.state.gate_ran` false instead keeps the shape
8414            // this function already treats as "still needs to run" (see the
8415            // early-return above), so the next call retries the command
8416            // rather than concluding anything.
8417            if verify_inconclusive(&outcomes) {
8418                self.state.save()?;
8419                return Ok(());
8420            }
8421            if outcomes.iter().all(CommandOutcome::ok) {
8422                break;
8423            }
8424            let fixes_before = self.state.gate_fixes.len();
8425            match self.gate_fix_round(&winner, &outcomes).await? {
8426                GateFix::Retry => {
8427                    outcomes = self.run_gate(&winner).await?;
8428                    attempted_head = self.state.gate_head.clone();
8429                }
8430                GateFix::Stop => {
8431                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8432                    if self.state.gate_fixes.len() > fixes_before
8433                        && attempted_head.as_deref() != Some(current.as_str())
8434                        && verify_current_head(&mut self.state, &winner, &wait).await?
8435                    {
8436                        // A new SHA owes a gate even when a spent fixer changed only
8437                        // commit metadata. This does not grant another repair round.
8438                        outcomes = self.run_gate(&winner).await?;
8439                        attempted_head = self.state.gate_head.clone();
8440                    } else {
8441                        break;
8442                    }
8443                }
8444                GateFix::Defer => {
8445                    self.state.save()?;
8446                    return Ok(());
8447                }
8448            }
8449        }
8450        let passed = outcomes.iter().all(CommandOutcome::ok);
8451        self.state.gate = outcomes;
8452        self.state.gate_ran = true;
8453        self.state.gate_head = attempted_head;
8454        if !self.state.gate.is_empty()
8455            && !verify_inconclusive(&self.state.gate)
8456            && let Some(head) = self.state.gate_head.clone()
8457        {
8458            self.state.record_base_progress(&head);
8459        }
8460        self.state.save()?;
8461        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8462            return Ok(());
8463        }
8464        if !passed {
8465            self.state.status = RunStatus::Blocked;
8466            let spent = self.state.gate_fixes.len();
8467            self.state.event(
8468                "gate",
8469                if spent == 0 {
8470                    "gate failed; not merging".to_owned()
8471                } else {
8472                    format!("gate failed after {spent} gate-fix round(s); not merging")
8473                },
8474            );
8475        }
8476        self.state.save()?;
8477        Ok(())
8478    }
8479
8480    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8481    /// changed into one commit. Runs before `verify.e2e` in the review loop
8482    /// and again before the gate commands, always on the winner and never on
8483    /// a candidate awaiting judging.
8484    ///
8485    /// Before e2e as well as before the gate because command evidence belongs
8486    /// to a SHA ([`verify_current_head`]): a commit made only at the gate is a
8487    /// commit e2e never saw, and the suite ran again for a formatter's diff -
8488    /// 5 of the 22 gated runs on this repository in November 2026. The gate's
8489    /// own call then finds nothing left to change. The commands must be
8490    /// idempotent; one that is not makes a commit per call.
8491    ///
8492    /// Never fails the run: a non-zero exit or timeout is a warning and a
8493    /// recorded outcome, and the gate remains the single arbiter. Nothing
8494    /// configured means nothing happens - no event, no commit. `commit_all`
8495    /// commits any leftover change under the neutral identity and returns
8496    /// `false` when the tree is clean, so no empty commit is ever made.
8497    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8498        let commands = state.config.verify.pre_gate.clone();
8499        if commands.is_empty() {
8500            return;
8501        }
8502        let shell = state.config.shell();
8503        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8504        let (outcomes, _) = run_commands(
8505            state,
8506            "pre_gate",
8507            "pre_gate",
8508            0,
8509            &shell,
8510            &commands,
8511            &winner.worktree,
8512            timeout,
8513        )
8514        .await;
8515        for o in &outcomes {
8516            if !o.ok() {
8517                tracing::warn!(
8518                    "pre_gate `{}` failed ({:?}); the gate decides",
8519                    o.command,
8520                    o.code
8521                );
8522            }
8523            state.event(
8524                "pre_gate",
8525                format!(
8526                    "`{}` -> {}",
8527                    o.command,
8528                    if o.ok() {
8529                        "pass".to_owned()
8530                    } else {
8531                        format!(
8532                            "FAIL ({:?})\n{}",
8533                            o.code,
8534                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8535                        )
8536                    }
8537                ),
8538            );
8539        }
8540        state.pre_gate = outcomes;
8541        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8542            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8543                Ok(head) => {
8544                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8545                    state.pre_gate_commit = Some(head);
8546                }
8547                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8548            },
8549            Ok(false) => {}
8550            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8551        }
8552        if let Err(e) = state.save() {
8553            tracing::warn!("could not persist the pre_gate record: {e:#}");
8554        }
8555    }
8556
8557    /// Run `verify.gate` once against the winner's current tree, logging one
8558    /// event per command. Empty when nothing is configured.
8559    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8560        let wait = self.repository_wait();
8561        run_landing_gate(&mut self.state, winner, &wait).await
8562    }
8563
8564    /// One bounded fix round for a failing gate.
8565    ///
8566    /// The fixer is told the failure came from the gate itself, not from a
8567    /// reviewer, and is shown the failed commands, their exit codes and a tail
8568    /// of their output - whatever `[verify].gate` holds, nothing here knows
8569    /// what those commands run. Only a normal non-zero exit that printed
8570    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8571    /// command or a full disk says nothing about the code, and a fixer sent
8572    /// after it can only appease the machine. The round is judged by what git
8573    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8574    /// again before the gate does, so a fix cannot trade a green gate for a
8575    /// red e2e unnoticed.
8576    async fn gate_fix_round(
8577        &mut self,
8578        winner: &Candidate,
8579        outcomes: &[CommandOutcome],
8580    ) -> Result<GateFix> {
8581        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8582            g.phase == crate::repair_budget::Phase::Fixing
8583                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8584        }) {
8585            self.state.gate_fixes.len().saturating_add(1)
8586        } else {
8587            self.state.config.graph.gate_fix_rounds
8588        };
8589        let spent = self.state.gate_fixes.len();
8590        if spent >= cap {
8591            if cap > 0 {
8592                self.state.event(
8593                    "gate",
8594                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8595                );
8596            }
8597            return Ok(GateFix::Stop);
8598        }
8599        if !gate_fixable(outcomes) {
8600            self.state.event(
8601                "gate",
8602                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8603                 command or similar); not spending a fix round on it",
8604            );
8605            return Ok(GateFix::Stop);
8606        }
8607        let min_free = self.state.config.disk.min_free_bytes;
8608        if min_free > 0 {
8609            match crate::disk::free_bytes(&winner.worktree) {
8610                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8611                Ok(free) => {
8612                    self.state.event(
8613                        "gate",
8614                        format!(
8615                            "only {free} bytes free ({min_free} required by `[disk] \
8616                             min_free_bytes`); not spending a fix round on a failure the disk \
8617                             may explain"
8618                        ),
8619                    );
8620                    return Ok(GateFix::Stop);
8621                }
8622                Err(e) => {
8623                    self.state.event(
8624                        "gate",
8625                        format!("free disk space could not be measured ({e:#}); no fix round"),
8626                    );
8627                    return Ok(GateFix::Stop);
8628                }
8629            }
8630        }
8631
8632        let attempt = spent + 1;
8633        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8634        let base = self.landing_base();
8635        let fix_prompt = prompt::gate_fix_in(
8636            &self.state.instruction,
8637            &failed,
8638            attempt,
8639            cap,
8640            &self.state.config.graph.language,
8641            &self.state.config.graph.github_language,
8642        );
8643        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8644        let sessions = self.state.config.graph.sessions;
8645        let artifacts = self.merge_repair_artifacts();
8646        self.state.event(
8647            "gate",
8648            format!("gate failed; gate-fix round {attempt} of {cap}"),
8649        );
8650        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8651        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8652        // Persist before invoking the fixer: it may commit and then be
8653        // interrupted before its result (or the following e2e) is saved.
8654        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8655            head: before.clone(),
8656            passed: false,
8657            outcomes: Vec::new(),
8658        });
8659        self.state.save()?;
8660        let (_, seat, out) = self
8661            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8662                prompt: fix_prompt.clone(),
8663                spec,
8664                seat,
8665                cwd: winner.worktree.clone(),
8666                timeout,
8667                allow_write: true,
8668                sessions,
8669                artifacts: artifacts.clone(),
8670                stem: format!("gate-fix-{attempt}"),
8671                handover: None,
8672            })
8673            .await;
8674        // The fixer was never asked: no round is spent (nothing is recorded)
8675        // and a later reentry asks the same agent again.
8676        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8677            return Ok(GateFix::Defer);
8678        }
8679        let mut record = GateFixRecord {
8680            agent: seat.agent.clone(),
8681            failed,
8682            notes: String::new(),
8683            committed: false,
8684            error: None,
8685        };
8686        match out {
8687            AgentOutcome::Ok(o) => {
8688                // A missing report is not a failed fix: the round is judged
8689                // by the tree below, and the report only carries prose.
8690                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8691                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8692                }
8693            }
8694            AgentOutcome::Dropped(_) => {
8695                record.error = Some("the CLI dropped the stream".to_owned());
8696            }
8697            AgentOutcome::Quota(o) => {
8698                self.state.quota.push(QuotaLoss::from_report(
8699                    seat.key.clone(),
8700                    "gate-fix",
8701                    o.quota.as_ref(),
8702                ));
8703                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8704            }
8705            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8706        }
8707        self.state.seats.insert(seat.key.clone(), seat);
8708        let coordination = self.coordinate(&winner.worktree).await?;
8709        if let Ok(r) = git::rescue_commit(
8710            &winner.worktree,
8711            &format!("magi: gate fix {attempt} (uncommitted work)"),
8712        )
8713        .await
8714        {
8715            self.state.note_withheld("gate-fix", &r.withheld);
8716        }
8717        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8718        record.committed = after != before;
8719        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8720        let note = record.error.clone();
8721        self.state.gate_fixes.push(record);
8722        self.state.save()?;
8723        if !changed {
8724            self.state.event(
8725                "gate",
8726                match note {
8727                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8728                    None => format!("gate-fix round {attempt}: the tree did not change"),
8729                },
8730            );
8731            return Ok(GateFix::Stop);
8732        }
8733        self.state.event(
8734            "gate",
8735            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8736        );
8737
8738        drop(coordination);
8739        let wait = self.repository_wait();
8740        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8741    }
8742
8743    // ---------------------------------------------------------------- merge
8744
8745    /// One read-only rewrite by the summary's author; a text the gate still
8746    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8747    /// applies on their word or on silence. `None` means the run is parked on
8748    /// that question and nothing may be pushed or opened yet.
8749    async fn guarded_pr_message(
8750        &mut self,
8751        winner: &Candidate,
8752        facts: Option<&BranchFacts>,
8753        posting: bool,
8754    ) -> Result<Option<PrMessage>> {
8755        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8756        let head = match final_head.as_deref() {
8757            Some(h) => crate::pr_outcome::FinalHead::At(h),
8758            None => crate::pr_outcome::FinalHead::Unknown,
8759        };
8760        // Only a pull request that is about to be posted pays for a summary.
8761        let summarized = if posting {
8762            Some(self.summarize_change(winner, final_head.as_deref()).await)
8763        } else {
8764            None
8765        };
8766        // A busy shared resource is not a permanent cause: publishing an
8767        // "unavailable" description would stick (a later head move never
8768        // edits the body). Stop before anything is pushed or opened; the
8769        // run is `Blocked` and a resume asks the summary again.
8770        if self.state.resource_block.is_some() {
8771            self.state.event(
8772                "pr-summary",
8773                "a shared resource was busy; nothing was published, resume to summarize again",
8774            );
8775            return Ok(None);
8776        }
8777        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8778            anyhow::bail!(
8779                "the final commit moved during public description generation; retry before publishing"
8780            );
8781        }
8782        let note = match (&summarized, &self.state.change_summary) {
8783            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8784            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8785            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8786            (None, _) => ChangeNote::NotAsked,
8787        };
8788        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8789        if !posting {
8790            let identity = crate::scrub::Identity::current();
8791            return Ok(Some(PrMessage {
8792                title: crate::scrub::scrub(&pr.title, &identity),
8793                body: crate::scrub::scrub(&pr.body, &identity),
8794            }));
8795        }
8796        // This very text was already rejected and asked about: no second
8797        // judge call, rewrite or question, just carry the record on.
8798        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8799        if let Some(known) = self
8800            .state
8801            .github_text
8802            .clone()
8803            .filter(|g| g.fingerprint == raw_fp)
8804        {
8805            return self.settle_withheld(winner, &pr, known).await;
8806        }
8807        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8808        let mut violations =
8809            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8810        let identity = crate::scrub::Identity::current();
8811        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8812            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8813            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8814        {
8815            violations.push(crate::github_text::Violation::SensitiveData);
8816        }
8817        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8818        // only a language violation starts one; sensitive data alone goes
8819        // straight to the owner, with the redacted text as the fallback.
8820        let has_language = violations
8821            .iter()
8822            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8823        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8824            self.state.event(
8825                "github-text",
8826                if has_language {
8827                    format!("description rejected: {violations:?}; requesting one rewrite")
8828                } else {
8829                    format!("description rejected: {violations:?}; asking the owner")
8830                },
8831            );
8832            let mut rewritten = false;
8833            let mut rejected_rewrite = None;
8834            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8835            {
8836                let output = text;
8837                if let Some(title) = summary_title(&output) {
8838                    // Judge the redacted candidate that will actually be posted.
8839                    // Sensitive data alone must not discard a useful rewrite.
8840                    let raw_body = summary_without_title(&output);
8841                    let clean_title = crate::scrub::scrub(&title, &identity);
8842                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8843                    let redacted = clean_title != title || body != raw_body;
8844                    let title = clean_title;
8845                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8846                    if !complete {
8847                        self.state.event(
8848                            "github-text",
8849                            "description rewrite omitted required findings",
8850                        );
8851                    }
8852                    if !body.trim().is_empty() {
8853                        body.push_str("\n\n");
8854                        body.push_str(&public_pr_evidence(&self.state, head));
8855                    }
8856                    let marker = format!("magi:run/{}", self.state.id);
8857                    if !body.trim().is_empty() && !body.contains(&marker) {
8858                        body.push_str(&format!("\n\n{marker}"));
8859                    }
8860                    let rewrite = PrMessage {
8861                        title: title.clone(),
8862                        body: body.clone(),
8863                    };
8864                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8865                    let mut failed = crate::github_text::check_for(
8866                        &self.state.config.graph,
8867                        &title,
8868                        &body,
8869                        redecision,
8870                    );
8871                    if (!crate::github_text::shareable(&title)
8872                        || !crate::github_text::shareable(&body))
8873                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8874                    {
8875                        failed.push(crate::github_text::Violation::SensitiveData);
8876                    }
8877                    if complete
8878                        && !title.trim().is_empty()
8879                        && !body.trim().is_empty()
8880                        && !failed.is_empty()
8881                    {
8882                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8883                            title: title.clone(),
8884                            body: body.clone(),
8885                            failed: failed
8886                                .iter()
8887                                .map(|v| crate::github_text::category(*v).to_owned())
8888                                .collect(),
8889                        });
8890                    }
8891                    if complete
8892                        && !title.trim().is_empty()
8893                        && !raw_body.trim().is_empty()
8894                        && failed.is_empty()
8895                    {
8896                        decision = redecision;
8897                        pr = PrMessage { title, body };
8898                        rewritten = true;
8899                        self.state
8900                            .event("github-text", "description rewrite passed");
8901                        if redacted {
8902                            self.state.event(
8903                                "github-text",
8904                                "sensitive data removed from description rewrite",
8905                            );
8906                        }
8907                    }
8908                }
8909            }
8910            if !rewritten {
8911                self.state.event(
8912                    "github-text",
8913                    "description rewrite unavailable or rejected; asking the owner",
8914                );
8915                if let Some(w) =
8916                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8917                {
8918                    // A different text than the one asked about before: that
8919                    // question no longer describes anything.
8920                    self.retire_github_text_question("the withheld text changed");
8921                    let record = crate::run::GithubTextAsk {
8922                        rewrite: rejected_rewrite,
8923                        fingerprint: raw_fp,
8924                        title: w.title,
8925                        body: w.body,
8926                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8927                        question: None,
8928                        asks: 0,
8929                        resolved: false,
8930                        chosen_title: None,
8931                        choice: None,
8932                        publication: None,
8933                    };
8934                    return self.settle_withheld(winner, &pr, record).await;
8935                }
8936            }
8937        }
8938        let (title, body) =
8939            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8940        Ok(Some(PrMessage { title, body }))
8941    }
8942
8943    /// Describe the whole change from its final diff in either run mode.
8944    ///
8945    /// The summary is tied to the final commit and to the merge base of the
8946    /// resolved remote base; a record for any other pair is dropped and the
8947    /// agent asked again, so a description never outlives the diff it was
8948    /// written from. The seat is read-only and the prompt tells it to read the
8949    /// diff and the code around it. `[roles] synthesizer` selects the agents
8950    /// (unset: the default pick), each asked at most once, advancing on an
8951    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8952    /// carries the reason it is unavailable; nothing is invented then.
8953    async fn summarize_change(
8954        &mut self,
8955        winner: &Candidate,
8956        head: Option<&str>,
8957    ) -> std::result::Result<(), String> {
8958        let Some(head) = head else {
8959            return Err("the final commit could not be read".to_owned());
8960        };
8961        let remote = self.state.config.merge.remote.clone();
8962        let tracking = format!("{remote}/{}", self.state.base_branch);
8963        if !git::rev_exists(&winner.worktree, &tracking).await {
8964            return Err(format!("the remote base `{tracking}` could not be read"));
8965        }
8966        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8967            Ok(b) if !b.is_empty() => b,
8968            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8969        };
8970        let evidence = crate::pr_outcome::summary_evidence(
8971            &self.state,
8972            crate::pr_outcome::FinalHead::At(head),
8973        );
8974        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8975        if self
8976            .state
8977            .change_summary
8978            .as_ref()
8979            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8980        {
8981            return Ok(());
8982        }
8983        if self.state.change_summary.take().is_some() {
8984            self.state.event(
8985                "pr-summary",
8986                "the final diff or evidence changed; the earlier change summary was discarded",
8987            );
8988        }
8989        // Save the source snapshot even if no generation agent is available.
8990        if let Err(e) = write_artifact(
8991            &self.state,
8992            &format!("public-description-{fingerprint}.md"),
8993            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8994        ) {
8995            tracing::warn!("could not save public description evidence: {e:#}");
8996        }
8997        let chain = match agent::pick_chain(
8998            &self.state.config.agents,
8999            self.state.config.roles.synthesizer.as_ref(),
9000            &agent::installed,
9001            "synthesizer",
9002        ) {
9003            Ok(chain) => chain,
9004            Err(e) => {
9005                tracing::warn!("no agent for the change summary: {e:#}");
9006                self.state.event("pr-summary", "no summary agent available");
9007                return Err("no agent was available to summarize it".to_owned());
9008            }
9009        };
9010        let subjects = git::subjects(&winner.worktree, &base, head)
9011            .await
9012            .unwrap_or_default();
9013        let stat = git::diff_stat(&winner.worktree, &base, head)
9014            .await
9015            .unwrap_or_default();
9016        let stat: String = stat.chars().take(6_000).collect();
9017        let mut prompt = crate::prompt::change_summary_in(
9018            &base,
9019            head,
9020            &subjects,
9021            stat.trim(),
9022            &self.state.config.graph.github_language,
9023        );
9024        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
9025        let sessions = self.state.config.graph.sessions;
9026        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
9027        let prompts = self.state.config.prompts.clone();
9028        let cache = self.state.config.cache_dir();
9029        let run = self.state.id.clone();
9030        for (i, spec) in chain.into_iter().enumerate() {
9031            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
9032            let github_spec = match agent::for_github_text(
9033                &spec,
9034                &self.state.config.graph.github_language,
9035                &winner.worktree,
9036            ) {
9037                Ok(spec) => spec,
9038                Err(e) => {
9039                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9040                    continue;
9041                }
9042            };
9043            let job = SeatJob {
9044                spec: github_spec,
9045                seat,
9046                cwd: winner.worktree.clone(),
9047                prompt: prompt.clone(),
9048                timeout,
9049                allow_write: false,
9050                sessions,
9051                artifacts: agent::artifacts_dir(&self.state.dir()),
9052                stem: if i == 0 {
9053                    "change-summary".to_owned()
9054                } else {
9055                    format!("change-summary-{}", spec.id)
9056                },
9057                handover: None,
9058            };
9059            let ctx = WaveCtx {
9060                run: &run,
9061                node: "pr-summary",
9062                prompts: &prompts,
9063                cache: cache.as_deref(),
9064                round: None,
9065                carry_seats: false,
9066                pauses: Some((&self.pause, &self.interrupt)),
9067            };
9068            let (seat, outcome) =
9069                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9070            self.state.seats.insert(seat.key.clone(), seat);
9071            let text = match outcome {
9072                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9073                AgentOutcome::ResourceBlocked(why) => {
9074                    // No agent was asked: the run stops resumably before
9075                    // anything is published (see `guarded_pr_message`).
9076                    if self.state.resource_block.is_none() {
9077                        self.state.resource_block = Some(crate::run::ResourceBlock {
9078                            cause: crate::run::ResourceBlockCause::Other,
9079                            node: "pr-summary".to_owned(),
9080                            kind: "shared resource".to_owned(),
9081                            detail: why,
9082                            at: Timestamp::now(),
9083                        });
9084                    }
9085                    return Err("a shared resource was busy".to_owned());
9086                }
9087                _ => continue,
9088            };
9089            let blind = &self.state.config.blind;
9090            let Some(title) = summary_title(&text) else {
9091                continue;
9092            };
9093            let body: String = summary_without_title(&text)
9094                .lines()
9095                .filter(|l| !l.contains("magi:run/"))
9096                .collect::<Vec<_>>()
9097                .join("\n");
9098            let body = blind::sanitize_prose(body.trim(), blind);
9099            let title = blind::sanitize_prose(&title, blind);
9100            if body.trim().is_empty() || title.trim().is_empty() {
9101                continue;
9102            }
9103            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9104                continue;
9105            }
9106            // The branch must still be the one the agent was shown.
9107            if git::rev_parse(&winner.worktree, "HEAD")
9108                .await
9109                .ok()
9110                .as_deref()
9111                != Some(head)
9112            {
9113                return Err("the branch moved while the summary was written".to_owned());
9114            }
9115            if git::merge_base(&winner.worktree, &tracking, head)
9116                .await
9117                .ok()
9118                .as_deref()
9119                != Some(base.as_str())
9120                || crate::pr_outcome::summary_evidence(
9121                    &self.state,
9122                    crate::pr_outcome::FinalHead::At(head),
9123                ) != evidence
9124            {
9125                return Err("the summary evidence changed while it was written".to_owned());
9126            }
9127            self.state.change_summary = Some(crate::run::ChangeSummary {
9128                head: head.to_owned(),
9129                base,
9130                evidence: fingerprint,
9131                title,
9132                body,
9133            });
9134            self.state.event(
9135                "pr-summary",
9136                format!("change summary written by {} for the final diff", spec.id),
9137            );
9138            if let Err(e) = self.state.save() {
9139                tracing::warn!("could not save the change summary: {e:#}");
9140            }
9141            return Ok(());
9142        }
9143        self.state
9144            .event("pr-summary", "no agent produced a change summary");
9145        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9146    }
9147
9148    /// One read-only rewrite request for a withheld description. The
9149    /// implementer's own conversation is tried first when it exists and can be
9150    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9151    /// writes it in a separate conversation, skipping the implementer's agent.
9152    /// The prompt carries the whole text, so no agent needs any memory. Each
9153    /// agent is asked at most once and the chain advances on an error, quota,
9154    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9155    /// that has run out of quota must not end the rewrite. Nothing is recorded
9156    /// as a handover.
9157    async fn rewrite_pr_text(
9158        &mut self,
9159        winner: &Candidate,
9160        pr: &PrMessage,
9161        violations: &[crate::github_text::Violation],
9162    ) -> Option<String> {
9163        let prompt = format!(
9164            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9165            pr.title, pr.body
9166        );
9167        let sessions = self.state.config.graph.sessions;
9168        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9169        // (spec, the resumed implementer's seat). A fallback seat is created
9170        // only when its turn comes, so an unused fallback leaves no record.
9171        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9172        let key = format!("impl-{}", winner.label);
9173        if let Some(spec) = self
9174            .state
9175            .config
9176            .agents
9177            .iter()
9178            .find(|a| a.id == winner.agent)
9179            .cloned()
9180            && let Some(existing) = self.state.seats.get(&key)
9181            && existing.agent == spec.id
9182            && agent::has_session(spec.kind, existing, sessions)
9183        {
9184            attempts.push((spec, Some(existing.clone())));
9185        }
9186        match agent::pick_chain(
9187            &self.state.config.agents,
9188            self.state.config.roles.language_judge.as_ref(),
9189            &agent::installed,
9190            "language_judge",
9191        ) {
9192            Ok(chain) => {
9193                for spec in chain {
9194                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9195                        continue;
9196                    }
9197                    attempts.push((spec, None));
9198                }
9199            }
9200            Err(e) if attempts.is_empty() => {
9201                tracing::warn!("no agent for the description rewrite: {e:#}");
9202                self.state
9203                    .event("github-text", "no rewrite agent available");
9204                return None;
9205            }
9206            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9207        }
9208        let prompts = self.state.config.prompts.clone();
9209        let cache = self.state.config.cache_dir();
9210        let run = self.state.id.clone();
9211        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9212            let resumed = resumed_seat.is_some();
9213            let seat = resumed_seat.unwrap_or_else(|| {
9214                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9215            });
9216            let github_spec = match agent::for_github_text(
9217                &spec,
9218                &self.state.config.graph.github_language,
9219                &winner.worktree,
9220            ) {
9221                Ok(spec) => spec,
9222                Err(e) => {
9223                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9224                    continue;
9225                }
9226            };
9227            let job = SeatJob {
9228                spec: github_spec,
9229                seat,
9230                cwd: winner.worktree.clone(),
9231                prompt: prompt.clone(),
9232                timeout: retry_budget(timeout_full, resumed),
9233                allow_write: false,
9234                sessions,
9235                artifacts: agent::artifacts_dir(&self.state.dir()),
9236                stem: if i == 0 {
9237                    "github-text-rewrite".to_owned()
9238                } else {
9239                    format!("github-text-rewrite-{}", spec.id)
9240                },
9241                handover: None,
9242            };
9243            let ctx = WaveCtx {
9244                run: &run,
9245                node: "github-text",
9246                prompts: &prompts,
9247                cache: cache.as_deref(),
9248                round: None,
9249                carry_seats: false,
9250                pauses: Some((&self.pause, &self.interrupt)),
9251            };
9252            let (seat, outcome) =
9253                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9254            self.state.seats.insert(seat.key.clone(), seat);
9255            match outcome {
9256                AgentOutcome::Ok(o)
9257                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9258                {
9259                    return Some(o.text);
9260                }
9261                AgentOutcome::ResourceBlocked(_) => return None,
9262                _ => {}
9263            }
9264        }
9265        None
9266    }
9267
9268    /// Abandon the standing posting-gate question, if any.
9269    fn retire_github_text_question(&mut self, why: &str) {
9270        let Some(id) = self
9271            .state
9272            .github_text
9273            .as_ref()
9274            .and_then(|g| g.question.clone())
9275        else {
9276            return;
9277        };
9278        let _ = ask::Questions::open().update(&id, |q| {
9279            q.abandon(why);
9280            Ok(())
9281        });
9282    }
9283
9284    /// Carry a withheld text to its end: file the question, park while it is
9285    /// open, and on an answer, silence or timeout pick the text to post. The
9286    /// decision is saved before it is used, so it applies once.
9287    async fn settle_withheld(
9288        &mut self,
9289        winner: &Candidate,
9290        pr: &PrMessage,
9291        mut g: crate::run::GithubTextAsk,
9292    ) -> Result<Option<PrMessage>> {
9293        use crate::github_text as gt;
9294        let store = ask::Questions::open();
9295        let timeout = self.state.config.graph.answer_timeout;
9296        if !g.resolved {
9297            let mut standing = g
9298                .question
9299                .as_ref()
9300                .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9301            if let Some(q) = standing.as_ref()
9302                && q.status.open()
9303                && !store.held_reply(&q.id)
9304                && gt::expired(q, timeout)
9305            {
9306                let why = format!("no answer within {}s of asking", timeout.max(1));
9307                standing = store
9308                    .update(&q.id, |q| {
9309                        q.abandon(&why);
9310                        Ok(())
9311                    })
9312                    .ok()
9313                    .map(|(q, ())| q);
9314            }
9315            let mut ask_again: Option<Option<Vec<String>>> = None;
9316            match standing {
9317                // Recorded but never written (a stop between the two saves),
9318                // or removed: ask again without spending the retry.
9319                None if g.asks <= 1 => {
9320                    g.asks = 0;
9321                    ask_again = Some(None);
9322                }
9323                None => {
9324                    g.resolved = true;
9325                    g.chosen_title = None;
9326                    g.choice = Some(GithubTextChoice::Fallback);
9327                }
9328                Some(q) if q.status.open() => {
9329                    self.park_on_github_text(&g)?;
9330                    return Ok(None);
9331                }
9332                Some(q) => match gt::read_reply(&q) {
9333                    gt::Reply::Fallback => {
9334                        g.resolved = true;
9335                        g.chosen_title = None;
9336                        g.choice = Some(GithubTextChoice::Fallback);
9337                    }
9338                    gt::Reply::Original => {
9339                        g.resolved = true;
9340                        g.chosen_title = None;
9341                        g.choice = Some(GithubTextChoice::Original);
9342                    }
9343                    gt::Reply::Rewrite => {
9344                        g.resolved = true;
9345                        g.chosen_title = None;
9346                        g.choice = Some(if g.rewrite.is_some() {
9347                            GithubTextChoice::Rewrite
9348                        } else {
9349                            GithubTextChoice::Fallback
9350                        });
9351                    }
9352                    gt::Reply::Title(text) => {
9353                        let first = text
9354                            .lines()
9355                            .map(str::trim)
9356                            .find(|l| !l.is_empty())
9357                            .unwrap_or("");
9358                        let decision = if first.is_empty() {
9359                            None
9360                        } else {
9361                            let candidate = PrMessage {
9362                                title: first.to_owned(),
9363                                body: String::new(),
9364                            };
9365                            self.judge_pr_language(&winner.worktree, &candidate).await
9366                        };
9367                        match gt::vet_title(
9368                            &text,
9369                            decision,
9370                            self.state.config.graph.github_english(),
9371                        ) {
9372                            Ok(title) => {
9373                                g.resolved = true;
9374                                g.chosen_title = Some(title);
9375                                g.choice = Some(GithubTextChoice::MyTitle);
9376                            }
9377                            Err(cats) if g.asks < 2 => {
9378                                ask_again = Some(Some(
9379                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9380                                ))
9381                            }
9382                            Err(_) => {
9383                                g.resolved = true;
9384                                g.chosen_title = None;
9385                                g.choice = Some(GithubTextChoice::Fallback);
9386                            }
9387                        }
9388                    }
9389                },
9390            }
9391            if let Some(rejected) = ask_again {
9392                // A retry (after a rejected replacement title) is still about
9393                // the original text: the same ORIGINAL / FALLBACK comparison.
9394                let w = gt::Withheld {
9395                    title: g.title,
9396                    body: g.body,
9397                    categories: g.categories.clone(),
9398                };
9399                // Named by the original text's fingerprint alone, so a resume
9400                // rewrites the same file and the detail stays identical.
9401                let artifact = match crate::run::write_artifact(
9402                    &self.state,
9403                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9404                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9405                ) {
9406                    Ok(p) => Some(p),
9407                    Err(e) => {
9408                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9409                        None
9410                    }
9411                };
9412                let (detail, comparison) = gt::question_content(
9413                    &w,
9414                    &pr.title,
9415                    &pr.body,
9416                    &self.state.id,
9417                    rejected.as_deref(),
9418                    artifact.as_deref(),
9419                    g.rewrite.as_ref(),
9420                );
9421                let mut q = ask::Question::new(
9422                    self.state.id.clone(),
9423                    gt::ASK_NODE.to_owned(),
9424                    gt::ASK_SEAT.to_owned(),
9425                    gt::question_summary(&self.state.config.graph.language, &w),
9426                    detail,
9427                    gt::question_choices(&w, g.rewrite.as_ref()),
9428                );
9429                q.posting_comparison = Some(comparison);
9430                q.answer_timeout = timeout;
9431                // Recorded before it is written: a stop in between leaves a
9432                // record naming a question the resume then files again.
9433                g.question = Some(q.id.clone());
9434                g.asks += 1;
9435                self.state.github_text = Some(g.clone());
9436                self.state.event(
9437                    "github-text",
9438                    format!("asking the owner about the withheld text ({})", q.short()),
9439                );
9440                self.state.save()?;
9441                store
9442                    .put_executable(&mut q)
9443                    .context("file the posting-gate question")?;
9444                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9445                    tracing::warn!(
9446                        "could not notify about posting-gate question {}: {e:#}",
9447                        q.short()
9448                    );
9449                }
9450                self.park_on_github_text(&g)?;
9451                return Ok(None);
9452            }
9453        }
9454        let resolved = g.chosen_title.clone();
9455        let original = g.choice == Some(GithubTextChoice::Original);
9456        // Field names only: the event log never records the text.
9457        self.state.event(
9458            "github-text",
9459            if original {
9460                format!(
9461                    "posting the original text for: {}",
9462                    match (g.title, g.body) {
9463                        (true, true) => "title, description",
9464                        (true, false) => "title",
9465                        _ => "description",
9466                    }
9467                )
9468            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9469                "posting the saved rewrite title and description".to_owned()
9470            } else if resolved.is_some() {
9471                "posting the owner's replacement title".to_owned()
9472            } else {
9473                "posting the neutral text for the withheld fields".to_owned()
9474            },
9475        );
9476        self.state.github_text = Some(g.clone());
9477        self.state.save()?;
9478        if g.choice == Some(GithubTextChoice::Rewrite)
9479            && let Some(r) = g.rewrite.as_ref()
9480        {
9481            return Ok(Some(PrMessage {
9482                title: r.title.clone(),
9483                body: r.body.clone(),
9484            }));
9485        }
9486        let w = gt::Withheld {
9487            title: g.title,
9488            body: g.body,
9489            categories: g.categories.clone(),
9490        };
9491        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9492        if let Some(t) = resolved {
9493            title = t;
9494        }
9495        // Every withheld field is now neutral or vetted English.
9496        let english = gt::LanguageDecision {
9497            title_english: true,
9498            body_english: true,
9499            body_complete: true,
9500        };
9501        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9502        if original {
9503            // The owner's explicit choice overrides the gate for the fields it
9504            // was made for; the others keep what the gate produced.
9505            if g.title {
9506                title = pr.title.clone();
9507            }
9508            if g.body {
9509                body = pr.body.clone();
9510            }
9511        }
9512        Ok(Some(PrMessage { title, body }))
9513    }
9514
9515    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9516        self.state.github_text = Some(g.clone());
9517        self.state.end_pass_parked(ParkCause::Question);
9518        self.state.parked = true;
9519        self.state.event(
9520            "github-text",
9521            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9522        );
9523        self.state.save()
9524    }
9525
9526    /// Ask `[roles] language_judge` about this text and record which source
9527    /// decided; the text itself is never recorded.
9528    async fn judge_pr_language(
9529        &mut self,
9530        cwd: &Path,
9531        pr: &PrMessage,
9532    ) -> Option<crate::github_text::LanguageDecision> {
9533        self.state.config.roles.language_judge.as_ref()?;
9534        if !self.state.config.graph.language_gate() {
9535            return None;
9536        }
9537        let decision =
9538            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9539        self.state.event(
9540            "github-text",
9541            if decision.is_some() {
9542                "language decided by the language judge"
9543            } else {
9544                "language judge unavailable; using built-in heuristics"
9545            },
9546        );
9547        decision
9548    }
9549
9550    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9551        Box::pin(self.merge_inner())
9552    }
9553
9554    async fn merge_inner(&mut self) -> Result<()> {
9555        if self.resource_halt()? || self.hold_base_sync().await? {
9556            return Ok(());
9557        }
9558        // Same reasoning as `gate`: ask the review and gate records directly
9559        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9560        // skip can rewrite on reentry to something that no longer says
9561        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9562        // so a hand-off (open findings, green verification) reaches merge
9563        // exactly like a genuinely clean round does.
9564        //
9565        // A run resumed mid-`land` never reaches here at all: `execute`
9566        // recognises `RunStatus::Landing` before it even calls `prep`, and
9567        // routes straight to `run_land` instead. That has to happen a level
9568        // up from this function, not with a check in here, because
9569        // `review_loop`'s own status recomputation (see its doc) runs
9570        // *before* `merge` on every reentry and would otherwise overwrite
9571        // the `Landing` marker with `Gating` before this node ever saw it.
9572        if self
9573            .state
9574            .base_sync
9575            .as_ref()
9576            .is_some_and(|s| s.conflict.is_some())
9577            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9578                != Some(RunStatus::Gating)
9579            // `gate_ran == false` is not "passed" - `gate` leaves it false
9580            // both before it has ever run and when its last attempt was
9581            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9582            // permission to merge on nothing but the review record. Only a
9583            // gate that actually ran may proceed; an intentional local
9584            // review can also finish with verification explicitly not configured.
9585            // `RunState::gate_status` computes that distinction.
9586            || !self.state.gate_allows_progress()
9587        {
9588            return Ok(());
9589        }
9590        // This node's own record, not `status`: `status == Ready` is not
9591        // unique to the harmless `MergeMode::None` path this line was
9592        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9593        // run's PR was closed without merging — and on that run `mode` is
9594        // still `Pr`, so a reentry that fell through here would push and
9595        // open a second pull request. `self.state.merge` is set exactly once
9596        // this node (or `land`) has already produced a verdict, under every
9597        // mode, which is what "already done" actually means here.
9598        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9599            return Ok(());
9600        }
9601        let Some(winner) = self.state.winner().cloned() else {
9602            return Ok(());
9603        };
9604        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9605        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9606            self.state.status = RunStatus::Blocked;
9607            self.state.event(
9608                "gate",
9609                "winner changed after gate; fresh verification required",
9610            );
9611            self.state.save()?;
9612            return Ok(());
9613        }
9614        let wait = self.repository_wait();
9615        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9616            return Ok(());
9617        }
9618        let repo = self.state.repo.clone();
9619        let base = self.state.base_branch.clone();
9620        let mode = self.state.config.merge.mode;
9621        let style = self.state.config.merge.style;
9622        let facts = if is_review_run(&self.state) {
9623            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9624            let start = review_base(
9625                &repo,
9626                &self.state.config.merge.remote,
9627                &base,
9628                &self.state.base_commit,
9629                &winner.branch,
9630            )
9631            .await;
9632            branch_facts(&repo, &start, &winner.branch).await
9633        } else {
9634            None
9635        };
9636        // `None` when the base could not be freshly read: then an adopted
9637        // pull request's title is left alone.
9638        let leaked = if is_review_run(&self.state) {
9639            leaked_subjects(&self.state, &winner.branch).await
9640        } else {
9641            Some(Vec::new())
9642        };
9643        // `None`: parked on the owner's word about a withheld title or body.
9644        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9645        // the resume arrives here again.
9646        let Some(pr) = self
9647            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9648            .await?
9649        else {
9650            return Ok(());
9651        };
9652        let message = pr.commit_message();
9653
9654        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9655        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9656            return Ok(());
9657        }
9658        let outcome = match mode {
9659            MergeMode::None => MergeOutcome {
9660                mode,
9661                ok: true,
9662                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9663                empty: false,
9664            },
9665            MergeMode::Pr | MergeMode::Local
9666                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9667            {
9668                MergeOutcome {
9669                    mode,
9670                    ok: false,
9671                    detail: empty_candidate_detail(&self.state, &base),
9672                    empty: true,
9673                }
9674            }
9675            MergeMode::Local => {
9676                let on = git::current_branch(&repo).await?;
9677                if on.as_deref() != Some(base.as_str()) {
9678                    MergeOutcome {
9679                        mode,
9680                        ok: false,
9681                        detail: format!(
9682                            "{} has {} checked out, not the base branch {base}",
9683                            repo.display(),
9684                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9685                        ),
9686                        empty: false,
9687                    }
9688                } else if !git::is_clean(&repo).await? {
9689                    MergeOutcome {
9690                        mode,
9691                        ok: false,
9692                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9693                        empty: false,
9694                    }
9695                } else {
9696                    let out = match style {
9697                        MergeStyle::Merge => {
9698                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9699                        }
9700                        MergeStyle::Squash => {
9701                            git::merge_squash(&repo, &winner.branch, &message).await?
9702                        }
9703                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9704                    };
9705                    MergeOutcome {
9706                        mode,
9707                        ok: out.ok(),
9708                        detail: if out.ok() { out.stdout } else { out.stderr },
9709                        empty: false,
9710                    }
9711                }
9712            }
9713            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9714                self.state.pr_publication_pending = true;
9715                self.state.save()?;
9716                let out =
9717                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9718                self.state.pr_publication_pending = out.is_err();
9719                MergeOutcome {
9720                    mode,
9721                    ok: out.is_ok(),
9722                    detail: match out {
9723                        Ok(url) => url,
9724                        Err(e) => format!("{e:#}"),
9725                    },
9726                    empty: false,
9727                }
9728            }
9729            MergeMode::Pr => {
9730                let remote = self.state.config.merge.remote.clone();
9731                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9732                let pushed = if self.state.foreign_checkout.is_some() {
9733                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9734                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9735                } else {
9736                    git::push(&winner.worktree, &remote, &winner.branch).await?
9737                };
9738                if !pushed.ok() {
9739                    MergeOutcome {
9740                        mode,
9741                        ok: false,
9742                        detail: pushed.stderr,
9743                        empty: false,
9744                    }
9745                } else {
9746                    if self.state.foreign_checkout.is_some() {
9747                        self.state.pr_pushed_head = Some(delivery_head.clone());
9748                    }
9749                    // A retry or resume of a run whose branch already has an
9750                    // open pull request adopts it rather than failing on a
9751                    // duplicate. Only this winner branch into this base:
9752                    // `branch_for` derives the name from the run id, so a
9753                    // different run's pull request never matches.
9754                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9755                    let out = match pr_merge_plan(found) {
9756                        PrPlan::Create => {
9757                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9758                                .await
9759                        }
9760                        PrPlan::Adopt { url, title } => {
9761                            self.state
9762                                .event("merge", format!("Pr: adopted open pull request {url}"));
9763                            if self.state.explicit_github_text_selection() {
9764                                // An explicit publication selection applies the prepared
9765                                // payload; acceptance alone is not publication proof.
9766                                land::set_pr_text_verbatim(
9767                                    &winner.worktree,
9768                                    &url,
9769                                    &pr.title,
9770                                    &pr.body,
9771                                )
9772                                .await
9773                                .map(|()| url)
9774                            } else {
9775                                if title != pr.title
9776                                    && (!is_review_run(&self.state)
9777                                        || leaked
9778                                            .as_deref()
9779                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9780                                    && let Err(e) =
9781                                        if self.state.github_text.as_ref().is_some_and(|g| {
9782                                            g.resolved
9783                                                && g.title
9784                                                && g.choice == Some(GithubTextChoice::Original)
9785                                        }) {
9786                                            // The owner chose to post this title unmodified.
9787                                            land::set_pr_title_verbatim(
9788                                                &winner.worktree,
9789                                                &url,
9790                                                &pr.title,
9791                                            )
9792                                            .await
9793                                        } else {
9794                                            land::set_pr_title(
9795                                                &mut self.state,
9796                                                &winner.worktree,
9797                                                &url,
9798                                                &pr.title,
9799                                            )
9800                                            .await
9801                                        }
9802                                {
9803                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9804                                    self.state
9805                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9806                                }
9807                                Ok(url)
9808                            }
9809                        }
9810                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9811                    };
9812                    match out {
9813                        Ok(url) => MergeOutcome {
9814                            mode,
9815                            ok: true,
9816                            detail: url,
9817                            empty: false,
9818                        },
9819                        Err(e) => MergeOutcome {
9820                            mode,
9821                            ok: false,
9822                            detail: e.to_string(),
9823                            empty: false,
9824                        },
9825                    }
9826                }
9827            }
9828        };
9829
9830        self.state.status = match (mode, outcome.ok) {
9831            (MergeMode::None, _) => RunStatus::Ready,
9832            (_, true) if outcome.empty => RunStatus::Ready,
9833            (MergeMode::Pr, true) => RunStatus::Ready,
9834            (_, true) => RunStatus::Merged,
9835            (_, false) => RunStatus::Blocked,
9836        };
9837        self.state.event(
9838            "merge",
9839            format!(
9840                "{:?}: {}",
9841                mode,
9842                outcome.detail.lines().next().unwrap_or("")
9843            ),
9844        );
9845        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9846            self.state.record_github_text_publication(&outcome.detail);
9847        }
9848        self.state.merge = Some(outcome);
9849        self.state.save()?;
9850
9851        // The PR is open and the run would historically stop here, leaving the
9852        // operator to watch checks, feed review comments back to a fixer, and
9853        // merge. That was done by hand six times in one session before this
9854        // existed. Opt-in, because merging is the one irreversible thing magi
9855        // can do to a repository.
9856        if self.state.config.graph.land
9857            && mode == MergeMode::Pr
9858            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9859        {
9860            self.run_land().await?;
9861        }
9862        // `run_land` may have left `status` at `Landing` - still waiting on
9863        // CI or the owner's approval, not actually settled - so this has to
9864        // read whatever `status` ended up as here, not the `Merged` this
9865        // function set a few lines up.
9866        self.settle_questions();
9867        Ok(())
9868    }
9869
9870    /// Enter `land`.
9871    ///
9872    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9873    /// a resumed run's re-entry. `land::land` itself is what serialises the
9874    /// two git-mutating moments inside the loop — the rebase push and
9875    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9876    /// to hold a lock across the whole call, and doing so would serialise
9877    /// this run's CI wait against a *different* run's land-approval resume
9878    /// in the same repository, which is exactly the "must not wait on
9879    /// another task" property the daemon's slot-freeing exists to give.
9880    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9881        Box::pin(self.run_land_inner())
9882    }
9883
9884    async fn run_land_inner(&mut self) -> Result<()> {
9885        crate::pr_head::ensure_owner(&self.state)?;
9886        let url = self
9887            .state
9888            .pr
9889            .as_ref()
9890            .map(|pr| pr.url.clone())
9891            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9892            .unwrap_or_default();
9893        let url = url.lines().next().unwrap_or("").trim().to_owned();
9894        if !url.starts_with("http") {
9895            return Ok(());
9896        }
9897        // A land failure is not a lost run: the work is on a branch and the
9898        // pull request is open, which is exactly where a human takes over.
9899        loop {
9900            match land::land(&mut self.state, &url).await {
9901                Ok(_)
9902                    if self
9903                        .state
9904                        .merge_repairs
9905                        .last()
9906                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9907                {
9908                    if self.repair_merge_review().await? {
9909                        continue;
9910                    }
9911                }
9912                Ok(_) if self.state.resource_block.is_some() => {
9913                    return self.stop_for_resource();
9914                }
9915                Ok(pr) if self.state.parked => {
9916                    // `land` already saved the parked marker; nothing here
9917                    // overrides `status` back to a terminal value while an
9918                    // approval is still outstanding.
9919                    let _ = pr;
9920                }
9921                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9922                    self.state.status = RunStatus::Superseded;
9923                    self.state.save()?;
9924                }
9925                Ok(pr) => {
9926                    self.state.status = match pr.state {
9927                        land::PrLifecycle::Merged => RunStatus::Merged,
9928                        _ => RunStatus::Blocked,
9929                    };
9930                    // Downstream of a confirmed merge only - see
9931                    // `bump::should_release_bump`'s own doc for why this one
9932                    // check covers all three of `land`'s success paths.
9933                    // Best-effort: the run already landed, so a failure here
9934                    // (the decision call, `gh`, `cargo`) is recorded and never
9935                    // turns a landed run into a failed one.
9936                    if bump::should_release_bump(self.state.status)
9937                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9938                    {
9939                        // The event is the run's own record. Not-eligible cases
9940                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9941                        // `Err` is a bump that was tried and failed:
9942                        // `after_merge` itself raises the operator notice for
9943                        // that, whether or not a release PR exists yet.
9944                        self.state
9945                            .event("bump", format!("release bump skipped: {e:#}"));
9946                    }
9947                    // Independent of the bump, and best-effort in the same way:
9948                    // findings the merge left open become follow-up tasks.
9949                    if self.state.status == RunStatus::Merged {
9950                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9951                    }
9952                    self.state.save()?;
9953                }
9954                Err(e) => {
9955                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9956                        return Err(e);
9957                    }
9958                    self.state.status = RunStatus::Blocked;
9959                    self.state.event("land", format!("gave up: {e}"));
9960                    self.state.save()?;
9961                }
9962            }
9963            break;
9964        }
9965        Ok(())
9966    }
9967
9968    /// One repair cycle stays under the existing driver/task claim. An
9969    /// interrupted fixer is rolled back and held rather than run twice.
9970    async fn repair_merge_review(&mut self) -> Result<bool> {
9971        use crate::run::MergeRepairPhase as Phase;
9972        crate::pr_head::ensure_owner(&self.state)?;
9973        let i = self.state.merge_repairs.len() - 1;
9974        let request = self.state.merge_repairs[i].clone();
9975        if request.phase == Phase::Fixing {
9976            let restored = match self.state.winner().cloned() {
9977                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9978                None => Err(anyhow::anyhow!("repair has no winning branch")),
9979            };
9980            let why = match restored {
9981                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9982                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9983                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9984            };
9985            self.hold_merge_repair(i, why)?;
9986            return Ok(false);
9987        }
9988        // A completed repair gate is a publication checkpoint. On reentry,
9989        // reject external changes before the ordinary gate path can replace
9990        // that evidence by verifying a new head. Requested repairs still
9991        // start their own review generation below.
9992        if request.phase == Phase::Reviewing && self.state.gate_ran {
9993            let winner = self
9994                .state
9995                .winner()
9996                .context("repair has no winning branch")?;
9997            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9998            if self.state.gate_head.as_deref() != Some(head.as_str())
9999                || !git::is_clean(&winner.worktree).await?
10000            {
10001                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10002                return Ok(false);
10003            }
10004        }
10005        let failure = if i >= self.state.config.graph.land_rounds
10006            || self.state.config.graph.review_rounds == 0
10007        {
10008            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
10009        } else {
10010            None
10011        };
10012        if let Some(why) = failure {
10013            self.hold_merge_repair(i, why)?;
10014            return Ok(false);
10015        }
10016        if request.phase == Phase::Requested {
10017            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
10018                Ok(pr) => pr,
10019                Err(e) => {
10020                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
10021                    return Ok(false);
10022                }
10023            };
10024            self.state.merge_repairs[i].phase = Phase::Fixing;
10025            self.state.land_approval = None;
10026            self.state.land_armed_head = None;
10027            self.state.save()?;
10028            // Serialize the exact records, including finding ids/details,
10029            // reconsideration reasons, final votes and verification output.
10030            let evidence = serde_json::to_string_pretty(&request)?;
10031            let reason = format!(
10032                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
10033            );
10034            let budget = self.state.config.graph.land_rounds;
10035            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10036            match result {
10037                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10038                Ok(land::Fixed::Deferred) => {
10039                    // An unavailable fixer did not spend this repair request.
10040                    // Restore partial fallback work before retrying it, as for
10041                    // an interrupted repair, without closing the request.
10042                    let worktree = self
10043                        .state
10044                        .winner()
10045                        .context("repair has no winning branch")?
10046                        .worktree
10047                        .clone();
10048                    self.graft_restore(&worktree, &request.head).await?;
10049                    self.state.merge_repairs[i].phase = Phase::Requested;
10050                    self.stop_for_resource()?;
10051                    return Ok(false);
10052                }
10053                Ok(land::Fixed::Failed(why)) => {
10054                    self.hold_merge_repair(
10055                        i,
10056                        format!(
10057                            "repair failed: {why}; inspect the branch and run an explicit review"
10058                        ),
10059                    )?;
10060                    return Ok(false);
10061                }
10062                Err(e) => {
10063                    self.hold_merge_repair(
10064                        i,
10065                        format!(
10066                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10067                        ),
10068                    )?;
10069                    return Ok(false);
10070                }
10071            }
10072            // Even no change gets a real new reviewer wave. The previous
10073            // evidence remains in the request; a former clean/concluded round
10074            // cannot short-circuit the new review or gate.
10075            self.begin_merge_repair_review(i)?;
10076        }
10077        self.review_loop().await?;
10078        self.gate().await?;
10079        if self.resource_halt()? || self.state.parked {
10080            return Ok(false);
10081        }
10082        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10083            != Some(RunStatus::Gating)
10084            || !self.state.gate_allows_progress()
10085        {
10086            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10087            return Ok(false);
10088        }
10089        // Review and gate may have added commits after the initial repair.
10090        // Publish exactly their final head to the existing PR, with a lease.
10091        let winner = self
10092            .state
10093            .winner()
10094            .cloned()
10095            .context("repair has no winning branch")?;
10096        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10097        if self.state.gate_head.as_deref() != Some(after.as_str())
10098            || !git::is_clean(&winner.worktree).await?
10099        {
10100            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10101            return Ok(false);
10102        }
10103        let expected = self
10104            .state
10105            .pr_pushed_head
10106            .as_deref()
10107            .unwrap_or(&request.head)
10108            .to_owned();
10109        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10110        // Push the checked commit itself: a local ref moving after the check
10111        // must not change which commit the forge receives.
10112        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10113            return Ok(false);
10114        }
10115        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10116        let refspec = format!("{after}:refs/heads/{public}");
10117        let push = git::git_raw(
10118            &winner.worktree,
10119            &["push", &lease, &self.state.config.merge.remote, &refspec],
10120        )
10121        .await;
10122        match push {
10123            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10124            Ok(out) => {
10125                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10126                return Ok(false);
10127            }
10128            Err(e) => {
10129                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10130                return Ok(false);
10131            }
10132        }
10133        self.state.merge_repairs[i].phase = Phase::Complete;
10134        self.state.status = RunStatus::Landing;
10135        self.state.event(
10136            "repair",
10137            "new review and gate completed; fresh owner merge decision required",
10138        );
10139        self.state.save()?;
10140        Ok(true)
10141    }
10142
10143    fn merge_repair_artifacts(&self) -> PathBuf {
10144        let artifacts = agent::artifacts_dir(&self.state.dir());
10145        match self
10146            .state
10147            .merge_repairs
10148            .last()
10149            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10150        {
10151            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10152            None => artifacts,
10153        }
10154    }
10155
10156    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10157        if let Some(e2e) = self.state.head_e2e.take() {
10158            self.state
10159                .verification_history
10160                .push(crate::run::VerificationEvidence {
10161                    superseded_by: None,
10162                    archived_at: Some(Timestamp::now()),
10163                    fixes_spent: Some(self.state.gate_fixes.len()),
10164                    gate_head: None,
10165                    gate_ran: false,
10166                    gate: Vec::new(),
10167                    e2e: Some(e2e),
10168                });
10169        }
10170        self.state.reviews.clear();
10171        self.state.gate.clear();
10172        self.state.gate_ran = false;
10173        self.state.gate_head = None;
10174        self.state.gate_fixes.clear();
10175        self.state.gate_repair_e2e = None;
10176        self.state.pre_gate.clear();
10177        self.state.pre_gate_commit = None;
10178        self.state.contested_handoff = None;
10179        self.state.land_reruns = None;
10180        self.state.land_approval = None;
10181        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10182        self.state.status = RunStatus::Reviewing;
10183        self.state.save()
10184    }
10185
10186    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10187        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10188        self.state.merge_repairs[index].failure = Some(why.clone());
10189        self.state.status = RunStatus::Blocked;
10190        self.state.event("repair", why);
10191        self.state.save()
10192    }
10193
10194    // -------------------------------------------------------------- helpers
10195
10196    /// Fetch or create a seat, keeping its conversation across nodes.
10197    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10198        if let Some(existing) = self.state.seats.get(key)
10199            && existing.agent == agent
10200        {
10201            return existing.clone();
10202        }
10203        // A seat that changes agent mints its session id from the agent too,
10204        // like a handover: the old agent's uuid is already taken by the CLI.
10205        let fresh = if self.state.seats.contains_key(key) {
10206            handover_seat(key, agent, self.state.next_seat_seed())
10207        } else {
10208            SeatState::new(key, agent, self.state.seed)
10209        };
10210        self.state.seats.insert(key.to_owned(), fresh.clone());
10211        fresh
10212    }
10213
10214    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10215    /// seat to another roster agent, in which case that agent. Nodes that
10216    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10217    /// reconsideration) must keep talking to whoever answered it, not slip
10218    /// back to the agent that failed it.
10219    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10220        match self.state.seats.get(key) {
10221            Some(s) if s.agent != spec.id => {
10222                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10223            }
10224            _ => spec,
10225        }
10226    }
10227
10228    /// A candidate rendered for judging, with the leak policy applied.
10229    fn view(&self, c: &Candidate) -> CandidateView {
10230        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10231            .unwrap_or_default();
10232        let (patch, _) = blind::sanitize_patch(
10233            &format!("candidate {} patch", c.label),
10234            &raw,
10235            &self.state.config.blind,
10236        );
10237        CandidateView {
10238            label: c.label,
10239            branch: c.branch.clone(),
10240            summary: c.summary.clone(),
10241            stat: c.stat.clone(),
10242            patch,
10243        }
10244    }
10245
10246    /// The full candidate set as prompt text, for seats with no live session.
10247    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10248        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10249        prompt::judge(
10250            "(see above)",
10251            &views,
10252            self.roles.judges.len(),
10253            base_short,
10254            "en",
10255        )
10256    }
10257
10258    /// The final-vote prompt with everything a seat that has no session of its
10259    /// own needs: the candidates, the seat's own ranking and reasons, and the
10260    /// anonymised deliberation it took part in (only when there was one, so a
10261    /// handed-over seat never sees more than the seat it replaces did). The
10262    /// `Final vote` heading stays first.
10263    fn vote_prompt_full(
10264        &self,
10265        j: usize,
10266        labels: &[char],
10267        language: &str,
10268        candidates: &[Candidate],
10269        base_short: &str,
10270    ) -> String {
10271        let mut text = format!(
10272            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10273            prompt::final_vote(labels, language),
10274            self.state.instruction,
10275            self.candidate_block(candidates, base_short)
10276        );
10277        if let Some(own) = self
10278            .state
10279            .judgements
10280            .get(j)
10281            .filter(|r| !r.ranking.is_empty())
10282        {
10283            let reasons = own
10284                .reasons
10285                .iter()
10286                .map(|(k, v)| format!("- {k}: {v}"))
10287                .collect::<Vec<_>>()
10288                .join("\n");
10289            text.push_str(&format!(
10290                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10291                own.ranking.iter().collect::<String>(),
10292                if reasons.is_empty() {
10293                    ""
10294                } else {
10295                    ", because:\n"
10296                }
10297            ));
10298        }
10299        if !self.state.deliberation.is_empty() {
10300            text.push_str("\n# What was argued before this vote\n");
10301            for t in self.transcript(&[], j) {
10302                text.push_str(&format!(
10303                    "\n## {}{}\n\n{}\n",
10304                    t.who,
10305                    if t.is_self { " (you)" } else { "" },
10306                    t.body.trim()
10307                ));
10308            }
10309        }
10310        text
10311    }
10312
10313    /// Anonymised transcript for judge `self_idx`.
10314    ///
10315    /// The initial rankings are always the opening statements. Seeding them
10316    /// only when no turn had been taken yet meant every judge after the first
10317    /// argued against a single voice instead of against the actual split — the
10318    /// disagreement is the information, so it is always on the table.
10319    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10320        let mut turns = Vec::new();
10321        for j in &self.state.judgements {
10322            if j.ranking.is_empty() {
10323                continue;
10324            }
10325            let reasons = j
10326                .reasons
10327                .iter()
10328                .map(|(k, v)| format!("- {k}: {v}"))
10329                .collect::<Vec<_>>()
10330                .join("\n");
10331            turns.push(Turn {
10332                who: format!("Judge {} (opening ranking)", j.judge),
10333                is_self: j.judge == self_idx + 1,
10334                body: format!(
10335                    "Ranked {}{}{reasons}",
10336                    j.ranking.iter().collect::<String>(),
10337                    if reasons.is_empty() {
10338                        ""
10339                    } else {
10340                        ", because:\n"
10341                    }
10342                ),
10343            });
10344        }
10345        for t in self
10346            .state
10347            .deliberation
10348            .iter()
10349            .flat_map(|r| r.turns.iter())
10350            .chain(current)
10351        {
10352            turns.push(Turn {
10353                who: format!("Judge {}", t.judge),
10354                is_self: t.judge == self_idx + 1,
10355                body: t.body.clone(),
10356            });
10357        }
10358        turns
10359    }
10360}
10361
10362/// Does this seat still hold the context a follow-up prompt would rely on?
10363fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10364    agent::has_session(spec.kind, seat, sessions)
10365}
10366
10367/// The next entry in `roster` after `start`, never wrapping back to the
10368/// front, whose id is not in `tried` yet.
10369///
10370/// Starts one past `start` rather than at the front of `roster`: `start` is
10371/// the seat's own original position, and a seat whose candidate slot already
10372/// sits on the roster's second entry must fall through to the third next, not
10373/// restart at the first — which is very likely a different candidate's own
10374/// agent already. Never wraps back past `start`, for the same reason: an
10375/// entry earlier in the roster than the seat's own position is almost
10376/// certainly some *other* candidate slot's own agent, and once the tail of
10377/// the roster is exhausted there are no more untried agents for *this* seat
10378/// to fall through to — the caller's fallback chain ends there, exactly as
10379/// "no further untried agents remain in the list for that seat" asks for.
10380///
10381/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10382/// the same id twice (an operator's `roles.implementers` typo, or a
10383/// `[[agents]]` list reused across roles) must not let
10384/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10385/// over `roster` either finds an untried id or runs out, so this always
10386/// terminates regardless of duplicates.
10387fn next_untried_in_roster<'a>(
10388    roster: &'a [AgentSpec],
10389    start: usize,
10390    tried: &BTreeSet<String>,
10391) -> Option<&'a AgentSpec> {
10392    roster
10393        .get(start + 1..)?
10394        .iter()
10395        .find(|s| !tried.contains(&s.id))
10396}
10397
10398/// The successor for a seat, shared by all four seat kinds (implement, judge,
10399/// review, advise). `others` holds the ids that *currently* occupy the other
10400/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10401/// sees them), so roster entries beyond the seat count act as spares: a failed
10402/// seat goes to an agent no other seat holds whenever the roster permits.
10403///
10404/// `carried` is `Some` only for the review loop's carried failure history.
10405/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10406/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10407/// whole roster: untried and not another seat's occupant; (3) the plain walk
10408/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10409/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10410/// a better outcome than an empty seat, and it keeps the answer to "is there
10411/// a successor at all" exactly what it was before occupants were considered,
10412/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10413/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10414/// carried failure is a cheaper bet than doubling an agent on the panel, and
10415/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10416/// the same round are handled one at a time, so a seat later in the batch
10417/// sees an earlier one's new occupant but not yet its own freed agent.
10418fn pick_successor<'a>(
10419    roster: &'a [AgentSpec],
10420    start: usize,
10421    tried: &BTreeSet<String>,
10422    carried: Option<&BTreeSet<String>>,
10423    others: &BTreeSet<String>,
10424) -> Option<&'a AgentSpec> {
10425    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10426    roster
10427        .get(start + 1..)
10428        .and_then(|tail| {
10429            tail.iter()
10430                .filter(free)
10431                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10432        })
10433        .or_else(|| {
10434            carried
10435                .is_some()
10436                .then(|| roster.iter().find(free))
10437                .flatten()
10438        })
10439        .or_else(|| match carried {
10440            Some(c) => next_for_seat(roster, start, tried, c),
10441            None => next_untried_in_roster(roster, start, tried),
10442        })
10443}
10444
10445/// The next agent for a seat that carries its failure history across rounds
10446/// (the review loop). `round_tried` is this round's own bound and starts
10447/// empty every round; `carried_failed` only decides priority.
10448///
10449/// First: an id walking forward from `start`, never wrapping, that is neither
10450/// tried this round nor failed in an earlier one. Only when that is exhausted
10451/// does it rescue: the first roster id (in roster order, so this one *does*
10452/// look before `start`) not yet tried this round, which is by then a carried
10453/// failure. Each id is rescued at most once per round, so it cannot loop.
10454fn next_for_seat<'a>(
10455    roster: &'a [AgentSpec],
10456    start: usize,
10457    round_tried: &BTreeSet<String>,
10458    carried_failed: &BTreeSet<String>,
10459) -> Option<&'a AgentSpec> {
10460    roster
10461        .get(start + 1..)?
10462        .iter()
10463        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10464        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10465}
10466
10467/// Where a reviewer seat starts a round: the agent that last answered it when
10468/// it is still on the roster and not marked failed, else the spec's own agent
10469/// unless it failed, else the next roster agent that has not failed, else the
10470/// spec's own agent again (the whole roster failed). Ids no longer on the
10471/// roster are ignored. An empty roster has no handover, so the spec stands.
10472fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10473    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10474        return spec;
10475    };
10476    let ok = |id: &str| !h.failed.contains(id);
10477    if let Some(last) = h.last_ok.as_deref()
10478        && ok(last)
10479        && let Some(s) = roster.iter().find(|s| s.id == last)
10480    {
10481        return s.clone();
10482    }
10483    if ok(&spec.id) {
10484        return spec;
10485    }
10486    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10487    roster
10488        .get(from + 1..)
10489        .into_iter()
10490        .flatten()
10491        .chain(roster.iter())
10492        .find(|s| ok(&s.id))
10493        .cloned()
10494        .unwrap_or(spec)
10495}
10496
10497/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10498/// seed so a CLI that mints its session id up front (`--session-id`) never
10499/// reuses the uuid the previous agent already opened under the same seat key.
10500pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10501    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10502}
10503
10504/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10505/// so the classifier below and the code that builds the message cannot drift.
10506const TIMED_OUT: &str = "timed out";
10507
10508impl FailClass {
10509    /// `None` for an answer; otherwise how the turn failed.
10510    fn of(out: &AgentOutcome) -> Option<Self> {
10511        match out {
10512            AgentOutcome::Ok(_) => None,
10513            AgentOutcome::Quota(_) => Some(Self::Quota),
10514            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10515            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10516            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10517            // Not a failure of the agent. Callers check for it first; `None`
10518            // here means "no class to hand over on".
10519            AgentOutcome::ResourceBlocked(_) => None,
10520        }
10521    }
10522
10523    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10524    fn stem_word(&self) -> &'static str {
10525        match self {
10526            Self::Quota => "quota",
10527            _ => "handover",
10528        }
10529    }
10530}
10531
10532/// The message's first line with its variable parts removed — digit runs and
10533/// path-like tokens — so "exit code 2" and "exit code 7" read
10534/// as one kind of failure.
10535fn failure_signature(msg: &str) -> String {
10536    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10537    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10538    // the historical machine key while the process diagnosis uses readable prose.
10539    if let Some(code) = line
10540        .strip_prefix("process exited with no usable output (exit code ")
10541        .and_then(|s| s.strip_suffix(')'))
10542        .and_then(|s| s.parse::<i32>().ok())
10543    {
10544        line = format!("exited with some({code}) and no usable output");
10545    } else if matches!(
10546        line.as_str(),
10547        "process exited with no usable output (no exit code reported)"
10548            | "process exited with no usable output (exit code unavailable)"
10549    ) {
10550        line = "exited with none and no usable output".to_owned();
10551    }
10552    let mut out = Vec::new();
10553    for word in line.split_whitespace() {
10554        if word.contains('/') || word.contains('\\') {
10555            out.push("<path>".to_owned());
10556            continue;
10557        }
10558        let mut w = String::new();
10559        let mut in_digits = false;
10560        for c in word.chars() {
10561            if c.is_ascii_digit() {
10562                if !in_digits {
10563                    w.push('#');
10564                }
10565                in_digits = true;
10566            } else {
10567                in_digits = false;
10568                w.push(c);
10569            }
10570        }
10571        out.push(w);
10572    }
10573    out.join(" ").chars().take(120).collect()
10574}
10575
10576/// Whether a seat that just failed with `cur` may go to the next roster agent.
10577/// A quota or a timeout always may. Any other failure may not when the agent
10578/// before it failed the same way: an error the prompt causes would otherwise
10579/// walk the whole roster. `prev` is the class of the immediately preceding
10580/// agent's failure, so a quota or timeout in between breaks the run of
10581/// identical failures by itself.
10582fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10583    match cur {
10584        FailClass::Quota | FailClass::Timeout => true,
10585        FailClass::Other(_) => prev != Some(cur),
10586    }
10587}
10588
10589/// A short human reason for a failed outcome, for the handover record.
10590fn fail_reason(out: &AgentOutcome) -> String {
10591    match out {
10592        AgentOutcome::Ok(_) => String::new(),
10593        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10594        AgentOutcome::Dropped(o) => format!(
10595            "the CLI dropped the stream ({})",
10596            o.dropped
10597                .as_ref()
10598                .map(|d| d.why.as_str())
10599                .unwrap_or("it ended without delivering its answer")
10600        ),
10601        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10602            e.lines().next().unwrap_or("").chars().take(160).collect()
10603        }
10604    }
10605}
10606
10607/// Note one handover in the run: the structured record and, in the timeline,
10608/// the sentence a person reads. A quota keeps the wording it always had.
10609pub(crate) fn record_handover(
10610    state: &mut RunState,
10611    node: &str,
10612    seat: &str,
10613    from: &str,
10614    to: &str,
10615    class: &FailClass,
10616    reason: &str,
10617) {
10618    let message = if *class == FailClass::Quota {
10619        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10620    } else {
10621        format!("{seat}: handed over {from} -> {to} ({reason})")
10622    };
10623    state.event(node, message);
10624    state.handovers.push(Handover {
10625        at: Timestamp::now(),
10626        node: node.to_owned(),
10627        seat: seat.to_owned(),
10628        from: from.to_owned(),
10629        to: to.to_owned(),
10630        reason: reason.to_owned(),
10631    });
10632}
10633
10634/// Did this reply report running a command whose own CLI never confirmed an
10635/// exit status?
10636///
10637/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10638/// command *finished* (see that type's own doc), so this can only be `true`
10639/// for a command whose completion event carried no readable exit code — not
10640/// for one that simply is not mentioned at all. That is the one signal this
10641/// crate can read, from the same record `state.jobs` renders, about a reply
10642/// standing next to work its own CLI cannot vouch for finishing; it is
10643/// deliberately not a check on the exit code's *value* (a fixer legitimately
10644/// runs a command that fails mid-iteration before it succeeds) and not a
10645/// guess at a command still running in the background (which emits no event
10646/// at all, and so leaves no evidence here to find).
10647fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10648    commands.iter().any(|c| c.exit_code.is_none())
10649}
10650
10651/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10652/// trusted as a verified no-op — the adoption guard's own text-level half.
10653///
10654/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10655/// timed out): a marker only earns the benefit of the doubt from a turn the
10656/// CLI itself vouches for finishing properly, the same house style
10657/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10658/// *fix* report to for `commands`. A candidate that timed out, exited
10659/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10660/// is, whatever prose it wrote — this returns `None` before it ever looks at
10661/// `text`. The remaining guards (the tree really is empty, the evidence is
10662/// non-empty) are the caller's: this only reads what the reply *claimed*.
10663fn verified_noop_claim(
10664    usable: bool,
10665    commands: &[agent::CommandEvidence],
10666    text: &str,
10667) -> Option<String> {
10668    (usable && !has_unconfirmed_command(commands))
10669        .then(|| verdict::verified_noop(text))
10670        .flatten()
10671}
10672
10673fn short(commit: &str) -> String {
10674    commit.chars().take(7).collect()
10675}
10676
10677fn make_executable(path: &Path) -> Result<()> {
10678    #[cfg(unix)]
10679    {
10680        use std::os::unix::fs::PermissionsExt as _;
10681        let mut perms = std::fs::metadata(path)?.permissions();
10682        perms.set_mode(0o755);
10683        std::fs::set_permissions(path, perms)?;
10684    }
10685    #[cfg(not(unix))]
10686    {
10687        let _ = path;
10688    }
10689    Ok(())
10690}
10691
10692/// What every seat in one batch shares: where the answers are attributed, the
10693/// prompt overlay they inherit, and the build cache they are told to use.
10694///
10695/// A struct rather than four more parameters: `wave` also needs the run's
10696/// state (to record who is answering right now) and the attempt number, and
10697/// eight positional arguments is both unreadable and a clippy error.
10698struct WaveCtx<'a> {
10699    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10700    /// paid for it.
10701    run: &'a str,
10702    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10703    node: &'a str,
10704    prompts: &'a Prompts,
10705    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10706    cache: Option<&'a Path>,
10707    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10708    /// `JobRecord::round`. `None` for every other node.
10709    round: Option<usize>,
10710    /// Carry each seat's failed-agent history across waves (the review loop
10711    /// only): start-of-round priority and handover choice read
10712    /// [`RunState::seat_history`], and every answer or failure writes it.
10713    carry_seats: bool,
10714    /// The run's park / interrupt requests, so a wait for a shared resource
10715    /// can be abandoned the moment the operator asks to park.
10716    pauses: Option<(&'a Pause, &'a Pause)>,
10717}
10718
10719/// Shared scheduling policy for operations and spawned seat invocations.
10720fn repository_wait(
10721    state: &RunState,
10722    pauses: Option<(&Pause, &Pause)>,
10723) -> crate::repo_lock::WaitPolicy {
10724    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10725    let inherited = crate::repo_lock::WaitPolicy::current();
10726    crate::repo_lock::WaitPolicy::new(
10727        Duration::from_secs(state.config.graph.timeout_resource_wait),
10728        move || {
10729            inherited.cancelled()
10730                || pauses
10731                    .as_ref()
10732                    .is_some_and(|(a, b)| a.parked() || b.parked())
10733        },
10734    )
10735}
10736
10737fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10738    if error.is::<crate::repo_lock::ResourceBlocked>() {
10739        state.resource_block = Some(crate::run::ResourceBlock {
10740            cause: crate::run::ResourceBlockCause::Other,
10741            node: "repository".into(),
10742            kind: if error.to_string().contains("disk pressure") {
10743                "disk pressure"
10744            } else if error.to_string().contains("disk admission") {
10745                "disk admission"
10746            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10747                "incremental cache admission"
10748            } else {
10749                "repository coordination"
10750            }
10751            .into(),
10752            detail: error.to_string(),
10753            at: Timestamp::now(),
10754        });
10755        state.event("resource", error.to_string());
10756    }
10757}
10758
10759async fn coordinate_operation(
10760    state: &mut RunState,
10761    cwd: &Path,
10762    wait: &crate::repo_lock::WaitPolicy,
10763) -> Result<Option<crate::repo_lock::SeatLease>> {
10764    let result =
10765        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10766            .await;
10767    if let Err(e) = &result {
10768        record_repository_block(state, e);
10769    }
10770    result
10771}
10772
10773/// Run one job, honouring the parallelism budget.
10774async fn run_one(
10775    job: SeatJob,
10776    sem: Arc<Semaphore>,
10777    ctx: &WaveCtx<'_>,
10778    state: &mut RunState,
10779    attempt: usize,
10780) -> (SeatState, AgentOutcome) {
10781    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10782        .await
10783        .pop()
10784        .expect("one job in, one result out");
10785    (seat, out)
10786}
10787
10788/// Run every job concurrently, capped by the semaphore, preserving order.
10789///
10790/// Every seat in the batch is recorded into [`RunState::active`] before the
10791/// wave starts and cleared as each answer lands, so the run's own record says
10792/// who is still being waited on rather than only who finished.
10793async fn wave(
10794    jobs: Vec<SeatJob>,
10795    sem: Arc<Semaphore>,
10796    ctx: &WaveCtx<'_>,
10797    state: &mut RunState,
10798    attempt: usize,
10799) -> Vec<(usize, SeatState, AgentOutcome)> {
10800    let WaveCtx {
10801        run,
10802        node,
10803        prompts,
10804        cache,
10805        round,
10806        carry_seats: _,
10807        pauses,
10808    } = *ctx;
10809    for job in &jobs {
10810        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10811    }
10812    if let Err(e) = state.save() {
10813        // A failed persist of "who is answering right now" must not abort the
10814        // wave: the seats are already being asked, and the alternative is
10815        // losing the answers to save a status line nobody may even be
10816        // watching.
10817        tracing::warn!("could not persist in-progress seats: {e:#}");
10818    }
10819    // Hold the shared build cache's lease for the whole batch, not per job:
10820    // several candidates (an implement wave) or a fixer legitimately share
10821    // one cache concurrently within this run, and that stays untouched — a
10822    // single lease taken once for the whole wave and released once it is
10823    // done is what stops a *different* borrower (another run's own wave, its
10824    // e2e/gate, a human's `magi review`) from interleaving a build into the
10825    // same directory while this one is in flight. Best-effort, not
10826    // all-or-nothing: a wave that cannot get the lease within its own
10827    // longest job's budget still runs — an hour of paid implementer calls is
10828    // not thrown away over cache contention — but every write-allowed seat
10829    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10830    // below), the same fallback a read-only seat always gets, rather than
10831    // building into a directory this run was never granted. The identity
10832    // record is still invalidated below either way, so the next tracked
10833    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10834    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10835    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10836    // never borrowed from a job's agent budget (and never charged to it either,
10837    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10838    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10839    let cache_guard = if let Some(cache_dir) = cache {
10840        if jobs_had_a_writer {
10841            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10842            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10843            match acquire_cache_lease(
10844                state,
10845                cache_dir,
10846                &owner,
10847                resource_limit,
10848                node,
10849                node,
10850                &cancel,
10851            )
10852            .await
10853            {
10854                Ok(Some(g)) => Some(g),
10855                // Parking: leave without spawning anything. The run parks at
10856                // the next boundary and the seats are asked afresh on resume.
10857                Ok(None) => {
10858                    if let Some(block) = &mut state.resource_block {
10859                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10860                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10861                        };
10862                    }
10863                    return block_wave(
10864                        jobs,
10865                        state,
10866                        node,
10867                        attempt,
10868                        "parked while waiting for the shared build cache",
10869                    );
10870                }
10871                // An unreadable lease is not contention: keep the old
10872                // fallback (no shared cache for this wave's seats).
10873                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10874                Err(e) => {
10875                    // Writing seats may not build into a cache this run does
10876                    // not hold, and an agent that cannot be given one is not
10877                    // asked at all: the run stops resumably instead.
10878                    let why = format!("{e:#}");
10879                    state.event(
10880                        "resource",
10881                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10882                    );
10883                    return block_wave(jobs, state, node, attempt, &why);
10884                }
10885            }
10886        } else {
10887            None
10888        }
10889    } else {
10890        None
10891    };
10892    // Every job keeps its whole budget: the wait above was not the agent's.
10893    // Restart the seats' clocks so "started" means "being spawned now".
10894    state.seats_spawning(node, attempt);
10895    if let Err(e) = state.save() {
10896        tracing::warn!("could not persist the start of a wave: {e:#}");
10897    }
10898    // Queueing behind this wave's own seats for a parallelism slot is normal
10899    // (each of them is bounded by its own agent timeout); only a wait beyond
10900    // all of them plus the resource limit means the slots never came back.
10901    let slot_limit = resource_limit
10902        + jobs
10903            .iter()
10904            .map(|j| j.timeout)
10905            .max()
10906            .unwrap_or_default()
10907            .saturating_mul(jobs.len() as u32);
10908    let mut set = tokio::task::JoinSet::new();
10909    // Seats announce themselves here the moment they hold a parallelism
10910    // permit, i.e. just before their CLI is spawned.
10911    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10912    let overlay = prompts.overlay(node);
10913    let repository_wait = repository_wait(state, pauses);
10914    for (i, mut job) in jobs.into_iter().enumerate() {
10915        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10916        if cache.is_some() {
10917            job.prompt.push('\n');
10918            job.prompt
10919                .push_str(&prompt::build_cache_note(node, job.allow_write));
10920        }
10921        let sem = Arc::clone(&sem);
10922        let run = run.to_owned();
10923        let node = node.to_owned();
10924        // Only implementers were told about the task's attachments, so only
10925        // their seats get the directory widened for reading.
10926        let attachments = if node == "implement" {
10927            state.attachments.clone()
10928        } else {
10929            Vec::new()
10930        };
10931        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10932        // `prompt::build_cache_note`'s doc for why setting it anyway is
10933        // exactly how a sandboxed reviewer's write refusal got reported as a
10934        // defect in the patch, not a property of its own seat. And a
10935        // write-allowed one is handed it only when the lease above was
10936        // actually acquired: a wave that could not get it (`cache_guard` is
10937        // `None`, see its own comment) must not send seats to build into a
10938        // directory this run does not hold - that is the exact concurrent,
10939        // unmanaged-write race this module exists to prevent, not something
10940        // "proceeding anyway" is allowed to reintroduce.
10941        let cache = cache
10942            .filter(|_| job.allow_write && cache_guard.is_some())
10943            .map(Path::to_path_buf);
10944        let spawned_tx = spawned_tx.clone();
10945        let repository_wait = repository_wait.clone();
10946        let coordination = state.config.coordination.clone();
10947        let fetched_base = state.fetched_base.clone();
10948        set.spawn(async move {
10949            // A wait for a parallelism slot is scheduling, not the agent: it
10950            // is bounded by its own limit and ends as a resource block, never
10951            // as a timeout of the agent that was not started.
10952            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10953                Ok(p) => p,
10954                Err(_) => {
10955                    return (
10956                        i,
10957                        job.seat,
10958                        AgentOutcome::ResourceBlocked(format!(
10959                            "no parallel slot became free within {}s",
10960                            slot_limit.as_secs()
10961                        )),
10962                        false,
10963                    );
10964                }
10965            };
10966            spawned_tx.send(job.seat.key.clone()).ok();
10967            let mut seat = job.seat;
10968            let out = repository_wait
10969                .scope(agent::invoke(
10970                    &job.spec,
10971                    &mut seat,
10972                    &Invocation {
10973                        coordination: coordination.as_ref(),
10974                        fetched_base: fetched_base.as_ref(),
10975                        cwd: &job.cwd,
10976                        prompt: &job.prompt,
10977                        timeout: job.timeout,
10978                        allow_write: job.allow_write,
10979                        sessions: job.sessions,
10980                        artifacts: &job.artifacts,
10981                        stem: &job.stem,
10982                        run: &run,
10983                        node: &node,
10984                        cache_dir: cache.as_deref(),
10985                        attachments: &attachments,
10986                        writable: &[],
10987                    },
10988                ))
10989                .await;
10990            let interrupted = out
10991                .as_ref()
10992                .err()
10993                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10994            let out = match out {
10995                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10996                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10997                // Billed work the CLI failed to hand over is not an ordinary
10998                // failure, but its text is the CLI's raw error JSON, not an
10999                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
11000                // read it as one by forgetting to check. `usable()` is always
11001                // false here (dropped implies an empty response), so this has
11002                // to be checked before the catch-all `Failed` below or the
11003                // one shape this exists for is lost with the rest.
11004                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
11005                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
11006                Ok(o) => AgentOutcome::Failed(format!(
11007                    "process exited with no usable output ({})",
11008                    crate::notices::exit_status(o.exit_code)
11009                )),
11010                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
11011                    AgentOutcome::ResourceBlocked(e.to_string())
11012                }
11013                Err(e) => AgentOutcome::Failed(e.to_string()),
11014            };
11015            (i, seat, out, interrupted)
11016        });
11017    }
11018    drop(spawned_tx);
11019    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
11020    loop {
11021        let joined = tokio::select! {
11022            Some(key) = spawned_rx.recv() => {
11023                // The clock of this seat starts now, not when it queued.
11024                state.seat_spawned(&key);
11025                if let Err(e) = state.save() {
11026                    tracing::warn!("could not persist a seat's start: {e:#}");
11027                }
11028                continue;
11029            }
11030            joined = set.join_next() => joined,
11031        };
11032        let Some(joined) = joined else {
11033            break;
11034        };
11035        let (i, seat, out, interrupted) = match joined {
11036            Ok(v) => v,
11037            // No seat to clear: a panicked task never reported which one it
11038            // was. The defensive sweep below this loop is what stops that
11039            // seat's `active` entry from surviving forever.
11040            Err(e) => {
11041                tracing::error!("agent task panicked: {e}");
11042                continue;
11043            }
11044        };
11045        state.seat_finished(&seat.key);
11046        if interrupted
11047            && node == "implement"
11048            && let Some(label) = state
11049                .candidates
11050                .iter()
11051                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11052                .map(|candidate| candidate.label)
11053        {
11054            state.candidate_preservation.insert(
11055                label,
11056                "interrupted implementation awaiting persistence".into(),
11057            );
11058            state.interrupted_implementations.insert(label);
11059        }
11060        if let AgentOutcome::ResourceBlocked(why) = &out
11061            && state.resource_block.is_none()
11062        {
11063            state.resource_block = Some(crate::run::ResourceBlock {
11064                cause: crate::run::ResourceBlockCause::Other,
11065                node: node.to_owned(),
11066                kind: if why.contains("disk pressure") {
11067                    "disk pressure"
11068                } else if why.contains("disk admission") {
11069                    "disk admission"
11070                } else if why.contains(crate::incremental::BLOCKED) {
11071                    "incremental cache admission"
11072                } else if why.contains("repository coordination") {
11073                    "repository coordination"
11074                } else {
11075                    "parallel slot"
11076                }
11077                .to_owned(),
11078                detail: why.clone(),
11079                at: Timestamp::now(),
11080            });
11081            let activity = if interrupted {
11082                "agent execution interrupted"
11083            } else {
11084                "no agent was started"
11085            };
11086            state.event("resource", format!("{node}: {activity} - {why}"));
11087        }
11088        record_jobs(state, node, round, &seat.key, &out);
11089        if let Err(e) = state.save() {
11090            tracing::warn!("could not persist a seat's completion: {e:#}");
11091        }
11092        if collected.len() <= i {
11093            collected.resize_with(i + 1, || None);
11094        }
11095        collected[i] = Some((i, seat, out));
11096    }
11097    // Belt-and-braces for the panic branch above: every seat this exact batch
11098    // started shares this `(node, attempt)` pair, and every seat that finished
11099    // normally already cleared itself, so anything left tagged with it here
11100    // can only be a panicked task's leftover. Cleared unconditionally rather
11101    // than left to read as still answering forever.
11102    if state
11103        .active
11104        .values()
11105        .any(|a| a.node == node && a.attempt == attempt)
11106    {
11107        state
11108            .active
11109            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11110        if let Err(e) = state.save() {
11111            tracing::warn!("could not persist the end of a wave: {e:#}");
11112        }
11113    }
11114    // Whether or not the lease above was actually held, several worktrees
11115    // may just have built into the cache with nothing here able to name one
11116    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11117    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11118    // clean it might not have strictly needed; trusting a stale match would
11119    // cost it a wrong answer.
11120    if let Some(cache_dir) = cache
11121        && jobs_had_a_writer
11122    {
11123        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11124    }
11125    if let Some(guard) = cache_guard {
11126        guard.release();
11127    }
11128    collected.into_iter().flatten().collect()
11129}
11130
11131/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11132/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11133/// this is data collection, not the fix-specific completion contract in
11134/// [`Runner::continue_fix_report`], and applies regardless of which node
11135/// asked.
11136///
11137/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11138/// evidence from; `Failed` does not, and correctly contributes nothing — a
11139/// timeout or crash is not itself evidence about a command the seat may have
11140/// started.
11141fn record_jobs(
11142    state: &mut RunState,
11143    node: &str,
11144    round: Option<usize>,
11145    seat: &str,
11146    out: &AgentOutcome,
11147) {
11148    let commands: &[agent::CommandEvidence] = match out {
11149        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11150        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11151    };
11152    let checked_at = Timestamp::now();
11153    for c in commands {
11154        state.jobs.push(JobRecord {
11155            node: node.to_owned(),
11156            round,
11157            seat: seat.to_owned(),
11158            id: c.id.clone(),
11159            description: c.description.clone(),
11160            checked_at,
11161            status: match c.exit_code {
11162                Some(0) => JobStatus::Completed,
11163                Some(_) => JobStatus::Failed,
11164                None => JobStatus::Unknown,
11165            },
11166            exit_code: c.exit_code,
11167            result_summary: c.result_summary.clone(),
11168            source: c.source.clone(),
11169        });
11170    }
11171}
11172
11173/// Is a review round clean, given how many reviewer seats answered against
11174/// how many the round expected?
11175///
11176/// A seat that never answered (timeout, crash, unparsable output) is not a
11177/// seat that read the patch and found nothing — treating it as such is
11178/// exactly the bug this function exists to close. Under the default `block`
11179/// policy a missing seat can never be clean; `warn` still requires the seats
11180/// that *did* answer to have found nothing blocking and verification to be
11181/// green.
11182///
11183/// `quota_missing` narrows that `block` default for exactly one cause of
11184/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11185/// a session limit lifts by the very next round buys nothing — the seat is
11186/// asked again with the same quota — so once every missing seat is accounted
11187/// for by a quota loss (and at least one seat *did* answer, so a decision has
11188/// something to rest on) the round is decided on the panel that could answer,
11189/// same as `warn` would. A panel that lost every seat to quota is not
11190/// decided here: `answered == 0` falls through to the existing `block`
11191/// fallback so a fully collapsed panel still waits rather than landing on no
11192/// review at all.
11193fn round_is_clean(
11194    blocking: usize,
11195    e2e_ok: bool,
11196    answered: usize,
11197    expected: usize,
11198    quota_missing: usize,
11199    policy: IncompleteReviewPolicy,
11200) -> bool {
11201    if blocking != 0 || !e2e_ok {
11202        return false;
11203    }
11204    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11205        return true;
11206    }
11207    answered > 0 && expected - answered <= quota_missing
11208}
11209
11210/// The review loop's own conclusion, derived entirely from its persisted
11211/// round records and the round budget that produced them — never from
11212/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11213/// recomputes the identical answer regardless of what an earlier node in the
11214/// same walk, or a previous walk, did to `status`.
11215///
11216/// `None` while more rounds remain to try, including when review never ran
11217/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11218/// gone clean, or the budget is spent, or the tree has stopped moving (see
11219/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11220///
11221/// - An incomplete panel that raised nothing is missing input, not a
11222///   verified tree — never a hand-off candidate, whatever verification said
11223///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11224/// - Otherwise, green e2e on the last round hands off (see
11225///   [`Runner::stop_reviewing`]); red e2e blocks.
11226///
11227/// A last round whose own verification is still `ResourceBlocked` — magi
11228/// itself never got a command to run, not evidence the patch is broken —
11229/// is neither: this returns `None` for it too, the same as "more rounds
11230/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11231/// handling of that shape) instead of this cheap recomputation guessing a
11232/// verdict a real attempt never produced.
11233pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11234    if max_rounds == 0
11235        || reviews.last().is_some_and(|r| {
11236            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11237        })
11238    {
11239        return Some(RunStatus::Gating);
11240    }
11241    let last = reviews.last()?;
11242    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11243    if reviews.len() < max_rounds && !stagnant {
11244        return None;
11245    }
11246    if last.incomplete() && last.blocking == 0 {
11247        return Some(RunStatus::Blocked);
11248    }
11249    if last.e2e_status() == E2eStatus::ResourceBlocked
11250        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11251    {
11252        return None;
11253    }
11254    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11255        RunStatus::Gating
11256    } else {
11257        RunStatus::Blocked
11258    })
11259}
11260
11261/// How long a re-ask may take, given the budget the first attempt had.
11262///
11263/// A `nudged` retry is a request to restate an answer the seat has already
11264/// worked out: it carries no new work, so it does not deserve the original
11265/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11266/// 133 seconds while a third sat for over ten minutes on a resumed session
11267/// holding 410 KB of prior output - and because the retry had inherited the
11268/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11269/// judging round whose other seats were long finished.
11270///
11271/// A quarter of the budget, with a floor so that a deliberately short timeout
11272/// does not collapse to nothing. A retry that re-sends the whole prompt
11273/// (because the seat kept no context) is the original job again, and keeps the
11274/// original budget.
11275pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11276    if nudged {
11277        (full / 4).max(Duration::from_secs(120)).min(full)
11278    } else {
11279        full
11280    }
11281}
11282
11283/// Run a wave and parse each reply, re-asking the seats whose reply was
11284/// unusable.
11285///
11286/// The re-ask is a nudge rather than the whole prompt again when the seat still
11287/// holds its conversation, which is the difference between a cheap retry and
11288/// paying for the entire candidate set twice.
11289///
11290/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11291/// successor in `roster` is handed to it instead of being re-asked: the
11292/// handover is the retry. A seat with no successor left (a single-agent
11293/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11294/// the asks to one seat in one node number at most
11295/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11296/// is asked once (a dropped stream is nudged first), and only the last agent
11297/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11298/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11299/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11300/// other failure stops the chain when the previous agent failed the same way
11301/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11302/// [`has_context`] is false and the job's own full prompt and full budget are
11303/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11304/// intermediate ones are not losses) and is returned as a failure like any
11305/// other absent seat — the caller decides whether the panel still has a
11306/// quorum. An empty `roster` disables handover: failures are nudged as they
11307/// always were, and a quota is simply lost. A reply that fails to parse or
11308/// validate is the prompt's doing and is only ever nudged, never handed over.
11309///
11310/// The returned [`SeatState`] names the agent that answered (or tried last).
11311#[allow(clippy::too_many_arguments)]
11312async fn ask_json_wave<T>(
11313    jobs: Vec<SeatJob>,
11314    sem: Arc<Semaphore>,
11315    retries: usize,
11316    roster: &[AgentSpec],
11317    ctx: &WaveCtx<'_>,
11318    losses: &mut Vec<QuotaLoss>,
11319    state: &mut RunState,
11320    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11321) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11322where
11323    T: serde::de::DeserializeOwned + Send + 'static,
11324{
11325    ask_wave_with(
11326        jobs,
11327        sem,
11328        retries,
11329        roster,
11330        ctx,
11331        losses,
11332        state,
11333        &|text: &str| {
11334            let v = verdict::extract_json::<T>(text)?;
11335            validate(&v)?;
11336            Ok(v)
11337        },
11338    )
11339    .await
11340}
11341
11342/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11343/// a node whose answer is prose (deliberation) shares the same handover,
11344/// failure classification, quota bookkeeping and bounds instead of a copy.
11345///
11346/// A seat handed to another roster agent is sent the job's `handover` prompt
11347/// (when it has one) rather than `prompt`: the new agent has no session, so a
11348/// resume-style prompt would be incomplete. That holds for the handover ask
11349/// and for every nudge to that agent whose `has_context` is false.
11350#[allow(clippy::too_many_arguments)]
11351async fn ask_wave_with<T>(
11352    jobs: Vec<SeatJob>,
11353    sem: Arc<Semaphore>,
11354    retries: usize,
11355    roster: &[AgentSpec],
11356    ctx: &WaveCtx<'_>,
11357    losses: &mut Vec<QuotaLoss>,
11358    state: &mut RunState,
11359    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11360) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11361where
11362    T: Send + 'static,
11363{
11364    let n = jobs.len();
11365    let originals: Vec<SeatJob> = jobs;
11366    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11367    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11368    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11369    // answer, N once it has gone through N nudges. Read back once this
11370    // returns, so a caller building a history record (`ReviewRecord`) can
11371    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11372    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11373    // field's own doc.
11374    let mut nudges: Vec<usize> = vec![0; n];
11375    // The agent now occupying each seat, the ids it has already been through,
11376    // where in the roster the walk began, the class of the last failure, and
11377    // the stem word of a handover not yet asked (full prompt, full budget).
11378    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11379    let mut tried: Vec<BTreeSet<String>> = specs
11380        .iter()
11381        .map(|s| BTreeSet::from([s.id.clone()]))
11382        .collect();
11383    let starts: Vec<usize> = specs
11384        .iter()
11385        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11386        .collect();
11387    let carry = ctx.carry_seats && !roster.is_empty();
11388    // Carried across rounds: ids that failed the seat earlier, and how the
11389    // last failure went (so a repeat of it is not handed over again).
11390    let carried: Vec<BTreeSet<String>> = originals
11391        .iter()
11392        .map(|j| {
11393            state
11394                .seat_history
11395                .get(&j.seat.key)
11396                .filter(|_| carry)
11397                .map(|h| h.failed.clone())
11398                .unwrap_or_default()
11399        })
11400        .collect();
11401    let mut prev: Vec<Option<FailClass>> = originals
11402        .iter()
11403        .map(|j| {
11404            state
11405                .seat_history
11406                .get(&j.seat.key)
11407                .filter(|_| carry)
11408                .and_then(|h| h.last_fail.clone())
11409        })
11410        .collect();
11411    let next_agent =
11412        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11413            let others: BTreeSet<String> = specs
11414                .iter()
11415                .enumerate()
11416                .filter(|(j, _)| *j != i)
11417                .map(|(_, s)| s.id.clone())
11418                .collect();
11419            pick_successor(
11420                roster,
11421                starts[i],
11422                tried,
11423                carry.then(|| &carried[i]),
11424                &others,
11425            )
11426            .cloned()
11427        };
11428    let mut fresh: Vec<Option<String>> = vec![None; n];
11429    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11430    let mut pending: Vec<usize> = (0..n).collect();
11431
11432    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11433    // asks: an agent with a successor is asked once and handed over, and only
11434    // a seat with no successor spends `retries` nudges on the same agent. This
11435    // only guarantees the loop's own termination whatever those say.
11436    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11437    for round in 0..max_rounds {
11438        if pending.is_empty() {
11439            break;
11440        }
11441        let mut batch = Vec::with_capacity(pending.len());
11442        let mut renudged: Vec<&str> = Vec::new();
11443        for &i in &pending {
11444            let src = &originals[i];
11445            // A seat now held by another agent than the job named has no
11446            // session of its own: it gets the full-context prompt whenever
11447            // it is asked in full (the handover ask, a nudge it cannot
11448            // resume).
11449            let full: &str = match &src.handover {
11450                Some(h) if specs[i].id != src.spec.id => h,
11451                _ => &src.prompt,
11452            };
11453            // The prompt and the budget are one decision: a nudge restates
11454            // finished work, a re-sent prompt redoes it.
11455            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11456                (
11457                    full.to_owned(),
11458                    src.timeout,
11459                    format!("{}-{word}-{}", src.stem, specs[i].id),
11460                )
11461            } else if nudges[i] == 0 {
11462                (full.to_owned(), src.timeout, src.stem.clone())
11463            } else {
11464                renudged.push(src.seat.key.as_str());
11465                let why = done[i]
11466                    .as_ref()
11467                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11468                    .unwrap_or_else(|| "no parsable answer".to_owned());
11469                let nudge = prompt::nudge(&why);
11470                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11471                let prompt = if nudged {
11472                    nudge
11473                } else {
11474                    format!("{full}\n\n---\n\n{nudge}")
11475                };
11476                (
11477                    prompt,
11478                    retry_budget(src.timeout, nudged),
11479                    format!("{}-retry{}", src.stem, nudges[i]),
11480                )
11481            };
11482            batch.push(SeatJob {
11483                spec: specs[i].clone(),
11484                seat: seats[i].clone(),
11485                cwd: src.cwd.clone(),
11486                prompt,
11487                timeout,
11488                allow_write: src.allow_write,
11489                sessions: src.sessions,
11490                artifacts: src.artifacts.clone(),
11491                stem,
11492                handover: None,
11493            });
11494        }
11495
11496        if !renudged.is_empty() {
11497            state.event(
11498                ctx.node,
11499                format!("retry {round}: re-asking {}", renudged.join(", ")),
11500            );
11501        }
11502        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11503        let mut still = Vec::new();
11504        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11505            seats[i] = seat;
11506            // No agent was asked: nothing to hand over, nudge, remember or
11507            // charge. The caller sees `resource_block` and stops the node.
11508            if let AgentOutcome::ResourceBlocked(why) = &out {
11509                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11510                continue;
11511            }
11512            let class = FailClass::of(&out);
11513            // A dropped stream is nudged first (the conversation is still
11514            // there to pick up); only a seat whose nudges are spent hands over.
11515            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11516                && nudges[i] < retries
11517                && !roster.is_empty();
11518            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11519                let next = should_hand_over(prev[i].as_ref(), &cur)
11520                    .then(|| next_agent(i, &tried[i], &specs))
11521                    .flatten();
11522                if carry {
11523                    let h = state
11524                        .seat_history
11525                        .entry(originals[i].seat.key.clone())
11526                        .or_default();
11527                    h.failed.insert(specs[i].id.clone());
11528                    h.last_fail = Some(cur.clone());
11529                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11530                        h.last_ok = None;
11531                    }
11532                    // Saved before the next agent is asked, so a restart in
11533                    // between does not forget who failed.
11534                    if let Err(e) = state.save() {
11535                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11536                    }
11537                }
11538                if let Some(next) = next {
11539                    record_handover(
11540                        state,
11541                        ctx.node,
11542                        &originals[i].seat.key,
11543                        &specs[i].id,
11544                        &next.id,
11545                        &cur,
11546                        &fail_reason(&out),
11547                    );
11548                    tried[i].insert(next.id.clone());
11549                    prev[i] = Some(cur.clone());
11550                    seats[i] =
11551                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11552                    specs[i] = next;
11553                    fresh[i] = Some(cur.stem_word().to_owned());
11554                    nudges[i] = 0;
11555                    done[i] = Some(Err(anyhow::anyhow!(
11556                        "handed over after: {}",
11557                        fail_reason(&out)
11558                    )));
11559                    still.push(i);
11560                    continue;
11561                }
11562            }
11563            if carry
11564                && !nudge_first
11565                && let Some(cur) = class.clone()
11566            {
11567                // The chain ended here (no successor, or a repeated failure).
11568                let h = state
11569                    .seat_history
11570                    .entry(originals[i].seat.key.clone())
11571                    .or_default();
11572                h.failed.insert(specs[i].id.clone());
11573                h.last_fail = Some(cur);
11574            }
11575            let parsed = match out {
11576                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11577                AgentOutcome::Quota(o) => {
11578                    last_quota[i] = o.quota.clone();
11579                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11580                }
11581                // Not a parseable answer: the nudge loop re-asks it, which is
11582                // exactly what a dropped stream needs. Just don't hand its raw
11583                // error JSON to `extract_json`.
11584                AgentOutcome::Dropped(o) => {
11585                    let why = o
11586                        .dropped
11587                        .as_ref()
11588                        .map(|d| d.why.as_str())
11589                        .unwrap_or("the CLI ended the stream without delivering its answer");
11590                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11591                }
11592                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11593                    Err(anyhow::anyhow!(e))
11594                }
11595            };
11596            let quota = class == Some(FailClass::Quota);
11597            let failed = parsed.is_err();
11598            if !failed && matches!(ctx.node, "judge" | "vote") {
11599                // A validated answer restores this seat's quorum presence,
11600                // including on the normal path after an all-seat block.
11601                state
11602                    .quota
11603                    .retain(|loss| loss.seat != originals[i].seat.key);
11604            }
11605            done[i] = Some(parsed);
11606            if carry && !failed {
11607                let h = state
11608                    .seat_history
11609                    .entry(originals[i].seat.key.clone())
11610                    .or_default();
11611                h.failed.remove(&specs[i].id);
11612                h.last_ok = Some(specs[i].id.clone());
11613                h.last_fail = None;
11614            }
11615            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11616            // fail the same way; and never re-ask a seat that already parsed.
11617            // A failed agent that still has a successor is not re-asked
11618            // either: the handover was its remedy and has just been refused
11619            // (the chain stops on a repeated failure class). A seat with no
11620            // successor left (a single-agent roster, the roster's tail, or an
11621            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11622            let agent_failure = class.is_some()
11623                && !nudge_first
11624                && !roster.is_empty()
11625                && next_agent(i, &tried[i], &specs).is_some();
11626            if failed && !quota && !agent_failure && nudges[i] < retries {
11627                nudges[i] += 1;
11628                still.push(i);
11629            }
11630        }
11631        pending = still;
11632    }
11633
11634    // One loss per seat whose chain ended on a quota: the intermediate ones
11635    // were absorbed by a handover and are not losses.
11636    for (i, q) in last_quota.into_iter().enumerate() {
11637        if let Some(quota) = q {
11638            losses.push(QuotaLoss::from_report(
11639                originals[i].seat.key.clone(),
11640                ctx.node,
11641                Some(&quota),
11642            ));
11643        }
11644    }
11645
11646    // A stalled quorum already has its own recovery checkpoint. Refresh
11647    // its missing seats before deciding whether it can resume.
11648    if state.status != RunStatus::Stalled
11649        && ctx.node != "deliberate"
11650        && n > 0
11651        && losses.len() == n
11652        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11653    {
11654        block_machine_outage(state, ctx.node, losses);
11655        state.quota.append(losses);
11656    }
11657
11658    seats
11659        .into_iter()
11660        .zip(done)
11661        .zip(nudges)
11662        .map(|((seat, res), attempts)| {
11663            (
11664                seat,
11665                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11666                attempts,
11667            )
11668        })
11669        .collect()
11670}
11671
11672/// A node with no available agent has no task verdict. Preserve the node
11673/// checkpoint and record why a subsequent explicit resume is needed.
11674pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11675    let mut auth: Vec<_> = losses
11676        .iter()
11677        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11678        .map(|loss| loss.agent.clone())
11679        .collect();
11680    auth.sort();
11681    auth.dedup();
11682    let (kind, detail) = if auth.is_empty() {
11683        (
11684            "agent usage limit",
11685            "All seats reached their agent usage limits; resume after usage is available"
11686                .to_owned(),
11687        )
11688    } else {
11689        (
11690            "agent authentication",
11691            format!(
11692                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11693                auth.join(", ")
11694            ),
11695        )
11696    };
11697    state.resource_block = Some(crate::run::ResourceBlock {
11698        node: node.to_owned(),
11699        kind: kind.to_owned(),
11700        cause: crate::run::ResourceBlockCause::Other,
11701        detail: detail.clone(),
11702        at: Timestamp::now(),
11703    });
11704    state.event(node, detail);
11705}
11706
11707/// Did the wave end without asking any agent (see [`block_wave`])?
11708fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11709    results
11710        .iter()
11711        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11712}
11713
11714/// Settle a wave that never spawned anything: every job answers
11715/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11716/// else about the seat (session, history, quota) is touched.
11717fn block_wave(
11718    jobs: Vec<SeatJob>,
11719    state: &mut RunState,
11720    node: &str,
11721    attempt: usize,
11722    why: &str,
11723) -> Vec<(usize, SeatState, AgentOutcome)> {
11724    state.resource_wait = None;
11725    state
11726        .active
11727        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11728    if let Err(e) = state.save() {
11729        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11730    }
11731    jobs.into_iter()
11732        .enumerate()
11733        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11734        .collect()
11735}
11736
11737/// Acquire the shared build cache's lease, waiting out contention within
11738/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11739/// unbounded wait is never acceptable).
11740///
11741/// A first, non-blocking check happens before ever waiting; if it finds the
11742/// lease busy, that fact is logged as a `verify` event *and* flushed with
11743/// [`RunState::save`] immediately — not only once the wait finally succeeds
11744/// or gives up — so a `magi show` run by a different process while this one
11745/// is still waiting reads a `run.json` that says so, rather than whatever it
11746/// looked like before the wait started. The same applies to the terminal
11747/// failure: logged and saved before this returns `Err`, so a caller that
11748/// could not get the lease at all still leaves a legible record of why.
11749async fn acquire_cache_lease(
11750    state: &mut RunState,
11751    cache_dir: &Path,
11752    owner: &crate::cache::Owner,
11753    budget: Duration,
11754    node: &str,
11755    context: &str,
11756    cancel: &(dyn Fn() -> bool + Sync),
11757) -> Result<Option<crate::cache::Guard>> {
11758    let home = crate::run::home();
11759    let started = Instant::now();
11760    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11761        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11762        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11763        Err(e) => {
11764            state.event(
11765                "verify",
11766                format!("{context}: could not check the shared build cache: {e:#}"),
11767            );
11768            if let Err(e2) = state.save() {
11769                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11770            }
11771            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11772        }
11773    };
11774    state.event(
11775        "verify",
11776        format!(
11777            "{context}: waiting for the shared build cache at {} ({})",
11778            cache_dir.display(),
11779            busy.describe()
11780        ),
11781    );
11782    // Seats of this node are waiting, not running: say so, so the countdown a
11783    // reader sees only ever measures an agent that has been spawned.
11784    state.resource_wait = Some(crate::run::ResourceWait {
11785        node: context.to_owned(),
11786        kind: "shared build cache".to_owned(),
11787        holder: busy.describe(),
11788        since: Timestamp::now(),
11789        limit_secs: budget.as_secs(),
11790    });
11791    if let Err(e) = state.save() {
11792        tracing::warn!("could not persist a cache wait: {e:#}");
11793    }
11794    let remaining = budget.saturating_sub(started.elapsed());
11795    let waited = crate::cache::wait_for_unless(
11796        &home,
11797        cache_dir,
11798        owner,
11799        remaining,
11800        Duration::from_secs(5),
11801        cancel,
11802    )
11803    .await;
11804    state.resource_wait = None;
11805    let cause = match &waited {
11806        Ok(Some(_)) => None,
11807        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11808        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11809            Some(crate::run::ResourceBlockCause::Timeout {
11810                waited_secs: started.elapsed().as_secs(),
11811                limit_secs: budget.as_secs(),
11812            })
11813        }
11814        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11815    };
11816    if let Some(cause) = cause {
11817        let detail = match &waited {
11818            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11819            Err(error) => format!("{error:#}"),
11820        };
11821        state.resource_block = Some(crate::run::ResourceBlock {
11822            cause,
11823            node: node.to_owned(),
11824            kind: "shared build cache".into(),
11825            detail,
11826            at: Timestamp::now(),
11827        });
11828        state.event(
11829            "resource",
11830            state.resource_block.as_ref().unwrap().description(),
11831        );
11832    }
11833    if let Err(error) = state.save() {
11834        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11835    }
11836    waited
11837}
11838
11839/// The lease state could not be read at all (unreadable lease file, a
11840/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11841/// known to be contended, so a wave degrades to running without a shared cache
11842/// instead of stopping on a hold that [`crate::triage`] could never release.
11843#[derive(Debug)]
11844struct CacheCheckError(String);
11845
11846impl std::fmt::Display for CacheCheckError {
11847    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11848        f.write_str(&self.0)
11849    }
11850}
11851
11852impl std::error::Error for CacheCheckError {}
11853
11854/// Keep a specific acquisition outcome when the caller adds command evidence.
11855fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11856    state
11857        .resource_block
11858        .get_or_insert_with(|| crate::run::ResourceBlock {
11859            cause: crate::run::ResourceBlockCause::Other,
11860            node: node.to_owned(),
11861            kind: "shared build cache".into(),
11862            detail: format!("{error:#}"),
11863            at: Timestamp::now(),
11864        });
11865    if let Err(error) = state.save() {
11866        tracing::warn!("could not persist cache resource block: {error:#}");
11867    }
11868}
11869
11870/// Run `body` — a verify command batch — while holding the shared build
11871/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11872/// never interleave with another borrower's build against the same
11873/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11874/// timeout, or a human's own `magi review`. See the `cache` module doc for
11875/// why this matters more than Cargo's own per-target locking covers — two
11876/// *different* worktrees building the same package name/version into one
11877/// cache directory is a staleness bug, not a lock contention one.
11878///
11879/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11880/// agent waves do. Repository scheduling also has its own bounded,
11881/// cancellable resource-wait policy. Contention must not consume the
11882/// verification budget: `body` receives the full `budget` after both leases
11883/// are acquired and the cache is fresh. Without a shared cache, only
11884/// repository coordination is acquired before handing over the full budget.
11885///
11886/// A lease that cannot be acquired within the resource wait is reported as a single
11887/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11888/// verification — the same shape a spawn failure already takes in
11889/// [`run_commands`], so a caller need not special-case it.
11890#[allow(clippy::too_many_arguments)]
11891async fn with_cache_lease<'s, F, Fut>(
11892    state: &'s mut RunState,
11893    cache_dir: Option<&Path>,
11894    node: &str,
11895    seat: &str,
11896    worktree: &Path,
11897    head: &str,
11898    budget: Duration,
11899    context: &str,
11900    wait: &crate::repo_lock::WaitPolicy,
11901    body: F,
11902) -> Result<(Vec<CommandOutcome>, bool)>
11903where
11904    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11905    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11906{
11907    // Match editing waves: cache first, repository second. In particular,
11908    // pre_gate must never hold the repository while waiting for a wave's cache.
11909    let started = Instant::now();
11910    let home = crate::run::home();
11911    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11912    let guard = if let Some(cache_dir) = cache_dir {
11913        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11914        match acquire_cache_lease(
11915            state,
11916            cache_dir,
11917            &owner,
11918            resource_limit,
11919            node,
11920            context,
11921            &|| wait.cancelled(),
11922        )
11923        .await
11924        .and_then(|g| {
11925            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11926        }) {
11927            Ok(g) => Some(g),
11928            Err(e) => {
11929                record_cache_error(state, node, &e);
11930                return Ok((
11931                    vec![CommandOutcome {
11932                        command: "(waiting for the shared build cache)".into(),
11933                        code: None,
11934                        output_tail: e.to_string(),
11935                        duration_ms: started.elapsed().as_millis() as u64,
11936                        resource_blocked: true,
11937                    }],
11938                    false,
11939                ));
11940            }
11941        }
11942    } else {
11943        None
11944    };
11945    let _coordination = coordinate_operation(state, worktree, wait).await?;
11946    if let Some(cache_dir) = cache_dir {
11947        let identity = crate::cache::Identity::new(worktree, head);
11948        if let Err(e) = crate::cache::ensure_fresh(
11949            &home,
11950            cache_dir,
11951            &identity,
11952            &state.config.verify.cargo_manifests(),
11953        ) {
11954            record_cache_error(state, node, &e);
11955            state.event(
11956                "verify",
11957                format!(
11958                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11959                    worktree.display(),
11960                    short(head)
11961                ),
11962            );
11963            if let Some(guard) = guard {
11964                guard.release();
11965            }
11966            return Ok((
11967                vec![CommandOutcome {
11968                    command: "(confirming the shared build cache is fresh)".into(),
11969                    code: None,
11970                    output_tail: e.to_string(),
11971                    duration_ms: started.elapsed().as_millis() as u64,
11972                    resource_blocked: true,
11973                }],
11974                false,
11975            ));
11976        }
11977    }
11978    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11979    if !timed_out_pids.is_empty() {
11980        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11981    }
11982    // Drop repository coordination before releasing the cache, reversing the
11983    // acquisition order. A cancelled/error future also drops both guards.
11984    drop(_coordination);
11985    if let Some(guard) = guard {
11986        guard.release();
11987    }
11988    Ok((outcomes, retried))
11989}
11990
11991/// Poll `pids` — commands [`run_commands`] reports as still running when its
11992/// own timeout elapsed — until every one is confirmed gone, or
11993/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11994///
11995/// Real confirmation where confirmation is possible, not a substitute for
11996/// full process-tree observation: a grandchild the timed-out process spawned
11997/// and that survives independently of it is invisible to a pid check the
11998/// same way it always was, and continuing to observe and collect *that*
11999/// stays a different piece of work with its own owner. This only narrows a
12000/// fixed blind wait into an actual check of the pids this process does know
12001/// about.
12002async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
12003    wait_for_pids_with(
12004        pids,
12005        crate::proc::pid_alive,
12006        LEASE_RELEASE_POLL,
12007        LEASE_RELEASE_MAX_WAIT,
12008    )
12009    .await;
12010}
12011
12012/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
12013/// interval and ceiling supplied by the caller, so the polling *logic* -
12014/// returns as soon as every pid reports dead, gives up at the ceiling
12015/// otherwise - is testable on millisecond durations without asking the real
12016/// OS about a pid at all.
12017async fn wait_for_pids_with<F: Fn(u32) -> bool>(
12018    pids: &[u32],
12019    alive: F,
12020    poll: Duration,
12021    max_wait: Duration,
12022) {
12023    let deadline = Instant::now() + max_wait;
12024    loop {
12025        if pids.iter().all(|&pid| !alive(pid)) {
12026            return;
12027        }
12028        if Instant::now() >= deadline {
12029            return;
12030        }
12031        tokio::time::sleep(poll).await;
12032    }
12033}
12034
12035/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
12036/// admission that it could not even get a verify command to run, as opposed
12037/// to evidence the command actually produced? A caller that would otherwise
12038/// read a resource-blocked outcome as a red command must check this first:
12039/// see [`Runner::gate`], which retries rather than records `Blocked` when
12040/// this is true.
12041fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12042    outcomes.iter().any(|o| o.resource_blocked)
12043}
12044
12045/// What [`Runner::gate_fix_round`] decided.
12046pub(crate) enum GateFix {
12047    /// The tree changed and `verify.e2e` is still green: run the gate again.
12048    Retry,
12049    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12050    /// last failure stands and the run ends blocked.
12051    Stop,
12052    /// `verify.e2e` could not run after the fix (magi's own contention):
12053    /// decide nothing now, a later reentry retries.
12054    Defer,
12055}
12056
12057/// Is every red command in `outcomes` an ordinary failure the code could
12058/// explain: it ran, exited non-zero, and said something?
12059///
12060/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12061/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12062/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12063/// what the command is stays the gate's business.
12064pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12065    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12066    red.peek().is_some()
12067        && red.all(|o| {
12068            !o.resource_blocked
12069                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12070                && !o.output_tail.trim().is_empty()
12071        })
12072}
12073
12074/// Describe one verify command's outcome for the event log, distinguishing a
12075/// build/link failure — the toolchain never produced a binary to run — from
12076/// an actual test failure, since only the latter is a verdict on the patch.
12077fn e2e_outcome_label(o: &CommandOutcome) -> String {
12078    if o.ok() {
12079        return "pass".to_owned();
12080    }
12081    let reason = if o.build_failed() {
12082        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12083    } else {
12084        format!("FAIL ({:?})", o.code)
12085    };
12086    format!(
12087        "{reason}\n{}",
12088        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12089    )
12090}
12091
12092/// Does `outcomes` hold a failure worth one more run on the same commit?
12093///
12094/// Only an ordinary test result qualifies: every red command ran, exited
12095/// non-zero and said which way (not a build/link failure, a timeout or kill,
12096/// a spawn failure, the shell's own 126 / 127, or a resource block). Those are
12097/// evidence about the machine or the toolchain, never about a flaky test.
12098fn rerunnable_failure(outcomes: &[CommandOutcome]) -> bool {
12099    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12100    red.peek().is_some()
12101        && !verify_inconclusive(outcomes)
12102        && red.all(|o| {
12103            !o.build_failed() && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12104        })
12105}
12106
12107/// Said at the top of the output a fixer gets when the same command failed on
12108/// both attempts, so it reads as a reproduced failure and not a one-off.
12109const FAILED_TWICE: &str = "[this command failed twice on the same commit; the output below is \
12110                            the second attempt's]";
12111
12112/// Run `verify.e2e`, retrying once if the first attempt could not build or
12113/// link — a build/link failure is frequently a race against a shared
12114/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12115/// `verify` event per command, tagged with `context` (normally `"round N"`)
12116/// so the call sites that need this — the ordinary per-round leg in
12117/// `review_loop`, the deferred catch-up run `stop_reviewing` makes before it
12118/// will ever call a round green, `verify_current_head` and the gate-repair
12119/// check — read identically in the event log.
12120///
12121/// A failure that is a test result (see [`rerunnable_failure`]) is believed
12122/// only after the failed commands have run once more on the same commit, in
12123/// the same worktree and — because the caller holds it around this whole
12124/// function — under the same cache lease, with a full `timeout` of their own
12125/// (`[graph] rerun_failed_e2e`). A command that passes the second time is a
12126/// flake: the head is verified, the first failure is recorded and filed
12127/// (`crate::flake`, `RunState::verify_flakes`). A command that fails again
12128/// stands, carrying the second attempt's output. One verification is therefore
12129/// run at most three times: the build/link retry, then this one rerun.
12130/// `round` is the review round the verification belongs to, if any.
12131async fn run_e2e_with_retry(
12132    state: &mut RunState,
12133    shell: &[String],
12134    commands: &[String],
12135    worktree: &Path,
12136    timeout: Duration,
12137    context: &str,
12138    round: Option<usize>,
12139) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12140    let (mut e2e, mut timed_out_pids) = run_commands(
12141        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12142    )
12143    .await;
12144    for o in &e2e {
12145        state.event(
12146            "verify",
12147            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12148        );
12149    }
12150    // A build/link failure is not a verdict on the patch — it is frequently a
12151    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12152    // one retry before letting a red like that decide the round.
12153    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12154    if verify_retried {
12155        state.event(
12156            "verify",
12157            format!(
12158                "{context}: verify could not build/link, not a test result — retrying once \
12159                 before concluding"
12160            ),
12161        );
12162        let retried = run_commands(
12163            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12164        )
12165        .await;
12166        e2e = retried.0;
12167        // Both attempts' timeouts matter, not just the last one: the first
12168        // attempt's descendants may still be alive alongside the retry's.
12169        timed_out_pids.extend(retried.1);
12170        for o in &e2e {
12171            state.event(
12172                "verify",
12173                format!(
12174                    "{context}: retry `{}` -> {}",
12175                    o.command,
12176                    e2e_outcome_label(o)
12177                ),
12178            );
12179        }
12180    }
12181    if state.config.graph.rerun_failed_e2e && rerunnable_failure(&e2e) {
12182        let failed: Vec<usize> = (0..e2e.len()).filter(|&i| !e2e[i].ok()).collect();
12183        let rerun_commands: Vec<String> = failed.iter().map(|&i| e2e[i].command.clone()).collect();
12184        let head = git::rev_parse(worktree, "HEAD").await.unwrap_or_default();
12185        state.event(
12186            "verify",
12187            format!(
12188                "{context}: {} failed with a test result — running it once more on the same \
12189                 commit {} before deciding",
12190                rerun_commands
12191                    .iter()
12192                    .map(|c| format!("`{c}`"))
12193                    .collect::<Vec<_>>()
12194                    .join(", "),
12195                short(&head)
12196            ),
12197        );
12198        let (second, pids) = run_commands(
12199            state,
12200            "verify",
12201            "e2e",
12202            1 + usize::from(verify_retried),
12203            shell,
12204            &rerun_commands,
12205            worktree,
12206            timeout,
12207        )
12208        .await;
12209        timed_out_pids.extend(pids);
12210        for o in &second {
12211            state.event(
12212                "verify",
12213                format!(
12214                    "{context}: rerun `{}` -> {}",
12215                    o.command,
12216                    e2e_outcome_label(o)
12217                ),
12218            );
12219        }
12220        let mut reproduced = false;
12221        for (slot, &i) in failed.iter().enumerate() {
12222            let Some(again) = second.get(slot) else {
12223                break;
12224            };
12225            let first = std::mem::replace(&mut e2e[i], again.clone());
12226            if again.ok() {
12227                let signature = crate::flake::signature(&first.output_tail, first.code);
12228                let flake = crate::run::VerifyFlake {
12229                    round,
12230                    context: context.to_owned(),
12231                    head: head.clone(),
12232                    command: first.command.clone(),
12233                    first_failure: summarize_verify(&first.output_tail, EVENT_OUTPUT_TAIL),
12234                    fingerprint: crate::flake::fingerprint(&state.repo, &signature),
12235                    at: Timestamp::now(),
12236                    task: None,
12237                    filed: false,
12238                    filing_error: None,
12239                };
12240                crate::flake::record(state, flake, &signature.label, &crate::queue::Queue::open());
12241            } else if !again.resource_blocked {
12242                reproduced = true;
12243                e2e[i].output_tail = format!("{FAILED_TWICE}\n{}", again.output_tail);
12244            }
12245        }
12246        if reproduced {
12247            state.event(
12248                "verify",
12249                format!("{context}: the failure reproduced on the second run — it stands"),
12250            );
12251        }
12252    }
12253    (e2e, verify_retried, timed_out_pids)
12254}
12255
12256/// Run configured shell commands in `cwd`, in order. The second element is
12257/// the pid of every command that hit `timeout` and was still running when
12258/// this stopped waiting on it (best-effort: `None` when the platform did not
12259/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12260/// that releases a shared resource afterward needs to know.
12261///
12262/// Records `task` into [`RunState::active`] at every command boundary
12263/// (`RunState::task_command`) and clears it once the whole list has run
12264/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12265/// for minutes with no seat and no output of its own to show for it (see
12266/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12267/// apart from "not yet run" without this), and this is the only place that
12268/// knows which command is running right now and how many are left. Three
12269/// saves per command — start, not per second — matching the same "only at a
12270/// boundary" rule [`wave`] already follows for seats.
12271#[allow(clippy::too_many_arguments)]
12272async fn run_commands(
12273    state: &mut RunState,
12274    node: &str,
12275    task: &str,
12276    attempt: usize,
12277    shell: &[String],
12278    commands: &[String],
12279    cwd: &Path,
12280    timeout: Duration,
12281) -> (Vec<CommandOutcome>, Vec<u32>) {
12282    if commands.is_empty() {
12283        // Nothing to mark as running and nothing to clear — an empty list
12284        // means "not configured", and touching `active` (or the disk) over
12285        // that would be a write for every round of a repo with no
12286        // `verify.e2e` / `verify.gate` commands at all.
12287        return (Vec::new(), Vec::new());
12288    }
12289    let mut out = Vec::new();
12290    let mut timed_out_pids = Vec::new();
12291    let total = commands.len();
12292    for (idx, command) in commands.iter().enumerate() {
12293        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12294        if let Err(e) = state.save() {
12295            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12296        }
12297        let started = Instant::now();
12298        let mut cmd = tokio::process::Command::new(&shell[0]);
12299        cmd.quiet();
12300        // Verify the checkout as a standalone caller. Worker ownership and
12301        // pinned executable selection belong to this driver, not to tests or
12302        // tools it runs. Preserve home/seat context and Running's containment.
12303        for key in crate::worker::INVOCATION_ENV {
12304            cmd.env_remove(key);
12305        }
12306        cmd.args(&shell[1..])
12307            .arg(command)
12308            .current_dir(cwd)
12309            .stdin(std::process::Stdio::null())
12310            .stdout(std::process::Stdio::piped())
12311            .stderr(std::process::Stdio::piped())
12312            .kill_on_drop(true);
12313        let mut outputs = Vec::new();
12314        let protected = (|| -> Result<()> {
12315            crate::incremental::configure(&mut cmd, cwd)?;
12316            let mut paths = vec![cwd.to_owned()];
12317            if let Some(cache) = state.config.cache_dir() {
12318                paths.push(cache);
12319            }
12320            outputs.extend(crate::retention::OutputLease::admit(
12321                &crate::run::home(),
12322                &paths,
12323                state.config.disk.min_free_bytes,
12324            )?);
12325            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12326                outputs.push(guard);
12327            }
12328            if let Some(cache) = state.config.cache_dir() {
12329                outputs.push(crate::retention::OutputLease::cache(
12330                    &crate::run::home(),
12331                    &cache,
12332                    false,
12333                )?);
12334            }
12335            Ok(())
12336        })();
12337        if let Err(e) = protected {
12338            let detail = format!("execution admission unavailable: {e:#}");
12339            state.resource_block = Some(crate::run::ResourceBlock {
12340                cause: crate::run::ResourceBlockCause::Other,
12341                node: node.to_owned(),
12342                kind: if detail.contains("disk admission") {
12343                    "disk admission"
12344                } else {
12345                    "output ownership"
12346                }
12347                .to_owned(),
12348                detail: detail.clone(),
12349                at: Timestamp::now(),
12350            });
12351            out.push(CommandOutcome {
12352                command: command.clone(),
12353                code: None,
12354                output_tail: detail,
12355                duration_ms: 0,
12356                resource_blocked: true,
12357            });
12358            break;
12359        }
12360        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12361        let (code, body) = match spawned {
12362            Ok(mut running) => {
12363                let pid = running.child().id();
12364                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12365                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12366                let mut paths = vec![cwd.to_owned()];
12367                if let Some(cache) = state.config.cache_dir() {
12368                    paths.push(cache);
12369                }
12370                let mut pressure = None;
12371                let status = tokio::select! {
12372                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12373                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12374                        pressure = Some(reason); None
12375                    }
12376                };
12377                let (code, failure) = match status {
12378                    Some(Ok(Ok(status))) => (
12379                        status.code(),
12380                        status
12381                            .code()
12382                            .is_none()
12383                            .then(|| format!("command terminated: {status}")),
12384                    ),
12385                    Some(Ok(Err(error))) => {
12386                        (None, Some(format!("failed waiting for command: {error}")))
12387                    }
12388                    Some(Err(_)) => {
12389                        // Leased cleanup confirms process-tree death itself.
12390                        // Unleased commands still need the caller's PID check
12391                        // after kill-on-drop, before releasing cache ownership.
12392                        if !running.coordinated()
12393                            && let Some(pid) = pid
12394                        {
12395                            timed_out_pids.push(pid);
12396                        }
12397                        (
12398                            None,
12399                            Some(format!("timed out after {}s", timeout.as_secs())),
12400                        )
12401                    }
12402                    None => (None, None),
12403                };
12404                running.finish().await;
12405                if let Some(reason) = pressure {
12406                    state.resource_block = Some(crate::run::ResourceBlock {
12407                        cause: crate::run::ResourceBlockCause::Other,
12408                        node: node.to_owned(),
12409                        kind: "disk pressure".to_owned(),
12410                        detail: reason.clone(),
12411                        at: Timestamp::now(),
12412                    });
12413                    out.push(CommandOutcome {
12414                        command: command.clone(),
12415                        code: None,
12416                        output_tail: reason,
12417                        duration_ms: started.elapsed().as_millis() as u64,
12418                        resource_blocked: true,
12419                    });
12420                    break;
12421                }
12422                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12423                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12424                if let Some(failure) = failure {
12425                    body.push('\n');
12426                    body.push_str(&failure);
12427                }
12428                (code, body)
12429            }
12430            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12431        };
12432        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12433        if !body.is_empty() {
12434            // The full text is kept whatever the summary dropped. A fresh
12435            // name per command so a resume or retry never overwrites an
12436            // earlier run's output. A failed write changes no verdict.
12437            match crate::run::write_verification_artifact(state, task, &body) {
12438                Ok(name) if summary != body => {
12439                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12440                }
12441                Ok(_) => {}
12442                Err(e) => {
12443                    let warning = format!("could not save the full {task} output: {e:#}");
12444                    tracing::warn!("{warning}");
12445                    state.event(node, warning);
12446                    if summary != body {
12447                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12448                    }
12449                }
12450            }
12451        }
12452        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12453        out.push(CommandOutcome {
12454            command: command.clone(),
12455            code,
12456            output_tail: summary,
12457            duration_ms: started.elapsed().as_millis() as u64,
12458            resource_blocked,
12459        });
12460        if resource_blocked {
12461            state.resource_block = Some(crate::run::ResourceBlock {
12462                cause: crate::run::ResourceBlockCause::Other,
12463                node: node.to_owned(),
12464                kind: "incremental cache admission".into(),
12465                detail: body,
12466                at: Timestamp::now(),
12467            });
12468            break;
12469        }
12470    }
12471    state.task_finished(task);
12472    if let Err(e) = state.save() {
12473        tracing::warn!("could not persist the end of {task}: {e:#}");
12474    }
12475    (out, timed_out_pids)
12476}
12477
12478/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12479/// section (`report::run`) and in the `merge` event this node records — for
12480/// the operator to run by hand.
12481///
12482/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12483/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12484/// merge commits", or "require linear history") rejects the push a `--no-ff`
12485/// merge would produce, which is exactly the guidance this function replaces.
12486/// `message`'s first line becomes the squash commit's subject, matching the
12487/// note `report::run` prints alongside this command — see that function for
12488/// why an explicit subject is not optional there.
12489fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12490    let repo = repo.display();
12491    match style {
12492        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12493        MergeStyle::Squash => {
12494            // The subject sits inside double quotes, and a title an agent
12495            // wrote may carry the characters that break out of them.
12496            let subject = message
12497                .lines()
12498                .next()
12499                .unwrap_or(branch)
12500                .replace(['\\', '"', '$', '`'], "");
12501            format!(
12502                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12503            )
12504        }
12505        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12506    }
12507}
12508
12509/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12510/// under the hood, rejects a `title` over 256 characters and the whole
12511/// command fails — no PR at all, for a run whose body was otherwise fine
12512/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12513/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12514/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12515/// this bound. It is a margin, not a guarantee — a title packed
12516/// with multi-unit characters could still in principle land close to the
12517/// edge, but a real task title's occasional emoji or accented letter fits
12518/// comfortably inside it.
12519const PR_TITLE_MAX: usize = 240;
12520
12521/// A pull request title from the opening line of `text`, or `None` when that
12522/// line is not English (GitHub text is) or has no letters.
12523///
12524/// A line within `max` is kept as is. A longer one is cut at the end of its
12525/// first sentence when that falls inside `max`, else at a word boundary with a
12526/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12527/// make every merely-truncated title look non-English). The language check
12528/// runs on the kept text before any mark is added, so only what GitHub will
12529/// show is judged: an English opening followed by non-ASCII far past the cut
12530/// still passes.
12531fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12532    let line = queue::first_line(text)?;
12533    let chars: Vec<char> = line.chars().collect();
12534    let (kept, mark) = if chars.len() <= max {
12535        (line.to_owned(), "")
12536    } else if let Some(end) = sentence_end(&chars, max) {
12537        (chars[..end].iter().collect::<String>(), "")
12538    } else {
12539        let room = max.saturating_sub(3);
12540        // Cut at the last space inside the room; when the char just past the
12541        // room is a space the room already ends on a word.
12542        let cut = if chars[room].is_whitespace() {
12543            room
12544        } else {
12545            chars[..room]
12546                .iter()
12547                .rposition(|c| c.is_whitespace())
12548                .unwrap_or(room)
12549        };
12550        let head: String = chars[..cut].iter().collect();
12551        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12552        (head.to_owned(), "...")
12553    };
12554    if kept.is_empty() {
12555        return None;
12556    }
12557    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12558        return None;
12559    }
12560    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12561        return None;
12562    }
12563    Some(format!("{kept}{mark}"))
12564}
12565
12566/// The char length of the first sentence of `chars` when it ends within `max`
12567/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12568/// abbreviations so `e.g. foo` does not end a title early.
12569fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12570    const MIN: usize = 20;
12571    for i in MIN..max.min(chars.len()) {
12572        if !matches!(chars[i], '.' | '!' | '?') {
12573            continue;
12574        }
12575        let Some(&next) = chars.get(i + 1) else {
12576            continue;
12577        };
12578        if !next.is_whitespace() {
12579            continue;
12580        }
12581        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12582        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12583            continue;
12584        }
12585        let word: String = chars[..i]
12586            .iter()
12587            .rev()
12588            .take_while(|c| !c.is_whitespace())
12589            .collect::<Vec<_>>()
12590            .into_iter()
12591            .rev()
12592            .collect();
12593        let word = word.to_ascii_lowercase();
12594        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12595            continue;
12596        }
12597        let end = chars[..i]
12598            .iter()
12599            .rposition(|c| !c.is_whitespace())
12600            .map_or(i, |p| p + 1);
12601        return Some(end);
12602    }
12603    None
12604}
12605
12606/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12607/// change: a title and a body describing what was *implemented*, not the task
12608/// that asked for it. A task reads as a request; a reader of the merged
12609/// history wants the change.
12610struct PrMessage {
12611    title: String,
12612    body: String,
12613}
12614
12615impl PrMessage {
12616    /// Title, blank line, body. The first line is the squash/merge commit
12617    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12618    /// has to stay one sensible line.
12619    fn commit_message(&self) -> String {
12620        format!("{}\n\n{}", self.title, self.body)
12621    }
12622}
12623
12624/// The text after a leading `TITLE:` (any case) on `line`.
12625fn title_marker(line: &str) -> Option<&str> {
12626    let line = line.trim();
12627    let head = line.get(..6)?;
12628    head.eq_ignore_ascii_case("title:")
12629        .then(|| line[6..].trim())
12630}
12631
12632/// The implementer's own one-line title: the `TITLE:` line the implement
12633/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12634/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12635/// source, and a title that says as much is refused here too.
12636fn summary_title(summary: &str) -> Option<String> {
12637    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12638    let raw = title_marker(first)?;
12639    if raw.is_empty() {
12640        return None;
12641    }
12642    let title = queue::title_from(raw, PR_TITLE_MAX);
12643    let lower = title.to_ascii_lowercase();
12644    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12645        return None;
12646    }
12647    Some(title)
12648}
12649
12650/// How `open_review`'s instruction begins; see [`landing_title`].
12651const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12652
12653/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12654pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12655
12656/// Does this run review work that already existed, rather than implement a
12657/// task? Runs recorded before `reviewed_commits` existed carry only the
12658/// candidate marker.
12659pub(crate) fn is_review_run(state: &RunState) -> bool {
12660    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12661}
12662
12663/// The title of a review-only run: the subject of the oldest commit under
12664/// review. Later commits are usually fixups, and `instruction` is the review
12665/// prompt, which says nothing about the change. GitHub text is English, so a
12666/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12667fn review_title(state: &RunState) -> Option<String> {
12668    english_subject(
12669        state.reviewed_commits.as_ref()?.first()?,
12670        state.config.graph.github_english(),
12671    )
12672}
12673
12674/// `raw` as a pull request title, or `None` when it is blank, not English
12675/// (GitHub text is), or one of magi's own candidate commit subjects.
12676fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12677    let raw = raw.trim();
12678    if raw.is_empty() {
12679        return None;
12680    }
12681    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12682        return None;
12683    }
12684    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12685        return None;
12686    }
12687    let title = queue::title_from(raw, PR_TITLE_MAX);
12688    let lower = title.to_ascii_lowercase();
12689    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12690        return None;
12691    }
12692    Some(title)
12693}
12694
12695/// What a review-only run's branch says about itself, read at the moment the
12696/// pull request is opened.
12697#[derive(Debug, Clone, PartialEq, Eq)]
12698struct BranchFacts {
12699    /// `(subject, body)` of each commit, oldest first.
12700    commits: Vec<(String, String)>,
12701    /// Trimmed `git diff --stat`.
12702    stat: String,
12703}
12704
12705/// Longest diff stat shown: this many file lines plus the summary line.
12706const STAT_FILE_LINES: usize = 25;
12707/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12708const COMMITS_MAX_CHARS: usize = 20_000;
12709
12710/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12711/// say or finds nothing, so the caller falls back to what the run recorded.
12712async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12713    let commits = git::commit_log(repo, base, branch).await.ok()?;
12714    if commits.is_empty() {
12715        return None;
12716    }
12717    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12718    let lines: Vec<&str> = stat.lines().collect();
12719    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12720        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12721        let more = format!(" ... {omitted} more file(s)");
12722        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12723        kept.push(&more);
12724        kept.push(lines[lines.len() - 1]);
12725        kept.join("\n")
12726    } else {
12727        lines.join("\n")
12728    };
12729    Some(BranchFacts { commits, stat })
12730}
12731
12732/// Defang what would break the surrounding markdown: a closing `</details>`
12733/// and a code fence.
12734fn markdown_safe(text: &str) -> String {
12735    text.replace("</details>", "&lt;/details&gt;")
12736        .replace("\x60\x60\x60", "~~~")
12737}
12738
12739fn neutral_title(state: &RunState, winner: char) -> String {
12740    format!(
12741        "chore: land candidate {} of run {}",
12742        winner.to_ascii_uppercase(),
12743        state.id
12744    )
12745}
12746
12747/// Reuse the graph's bounded fixer without starting another graph or driver.
12748/// Copy the working state back even on errors so spent rounds remain visible.
12749pub(crate) async fn repair_landing_gate(
12750    state: &mut RunState,
12751    winner: &Candidate,
12752    outcomes: &[CommandOutcome],
12753) -> Result<GateFix> {
12754    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12755        return Ok(GateFix::Stop);
12756    }
12757    let mut runner = Runner {
12758        roles: state.config.resolve_roles()?,
12759        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12760        state: state.clone(),
12761        pause: Pause::new(),
12762        interrupt: Pause::new(),
12763    };
12764    let result = runner.gate_fix_round(winner, outcomes).await;
12765    *state = runner.state;
12766    result
12767}
12768
12769/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12770/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12771pub(crate) async fn verify_current_head(
12772    state: &mut RunState,
12773    winner: &Candidate,
12774    wait: &crate::repo_lock::WaitPolicy,
12775) -> Result<bool> {
12776    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12777    // Preserve the previous record before replacing it, including on crash recovery.
12778    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12779        state
12780            .verification_history
12781            .push(crate::run::VerificationEvidence {
12782                superseded_by: Some(head.clone()),
12783                archived_at: Some(Timestamp::now()),
12784                fixes_spent: Some(state.gate_fixes.len()),
12785                gate_head: None,
12786                gate_ran: false,
12787                gate: Vec::new(),
12788                e2e: state.head_e2e.take(),
12789            });
12790    }
12791    if let Some(e) = &state.head_e2e
12792        && e.head == head
12793    {
12794        if e.passed {
12795            return Ok(true);
12796        }
12797        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12798            state.status = RunStatus::Blocked;
12799            state.save()?;
12800            return Ok(false);
12801        }
12802    } else if state.reviews.last().is_some_and(|r| {
12803        r.verified_head.as_deref() == Some(head.as_str())
12804            && (r.e2e_status() == E2eStatus::Passed
12805                || (r.e2e_status() == E2eStatus::NotConfigured
12806                    && state.config.verify.e2e.is_empty()))
12807    }) || state
12808        .gate_repair_e2e
12809        .as_ref()
12810        .is_some_and(|e| e.head == head && e.passed)
12811    {
12812        return Ok(true);
12813    }
12814    if let Some(e) = &state.gate_repair_e2e
12815        && e.head == head
12816        && !e.passed
12817        && !e.outcomes.is_empty()
12818        && !verify_inconclusive(&e.outcomes)
12819    {
12820        state.status = RunStatus::Blocked;
12821        state.save()?;
12822        return Ok(false);
12823    }
12824    state.head_e2e = Some(crate::run::GateRepairE2e {
12825        head: head.clone(),
12826        passed: false,
12827        outcomes: Vec::new(),
12828    });
12829    state.save()?;
12830    let commands = state.config.verify.e2e.clone();
12831    let shell = state.config.shell();
12832    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12833    let cache_dir = state.config.cache_dir();
12834    let outcomes = if commands.is_empty() {
12835        Vec::new()
12836    } else {
12837        let (outcomes, _) = with_cache_lease(
12838            state,
12839            cache_dir.as_deref(),
12840            "e2e",
12841            "e2e",
12842            &winner.worktree,
12843            &head,
12844            timeout,
12845            "current head e2e",
12846            wait,
12847            |state, budget| async move {
12848                run_e2e_with_retry(
12849                    state,
12850                    &shell,
12851                    &commands,
12852                    &winner.worktree,
12853                    budget,
12854                    "current head e2e",
12855                    None,
12856                )
12857                .await
12858            },
12859        )
12860        .await?;
12861        outcomes
12862    };
12863    let unresolved = verify_inconclusive(&outcomes);
12864    let passed = !unresolved
12865        && outcomes.iter().all(CommandOutcome::ok)
12866        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12867    if !outcomes.is_empty() && !unresolved {
12868        state.record_base_progress(&head);
12869    }
12870    state.head_e2e = Some(crate::run::GateRepairE2e {
12871        head: head.clone(),
12872        passed,
12873        outcomes,
12874    });
12875    if !passed && !unresolved {
12876        state.status = RunStatus::Blocked;
12877        state.event(
12878            "e2e",
12879            format!("verify.e2e failed on head {head}; nothing published"),
12880        );
12881    }
12882    state.save()?;
12883    Ok(passed)
12884}
12885
12886/// Recheck a repair's e2e requirement across failures, resource deferrals and
12887/// process interruptions. A successful result only applies to its exact SHA.
12888pub(crate) async fn check_gate_repair_e2e(
12889    state: &mut RunState,
12890    winner: &Candidate,
12891    wait: &crate::repo_lock::WaitPolicy,
12892) -> Result<GateFix> {
12893    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12894        return Ok(GateFix::Retry);
12895    }
12896    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12897    if state
12898        .gate_repair_e2e
12899        .as_ref()
12900        .is_some_and(|r| r.head == head && r.passed)
12901    {
12902        return Ok(GateFix::Retry);
12903    }
12904    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12905        head: head.clone(),
12906        passed: false,
12907        outcomes: Vec::new(),
12908    });
12909    state.save()?;
12910    let commands = state.config.verify.e2e.clone();
12911    let outcomes = if commands.is_empty() {
12912        Vec::new()
12913    } else {
12914        let shell = state.config.shell();
12915        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12916        let cache_dir = state.config.cache_dir();
12917        let (outcomes, _) = with_cache_lease(
12918            state,
12919            cache_dir.as_deref(),
12920            "e2e",
12921            "e2e",
12922            &winner.worktree,
12923            &head,
12924            timeout,
12925            "gate repair e2e",
12926            wait,
12927            |state, budget| async move {
12928                run_e2e_with_retry(
12929                    state,
12930                    &shell,
12931                    &commands,
12932                    &winner.worktree,
12933                    budget,
12934                    "gate repair e2e",
12935                    None,
12936                )
12937                .await
12938            },
12939        )
12940        .await?;
12941        outcomes
12942    };
12943    let deferred = verify_inconclusive(&outcomes);
12944    let passed = outcomes.iter().all(CommandOutcome::ok)
12945        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12946    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12947        head,
12948        passed,
12949        outcomes,
12950    });
12951    state.save()?;
12952    if deferred {
12953        Ok(GateFix::Defer)
12954    } else if !passed {
12955        state.event(
12956            "gate",
12957            "verify.e2e failed after gate repair; nothing published",
12958        );
12959        Ok(GateFix::Stop)
12960    } else {
12961        Ok(GateFix::Retry)
12962    }
12963}
12964
12965/// Run the same configured gate for a landing head and a fresh graph.
12966pub(crate) async fn run_landing_gate(
12967    state: &mut RunState,
12968    winner: &Candidate,
12969    wait: &crate::repo_lock::WaitPolicy,
12970) -> Result<Vec<CommandOutcome>> {
12971    let shell = state.config.shell();
12972    let gate_commands = state.config.verify.gate.clone();
12973    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12974    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12975        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12976        Vec::new()
12977    } else {
12978        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12979        let cache_dir = state.config.cache_dir();
12980        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12981        let (outcomes, _) = with_cache_lease(
12982            state,
12983            cache_dir.as_deref(),
12984            "gate",
12985            "gate",
12986            &winner.worktree,
12987            &head,
12988            timeout,
12989            "final gate",
12990            wait,
12991            |state, budget| {
12992                let shell = shell.clone();
12993                let gate_commands = gate_commands.clone();
12994                let winner = winner.clone();
12995                async move {
12996                    Runner::run_pre_gate(state, &winner).await;
12997                    // Bind command evidence after mechanical fixes, before commands start.
12998                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12999                    let (outcomes, timed_out_pids) = run_commands(
13000                        state,
13001                        "gate",
13002                        "gate",
13003                        0,
13004                        &shell,
13005                        &gate_commands,
13006                        &winner.worktree,
13007                        budget,
13008                    )
13009                    .await;
13010                    (outcomes, false, timed_out_pids)
13011                }
13012            },
13013        )
13014        .await?;
13015        outcomes
13016    };
13017    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
13018    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
13019        let mut outcomes = outcomes;
13020        outcomes.push(CommandOutcome {
13021            command: "gate head stability".into(), code: None,
13022            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
13023            duration_ms: 0, resource_blocked: false,
13024        });
13025        return Ok(outcomes);
13026    }
13027    if outcomes.is_empty() {
13028        // Nothing configured to check — distinct from every other
13029        // silence in this run's event log, since an empty `gate` alone
13030        // no longer says whether the gate ran at all (see
13031        // `RunState::gate_ran`'s own doc).
13032        state.event("gate", "no gate commands configured; not tested");
13033    }
13034    for o in &outcomes {
13035        state.event(
13036            "gate",
13037            format!(
13038                "`{}` -> {}",
13039                o.command,
13040                if o.ok() {
13041                    "pass".to_owned()
13042                } else {
13043                    format!(
13044                        "FAIL ({:?})\n{}",
13045                        o.code,
13046                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
13047                    )
13048                }
13049            ),
13050        );
13051    }
13052    Ok(outcomes)
13053}
13054
13055/// The pull request title to hand to `land::merge_subject`. A review-only run
13056/// opened by an earlier build titled its pull request with the review prompt;
13057/// that title is dropped (empty, so the fallback applies) rather than landed.
13058/// Any other title, including an operator's rename, passes through untouched,
13059/// and so does every title of a run that implements a task.
13060pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
13061    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
13062        ""
13063    } else {
13064        pr_title
13065    }
13066}
13067
13068/// What the squash subject falls back to when the pull request title is empty
13069/// or candidate-shaped: for a review-only run the derived title, never the
13070/// review prompt held in `instruction`.
13071pub fn landing_subject_source(state: &RunState) -> String {
13072    if is_review_run(state) {
13073        let winner = state.candidates.first().map_or('A', |c| c.label);
13074        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
13075    }
13076    state.instruction.clone()
13077}
13078
13079/// `summary` without its `TITLE:` line, which the pull request title already
13080/// carries.
13081fn summary_without_title(summary: &str) -> String {
13082    let mut lines = summary.trim().lines().peekable();
13083    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
13084        lines.next();
13085    }
13086    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
13087}
13088
13089/// The pull request title and body for the winning candidate.
13090///
13091/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
13092/// the task's own opening line via [`queue::title_from`] when there is none.
13093/// `state.instruction` can open with blank lines (`task_text` only rejects a
13094/// body that is blank *entirely*), which `title_from` skips.
13095///
13096/// Publishing uses an independent description generated from the final diff
13097/// and structured evidence. Internal summaries and reports remain local.
13098/// Non-publishing previews may display the original task and candidate summary.
13099/// The footer retains the run and candidate provenance.
13100#[cfg(test)]
13101fn pr_message(state: &RunState, winner: char) -> PrMessage {
13102    pr_message_with(state, winner, None)
13103}
13104
13105/// [`pr_message`] with what the branch of a review-only run says about itself.
13106/// `facts` is ignored for a run that implements a task.
13107#[cfg(test)]
13108fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
13109    let raw = pr_message_raw(
13110        state,
13111        winner,
13112        facts,
13113        crate::pr_outcome::FinalHead::Unreported,
13114        ChangeNote::NotAsked,
13115    );
13116    let id = crate::scrub::Identity::current();
13117    PrMessage {
13118        title: crate::scrub::scrub(&raw.title, &id),
13119        body: crate::scrub::scrub(&raw.body, &id),
13120    }
13121}
13122
13123/// What the public description may say about the change itself.
13124#[derive(Debug, Clone, Copy)]
13125enum ChangeNote<'a> {
13126    /// A local preview or non-publishing merge mode: no public generation needed.
13127    NotAsked,
13128    /// A summary written from the final diff.
13129    Summary(&'a crate::run::ChangeSummary),
13130    /// A summary was asked for and could not be produced; the text says why.
13131    Unavailable(&'a str),
13132}
13133
13134fn pr_message_raw(
13135    state: &RunState,
13136    winner: char,
13137    facts: Option<&BranchFacts>,
13138    head: crate::pr_outcome::FinalHead<'_>,
13139    change: ChangeNote<'_>,
13140) -> PrMessage {
13141    if !matches!(change, ChangeNote::NotAsked) {
13142        let (title, mut body) = match change {
13143            ChangeNote::Summary(c) => (
13144                c.title.clone(),
13145                format!("## Summary\n\n{}\n\n", c.body.trim()),
13146            ),
13147            ChangeNote::Unavailable(why) => (
13148                neutral_title(state, winner),
13149                format!(
13150                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13151                ),
13152            ),
13153            ChangeNote::NotAsked => unreachable!(),
13154        };
13155        body.push_str(&public_pr_evidence(state, head));
13156        body.push_str(&format!(
13157            "\n---\nmagi:run/{} magi:candidate-{}\n",
13158            state.id,
13159            winner.to_ascii_lowercase()
13160        ));
13161        return PrMessage { title, body };
13162    }
13163    pr_preview(state, winner, facts, head)
13164}
13165
13166/// Append public facts from the run records after every prose generation,
13167/// including posting-gate rewrites. The agent cannot drop this provenance.
13168fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13169    let mut body = crate::pr_outcome::review_rounds(state);
13170    body.push_str(&crate::pr_outcome::final_verification(state, head));
13171    let open = state.last_round_findings();
13172    if !open.is_empty() {
13173        body.push_str("## Open review findings\n\n");
13174        for f in open {
13175            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13176        }
13177        body.push('\n');
13178    }
13179    if let Some(c) = state.change_summary.as_ref()
13180        && head == crate::pr_outcome::FinalHead::At(&c.head)
13181    {
13182        body.push_str(&format!(
13183            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13184            c.base, c.head
13185        ));
13186    }
13187    body
13188}
13189
13190/// Local-only preview of the operator's task and candidate summary.
13191fn pr_preview(
13192    state: &RunState,
13193    winner: char,
13194    facts: Option<&BranchFacts>,
13195    head: crate::pr_outcome::FinalHead<'_>,
13196) -> PrMessage {
13197    let summary = state
13198        .candidates
13199        .iter()
13200        .find(|c| c.label == winner)
13201        .map(|c| c.summary.as_str())
13202        .unwrap_or_default();
13203    // The fallback is the operator's own words and may not be English; GitHub
13204    // text always is, so a non-English task gets a neutral title instead.
13205    let review = is_review_run(state);
13206    let title = if review {
13207        facts
13208            .and_then(|f| {
13209                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13210            })
13211            .or_else(|| review_title(state))
13212            .or_else(|| {
13213                state
13214                    .candidates
13215                    .iter()
13216                    .find(|c| c.label == winner)
13217                    .filter(|c| !c.branch.starts_with("magi/"))
13218                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13219            })
13220            .unwrap_or_else(|| neutral_title(state, winner))
13221    } else {
13222        summary_title(summary).unwrap_or_else(|| {
13223            english_title(
13224                &state.instruction,
13225                PR_TITLE_MAX,
13226                state.config.graph.github_english(),
13227            )
13228            .unwrap_or_else(|| neutral_title(state, winner))
13229        })
13230    };
13231
13232    let mut body = String::new();
13233    let what = summary_without_title(summary);
13234    if !what.is_empty() {
13235        body.push_str("## Summary\n\n");
13236        body.push_str(&what);
13237        body.push_str("\n\n");
13238    }
13239
13240    // Review history uses structured facts across all rounds, never reports.
13241    body.push_str(&crate::pr_outcome::review_rounds(state));
13242    body.push_str(&crate::pr_outcome::final_verification(state, head));
13243    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13244
13245    let open = state.open_findings();
13246    if !open.is_empty() {
13247        body.push_str("## Open review findings\n\n");
13248        for f in &open {
13249            body.push_str(&format!(
13250                "- `{}` [{:?}] {}\n",
13251                f.id, f.severity, "Context retained in local review records."
13252            ));
13253        }
13254        body.push('\n');
13255    }
13256
13257    if let Some(fix) = fix
13258        && !fix.rejected.is_empty()
13259    {
13260        body.push_str("## Declined by the fixer\n\n");
13261        for r in &fix.rejected {
13262            body.push_str(&format!(
13263                "- `{}`: {}\n",
13264                r.id, "Reason retained in local review records."
13265            ));
13266        }
13267        body.push('\n');
13268    }
13269
13270    if review {
13271        // The review prompt is not the task; list what the branch carries.
13272        body.push_str("## Commits under review\n\n");
13273        if let Some(facts) = facts {
13274            let mut left = COMMITS_MAX_CHARS;
13275            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13276                let mut entry = format!("- {}\n", markdown_safe(subject));
13277                for l in markdown_safe(text).lines() {
13278                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13279                }
13280                if left == 0 {
13281                    body.push_str(&format!(
13282                        "- ... {} more commit(s)\n",
13283                        facts.commits.len() - i
13284                    ));
13285                    break;
13286                }
13287                if entry.len() > left {
13288                    // Even the first commit is cut: one huge body must not
13289                    // push the whole description past GitHub's limit.
13290                    let mut end = left;
13291                    while !entry.is_char_boundary(end) {
13292                        end -= 1;
13293                    }
13294                    entry.truncate(end);
13295                    entry.push_str("\n  ... (truncated)\n");
13296                    left = 0;
13297                } else {
13298                    left -= entry.len();
13299                }
13300                body.push_str(&entry);
13301            }
13302            if !facts.stat.trim().is_empty() {
13303                body.push_str(&format!(
13304                    "\n## Diff stat\n\n```\n{}\n```\n",
13305                    markdown_safe(facts.stat.trim())
13306                ));
13307            }
13308        } else {
13309            match &state.reviewed_commits {
13310                Some(subjects) => {
13311                    for s in subjects {
13312                        body.push_str(&format!("- {}\n", s.trim()));
13313                    }
13314                }
13315                None => {
13316                    // An older run kept only the prompt, with the commit list
13317                    // after its first paragraph.
13318                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13319                    body.push_str(rest.trim());
13320                    body.push('\n');
13321                }
13322            }
13323        }
13324    } else {
13325        let task = state.instruction.trim();
13326        let task = if task.is_empty() {
13327            "(empty task)"
13328        } else {
13329            task
13330        };
13331        body.push_str(&format!(
13332            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13333            task.replace("</details>", "&lt;/details&gt;")
13334        ));
13335    }
13336
13337    body.push_str(&format!(
13338        "\n---\nmagi:run/{} magi:candidate-{}\n",
13339        state.id,
13340        winner.to_ascii_lowercase()
13341    ));
13342
13343    PrMessage { title, body }
13344}
13345
13346/// Source imports have already been prepared from structured inputs. Legacy
13347/// inferred references must never instruct a seat to reconstruct those imports.
13348fn seeded_instruction(state: &RunState) -> String {
13349    let mut inputs = Vec::new();
13350    if let Some(start) = &state.sources.start_from {
13351        inputs.push(format!(
13352            "Start from `{}` at `{}` (including ancestry).",
13353            start.token, start.sha
13354        ));
13355    }
13356    for pick in &state.sources.pick_commits {
13357        inputs.push(format!(
13358            "Pick `{}` at `{}` (single commit).",
13359            pick.token, pick.sha
13360        ));
13361    }
13362    if inputs.is_empty() {
13363        state.instruction.clone()
13364    } else {
13365        format!(
13366            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13367            state.instruction,
13368            inputs.join("\n")
13369        )
13370    }
13371}
13372
13373/// Does the winner have no commits ahead of the base it would land on?
13374/// Any failure to find out reads as "not empty": the merge then behaves as it
13375/// always did rather than refusing on a guess.
13376async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13377    let base = &state.base_branch;
13378    let mut against = base.clone();
13379    if mode == MergeMode::Pr {
13380        // A pull request lands on the remote's base, never the local branch
13381        // of the same name: if that cannot be read, "not empty" is the safe
13382        // answer.
13383        let remote = &state.config.merge.remote;
13384        let tracking = format!("{remote}/{base}");
13385        let fetched = git::fetch(repo, remote, base).await;
13386        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13387            against = tracking;
13388        } else {
13389            return false;
13390        }
13391    }
13392    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13393}
13394
13395/// Why nothing was opened for an empty winner, with what the task's own
13396/// references resolved to.
13397fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13398    let mut detail = format!(
13399        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13400         nothing to open a pull request for"
13401    );
13402    detail.push_str(
13403        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13404    );
13405    detail
13406}
13407
13408/// What the `Pr` merge does once it knows whether the branch already has an
13409/// open pull request.
13410#[derive(Debug, PartialEq, Eq)]
13411enum PrPlan {
13412    Create,
13413    Adopt { url: String, title: String },
13414    Stop(String),
13415}
13416
13417/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13418/// a failed lookup -> stop with the real reason. Never guesses.
13419fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13420    match found {
13421        Ok(land::OpenPr::None) => PrPlan::Create,
13422        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13423        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13424            "several open pull requests exist for this branch, not picking one: {}",
13425            urls.join(" ")
13426        )),
13427        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13428    }
13429}
13430
13431/// `gh pr create`, returning the PR url.
13432pub(crate) async fn gh_pr_create(
13433    cwd: &Path,
13434    base: &str,
13435    head: &str,
13436    title: &str,
13437    body: &str,
13438) -> Result<String> {
13439    let out = tokio::process::Command::new("gh")
13440        .args([
13441            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13442        ])
13443        .current_dir(cwd)
13444        .quiet()
13445        .stdin(std::process::Stdio::null())
13446        .output()
13447        .await
13448        .context("spawn gh")?;
13449    if out.status.success() {
13450        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13451    } else {
13452        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13453    }
13454}
13455
13456/// Never force deletion of uncommitted work, including unknown legacy trees.
13457async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13458    match std::fs::symlink_metadata(path) {
13459        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13460            // A previous cleanup may have removed the tree before a branch
13461            // deletion or state save failed. Unregister it if still present.
13462            git::worktree_remove_clean(repo, path).await?;
13463            return Ok(());
13464        }
13465        Err(e) => return Err(e.into()),
13466        Ok(_) => {}
13467    }
13468    if !git::is_clean(path).await? {
13469        bail!(
13470            "worktree has uncommitted work; refusing removal: {}",
13471            path.display()
13472        );
13473    }
13474    if !git::worktree_remove_clean(repo, path).await? {
13475        bail!("worktree removal refused: {}", path.display());
13476    }
13477    Ok(())
13478}
13479
13480/// Read seats return replies, not working-tree changes. Only registered,
13481/// detached seats directly inside this run's root may discard those changes.
13482async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13483    if !crate::retention::read_seat(path) || !path.exists() {
13484        return remove_preserved_worktree(&state.repo, path).await;
13485    }
13486    let owned = std::fs::symlink_metadata(path)?.is_dir()
13487        && path.parent().map(Path::canonicalize).transpose()?
13488            == Some(state.worktree_root().canonicalize()?)
13489        && crate::retention::worktrees(state)
13490            .await?
13491            .contains(&path.canonicalize()?);
13492    let common = crate::repo_lock::common_dir(&state.repo).await?;
13493    if !owned
13494        || common.is_none()
13495        || crate::repo_lock::common_dir(path).await? != common
13496        || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13497            .await?
13498            .ok()
13499        || state.candidates.iter().any(|c| c.worktree == path)
13500    {
13501        return remove_preserved_worktree(&state.repo, path).await;
13502    }
13503    let changes = git::status_porcelain(path).await?;
13504    if !git::worktree_remove(&state.repo, path).await? {
13505        bail!(
13506            "read-only seat worktree removal refused: {}",
13507            path.display()
13508        );
13509    }
13510    if !changes.is_empty() {
13511        let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13512        let paths: String = paths.chars().take(1024).collect();
13513        state.event(
13514            "fold",
13515            format!(
13516                "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13517                path.file_name().unwrap_or_default().to_string_lossy(),
13518                changes.lines().count()
13519            ),
13520        );
13521    }
13522    Ok(())
13523}
13524
13525/// Tear a run's worktrees and branches down.
13526///
13527/// `home` is where the updated `run.json` is saved (via
13528/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13529/// a housekeeping pass already has its own honest `home` handed to it, and
13530/// falling through to the global here would write back through whichever
13531/// directory some other process or test pinned into that `OnceLock` first,
13532/// not the one the caller actually resolved its `runs` and `state` from.
13533pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13534    fold_run_with_policy(state, drop_winner, home, false).await
13535}
13536
13537/// Explicit CLI-only retention override; automatic callers use [`fold_run`].
13538pub async fn fold_run_overriding_recovery_retention(
13539    state: &mut RunState,
13540    drop_winner: bool,
13541    home: &Path,
13542) -> Result<Vec<String>> {
13543    fold_run_with_policy(state, drop_winner, home, true).await
13544}
13545
13546async fn fold_run_with_policy(
13547    state: &mut RunState,
13548    drop_winner: bool,
13549    home: &Path,
13550    override_recovery_retention: bool,
13551) -> Result<Vec<String>> {
13552    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13553        && !override_recovery_retention
13554    {
13555        bail!("refusing fold: {reason}");
13556    }
13557    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13558        bail!("candidate preservation incomplete; refusing cleanup");
13559    }
13560    // Serialize ordinary folding with the same starts/resumes as target-only
13561    // eviction. Otherwise a second janitor could remove the whole worktree
13562    // around a live output borrower even though target eviction was fenced.
13563    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13564    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13565    // An override must use current durable ownership, not a pre-lock snapshot.
13566    if override_recovery_retention {
13567        *state = RunState::load_under(&state.id, home)?;
13568        anyhow::ensure!(
13569            state.status.done() && !state.status.resumable(),
13570            "recovery retention override requires a finished, non-resumable run"
13571        );
13572        let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13573        anyhow::ensure!(
13574            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13575            "run is live or its ownership is unknown"
13576        );
13577        anyhow::ensure!(
13578            state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13579            "candidate preservation incomplete; refusing cleanup"
13580        );
13581    }
13582    let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13583    if let Some(reason) = &retention_reason
13584        && !override_recovery_retention
13585    {
13586        bail!("refusing fold: {reason}");
13587    }
13588    let _driver = crate::driver::maintenance(home, &state.id)?;
13589    let mut output_guards = Vec::new();
13590    for entry in std::fs::read_dir(state.worktree_root())
13591        .into_iter()
13592        .flatten()
13593        .flatten()
13594    {
13595        if entry.path().join(".git").is_file() {
13596            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13597        }
13598    }
13599    let repo = state.repo.clone();
13600    // CLI, Web and automatic cleanup all enter here, independently of
13601    // Runner::fold_losers. Protect deletion and shared config release too.
13602    let wait = repository_wait(state, None);
13603    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13604        Ok(lease) => lease,
13605        Err(e) => {
13606            state.save_under(home)?;
13607            return Err(e);
13608        }
13609    };
13610    // Confirmed merges dispose of their owned sources after durable evidence
13611    // capture. Preflight every source before removing anything.
13612    let _source_outputs = if retention_reason.is_some()
13613        || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13614    {
13615        verify_merged_fold_sources(state, drop_winner, home).await?
13616    } else {
13617        Vec::new()
13618    };
13619    if let Some(reason) = retention_reason {
13620        state.event(
13621            "fold",
13622            format!(
13623                "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13624                state.id
13625            ),
13626        );
13627        state.save_under(home)?;
13628    }
13629    let root = state.worktree_root();
13630    let winner = state.tally.as_ref().map(|t| t.winner);
13631    let mut removed = Vec::new();
13632
13633    for i in 0..state.candidates.len() {
13634        let c = state.candidates[i].clone();
13635        let is_winner = Some(c.label) == winner;
13636        if is_winner && !drop_winner {
13637            continue;
13638        }
13639        let existed = c.worktree.exists();
13640        // A vanished directory can still have a Git registration that keeps
13641        // its branch checked out. Remove that registration before the branch.
13642        remove_preserved_worktree(&repo, &c.worktree).await?;
13643        if existed {
13644            removed.push(c.worktree.to_string_lossy().into_owned());
13645        }
13646        // A branch handed to a later run (and its pull request) is not this
13647        // run's to delete.
13648        let handed_over = state.released_branches.contains(&c.branch);
13649        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13650            if !git::branch_delete(&repo, &c.branch).await? {
13651                bail!("candidate branch removal refused");
13652            }
13653            removed.push(c.branch.clone());
13654        }
13655        state.candidates[i].folded = true;
13656    }
13657
13658    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13659        let path = name.path();
13660        let keep = !drop_winner
13661            && winner.is_some_and(|w| {
13662                path.file_name()
13663                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13664            });
13665        if keep {
13666            continue;
13667        }
13668        remove_read_seat_worktree(state, &path).await?;
13669        removed.push(path.to_string_lossy().into_owned());
13670    }
13671
13672    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13673    // judge worktrees, so once the loop above has cleared all of them out,
13674    // the parent is a bare directory nobody else was ever going to remove -
13675    // git only ever managed what was inside it. Left alone, one of these
13676    // accumulates per fully-folded run; the operator's own machine had 74.
13677    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13678    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13679    // so does anything a run never claimed that happens to share the bay.
13680    remove_if_empty(&root);
13681
13682    if state.enabled_worktree_config && drop_winner {
13683        // A release, not a raw disable: some sibling run in this repository
13684        // may still hold its own reference (see `git::acquire_worktree_config`),
13685        // and only the last release actually turns the setting back off.
13686        git::release_worktree_config(&repo).await.ok();
13687        state.enabled_worktree_config = false;
13688    }
13689    state.save_under(home)?;
13690    Ok(removed)
13691}
13692
13693async fn verify_merged_fold_sources(
13694    state: &RunState,
13695    drop_winner: bool,
13696    home: &Path,
13697) -> Result<Vec<std::fs::File>> {
13698    let winner = state.tally.as_ref().map(|t| t.winner);
13699    let mut heads = BTreeSet::new();
13700    for candidate in &state.candidates {
13701        if Some(candidate.label) == winner && !drop_winner {
13702            continue;
13703        }
13704        anyhow::ensure!(
13705            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13706            "candidate recovery source is outside the inventoried run root: {}",
13707            candidate.worktree.display()
13708        );
13709        match std::fs::symlink_metadata(&candidate.worktree) {
13710            Ok(_) => {}
13711            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13712            Err(error) => return Err(error.into()),
13713        }
13714        if !state.released_branches.contains(&candidate.branch)
13715            && git::branch_exists(&state.repo, &candidate.branch).await?
13716        {
13717            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13718        }
13719    }
13720    let mut cargo_guards = Vec::new();
13721    let entries = match std::fs::read_dir(state.worktree_root()) {
13722        Ok(entries) => Some(entries),
13723        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13724        Err(error) => return Err(error.into()),
13725    };
13726    for entry in entries.into_iter().flatten() {
13727        let path = entry?.path();
13728        let metadata = std::fs::symlink_metadata(&path)?;
13729        anyhow::ensure!(
13730            metadata.is_dir() && !metadata.file_type().is_symlink(),
13731            "non-directory or symlink recovery source retained: {}",
13732            path.display()
13733        );
13734        if !drop_winner
13735            && winner.is_some_and(|w| {
13736                path.file_name()
13737                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13738            })
13739        {
13740            continue;
13741        }
13742        anyhow::ensure!(
13743            path.join(".git").is_file(),
13744            "unregistered recovery source retained: {}",
13745            path.display()
13746        );
13747        anyhow::ensure!(
13748            git::git(
13749                &path,
13750                &[
13751                    "status",
13752                    "--porcelain",
13753                    "--untracked-files=all",
13754                    "--ignored",
13755                    "--",
13756                    ".",
13757                    ":(exclude)target"
13758                ]
13759            )
13760            .await?
13761            .is_empty(),
13762            "source changes or ignored recovery files retained: {}",
13763            path.display()
13764        );
13765        match std::fs::symlink_metadata(path.join("target")) {
13766            Ok(_) => {
13767                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13768            }
13769            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13770            Err(error) => return Err(error.into()),
13771        }
13772        for checkpoint in [
13773            "rebase-merge",
13774            "rebase-apply",
13775            "sequencer",
13776            "CHERRY_PICK_HEAD",
13777            "MERGE_HEAD",
13778            "REBASE_HEAD",
13779            "REVERT_HEAD",
13780            "BISECT_LOG",
13781            "AUTO_MERGE",
13782            "MERGE_MSG",
13783        ] {
13784            let location = git::git(
13785                &path,
13786                &[
13787                    "rev-parse",
13788                    "--path-format=absolute",
13789                    "--git-path",
13790                    checkpoint,
13791                ],
13792            )
13793            .await?;
13794            anyhow::ensure!(
13795                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13796                if e.kind() == std::io::ErrorKind::NotFound),
13797                "Git recovery checkpoint retained: {checkpoint}"
13798            );
13799        }
13800        heads.insert(git::rev_parse(&path, "HEAD").await?);
13801    }
13802    crate::fold_evidence::preserve(state, home, &heads).await?;
13803
13804    Ok(cargo_guards)
13805}
13806
13807/// Remove `dir` if it exists and has nothing in it.
13808///
13809/// Best-effort and silent by design: a directory that is not empty (a run
13810/// whose winner is still parked there, a stray file some other process left)
13811/// is exactly the case this must refuse, and a directory that is already gone
13812/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13813/// already refuses a non-empty directory, so the emptiness check below is
13814/// belt, not suspenders - it is what keeps this from ever attempting the
13815/// removal in the case that matters, rather than trusting `remove_dir`'s
13816/// error path to have no side effects if it ever changed.
13817fn remove_if_empty(dir: &Path) {
13818    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13819        std::fs::remove_dir(dir).ok();
13820    }
13821}
13822
13823/// Severity of the worst open finding in the last review round, for reporting.
13824pub fn worst_open(state: &RunState) -> Option<Severity> {
13825    state
13826        .reviews
13827        .last()?
13828        .reviews
13829        .iter()
13830        .flat_map(|r| r.findings.iter())
13831        .map(|f| f.severity)
13832        .max()
13833}
13834
13835/// `[worktree] setup` hides its products per worktree, which needs
13836/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13837/// worktrees, like the hook's; every place that runs setup calls this first.
13838async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13839    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13840        git::acquire_worktree_config(repo).await?;
13841        state.enabled_worktree_config = true;
13842    }
13843    Ok(())
13844}
13845
13846#[cfg(test)]
13847mod tests {
13848    #[tokio::test]
13849    async fn launcher_base_fetch_requires_remote_success() {
13850        let tmp = tempfile::tempdir().unwrap();
13851        let repo = tmp.path().join("repo");
13852        let remote = tmp.path().join("remote");
13853        std::fs::create_dir(&repo).unwrap();
13854        std::fs::create_dir(&remote).unwrap();
13855        git::git(&remote, &["init", "--bare", "-b", "main"])
13856            .await
13857            .unwrap();
13858        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13859        git::git(
13860            &repo,
13861            &[
13862                "-c",
13863                "user.name=Fixture",
13864                "-c",
13865                "user.email=fixture@example.invalid",
13866                "commit",
13867                "--allow-empty",
13868                "-m",
13869                "fixture",
13870            ],
13871        )
13872        .await
13873        .unwrap();
13874        git::git(
13875            &repo,
13876            &["remote", "add", "origin", remote.to_str().unwrap()],
13877        )
13878        .await
13879        .unwrap();
13880        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13881        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13882        assert_eq!(
13883            resolve_base(&repo, "main", "origin", None)
13884                .await
13885                .unwrap()
13886                .commit,
13887            head
13888        );
13889        #[cfg(unix)]
13890        {
13891            let policy: crate::config::Coordination = toml::from_str(
13892                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13893            ).unwrap();
13894            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13895            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13896            let held = crate::repo_lock::SeatLease::operation_with(
13897                &repo,
13898                Some(&policy),
13899                &crate::repo_lock::WaitPolicy::default(),
13900            )
13901            .await
13902            .unwrap();
13903            assert!(held.is_some());
13904            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13905            tokio::pin!(fetch);
13906            assert!(
13907                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13908                    .await
13909                    .is_err()
13910            );
13911            assert!(
13912                !repo.join(".git/FETCH_HEAD").exists(),
13913                "no fetch before admission"
13914            );
13915            drop(held);
13916            let fetched = fetch.await.unwrap();
13917            assert_eq!(fetched.commit, head);
13918            assert_eq!(fetched.remote, "origin");
13919            assert_eq!(fetched.branch, "main");
13920        }
13921        // A surviving tracking ref is not successful fetch evidence.
13922        git::git(
13923            &repo,
13924            &[
13925                "remote",
13926                "set-url",
13927                "origin",
13928                tmp.path().join("missing").to_str().unwrap(),
13929            ],
13930        )
13931        .await
13932        .unwrap();
13933        let error = resolve_base(&repo, "main", "origin", None)
13934            .await
13935            .unwrap_err()
13936            .to_string();
13937        assert!(error.contains("cannot read origin/main"));
13938        assert!(error.contains("refusing to branch off"));
13939    }
13940
13941    #[test]
13942    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13943        let leaked = vec!["chore(deps): update a crate".to_owned()];
13944        let own = "fix(daemon): apply a chosen action";
13945        assert!(should_retitle("", own, &leaked));
13946        assert!(should_retitle(
13947            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13948            own,
13949            &leaked
13950        ));
13951        assert!(should_retitle(
13952            "chore: land candidate A of run 1",
13953            own,
13954            &leaked
13955        ));
13956        assert!(should_retitle(
13957            "magi: candidate A (uncommitted work)",
13958            own,
13959            &leaked
13960        ));
13961        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13962        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13963        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13964    }
13965
13966    #[test]
13967    fn pr_merge_plan_creates_adopts_or_stops() {
13968        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13969        assert_eq!(
13970            pr_merge_plan(Ok(land::OpenPr::One {
13971                url: "u".into(),
13972                title: "t".into()
13973            })),
13974            PrPlan::Adopt {
13975                url: "u".into(),
13976                title: "t".into()
13977            }
13978        );
13979        let PrPlan::Stop(many) =
13980            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13981        else {
13982            panic!("many must stop");
13983        };
13984        assert!(many.contains('a') && many.contains('b'));
13985        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13986            panic!("a failed lookup must stop");
13987        };
13988        assert!(err.contains("bad token"));
13989    }
13990
13991    use super::*;
13992    use crate::run::GateStatus;
13993    use std::collections::BTreeMap;
13994    use std::time::Duration;
13995
13996    fn conductor() -> AgentSpec {
13997        AgentSpec {
13998            id: "conductor".to_owned(),
13999            kind: crate::config::AgentKind::Command,
14000            model: None,
14001            command: vec!["true".to_owned()],
14002            extra_args: Vec::new(),
14003            env: BTreeMap::new(),
14004            prompt_delivery: None,
14005            max_turn_secs: None,
14006        }
14007    }
14008
14009    fn spec(id: &str) -> AgentSpec {
14010        AgentSpec {
14011            id: id.to_owned(),
14012            kind: crate::config::AgentKind::Command,
14013            model: None,
14014            command: vec!["true".to_owned()],
14015            extra_args: Vec::new(),
14016            env: BTreeMap::new(),
14017            prompt_delivery: None,
14018            max_turn_secs: None,
14019        }
14020    }
14021
14022    fn ids(xs: &[&str]) -> BTreeSet<String> {
14023        xs.iter().map(|s| (*s).to_owned()).collect()
14024    }
14025
14026    #[test]
14027    fn pick_successor_skips_an_agent_another_seat_holds() {
14028        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
14029        let roster = [spec("a"), spec("b"), spec("c")];
14030        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14031        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14032    }
14033
14034    #[test]
14035    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
14036        // The other seat started on c but was handed to d; c is free again.
14037        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
14038        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14039        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14040        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
14041        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
14042    }
14043
14044    #[test]
14045    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
14046        let roster = [spec("a"), spec("b")];
14047        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14048        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14049        let carried = ids(&["b"]);
14050        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
14051        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14052    }
14053
14054    #[test]
14055    fn pick_successor_returns_none_without_any_untried_successor() {
14056        let roster = [spec("a"), spec("b")];
14057        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
14058        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
14059        let carried = ids(&["a"]);
14060        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
14061    }
14062
14063    #[test]
14064    fn pick_successor_rescue_avoids_another_seats_occupant() {
14065        // b is a carried failure and free; a is held by the other seat.
14066        let roster = [spec("a"), spec("b"), spec("c")];
14067        let carried = ids(&["b", "c"]);
14068        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
14069        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14070    }
14071
14072    #[test]
14073    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
14074        let roster = [spec("a"), spec("b"), spec("c")];
14075        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
14076        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14077    }
14078
14079    #[test]
14080    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
14081        let roster = [spec("a"), spec("b"), spec("c")];
14082        let failed = ids(&["a", "b", "c"]);
14083        // Rescue looks at the whole roster, once per id, then runs out.
14084        let mut tried = ids(&["b"]);
14085        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14086        assert_eq!(first.id, "a");
14087        tried.insert(first.id.clone());
14088        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14089        assert_eq!(second.id, "c");
14090        tried.insert(second.id.clone());
14091        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
14092    }
14093
14094    #[test]
14095    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
14096        let roster = [spec("a"), spec("b")];
14097        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
14098        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14099    }
14100
14101    #[test]
14102    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
14103        let roster = [spec("a"), spec("b"), spec("c")];
14104        let h = SeatHistory {
14105            failed: ids(&["a"]),
14106            last_ok: Some("b".to_owned()),
14107            last_fail: None,
14108        };
14109        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
14110        // A last answerer that left the roster, or later failed, is ignored.
14111        let gone = SeatHistory {
14112            last_ok: Some("zzz".to_owned()),
14113            ..h.clone()
14114        };
14115        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
14116        let failed = SeatHistory {
14117            failed: ids(&["a", "b"]),
14118            last_ok: Some("b".to_owned()),
14119            last_fail: None,
14120        };
14121        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
14122    }
14123
14124    #[test]
14125    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
14126        let first = SeatState::new("review-1", "alpha", 7);
14127        let next = handover_seat("review-1", "gamma", 7);
14128        assert_ne!(first.claude_session, next.claude_session);
14129    }
14130
14131    #[test]
14132    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
14133        let mut state = state_with_summary("x", "y");
14134        let a = handover_seat("review-1", "beta", state.next_seat_seed());
14135        let b = handover_seat("review-1", "beta", state.next_seat_seed());
14136        assert_ne!(a.claude_session, b.claude_session);
14137    }
14138
14139    #[test]
14140    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14141        let roster = [spec("a"), spec("b")];
14142        let h = SeatHistory {
14143            failed: ids(&["a", "b"]),
14144            ..SeatHistory::default()
14145        };
14146        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14147        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14148        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14149    }
14150
14151    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
14152    // fallback loop depends on to terminate: it must walk forward from the
14153    // seat's own position, never restart at the front of the roster, and it
14154    // must never hand back an id already tried, however many times that id
14155    // happens to appear.
14156
14157    #[test]
14158    fn failure_signature_ignores_numbers_and_paths() {
14159        assert_eq!(
14160            failure_signature("exited with Some(2) and no usable output"),
14161            failure_signature("exited with Some(137) and no usable output")
14162        );
14163        assert_eq!(
14164            failure_signature("process exited with no usable output (exit code 2)"),
14165            failure_signature("process exited with no usable output (exit code 137)")
14166        );
14167        for (old, new) in [
14168            (
14169                "exited with Some(2) and no usable output",
14170                "process exited with no usable output (exit code 2)",
14171            ),
14172            (
14173                "exited with None and no usable output",
14174                "process exited with no usable output (exit code unavailable)",
14175            ),
14176        ] {
14177            let previous = FailClass::Other(failure_signature(old));
14178            let current = FailClass::Other(failure_signature(new));
14179            assert_eq!(previous, current, "resumption retains the failure class");
14180            assert!(!should_hand_over(Some(&previous), &current));
14181        }
14182        assert_eq!(
14183            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14184            failure_signature("cannot open /var/x.txt: denied")
14185        );
14186        assert_ne!(failure_signature("boom"), failure_signature("bang"));
14187    }
14188
14189    #[test]
14190    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14191        let other = FailClass::Other("x".into());
14192        assert!(should_hand_over(None, &FailClass::Quota));
14193        assert!(should_hand_over(Some(&other), &FailClass::Quota));
14194        assert!(should_hand_over(
14195            Some(&FailClass::Timeout),
14196            &FailClass::Timeout
14197        ));
14198        assert!(should_hand_over(None, &other));
14199        assert!(!should_hand_over(Some(&other), &other));
14200        assert!(should_hand_over(
14201            Some(&other),
14202            &FailClass::Other("y".into())
14203        ));
14204        // A quota or timeout in between ends the run of identical failures.
14205        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14206        assert!(should_hand_over(Some(&FailClass::Quota), &other));
14207    }
14208
14209    #[test]
14210    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14211        let a = SeatState::new("judge-1", "alpha", 7);
14212        let b = handover_seat("judge-1", "beta", 7);
14213        assert_ne!(a.claude_session, b.claude_session);
14214        assert_eq!(b.turns, 0);
14215    }
14216
14217    #[test]
14218    fn a_timeout_is_classified_apart_from_other_failures() {
14219        assert_eq!(
14220            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14221            Some(FailClass::Timeout)
14222        );
14223        assert!(matches!(
14224            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14225            Some(FailClass::Other(_))
14226        ));
14227    }
14228
14229    #[test]
14230    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14231        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14232        let tried = BTreeSet::from(["beta".to_owned()]);
14233        // beta sits at index 1; the next candidate is gamma, never alpha —
14234        // which is very likely a different candidate slot's own agent.
14235        let next = next_untried_in_roster(&roster, 1, &tried);
14236        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14237    }
14238
14239    #[test]
14240    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14241        let roster = vec![spec("alpha"), spec("beta")];
14242        let tried = BTreeSet::from(["beta".to_owned()]);
14243        // beta is the roster's last entry: nothing follows it, and alpha —
14244        // earlier in the roster, almost certainly a different candidate
14245        // slot's own agent — must not be reached by wrapping back to it.
14246        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14247    }
14248
14249    #[test]
14250    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14251        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14252        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14253        // beta (index 1) and gamma (index 2, the only entry after it) have
14254        // both been tried; alpha (index 0) never has, but it comes before
14255        // beta's own position, so there is nothing further for this seat.
14256        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14257    }
14258
14259    #[test]
14260    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14261        let roster = vec![spec("a"), spec("a"), spec("b")];
14262        let tried = BTreeSet::from(["a".to_owned()]);
14263        let next = next_untried_in_roster(&roster, 0, &tried);
14264        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14265    }
14266
14267    #[test]
14268    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14269        let roster = vec![spec("a"), spec("b")];
14270        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14271        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14272    }
14273
14274    #[test]
14275    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14276        let dir = tempfile::tempdir().unwrap();
14277        let bay = dir.path().join("ffff");
14278
14279        // Not there yet: nothing to do, nothing to panic on.
14280        remove_if_empty(&bay);
14281        assert!(!bay.exists());
14282
14283        // Something still inside - the winner's worktree, or a stray file -
14284        // keeps the directory standing.
14285        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14286        remove_if_empty(&bay);
14287        assert!(bay.exists(), "non-empty directory must survive");
14288
14289        // Once the last entry is gone, so is the directory itself.
14290        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14291        remove_if_empty(&bay);
14292        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14293    }
14294
14295    // `round_is_clean` is the exact decision this task fixed: a round with a
14296    // seat that never answered must not read the same as a round every seat
14297    // actually reviewed. These are deterministic and process-free by design —
14298    // the equivalent end-to-end check (a real reviewer timing out under a
14299    // live graph run) is a genuine race against wall-clock contention, and a
14300    // spawn slow enough to blow even a generous budget under a loaded test
14301    // run must not turn this specific regression check flaky.
14302
14303    #[test]
14304    fn a_full_panel_that_found_nothing_is_clean() {
14305        assert!(round_is_clean(
14306            0,
14307            true,
14308            2,
14309            2,
14310            0,
14311            IncompleteReviewPolicy::Block
14312        ));
14313    }
14314
14315    #[test]
14316    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14317        assert!(!round_is_clean(
14318            0,
14319            true,
14320            1,
14321            2,
14322            0,
14323            IncompleteReviewPolicy::Block
14324        ));
14325    }
14326
14327    #[test]
14328    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14329        assert!(!round_is_clean(
14330            1,
14331            true,
14332            1,
14333            2,
14334            0,
14335            IncompleteReviewPolicy::Warn
14336        ));
14337    }
14338
14339    #[test]
14340    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14341        assert!(round_is_clean(
14342            0,
14343            true,
14344            1,
14345            2,
14346            0,
14347            IncompleteReviewPolicy::Warn
14348        ));
14349    }
14350
14351    #[test]
14352    fn a_full_panel_with_an_open_finding_is_not_clean() {
14353        assert!(!round_is_clean(
14354            1,
14355            true,
14356            2,
14357            2,
14358            0,
14359            IncompleteReviewPolicy::Block
14360        ));
14361    }
14362
14363    #[test]
14364    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14365        assert!(!round_is_clean(
14366            0,
14367            false,
14368            2,
14369            2,
14370            0,
14371            IncompleteReviewPolicy::Block
14372        ));
14373    }
14374
14375    // The stall this task closes: under the default `block` policy, a seat
14376    // missing only because it was rate limited must not force a wait for a
14377    // session limit that will not lift by the next round. `round_is_clean`
14378    // is where that quorum carve-out lives; the review loop around it never
14379    // changes what a reviewer's vote or a finding's severity means.
14380
14381    #[test]
14382    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14383        // 1 of 2 answered, and the one missing was quota'd — the exact
14384        // "review-2 rate limited (quota)" shape from the field report.
14385        assert!(round_is_clean(
14386            0,
14387            true,
14388            1,
14389            2,
14390            1,
14391            IncompleteReviewPolicy::Block
14392        ));
14393    }
14394
14395    #[test]
14396    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14397        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14398        // not a quota loss (`quota_missing` stays 0) — worth another try.
14399        assert!(!round_is_clean(
14400            0,
14401            true,
14402            1,
14403            2,
14404            0,
14405            IncompleteReviewPolicy::Block
14406        ));
14407    }
14408
14409    #[test]
14410    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14411        assert!(!round_is_clean(
14412            1,
14413            true,
14414            1,
14415            2,
14416            1,
14417            IncompleteReviewPolicy::Block
14418        ));
14419        assert!(!round_is_clean(
14420            0,
14421            false,
14422            1,
14423            2,
14424            1,
14425            IncompleteReviewPolicy::Block
14426        ));
14427    }
14428
14429    #[test]
14430    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14431        // Every seat quota'd, nobody answered: there is no panel to decide
14432        // on, so this must fall through to the existing block-and-retry
14433        // fallback rather than call an unreviewed patch clean.
14434        assert!(!round_is_clean(
14435            0,
14436            true,
14437            0,
14438            2,
14439            2,
14440            IncompleteReviewPolicy::Block
14441        ));
14442    }
14443
14444    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14445        CommandOutcome {
14446            command: "test".to_owned(),
14447            code,
14448            output_tail: String::new(),
14449            duration_ms: 0,
14450            resource_blocked,
14451        }
14452    }
14453
14454    #[cfg(unix)]
14455    #[tokio::test]
14456    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14457        ask_test_home();
14458        let temp = tempfile::tempdir().unwrap();
14459        let mut runner = runner_at(RunStatus::Gating);
14460        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14461        let commands = vec![
14462            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14463            "touch must-not-run".to_owned(),
14464        ];
14465        let (outcomes, retried, pids) = run_e2e_with_retry(
14466            &mut runner.state,
14467            &shell,
14468            &commands,
14469            temp.path(),
14470            Duration::from_secs(5),
14471            "fixture",
14472            None,
14473        )
14474        .await;
14475        assert_eq!(outcomes.len(), 1);
14476        assert!(outcomes[0].resource_blocked);
14477        assert!(!outcomes[0].build_failed());
14478        assert!(!retried);
14479        assert!(pids.is_empty());
14480        assert!(!temp.path().join("must-not-run").exists());
14481        assert_eq!(
14482            runner.state.resource_block.as_ref().unwrap().kind,
14483            "incremental cache admission"
14484        );
14485    }
14486
14487    #[test]
14488    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14489        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14490        assert!(
14491            !verify_inconclusive(&[outcome(Some(1), false)]),
14492            "an ordinary failure is still evidence about the patch"
14493        );
14494        assert!(verify_inconclusive(&[outcome(None, true)]));
14495        assert!(
14496            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14497            "one inconclusive outcome taints the whole batch"
14498        );
14499        assert!(!verify_inconclusive(&[]));
14500    }
14501
14502    #[tokio::test]
14503    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14504        // Alive for the first two checks, then dead - confirms the loop
14505        // actually re-polls rather than deciding once and sleeping out the
14506        // ceiling regardless.
14507        let calls = std::sync::atomic::AtomicUsize::new(0);
14508        let started = Instant::now();
14509        wait_for_pids_with(
14510            &[123],
14511            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14512            Duration::from_millis(5),
14513            Duration::from_secs(5),
14514        )
14515        .await;
14516        assert!(
14517            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14518            "must keep checking rather than deciding on the first answer"
14519        );
14520        assert!(
14521            started.elapsed() < Duration::from_secs(1),
14522            "must return the moment it is confirmed dead, not wait out the ceiling"
14523        );
14524    }
14525
14526    #[tokio::test]
14527    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14528        let started = Instant::now();
14529        wait_for_pids_with(
14530            &[123],
14531            |_| true, // never reports dead
14532            Duration::from_millis(5),
14533            Duration::from_millis(30),
14534        )
14535        .await;
14536        let elapsed = started.elapsed();
14537        assert!(
14538            elapsed >= Duration::from_millis(30),
14539            "must not give up before its own ceiling: {elapsed:?}"
14540        );
14541        assert!(
14542            elapsed < Duration::from_secs(1),
14543            "must not wait past its own ceiling either: {elapsed:?}"
14544        );
14545    }
14546
14547    #[tokio::test]
14548    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14549        let started = Instant::now();
14550        wait_for_pids_with(
14551            &[],
14552            |_| true,
14553            Duration::from_secs(5),
14554            Duration::from_secs(5),
14555        )
14556        .await;
14557        assert!(
14558            started.elapsed() < Duration::from_millis(200),
14559            "an empty pid list has nothing to confirm"
14560        );
14561    }
14562
14563    // `review_conclusion` is the exact decision the review hand-off task
14564    // fixed: a round budget spent (or a tree that stopped moving) must not
14565    // collapse into `Blocked` regardless of what verification actually
14566    // said. Deterministic and process-free for the same reason the
14567    // `round_is_clean` family above is.
14568    fn review_round(
14569        clean: bool,
14570        blocking: usize,
14571        answered: usize,
14572        expected: usize,
14573        progressed: bool,
14574        e2e_ok: bool,
14575    ) -> ReviewRound {
14576        ReviewRound {
14577            round: 1,
14578            head: "h".to_owned(),
14579            verified_head: None,
14580            verified_at: None,
14581            reviews: Vec::new(),
14582            e2e: vec![CommandOutcome {
14583                command: "test".to_owned(),
14584                code: Some(if e2e_ok { 0 } else { 1 }),
14585                output_tail: String::new(),
14586                duration_ms: 0,
14587                resource_blocked: false,
14588            }],
14589            verify_retried: false,
14590            e2e_deferred: false,
14591            e2e_defer_reason: None,
14592            fix: None,
14593            blocking,
14594            answered,
14595            expected,
14596            clean,
14597            progressed,
14598            vote_split: false,
14599            reconsideration: Vec::new(),
14600            verdict: None,
14601        }
14602    }
14603
14604    #[test]
14605    fn review_conclusion_is_none_when_nothing_has_run() {
14606        assert_eq!(review_conclusion(&[], 3), None);
14607    }
14608
14609    #[test]
14610    fn review_conclusion_is_none_while_rounds_remain() {
14611        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14612        assert_eq!(review_conclusion(&rounds, 3), None);
14613    }
14614
14615    #[test]
14616    fn review_conclusion_is_gating_once_a_round_is_clean() {
14617        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14618        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14619    }
14620
14621    #[test]
14622    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14623        let rounds = vec![
14624            review_round(false, 1, 2, 2, true, true),
14625            review_round(false, 1, 2, 2, true, true),
14626        ];
14627        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14628    }
14629
14630    #[test]
14631    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14632        let rounds = vec![
14633            review_round(false, 1, 2, 2, true, true),
14634            review_round(false, 1, 2, 2, true, false),
14635        ];
14636        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14637    }
14638
14639    #[test]
14640    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14641        // Magi never got a command to run against this round's own head — a
14642        // resource-blocked attempt, not a red one — so this must never
14643        // settle on `Blocked` the way a genuine e2e failure would. `None`
14644        // here is what tells `Runner::review_loop` to retry the check
14645        // itself rather than trust this cheap recomputation with a verdict
14646        // it cannot actually produce.
14647        let mut blocked = review_round(false, 1, 2, 2, true, false);
14648        blocked.e2e[0].resource_blocked = true;
14649        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14650        assert_eq!(review_conclusion(&rounds, 2), None);
14651    }
14652
14653    #[test]
14654    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14655        // Missing input, not a verified tree — never a hand-off candidate.
14656        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14657        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14658    }
14659
14660    #[test]
14661    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14662        let rounds = vec![
14663            review_round(false, 1, 2, 2, false, true),
14664            review_round(false, 1, 2, 2, false, true),
14665        ];
14666        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14667    }
14668
14669    fn secs(n: u64) -> Duration {
14670        Duration::from_secs(n)
14671    }
14672
14673    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14674    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14675    #[tokio::test]
14676    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14677        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14678        if std::env::var_os(CHILD).is_none() {
14679            // Other tests reconcile every run in the process-global home,
14680            // briefly taking owner locks. This ancestry test expects an idle
14681            // predecessor, so give it its own process and pinned test home.
14682            let output = std::process::Command::new(std::env::current_exe().unwrap())
14683                .args([
14684                    "--exact",
14685                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14686                    "--nocapture",
14687                ])
14688                .env(CHILD, "1")
14689                .env_remove("MAGI_WORKER_HOME")
14690                .env_remove("MAGI_WORKER_ID")
14691                .output()
14692                .unwrap();
14693            assert!(
14694                output.status.success()
14695                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14696                "{}\n{}",
14697                String::from_utf8_lossy(&output.stdout),
14698                String::from_utf8_lossy(&output.stderr)
14699            );
14700            return;
14701        }
14702        ask_test_home();
14703        let tmp = tempfile::tempdir().unwrap();
14704        let repo = tmp.path().join("repo");
14705        std::fs::create_dir_all(&repo).unwrap();
14706        init_repo(&repo);
14707        let g = |args: &[&str]| {
14708            let out = std::process::Command::new("git")
14709                .args(args)
14710                .current_dir(&repo)
14711                .quiet()
14712                .output()
14713                .unwrap();
14714            assert!(
14715                out.status.success(),
14716                "{}",
14717                String::from_utf8_lossy(&out.stderr)
14718            );
14719        };
14720        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14721        g(&["checkout", "-b", "historical"]);
14722        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14723        g(&["add", "."]);
14724        g(&["commit", "-m", "old alternative"]);
14725        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14726        g(&["checkout", "-b", "magi/task-selected", &base]);
14727        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14728        g(&["add", "."]);
14729        g(&["commit", "-m", "measurement infrastructure"]);
14730        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14731        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14732        g(&["add", "."]);
14733        g(&["commit", "-m", "dependent implementation"]);
14734        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14735        g(&["checkout", "main"]);
14736        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14737        g(&["add", "."]);
14738        g(&["commit", "-m", "unrelated base change"]);
14739        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14740        let origin = tmp.path().join("origin.git");
14741        g(&[
14742            "clone",
14743            "--bare",
14744            repo.to_str().unwrap(),
14745            origin.to_str().unwrap(),
14746        ]);
14747        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14748        g(&["fetch", "origin"]);
14749        let selection = crate::pr_recovery::Selection {
14750            url: "https://example.test/project/pull/1".into(),
14751            branch: "magi/task-selected".into(),
14752            head: head.clone(),
14753            source_head: None,
14754            source_run: None,
14755            source_proof: None,
14756            disposition: Some(crate::pr_recovery::Disposition::Review),
14757            instruction: "preserve measured behavior".into(),
14758            approved_at: jiff::Timestamp::now(),
14759        };
14760        let mut template = runner_at(RunStatus::Reviewing);
14761        template.state.repo = repo.clone();
14762        template.state.base_commit = current.clone();
14763        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14764        template.state.origin = Some(crate::run::Origin::queue("selected"));
14765        template.state.pr_head = Some(selection.branch.clone());
14766        template.state.pr_recovery = Some(selection.clone());
14767        let mut runner =
14768            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14769                .await
14770                .unwrap();
14771        let wt = runner.state.winner().unwrap().worktree.clone();
14772        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14773        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14774        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14775        assert!(runner.state.seeds.is_empty());
14776        assert_eq!(runner.state.instruction, "preserve measured behavior");
14777        runner.sync_to_base().await.unwrap();
14778        assert_eq!(
14779            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14780            "prerequisite"
14781        );
14782        assert_eq!(
14783            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14784            "measured implementation"
14785        );
14786        assert_eq!(
14787            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14788            "current main change"
14789        );
14790        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14791        assert_eq!(
14792            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14793            head
14794        );
14795
14796        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14797        let mut partial = runner_at(RunStatus::Reviewing);
14798        partial.state.repo = repo.clone();
14799        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14800        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14801            run: Some("exit 1".into()),
14802            ..Default::default()
14803        }];
14804        let selection = runner.state.pr_recovery.clone().unwrap();
14805        partial.state.pr_recovery = Some(selection.clone());
14806        partial.state.pr_head = Some(selection.branch.clone());
14807        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14808        let id = partial.state.id.clone();
14809        let error = match Runner::open_selected(
14810            &repo,
14811            partial.state,
14812            partial.roles,
14813            1,
14814            runner.state.base_commit.clone(),
14815            selection,
14816        )
14817        .await
14818        {
14819            Ok(_) => panic!("setup must fail"),
14820            Err(e) => e,
14821        };
14822        assert_eq!(
14823            error
14824                .downcast_ref::<crate::run::StartupFailure>()
14825                .unwrap()
14826                .run,
14827            id
14828        );
14829        let failed = RunState::load(&id).unwrap();
14830        assert_eq!(failed.status, RunStatus::Failed);
14831        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14832        assert!(Runner::resume(&id).is_err());
14833        runner.state.status = RunStatus::Ready;
14834        runner.state.driver_exited = true;
14835        runner.state.save().unwrap();
14836        let mut retry = runner_at(RunStatus::Reviewing);
14837        retry.state.repo = repo.clone();
14838        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14839        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14840        reselected.approved_at = jiff::Timestamp::now();
14841        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14842        retry.state.pr_head = Some(reselected.branch.clone());
14843        retry.state.pr_recovery = Some(reselected.clone());
14844        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14845            .await
14846            .unwrap();
14847        let restarted = Runner::open_selected(
14848            &repo,
14849            retry.state,
14850            retry.roles,
14851            1,
14852            runner.state.base_commit.clone(),
14853            reselected,
14854        )
14855        .await
14856        .unwrap();
14857        assert_eq!(
14858            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14859                .await
14860                .unwrap(),
14861            head
14862        );
14863        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14864        assert_eq!(
14865            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14866            head
14867        );
14868    }
14869
14870    #[tokio::test]
14871    #[cfg(unix)]
14872    async fn fallback_and_interrupted_restoration_require_coordination() {
14873        use std::os::fd::AsRawFd as _;
14874        ask_test_home();
14875        let tmp = tempfile::tempdir().unwrap();
14876        let mut runner = registered_runner(tmp.path()).await;
14877        runner.state.config.graph.timeout_resource_wait = 1;
14878        let winner = runner.state.candidates[0].clone();
14879        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14880        let leftover = winner.worktree.join("leftover");
14881        std::fs::write(&leftover, "untrusted edits").unwrap();
14882        let next = tmp.path().join("next-fixer");
14883        let mut failed = spec("first");
14884        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14885        let mut successor = spec("second");
14886        successor.command = vec![
14887            "sh".into(),
14888            "-c".into(),
14889            format!("touch '{}'; echo done", next.display()),
14890        ];
14891        let attempts = vec![
14892            (failed.clone(), "first".into()),
14893            (successor.clone(), "second".into()),
14894        ];
14895        let lock_path = runner.state.repo.join(".git/updater.lock");
14896        let external = std::fs::File::create(&lock_path).unwrap();
14897        assert_eq!(
14898            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14899            0
14900        );
14901        let artifacts = tmp.path().join("artifacts");
14902        let build = |spec, seat| SeatJob {
14903            spec,
14904            seat,
14905            cwd: winner.worktree.clone(),
14906            prompt: "fixture".into(),
14907            timeout: Duration::from_secs(10),
14908            allow_write: true,
14909            sessions: false,
14910            artifacts: artifacts.clone(),
14911            stem: "fix".into(),
14912            handover: None,
14913        };
14914        let (_, _, out) = runner
14915            .ask_fixer_among(
14916                attempts.clone(),
14917                "graft",
14918                None,
14919                Some((&winner.worktree, &head)),
14920                build,
14921            )
14922            .await;
14923        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14924        assert!(leftover.exists(), "contention must prevent clean");
14925        assert!(!next.exists());
14926        assert!(runner.state.handovers.is_empty());
14927        // Early interrupted-repair rollback uses the same protected route.
14928        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14929            from_run: "earlier".into(),
14930            generation: 1,
14931            mode: crate::recovery::Mode::VerifyRepair,
14932            reasons: Vec::new(),
14933            reviewed_head: None,
14934            rounds: Vec::new(),
14935            open_findings: Vec::new(),
14936            evidence: None,
14937            criteria_inherited: true,
14938            repair: crate::recovery::Repair::Started {
14939                before: head.clone(),
14940            },
14941        });
14942        assert!(
14943            runner
14944                .recovery_repair(&winner)
14945                .await
14946                .unwrap_err()
14947                .is::<crate::repo_lock::ResourceBlocked>()
14948        );
14949        assert!(matches!(
14950            runner.state.recovery.as_ref().unwrap().repair,
14951            crate::recovery::Repair::Started { .. }
14952        ));
14953        assert!(leftover.exists());
14954        drop(external);
14955        runner.recovery_repair(&winner).await.unwrap();
14956        assert!(!leftover.exists());
14957        assert!(matches!(
14958            runner.state.recovery.as_ref().unwrap().repair,
14959            crate::recovery::Repair::Done { .. }
14960        ));
14961        // A hard lock-open failure also leaves HEAD and untracked work intact.
14962        std::fs::remove_file(&lock_path).unwrap();
14963        std::fs::create_dir(&lock_path).unwrap();
14964        std::fs::write(&leftover, "untrusted edits").unwrap();
14965        let (_, _, out) = runner
14966            .ask_fixer_among(
14967                attempts,
14968                "graft",
14969                None,
14970                Some((&winner.worktree, &head)),
14971                build,
14972            )
14973            .await;
14974        assert!(matches!(out, AgentOutcome::Failed(_)));
14975        assert!(leftover.exists());
14976        assert!(!next.exists());
14977        assert_eq!(
14978            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14979            head
14980        );
14981    }
14982
14983    #[cfg(unix)]
14984    async fn registered_runner(root: &Path) -> Runner {
14985        repository_runner(root, crate::config::test_coordination()).await
14986    }
14987
14988    async fn repository_runner(
14989        root: &Path,
14990        coordination: Option<crate::config::Coordination>,
14991    ) -> Runner {
14992        ask_test_home();
14993        let repo = root.join("repo");
14994        std::fs::create_dir_all(&repo).unwrap();
14995        init_repo(&repo);
14996        if coordination.is_some() {
14997            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14998        }
14999        let mut runner = runner_at(RunStatus::Prep);
15000        runner.state.repo = repo.clone();
15001        runner.state.config.coordination = coordination;
15002        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15003        runner.state.config.graph.worktree_root = Some(root.join("wt"));
15004        runner.state.config.graph.advise = false;
15005        runner.roles.implementers = vec![spec("implement")];
15006        runner.roles.judges = vec![spec("judge")];
15007        runner.prep().await.unwrap();
15008        runner
15009    }
15010
15011    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
15012
15013    #[tokio::test]
15014    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
15015        let tmp = tempfile::tempdir().unwrap();
15016        let mut runner = repository_runner(tmp.path(), None).await;
15017        let prompts = runner.state.config.prompts.clone();
15018        let run = runner.state.id.clone();
15019        let ctx = WaveCtx {
15020            carry_seats: false,
15021            pauses: None,
15022            run: &run,
15023            node: "advise",
15024            prompts: &prompts,
15025            cache: None,
15026            round: None,
15027        };
15028        let jobs: Vec<_> = ["alpha", "beta"]
15029            .into_iter()
15030            .map(|id| SeatJob {
15031                spec: scripted(id, AUTH_OUTAGE),
15032                seat: SeatState::new(id, id, 7),
15033                prompt: "propose".into(),
15034                cwd: runner.state.candidates[0].worktree.clone(),
15035                timeout: Duration::from_secs(10),
15036                allow_write: false,
15037                sessions: false,
15038                artifacts: agent::artifacts_dir(&runner.state.dir()),
15039                stem: id.into(),
15040                handover: None,
15041            })
15042            .collect();
15043        let mut losses = Vec::new();
15044        let results = ask_wave_with(
15045            jobs.clone(),
15046            runner.sem.clone(),
15047            2,
15048            &[],
15049            &ctx,
15050            &mut losses,
15051            &mut runner.state,
15052            &|_: &str| Ok(()),
15053        )
15054        .await;
15055        assert!(
15056            results
15057                .iter()
15058                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
15059        );
15060        assert_eq!(runner.state.quota.len(), 2);
15061        assert!(
15062            runner
15063                .state
15064                .quota
15065                .iter()
15066                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
15067        );
15068        assert!(!runner.state.advise_attempted);
15069        runner.stop_for_resource().unwrap();
15070        assert_eq!(runner.state.status, RunStatus::Blocked);
15071        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
15072        assert_eq!(
15073            runner.state.resource_block.as_ref().unwrap().kind,
15074            "agent authentication"
15075        );
15076        // A machine loss followed by an ordinary failure is not evidence
15077        // that the whole node was unavailable, and must not earn a refund.
15078        runner.state.resource_block = None;
15079        runner.state.quota.clear();
15080        let mut mixed = jobs;
15081        mixed[1].spec.command = vec!["false".into()];
15082        let results = ask_wave_with(
15083            mixed,
15084            runner.sem.clone(),
15085            0,
15086            &[],
15087            &ctx,
15088            &mut losses,
15089            &mut runner.state,
15090            &|_: &str| Ok(()),
15091        )
15092        .await;
15093        assert!(results.iter().all(|(_, res, _)| res.is_err()));
15094        assert!(runner.state.resource_block.is_none());
15095        assert_eq!(losses.len(), 1);
15096    }
15097
15098    #[tokio::test]
15099    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
15100        let tmp = tempfile::tempdir().unwrap();
15101        let mut runner = repository_runner(tmp.path(), None).await;
15102        let implementer = scripted("implement", AUTH_OUTAGE);
15103        runner.roles.implementers = vec![implementer.clone()];
15104        runner.state.config.agents = vec![implementer];
15105        runner.implement().await.unwrap();
15106        assert_eq!(runner.state.status, RunStatus::Blocked);
15107        assert_eq!(runner.state.quota.len(), 1);
15108        assert_eq!(
15109            runner.state.quota[0].cause,
15110            agent::QuotaCause::Authentication
15111        );
15112        let candidate = &runner.state.candidates[0];
15113        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
15114        assert_eq!(runner.state.seats["impl-A"].turns, 1);
15115    }
15116
15117    #[tokio::test]
15118    #[cfg(unix)]
15119    async fn parked_implementer_persists_completed_work() {
15120        for registered in [false, true] {
15121            let tmp = tempfile::tempdir().unwrap();
15122            let mut runner = registered_runner(tmp.path()).await;
15123            if !registered {
15124                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
15125            }
15126            let marker = tmp.path().join("started");
15127            let release = tmp.path().join("release");
15128            let mut implementer = spec("implement");
15129            implementer.command = vec![
15130                "sh".into(),
15131                "-c".into(),
15132                format!(
15133                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
15134                    marker.display(),
15135                    release.display()
15136                ),
15137            ];
15138            runner.roles.implementers = vec![implementer.clone()];
15139            runner.state.config.agents = vec![implementer];
15140            let pause = runner.pause.clone();
15141            let (result, ()) = tokio::join!(runner.implement(), async {
15142                tokio::time::timeout(Duration::from_secs(10), async {
15143                    while !marker.exists() {
15144                        tokio::time::sleep(Duration::from_millis(10)).await;
15145                    }
15146                })
15147                .await
15148                .unwrap();
15149                pause.park();
15150                std::fs::write(&release, "go").unwrap();
15151            });
15152            result.unwrap();
15153            let candidate = &runner.state.candidates[0];
15154            assert!(candidate.commits > 0, "{candidate:?}");
15155            assert!(candidate.summary.contains("saved-result"));
15156            assert_eq!(candidate.files, 1);
15157            let before = candidate.commits;
15158            runner.implement().await.unwrap();
15159            assert_eq!(runner.state.candidates[0].commits, before);
15160        }
15161    }
15162
15163    #[tokio::test]
15164    #[cfg(unix)]
15165    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15166        let tmp = tempfile::tempdir().unwrap();
15167        let mut runner = registered_runner(tmp.path()).await;
15168        let winner = runner.state.candidates[0].clone();
15169        let cache = tmp.path().join("cache");
15170        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15171        runner.state.config.graph.timeout_resource_wait = 0;
15172        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15173        let crate::cache::AcquireOutcome::Acquired(held) =
15174            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15175        else {
15176            panic!("cache must be free")
15177        };
15178        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15179            from_run: "earlier".into(),
15180            generation: 1,
15181            mode: crate::recovery::Mode::VerifyRepair,
15182            reasons: Vec::new(),
15183            reviewed_head: None,
15184            rounds: Vec::new(),
15185            open_findings: Vec::new(),
15186            evidence: None,
15187            criteria_inherited: true,
15188            repair: crate::recovery::Repair::NotStarted,
15189        });
15190        runner.recovery_repair(&winner).await.unwrap();
15191        assert!(matches!(
15192            runner.state.recovery.as_ref().unwrap().repair,
15193            crate::recovery::Repair::WaitingForRestore { .. }
15194        ));
15195        drop(held);
15196        runner.state.resource_block = None;
15197        runner.state.config.graph.timeout_resource_wait = 1;
15198        runner.recovery_repair(&winner).await.unwrap();
15199        assert!(matches!(
15200            runner.state.recovery.as_ref().unwrap().repair,
15201            crate::recovery::Repair::Done { .. }
15202        ));
15203        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15204    }
15205
15206    #[tokio::test]
15207    #[cfg(unix)]
15208    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15209        use std::os::fd::AsRawFd as _;
15210        for graft in [false, true] {
15211            let tmp = tempfile::tempdir().unwrap();
15212            let mut runner = registered_runner(tmp.path()).await;
15213            runner.state.config.graph.timeout_resource_wait = 1;
15214            let winner = runner.state.candidates[0].clone();
15215            let marker = tmp.path().join("fixer-started");
15216            let mut fixer = spec("fixer");
15217            fixer.command = vec![
15218                "sh".into(),
15219                "-c".into(),
15220                format!("echo started >> '{}'; echo done", marker.display()),
15221            ];
15222            runner.roles.fixer = Some(vec![fixer]);
15223            if graft {
15224                let mut loser = winner.clone();
15225                loser.label = 'B';
15226                loser.index = 1;
15227                runner.state.candidates.push(loser);
15228                runner.state.config.graph.graft_runner_up = true;
15229                runner.state.tally = Some(Tally {
15230                    first_choice: BTreeMap::new(),
15231                    borda: BTreeMap::new(),
15232                    winner: winner.label,
15233                    rankings: 1,
15234                    unanimous_initial: true,
15235                    deliberated: false,
15236                    changed_votes: 0,
15237                    unanimous_final: true,
15238                    tie_break: None,
15239                    judges: 1,
15240                    present: 1,
15241                    quorum: 1,
15242                    met_quorum: true,
15243                    uncontested: None,
15244                });
15245                runner.state.judgements.push(crate::run::Judgement {
15246                    judge: 1,
15247                    seat: "judge-1".into(),
15248                    agent: "judge".into(),
15249                    ranking: vec![winner.label, 'B'],
15250                    reasons: BTreeMap::new(),
15251                    confidence: None,
15252                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15253                    order: vec![0, 1],
15254                    failed: None,
15255                    duration_ms: 0,
15256                });
15257            } else {
15258                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15259                    from_run: "earlier".into(),
15260                    generation: 1,
15261                    mode: crate::recovery::Mode::VerifyRepair,
15262                    reasons: Vec::new(),
15263                    reviewed_head: None,
15264                    rounds: Vec::new(),
15265                    open_findings: Vec::new(),
15266                    evidence: None,
15267                    criteria_inherited: true,
15268                    repair: crate::recovery::Repair::NotStarted,
15269                });
15270            }
15271            let reader = crate::repo_lock::SeatLease::for_checkout(
15272                &winner.worktree,
15273                false,
15274                crate::config::test_coordination().as_ref(),
15275            )
15276            .await
15277            .unwrap();
15278            let pause = runner.pause.clone();
15279            let (result, ()) = tokio::join!(
15280                async {
15281                    if graft {
15282                        runner.graft().await
15283                    } else {
15284                        runner.recovery_repair(&winner).await
15285                    }
15286                },
15287                async {
15288                    tokio::time::sleep(Duration::from_millis(100)).await;
15289                    pause.park();
15290                }
15291            );
15292            result.unwrap();
15293            assert!(!marker.exists(), "a blocked seat must not spawn");
15294            // Exercise the persisted state, as a binary replacement/resume does.
15295            runner.state = RunState::load(&runner.state.id).unwrap();
15296            if graft {
15297                assert_eq!(
15298                    runner.state.graft.as_ref().unwrap().outcome,
15299                    GraftOutcome::WaitingForRestore
15300                );
15301            } else {
15302                assert!(matches!(
15303                    runner.state.recovery.as_ref().unwrap().repair,
15304                    crate::recovery::Repair::WaitingForRestore { .. }
15305                ));
15306            }
15307            drop(reader);
15308            runner.pause = Pause::new();
15309            runner.state.resource_block = None;
15310            // Deferred restoration must still respect an external lock owner.
15311            let external =
15312                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15313            assert_eq!(
15314                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15315                0
15316            );
15317            let leftover = winner.worktree.join("leftover");
15318            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15319            let result = if graft {
15320                runner.graft().await
15321            } else {
15322                runner.recovery_repair(&winner).await
15323            };
15324            assert!(
15325                result
15326                    .unwrap_err()
15327                    .is::<crate::repo_lock::ResourceBlocked>()
15328            );
15329            assert!(leftover.exists());
15330            assert!(!marker.exists());
15331            drop(external);
15332            runner.state.resource_block = None;
15333            let result = if graft {
15334                runner.graft().await
15335            } else {
15336                runner.recovery_repair(&winner).await
15337            };
15338            result.unwrap();
15339            assert!(!leftover.exists(), "restore must precede retry");
15340            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15341            if graft {
15342                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15343            } else {
15344                assert!(matches!(
15345                    runner.state.recovery.as_ref().unwrap().repair,
15346                    crate::recovery::Repair::Done { .. }
15347                ));
15348            }
15349        }
15350    }
15351
15352    #[tokio::test]
15353    #[cfg(unix)]
15354    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15355        use std::os::fd::AsRawFd as _;
15356        let tmp = tempfile::tempdir().unwrap();
15357        let mut runner = registered_runner(tmp.path()).await;
15358        runner.state.config.graph.timeout_resource_wait = 1;
15359        let winner = runner.state.candidates[0].clone();
15360        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15361        runner.state.tally = Some(Tally {
15362            first_choice: BTreeMap::new(),
15363            borda: BTreeMap::new(),
15364            winner: winner.label,
15365            rankings: 0,
15366            unanimous_initial: false,
15367            deliberated: false,
15368            changed_votes: 0,
15369            unanimous_final: false,
15370            tie_break: None,
15371            judges: 0,
15372            present: 0,
15373            quorum: 0,
15374            met_quorum: true,
15375            uncontested: Some("fixture".into()),
15376        });
15377        runner.state.graft = Some(GraftRecord {
15378            pre: pre.clone(),
15379            post: None,
15380            lines: None,
15381            limit: None,
15382            agent: None,
15383            outcome: GraftOutcome::Running,
15384            requested: Vec::new(),
15385            adoption: None,
15386            adoption_error: None,
15387        });
15388        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15389        git::commit_all(&winner.worktree, "unfinished graft")
15390            .await
15391            .unwrap();
15392        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15393        let leftover = winner.worktree.join("leftover");
15394        std::fs::write(&leftover, "untrusted edits").unwrap();
15395        let path = runner.state.repo.join(".git/updater.lock");
15396        let external = std::fs::File::create(&path).unwrap();
15397        assert_eq!(
15398            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15399            0
15400        );
15401        assert!(
15402            runner
15403                .graft()
15404                .await
15405                .unwrap_err()
15406                .is::<crate::repo_lock::ResourceBlocked>()
15407        );
15408        assert!(matches!(
15409            runner.state.graft.as_ref().unwrap().outcome,
15410            GraftOutcome::Running
15411        ));
15412        assert_eq!(
15413            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15414            unfinished
15415        );
15416        assert!(leftover.exists());
15417        drop(external);
15418        std::fs::remove_file(&path).unwrap();
15419        std::fs::create_dir(&path).unwrap();
15420        assert!(runner.graft().await.is_err());
15421        assert!(matches!(
15422            runner.state.graft.as_ref().unwrap().outcome,
15423            GraftOutcome::Running
15424        ));
15425        assert_eq!(
15426            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15427            unfinished
15428        );
15429        assert!(leftover.exists());
15430        std::fs::remove_dir(&path).unwrap();
15431        runner.graft().await.unwrap();
15432        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15433        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15434        assert!(!leftover.exists());
15435    }
15436
15437    #[tokio::test]
15438    #[cfg(unix)]
15439    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15440        use std::os::fd::AsRawFd as _;
15441        let tmp = tempfile::tempdir().unwrap();
15442        let mut runner = registered_runner(tmp.path()).await;
15443        runner.state.prep_complete = Some(false);
15444        runner.state.config.graph.timeout_resource_wait = 1;
15445        runner.state.save().unwrap();
15446        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15447        assert_eq!(
15448            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15449            0
15450        );
15451        runner.execute().await.unwrap();
15452        let disk = RunState::load(&runner.state.id).unwrap();
15453        assert!(disk.driver_exited);
15454        assert_eq!(disk.driver_pid, None);
15455        assert_eq!(runner.state.status, RunStatus::Blocked);
15456        assert_eq!(
15457            runner.state.resource_block.as_ref().unwrap().kind,
15458            "repository coordination"
15459        );
15460        assert!(runner.state.prep_failure.is_none());
15461        assert!(runner.state.seats.is_empty());
15462        runner.pause.park();
15463        // Include headroom for Git subprocesses under parallel test load;
15464        // the repository contention deadline itself remains one second.
15465        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15466            .await
15467            .unwrap()
15468            .unwrap();
15469        assert!(runner.state.parked);
15470        assert!(runner.state.prep_failure.is_none());
15471        let disk = RunState::load(&runner.state.id).unwrap();
15472        assert!(disk.driver_exited);
15473        assert_eq!(disk.driver_pid, None);
15474    }
15475
15476    #[tokio::test]
15477    #[cfg(unix)]
15478    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15479        use std::os::fd::AsRawFd as _;
15480        let tmp = tempfile::tempdir().unwrap();
15481        let mut runner = registered_runner(tmp.path()).await;
15482        runner.state.prep_complete = Some(false);
15483        runner.state.config.graph.timeout_resource_wait = 30;
15484        runner.state.save().unwrap();
15485        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15486        assert_eq!(
15487            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15488            0
15489        );
15490        assert!(
15491            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15492                .await
15493                .is_err()
15494        );
15495        let disk = RunState::load(&runner.state.id).unwrap();
15496        assert!(disk.driver_exited);
15497        assert_eq!(disk.driver_pid, None);
15498        assert!(disk.status.resumable());
15499        assert!(runner.state.driver_exited);
15500        assert_eq!(runner.state.driver_pid, None);
15501        runner.state.save().unwrap();
15502        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15503    }
15504
15505    #[tokio::test]
15506    #[cfg(unix)]
15507    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15508        let tmp = tempfile::tempdir().unwrap();
15509        let mut runner = registered_runner(tmp.path()).await;
15510        let winner = runner.state.candidates[0].clone();
15511        let cache = tmp.path().join("cache");
15512        let started = tmp.path().join("pre-gate-started");
15513        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15514        runner.state.config.verify.gate =
15515            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15516        let home = crate::run::home();
15517        let owner =
15518            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15519        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15520            crate::cache::AcquireOutcome::Acquired(g) => g,
15521            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15522        };
15523        let mut gate = Box::pin(runner.run_gate(&winner));
15524        assert!(
15525            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15526                .await
15527                .is_err()
15528        );
15529        assert!(!started.exists(), "pre-gate must wait for the cache too");
15530        // Simulate the wave that already owns the cache entering invoke.
15531        // It must obtain the editor reservation and finish without waiting
15532        // for the gate that is queued behind its cache.
15533        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15534        let editor = crate::repo_lock::SeatLease::checkout_with(
15535            &winner.worktree,
15536            true,
15537            crate::config::test_coordination().as_ref(),
15538            &wait,
15539        )
15540        .await
15541        .unwrap();
15542        drop(editor);
15543        guard.release();
15544        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15545            .await
15546            .unwrap()
15547            .unwrap();
15548        drop(gate);
15549        assert!(outcomes.iter().all(CommandOutcome::ok));
15550        assert!(started.exists());
15551    }
15552
15553    #[tokio::test]
15554    #[cfg(unix)]
15555    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15556        use std::os::fd::AsRawFd as _;
15557
15558        ask_test_home();
15559        let tmp = tempfile::tempdir().unwrap();
15560        let repo = tmp.path().join("repo");
15561        std::fs::create_dir_all(&repo).unwrap();
15562        init_repo(&repo);
15563        let common = repo.join(".git");
15564        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15565        let mut runner = runner_at(RunStatus::Prep);
15566        runner.state.repo = repo.clone();
15567        runner.state.config.coordination = crate::config::test_coordination();
15568        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15569        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15570        runner.state.config.graph.advise = false;
15571        runner.roles.implementers = vec![spec("implement")];
15572        runner.roles.judges = vec![spec("judge")];
15573        runner.prep().await.unwrap();
15574        let winner = runner.state.candidates[0].clone();
15575        // Emulate edits already left by a completed seat. The fixture CLI
15576        // only reports completion; it performs no repository operations.
15577        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15578        let completed = tmp.path().join("seat-completed");
15579        runner.roles.implementers[0].command = vec![
15580            "sh".into(),
15581            "-c".into(),
15582            format!("touch '{}'; echo completed", completed.display()),
15583        ];
15584        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15585        assert_eq!(
15586            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15587            0
15588        );
15589        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15590        let mut implement = Box::pin(runner.implement());
15591        tokio::select! {
15592            result = &mut implement => panic!("rescue must wait: {result:?}"),
15593            result = tokio::time::timeout(Duration::from_secs(10), async {
15594                while !completed.exists() {
15595                    tokio::time::sleep(Duration::from_millis(20)).await;
15596                }
15597            }) => result.unwrap(),
15598        }
15599        assert!(
15600            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15601                .await
15602                .is_err()
15603        );
15604        assert_eq!(
15605            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15606            before
15607        );
15608        assert_eq!(
15609            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15610            0
15611        );
15612        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15613            .await
15614            .unwrap()
15615            .unwrap();
15616        drop(implement);
15617        assert_ne!(
15618            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15619            before
15620        );
15621        assert!(git::is_clean(&winner.worktree).await.unwrap());
15622
15623        let started = tmp.path().join("pre-gate-started");
15624        let proceed = tmp.path().join("pre-gate-proceed");
15625        let checked = tmp.path().join("gate-ran");
15626        runner.state.config.verify.pre_gate = vec![format!(
15627            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15628            started.display(),
15629            proceed.display()
15630        )];
15631        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15632        assert_eq!(
15633            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15634            0
15635        );
15636        let mut gate = Box::pin(runner.run_gate(&winner));
15637        assert!(
15638            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15639                .await
15640                .is_err()
15641        );
15642        assert!(!started.exists());
15643        assert!(!checked.exists());
15644        assert!(git::is_clean(&winner.worktree).await.unwrap());
15645        assert_eq!(
15646            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15647            0
15648        );
15649        tokio::select! {
15650            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15651            result = tokio::time::timeout(Duration::from_secs(10), async {
15652                while !started.exists() {
15653                    tokio::time::sleep(Duration::from_millis(20)).await;
15654                }
15655            }) => result.unwrap(),
15656        }
15657        assert_eq!(
15658            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15659            -1,
15660            "mechanical edits must hold the real lock"
15661        );
15662        std::fs::write(proceed, "ready").unwrap();
15663        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15664            .await
15665            .unwrap()
15666            .unwrap();
15667        drop(gate);
15668        assert!(outcomes.iter().all(CommandOutcome::ok));
15669        assert!(checked.exists());
15670        assert!(runner.state.pre_gate_commit.is_some());
15671        assert!(git::is_clean(&winner.worktree).await.unwrap());
15672        assert_eq!(
15673            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15674            0,
15675            "gate completion must release coordination"
15676        );
15677        drop(outsider);
15678        std::fs::remove_file(common.join("updater.lock")).unwrap();
15679        std::fs::create_dir(common.join("updater.lock")).unwrap();
15680        std::fs::remove_file(&started).unwrap();
15681        std::fs::remove_file(&checked).unwrap();
15682        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15683        let error = runner.run_gate(&winner).await.unwrap_err();
15684        assert!(
15685            error
15686                .to_string()
15687                .contains("acquire mandatory repository lock")
15688        );
15689        assert!(
15690            !started.exists(),
15691            "failed acquisition must prevent pre-gate"
15692        );
15693        assert!(!checked.exists(), "failed acquisition must prevent gate");
15694        assert!(git::is_clean(&winner.worktree).await.unwrap());
15695        assert_eq!(
15696            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15697            head
15698        );
15699    }
15700
15701    #[tokio::test]
15702    #[cfg(unix)]
15703    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15704        use std::os::fd::AsRawFd as _;
15705        use std::time::Duration;
15706
15707        ask_test_home();
15708        let tmp = tempfile::tempdir().unwrap();
15709        let repo = tmp.path().join("repo");
15710        std::fs::create_dir_all(&repo).unwrap();
15711        init_repo(&repo);
15712        let common = repo.join(".git");
15713        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15714        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15715        assert_eq!(
15716            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15717            0
15718        );
15719        let mut runner = runner_at(RunStatus::Prep);
15720        runner.state.repo = repo.clone();
15721        runner.state.config.coordination = crate::config::test_coordination();
15722        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15723        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15724        runner.state.config.graph.advise = true;
15725        runner.state.config.graph.advisors = 1;
15726        runner.roles.implementers = vec![spec("implement")];
15727        runner.roles.judges = vec![spec("judge")];
15728        let root = runner.state.worktree_root();
15729        let started = tmp.path().join("setup-started");
15730        let proceed = tmp.path().join("setup-proceed");
15731        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15732            run: Some(format!(
15733                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15734                started.display(),
15735                proceed.display()
15736            )),
15737            ..Default::default()
15738        }];
15739        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15740        let config = git::git(&repo, &["config", "--local", "--list"])
15741            .await
15742            .unwrap();
15743        let mut prep = Box::pin(runner.prep());
15744        assert!(
15745            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15746                .await
15747                .is_err()
15748        );
15749        assert!(
15750            !root.exists(),
15751            "contention must prevent even checkout creation"
15752        );
15753        assert!(!started.exists(), "contention must prevent setup");
15754        assert_eq!(
15755            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15756            branches
15757        );
15758        assert_eq!(
15759            git::git(&repo, &["config", "--local", "--list"])
15760                .await
15761                .unwrap(),
15762            config
15763        );
15764        drop(lock);
15765        tokio::select! {
15766            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15767            result = tokio::time::timeout(Duration::from_secs(10), async {
15768                while !started.exists() {
15769                    tokio::time::sleep(Duration::from_millis(20)).await;
15770                }
15771            }) => result.unwrap(),
15772        }
15773        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15774        assert_eq!(
15775            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15776            -1
15777        );
15778        assert!(
15779            tokio::time::timeout(
15780                Duration::from_millis(100),
15781                crate::repo_lock::SeatLease::for_checkout(
15782                    &repo,
15783                    true,
15784                    crate::config::test_coordination().as_ref()
15785                )
15786            )
15787            .await
15788            .is_err(),
15789            "preparation must also exclude local editors"
15790        );
15791        std::fs::write(proceed, "ready").unwrap();
15792        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15793            .await
15794            .unwrap()
15795            .unwrap();
15796        drop(prep);
15797        assert_eq!(runner.state.prep_complete, Some(true));
15798        assert!(root.join("judge-1").exists());
15799        assert!(root.join("advisor-1").exists());
15800        assert_eq!(
15801            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15802            0
15803        );
15804    }
15805
15806    #[tokio::test]
15807    #[cfg(unix)]
15808    async fn preparation_lock_failure_prevents_repository_changes() {
15809        ask_test_home();
15810        let tmp = tempfile::tempdir().unwrap();
15811        let repo = tmp.path().join("repo");
15812        std::fs::create_dir_all(&repo).unwrap();
15813        init_repo(&repo);
15814        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15815        // A directory at the required lock path makes opening the real lock
15816        // fail, rather than merely exercising an invalid registration.
15817        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15818        let mut runner = runner_at(RunStatus::Prep);
15819        runner.state.repo = repo.clone();
15820        runner.state.config.coordination = crate::config::test_coordination();
15821        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15822        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15823        runner.roles.implementers = vec![spec("implement")];
15824        let error = runner.prep().await.unwrap_err();
15825        assert!(
15826            error
15827                .to_string()
15828                .contains("acquire mandatory repository lock")
15829        );
15830        assert!(!runner.state.worktree_root().exists());
15831        assert!(runner.state.candidates.is_empty());
15832        assert!(!repo.join(".git/config.worktree").exists());
15833        assert_eq!(
15834            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15835            "* main"
15836        );
15837    }
15838
15839    #[tokio::test]
15840    async fn preparation_records_creation_intent_before_git_can_fail() {
15841        ask_test_home();
15842        let tmp = tempfile::tempdir().unwrap();
15843        let repo = tmp.path().join("repo");
15844        std::fs::create_dir_all(&repo).unwrap();
15845        init_repo(&repo);
15846        let mut runner = runner_at(RunStatus::Prep);
15847        runner.state.repo = repo.clone();
15848        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15849        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15850        runner.state.config.blind.commit_msg_hook = false;
15851        runner.roles.implementers = vec![spec("one")];
15852        let label = blind::assign_labels(1, runner.state.seed)[0];
15853        let branch = runner.state.branch_for(label);
15854        // A conflicting branch makes Git refuse creation before a candidate
15855        // can be registered. The durable intent must still identify it.
15856        git::git(&repo, &["branch", &branch]).await.unwrap();
15857        assert!(runner.prep().await.is_err());
15858        let saved = RunState::load(&runner.state.id).unwrap();
15859        assert!(saved.candidates.is_empty());
15860        let event = saved
15861            .events
15862            .iter()
15863            .find(|e| e.node == "prep-intent")
15864            .unwrap();
15865        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15866        assert_eq!(intent["branch"], branch);
15867        assert_eq!(intent["start"], runner.state.base_commit);
15868        assert_eq!(
15869            intent["worktree"],
15870            runner
15871                .state
15872                .worktree_root()
15873                .join(format!("cand-{label}"))
15874                .to_string_lossy()
15875                .as_ref()
15876        );
15877        assert_eq!(intent["picks"], serde_json::json!([]));
15878    }
15879
15880    #[tokio::test]
15881    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15882        ask_test_home();
15883        let tmp = tempfile::tempdir().unwrap();
15884        let repo = tmp.path().join("repo");
15885        std::fs::create_dir_all(&repo).unwrap();
15886        init_repo(&repo);
15887        let mut runner = runner_at(RunStatus::Prep);
15888        runner.state.repo = repo.clone();
15889        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15890        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15891        runner.state.sources.pick_commits.push(refs::Input {
15892            token: "missing".into(),
15893            sha: "0".repeat(40),
15894            provenance: refs::Provenance::ExplicitCli,
15895        });
15896        assert!(runner.prep().await.is_err());
15897        assert!(runner.state.candidates.is_empty());
15898        assert!(!runner.state.worktree_root().exists());
15899    }
15900
15901    #[tokio::test]
15902    async fn prose_and_legacy_inference_never_import_into_candidates() {
15903        ask_test_home();
15904        let tmp = tempfile::tempdir().unwrap();
15905        let repo = tmp.path().join("repo");
15906        std::fs::create_dir_all(&repo).unwrap();
15907        init_repo(&repo);
15908        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15909        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15910            .await
15911            .unwrap();
15912        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15913        git::git(&repo, &["add", "."]).await.unwrap();
15914        git::git(&repo, &["commit", "-m", "historical implementation"])
15915            .await
15916            .unwrap();
15917        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15918        git::git(&repo, &["checkout", "main"]).await.unwrap();
15919        for instruction in [
15920            format!("Example: magi/history/A at {source}"),
15921            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15922            format!("Never import magi/history/A or {source}"),
15923            format!("Conversation answer: I mentioned {source} only as background"),
15924        ] {
15925            let mut runner = runner_at(RunStatus::Prep);
15926            runner.state.repo = repo.clone();
15927            runner.state.base_commit = base.clone();
15928            runner.state.instruction = instruction.clone();
15929            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15930            runner.state.config.blind.commit_msg_hook = false;
15931            runner.state.seeds = vec![refs::Seed {
15932                token: "magi/history/A".into(),
15933                sha: source.clone(),
15934                kind: refs::SeedKind::Unmerged,
15935                branch: true,
15936                detail: String::new(),
15937            }];
15938            runner.roles.implementers = vec![spec("one")];
15939            assert_eq!(seeded_instruction(&runner.state), instruction);
15940            // Model an old record with inferred seeds but no structured input field.
15941            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15942            legacy.as_object_mut().unwrap().remove("sources");
15943            runner.state = serde_json::from_value(legacy).unwrap();
15944            assert_eq!(runner.state.sources, refs::Sources::default());
15945            runner.state.save().unwrap();
15946            runner.state = RunState::load(&runner.state.id).unwrap();
15947            runner.prep().await.unwrap();
15948            for candidate in &runner.state.candidates {
15949                assert_eq!(
15950                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15951                    base
15952                );
15953                assert!(!candidate.worktree.join("unwanted").exists());
15954            }
15955        }
15956    }
15957
15958    #[tokio::test]
15959    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15960        ask_test_home();
15961        let tmp = tempfile::tempdir().unwrap();
15962        let repo = tmp.path().join("repo");
15963        std::fs::create_dir_all(&repo).unwrap();
15964        init_repo(&repo);
15965        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15966        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15967        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15968        git::git(&repo, &["add", "."]).await.unwrap();
15969        git::git(&repo, &["commit", "-m", "required seed"])
15970            .await
15971            .unwrap();
15972        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15973        git::git(&repo, &["checkout", "main"]).await.unwrap();
15974        let mut runner = runner_at(RunStatus::Prep);
15975        runner.state.repo = repo.clone();
15976        runner.state.base_commit = base.clone();
15977        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15978        runner.state.config.graph.advise = false;
15979        runner.state.config.blind.commit_msg_hook = false;
15980        runner.state.sources =
15981            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15982                .await
15983                .unwrap();
15984        runner.roles.implementers = vec![spec("one"), spec("two")];
15985        runner.roles.judges = vec![spec("judge")];
15986        let label = blind::assign_labels(2, runner.state.seed)[0];
15987        let branch = runner.state.branch_for(label);
15988        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15989        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15990            .await
15991            .unwrap();
15992        runner.state.candidates.push(Candidate {
15993            index: 0,
15994            label,
15995            agent: "one".into(),
15996            branch,
15997            worktree,
15998            summary: String::new(),
15999            stat: String::new(),
16000            files: 0,
16001            commits: 0,
16002            empty: false,
16003            failed: None,
16004            verified_noop: None,
16005            duration_ms: 0,
16006            folded: false,
16007        });
16008        runner.state.save().unwrap();
16009        // Reload the checkpoint immediately before its first cherry-pick.
16010        runner.state = RunState::load(&runner.state.id).unwrap();
16011        assert_eq!(runner.state.prep_complete, Some(false));
16012        runner.prep().await.unwrap();
16013        assert_eq!(runner.state.prep_complete, Some(true));
16014        assert_eq!(runner.state.candidates.len(), 2);
16015        for candidate in &runner.state.candidates {
16016            assert_eq!(
16017                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16018                git::tree_of(&repo, &seed).await.unwrap()
16019            );
16020            assert_eq!(
16021                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
16022                "required commit"
16023            );
16024        }
16025        assert!(runner.state.worktree_root().join("judge-1").exists());
16026        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
16027        runner.prep().await.unwrap();
16028        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
16029
16030        runner.state.prep_complete = Some(false);
16031        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
16032            run: Some(format!(
16033                "test -f '{}'",
16034                tmp.path().join("dependency-ready").display()
16035            )),
16036            ..Default::default()
16037        }];
16038        assert!(runner.prep().await.is_err());
16039        assert!(runner.state.prep_failure.is_some());
16040        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
16041        runner.state = RunState::load(&runner.state.id).unwrap();
16042        assert!(
16043            runner.prep().await.is_err(),
16044            "correcting the cause alone must not auto-retry"
16045        );
16046        let failed_at = runner.state.events.last().unwrap().at;
16047        assert!(
16048            runner.retry_preparation(failed_at).is_err(),
16049            "old approvals cannot unlock the hold"
16050        );
16051        let mut task = crate::queue::Task::new(
16052            "retry".into(),
16053            "retry setup".into(),
16054            repo.clone(),
16055            crate::queue::Source::Human,
16056        );
16057        task.start(runner.state.id.clone());
16058        task.hold_machine(Some("setup failed".into()));
16059        task.release_operator("cli", None);
16060        runner
16061            .retry_preparation(task.resumption.as_ref().unwrap().at)
16062            .unwrap();
16063        runner.prep().await.unwrap();
16064        assert!(runner.state.prep_failure.is_none());
16065        assert_eq!(runner.state.prep_complete, Some(true));
16066        assert_eq!(runner.state.candidates.len(), 2);
16067        assert!(runner.state.worktree_root().join("judge-1").exists());
16068        for candidate in &runner.state.candidates {
16069            assert_eq!(
16070                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16071                git::tree_of(&repo, &seed).await.unwrap()
16072            );
16073        }
16074    }
16075
16076    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
16077        let mut heads = Vec::new();
16078        for candidate in candidates {
16079            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
16080        }
16081        heads
16082    }
16083
16084    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
16085        ask_test_home();
16086        let tmp = tempfile::tempdir().unwrap();
16087        let repo = tmp.path().join("repo");
16088        std::fs::create_dir(&repo).unwrap();
16089        init_repo(&repo);
16090        let mut runner = runner_at(RunStatus::Reviewing);
16091        runner.state.repo = repo.clone();
16092        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16093        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16094        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16095            let branch = format!("candidate-{label}");
16096            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
16097            git::worktree_add_branch(&repo, &worktree, &branch, "main")
16098                .await
16099                .unwrap();
16100            if label == 'B' {
16101                std::fs::write(worktree.join("change"), "winner").unwrap();
16102                git::rescue_commit(&worktree, "candidate change")
16103                    .await
16104                    .unwrap();
16105            }
16106            runner.state.candidates.push(Candidate {
16107                index,
16108                label,
16109                agent: "alpha".into(),
16110                branch,
16111                worktree,
16112                summary: "completed reply".into(),
16113                stat: String::new(),
16114                files: usize::from(label == 'B'),
16115                commits: usize::from(label == 'B'),
16116                empty: label == 'A',
16117                failed: None,
16118                verified_noop: None,
16119                duration_ms: 1,
16120                folded: false,
16121            });
16122        }
16123        runner.state.judge_skipped = true;
16124        runner.state.tally = Some(Tally {
16125            first_choice: BTreeMap::new(),
16126            borda: BTreeMap::new(),
16127            winner: 'B',
16128            rankings: 0,
16129            unanimous_initial: false,
16130            deliberated: false,
16131            changed_votes: 0,
16132            unanimous_final: false,
16133            tie_break: None,
16134            judges: 0,
16135            present: 0,
16136            quorum: 0,
16137            met_quorum: true,
16138            uncontested: Some("only B produced a change".into()),
16139        });
16140        runner.state.save().unwrap();
16141        (tmp, runner)
16142    }
16143
16144    #[tokio::test]
16145    async fn fold_losers_persists_discard_before_later_teardown_failure() {
16146        let (_tmp, mut runner) = preservation_resume_fixture().await;
16147        runner.state.candidates[0].folded = true;
16148        runner.state.config.graph.advise = true;
16149        runner.state.config.graph.advisors = 2;
16150        let repo = runner.state.repo.clone();
16151        let root = runner.state.worktree_root();
16152        let discarded = root.join("advisor-1");
16153        git::worktree_add_detached(&repo, &discarded, "main")
16154            .await
16155            .unwrap();
16156        std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16157        let protected = root.join("advisor-2");
16158        git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16159            .await
16160            .unwrap();
16161        std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16162        runner.state.save().unwrap();
16163
16164        assert!(runner.fold_losers().await.is_err());
16165        assert!(!discarded.exists());
16166        assert!(protected.join("keep").exists());
16167        let saved = RunState::load(&runner.state.id).unwrap();
16168        assert!(saved.events.iter().any(|event| {
16169            event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16170        }));
16171    }
16172
16173    #[tokio::test]
16174    async fn read_seat_teardown_discards_only_owned_detached_seats() {
16175        let (tmp, mut runner) = preservation_resume_fixture().await;
16176        let repo = runner.state.repo.clone();
16177        let root = runner.state.worktree_root();
16178        for name in ["advisor-3", "judge-1", "review-2"] {
16179            let path = root.join(name);
16180            git::worktree_add_detached(&repo, &path, "main")
16181                .await
16182                .unwrap();
16183            std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16184            std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16185            remove_read_seat_worktree(&mut runner.state, &path)
16186                .await
16187                .unwrap();
16188            assert!(!path.exists());
16189        }
16190        let outside = tmp.path().join("advisor-1");
16191        let unknown = root.join("unknown");
16192        let attached = root.join("advisor-4");
16193        git::worktree_add_detached(&repo, &outside, "main")
16194            .await
16195            .unwrap();
16196        git::worktree_add_detached(&repo, &unknown, "main")
16197            .await
16198            .unwrap();
16199        git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16200            .await
16201            .unwrap();
16202        let foreign = tmp.path().join("foreign");
16203        std::fs::create_dir(&foreign).unwrap();
16204        init_repo(&foreign);
16205        let wrong_repo = root.join("judge-2");
16206        git::worktree_add_detached(&foreign, &wrong_repo, "main")
16207            .await
16208            .unwrap();
16209        let candidate = runner.state.candidates[0].worktree.clone();
16210        for path in [outside, unknown, attached, wrong_repo, candidate] {
16211            std::fs::write(path.join("uncommitted"), "keep").unwrap();
16212            assert!(
16213                remove_read_seat_worktree(&mut runner.state, &path)
16214                    .await
16215                    .is_err()
16216            );
16217            assert_eq!(
16218                std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16219                "keep"
16220            );
16221        }
16222    }
16223
16224    #[tokio::test]
16225    #[cfg(unix)]
16226    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16227        for (solo, failed_checkpoint) in
16228            [(true, false), (false, false), (true, true), (false, true)]
16229        {
16230            let (_tmp, mut runner) = preservation_resume_fixture().await;
16231            if solo {
16232                runner.state.candidates.retain(|c| c.label == 'A');
16233            }
16234            runner.state.candidates[0].empty = false;
16235            runner.state.candidates[0].summary.clear();
16236            runner.state.tally = None;
16237            runner.state.judge_skipped = false;
16238            runner.state.status = RunStatus::Implementing;
16239            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16240            let mut implementer = spec("alpha");
16241            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16242            runner.roles.implementers = vec![implementer];
16243            let patch = runner.state.dir().join("artifacts/cand-A.patch");
16244            if failed_checkpoint {
16245                std::fs::create_dir_all(&patch).unwrap();
16246            }
16247            runner.state.save().unwrap();
16248            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16249            drop(pressure);
16250            runner.state = RunState::load(&runner.state.id).unwrap();
16251            assert_eq!(runner.state.status, RunStatus::Blocked);
16252            assert!(runner.state.interrupted_implementations.contains(&'A'));
16253            assert_eq!(runner.state.candidates[0].commits, 0);
16254            assert!(!runner.state.candidates[0].empty);
16255            assert!(runner.state.candidates[0].failed.is_none());
16256            if failed_checkpoint {
16257                assert!(runner.state.candidate_preservation.contains_key(&'A'));
16258                std::fs::remove_dir(patch).unwrap();
16259            }
16260            runner.state.resource_block = None;
16261            runner.state.config.disk.min_free_bytes = 0;
16262            runner.roles.implementers[0].command = vec![
16263                "sh".into(),
16264                "-c".into(),
16265                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16266            ];
16267            runner.implement().await.unwrap();
16268            assert!(runner.state.interrupted_implementations.is_empty());
16269            assert!(runner.state.candidate_preservation.is_empty());
16270            assert_eq!(runner.state.candidates[0].commits, 1);
16271            assert!(!runner.state.candidates[0].empty);
16272            assert!(
16273                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16274                    .unwrap()
16275                    .contains("retried empty interrupted seat")
16276            );
16277            if !solo {
16278                assert_eq!(runner.state.candidates[1].commits, 1);
16279            }
16280        }
16281    }
16282
16283    #[tokio::test]
16284    #[cfg(unix)]
16285    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16286        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16287            let (_tmp, mut runner) = preservation_resume_fixture().await;
16288            runner.state.candidates.retain(|c| c.label == 'A');
16289            runner.state.candidates[0].empty = false;
16290            runner.state.candidates[0].summary.clear();
16291            runner.state.tally = None;
16292            runner.state.judge_skipped = false;
16293            runner.state.status = RunStatus::Implementing;
16294            // Force only the real post-spawn watchdog, without failing the
16295            // separate admission check. Existing edits predate its measurement.
16296            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16297            let wt = runner.state.candidates[0].worktree.clone();
16298            std::fs::write(
16299                wt.join("interrupted-source"),
16300                "retain implementation work\n",
16301            )
16302            .unwrap();
16303            if committed {
16304                git::rescue_commit(&wt, "implementation checkpoint")
16305                    .await
16306                    .unwrap();
16307            }
16308            let lock = git::git(
16309                &wt,
16310                &[
16311                    "rev-parse",
16312                    "--path-format=absolute",
16313                    "--git-path",
16314                    "index.lock",
16315                ],
16316            )
16317            .await
16318            .unwrap();
16319            if block_rescue {
16320                std::fs::write(&lock, "fixture-owned lock").unwrap();
16321            }
16322            let mut implementer = spec("alpha");
16323            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16324            runner.roles.implementers = vec![implementer];
16325            runner.state.save().unwrap();
16326            let result = runner.implement().await;
16327            drop(pressure);
16328            assert_eq!(result.is_err(), block_rescue);
16329            assert_eq!(
16330                runner.state.resource_block.as_ref().unwrap().kind,
16331                "disk pressure"
16332            );
16333            assert!(
16334                runner
16335                    .state
16336                    .events
16337                    .iter()
16338                    .any(|e| e.message.contains("agent execution interrupted"))
16339            );
16340            assert!(
16341                runner
16342                    .state
16343                    .dir()
16344                    .join("artifacts/candidate-A-recovery/staged.patch")
16345                    .exists()
16346            );
16347            if block_rescue {
16348                let saved = RunState::load(&runner.state.id).unwrap();
16349                assert!(saved.candidate_preservation.contains_key(&'A'));
16350                assert!(!saved.candidates[0].empty);
16351                assert_eq!(
16352                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16353                    "retain implementation work\n"
16354                );
16355                std::fs::remove_file(lock).unwrap();
16356                runner.state = saved;
16357            } else {
16358                assert_eq!(runner.state.status, RunStatus::Blocked);
16359                assert_eq!(runner.state.candidates[0].commits, 1);
16360            }
16361            // Recovery must save the candidate before deciding whether another
16362            // implementation is needed. No implementer is available on retry.
16363            runner.roles.implementers.clear();
16364            runner.state.config.disk.min_free_bytes = 0;
16365            runner.state.resource_block = None;
16366            runner.implement().await.unwrap();
16367            assert!(runner.state.candidate_preservation.is_empty());
16368            assert_eq!(runner.state.candidates[0].commits, 1);
16369            assert!(!runner.state.candidates[0].empty);
16370            assert!(git::is_clean(&wt).await.unwrap());
16371            assert!(
16372                crate::run::read_artifact(&runner.state, "cand-A.patch")
16373                    .unwrap()
16374                    .contains("retain implementation work")
16375            );
16376        }
16377    }
16378
16379    #[tokio::test]
16380    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16381        let (_tmp, mut runner) = preservation_resume_fixture().await;
16382        // Schema 15 already confirmed the unfolded no-change result.
16383        runner.implement().await.unwrap();
16384        assert!(runner.state.candidate_preservation.is_empty());
16385        assert!(
16386            !runner
16387                .state
16388                .dir()
16389                .join("artifacts/candidate-A-recovery")
16390                .exists()
16391        );
16392        runner.fold_losers().await.unwrap();
16393        assert!(!runner.state.candidates[0].worktree.exists());
16394        for schema in [14, crate::run::SCHEMA] {
16395            runner.state.schema = schema;
16396            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16397            runner.state.save().unwrap();
16398            runner.state = RunState::load(&runner.state.id).unwrap();
16399            runner.implement().await.unwrap();
16400            assert!(runner.state.candidate_preservation.is_empty());
16401            assert!(runner.state.candidates[0].folded);
16402            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16403        }
16404    }
16405
16406    #[tokio::test]
16407    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16408        let (_tmp, mut runner) = preservation_resume_fixture().await;
16409        runner.state.candidates.retain(|c| c.label == 'A');
16410        runner.state.tally = None;
16411        runner.state.judge_skipped = false;
16412        let candidate = &mut runner.state.candidates[0];
16413        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16414        candidate.failed = Some("deployment validation unavailable".into());
16415        std::fs::write(
16416            candidate.worktree.join("repair"),
16417            "future launcher policy\n",
16418        )
16419        .unwrap();
16420        runner
16421            .state
16422            .candidate_preservation
16423            .insert('A', "awaiting persistence".into());
16424        runner.persist_candidates().await.unwrap();
16425        let candidate = &runner.state.candidates[0];
16426        assert!(!candidate.empty);
16427        assert_eq!(candidate.commits, 1);
16428        assert_eq!(candidate.files, 1);
16429        assert!(candidate.failed.is_none());
16430        assert!(candidate.summary.contains("EPERM"));
16431        assert!(
16432            crate::run::read_artifact(&runner.state, "cand-A.patch")
16433                .unwrap()
16434                .contains("future launcher policy")
16435        );
16436        assert!(runner.state.candidate_preservation.is_empty());
16437    }
16438
16439    #[tokio::test]
16440    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16441        let (_tmp, mut runner) = preservation_resume_fixture().await;
16442        let wt = runner.state.candidates[0].worktree.clone();
16443        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16444        runner.state.schema = 14;
16445        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16446        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16447        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16448        for _ in 0..2 {
16449            assert!(runner.implement().await.is_err());
16450            assert_eq!(runner.state.status, RunStatus::Blocked);
16451            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16452            assert!(
16453                runner.state.candidates[0].empty,
16454                "old verdict must remain unusable"
16455            );
16456            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16457            assert!(runner.state.judge_skipped);
16458            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16459            assert!(
16460                crate::run::read_artifact(&runner.state, "cand-A.patch")
16461                    .unwrap()
16462                    .contains("owned edits")
16463            );
16464            assert!(wt.exists());
16465            assert_eq!(
16466                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16467                    .await
16468                    .unwrap(),
16469                1
16470            );
16471            runner.state = RunState::load(&runner.state.id).unwrap();
16472        }
16473    }
16474
16475    #[tokio::test]
16476    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16477        let (_tmp, mut runner) = preservation_resume_fixture().await;
16478        let wt = runner.state.candidates[0].worktree.clone();
16479        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16480        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16481        let index_lock = PathBuf::from(
16482            git::git(
16483                &wt,
16484                &[
16485                    "rev-parse",
16486                    "--path-format=absolute",
16487                    "--git-path",
16488                    "index.lock",
16489                ],
16490            )
16491            .await
16492            .unwrap(),
16493        );
16494        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16495        runner.state.schema = 14;
16496        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16497        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16498        assert!(runner.implement().await.is_err());
16499        assert!(runner.after_implement().is_err());
16500        assert!(runner.fold_losers().await.is_err());
16501        assert_eq!(
16502            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16503            "failed seat edits"
16504        );
16505        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16506        runner.state = RunState::load(&runner.state.id).unwrap();
16507        std::fs::remove_file(index_lock).unwrap();
16508        // Recovery finds changes that the old uncontested result never judged.
16509        assert!(runner.implement().await.is_err());
16510        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16511        assert!(runner.state.candidates[0].empty);
16512        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16513        assert!(
16514            crate::run::read_artifact(&runner.state, "cand-A.patch")
16515                .unwrap()
16516                .contains("failed seat edits")
16517        );
16518        assert!(wt.exists());
16519    }
16520
16521    #[tokio::test]
16522    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16523        let (_tmp, mut runner) = preservation_resume_fixture().await;
16524        let repo = runner.state.repo.clone();
16525        let common = git::git(
16526            &repo,
16527            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16528        )
16529        .await
16530        .unwrap();
16531        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16532        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16533        assert!(runner.fold_losers().await.is_err());
16534        assert!(!runner.state.candidates[0].worktree.exists());
16535        assert!(!runner.state.candidates[0].folded);
16536        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16537        runner.state = RunState::load(&runner.state.id).unwrap();
16538        std::fs::remove_file(branch_lock).unwrap();
16539        let judge = runner.state.worktree_root().join("judge-1");
16540        git::worktree_add_detached(&repo, &judge, "main")
16541            .await
16542            .unwrap();
16543        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16544        git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16545            .await
16546            .unwrap();
16547        runner.roles.judges = vec![spec("judge")];
16548        assert!(runner.fold_losers().await.is_err());
16549        runner.state = RunState::load(&runner.state.id).unwrap();
16550        assert!(
16551            runner.state.candidates[0].folded,
16552            "candidate progress saved before judge failure"
16553        );
16554        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16555        assert!(judge.join("uncommitted").exists());
16556        git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16557            .await
16558            .unwrap();
16559        runner.fold_losers().await.unwrap();
16560        assert!(!judge.exists());
16561        runner.fold_losers().await.unwrap();
16562    }
16563
16564    #[tokio::test]
16565    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16566        let (_tmp, mut runner) = preservation_resume_fixture().await;
16567        runner.state.candidates.retain(|c| c.label == 'B');
16568        runner.state.tally = None;
16569        runner.state.judge_skipped = false;
16570        runner.state.prep_complete = Some(true);
16571        runner.state.config.graph.advise = false;
16572        runner.state.config.graph.selected_implementer = Some("alpha".into());
16573        let wt = runner.state.candidates[0].worktree.clone();
16574        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16575        runner
16576            .state
16577            .candidate_preservation
16578            .insert('B', "awaiting persistence".into());
16579        let index_lock = git::git(
16580            &wt,
16581            &[
16582                "rev-parse",
16583                "--path-format=absolute",
16584                "--git-path",
16585                "index.lock",
16586            ],
16587        )
16588        .await
16589        .unwrap();
16590        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16591        runner.state.save().unwrap();
16592        assert!(runner.execute_graph().await.is_err());
16593        assert_eq!(runner.state.status, RunStatus::Blocked);
16594        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16595        std::fs::remove_file(index_lock).unwrap();
16596        // Park at the next graph boundary: this tests the real selected-seat
16597        // hold guard without dispatching reviewers or publishing anything.
16598        for _ in 0..2 {
16599            runner.state = RunState::load(&runner.state.id).unwrap();
16600            runner.pause.park();
16601            runner.execute_graph().await.unwrap();
16602            assert_eq!(runner.state.status, RunStatus::Implementing);
16603            assert!(runner.state.candidate_preservation.is_empty());
16604            assert_eq!(runner.state.candidates[0].commits, 2);
16605            assert!(runner.state.tally.is_none());
16606        }
16607        runner.implement().await.unwrap();
16608        assert_eq!(runner.state.status, RunStatus::Judging);
16609        // A genuine selected-implementer hold must still require release.
16610        runner.state.status = RunStatus::Blocked;
16611        runner.state.candidates[0].failed = Some("implementation failed".into());
16612        runner.state.save().unwrap();
16613        let error = runner.execute_graph().await.unwrap_err();
16614        assert!(
16615            error
16616                .to_string()
16617                .contains("selected implementation is held")
16618        );
16619    }
16620
16621    #[tokio::test]
16622    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16623        let (_tmp, mut runner) = preservation_resume_fixture().await;
16624        let repo = runner.state.repo.clone();
16625        let candidate = runner.state.candidates[0].clone();
16626        // Remove only the disposable fixture directory, leaving Git's linked
16627        // worktree registration and branch intact, as in a vanished checkout.
16628        let registered = git::worktree_holding(&repo, &candidate.branch)
16629            .await
16630            .unwrap()
16631            .unwrap();
16632        assert_eq!(
16633            registered.canonicalize().unwrap(),
16634            candidate.worktree.canonicalize().unwrap()
16635        );
16636        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16637        assert_eq!(
16638            git::worktree_holding(&repo, &candidate.branch)
16639                .await
16640                .unwrap(),
16641            Some(registered)
16642        );
16643        assert!(
16644            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16645            "the stale registration initially keeps the branch checked out"
16646        );
16647        runner.fold_losers().await.unwrap();
16648        assert!(runner.state.candidates[0].folded);
16649        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16650        assert_eq!(
16651            git::worktree_holding(&repo, &candidate.branch)
16652                .await
16653                .unwrap(),
16654            None
16655        );
16656        runner.state = RunState::load(&runner.state.id).unwrap();
16657        runner.fold_losers().await.unwrap();
16658    }
16659
16660    #[tokio::test]
16661    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16662        let (_tmp, mut runner) = preservation_resume_fixture().await;
16663        let repo = runner.state.repo.clone();
16664        let candidate = runner.state.candidates[0].clone();
16665        let winner = runner.state.candidates[1].clone();
16666        let registered = git::worktree_holding(&repo, &candidate.branch)
16667            .await
16668            .unwrap()
16669            .unwrap();
16670        assert_eq!(
16671            registered.canonicalize().unwrap(),
16672            candidate.worktree.canonicalize().unwrap()
16673        );
16674        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16675        assert_eq!(
16676            git::worktree_holding(&repo, &candidate.branch)
16677                .await
16678                .unwrap(),
16679            Some(registered)
16680        );
16681        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16682
16683        for _ in 0..2 {
16684            fold_run(&mut runner.state, false, &crate::run::home())
16685                .await
16686                .unwrap();
16687            runner.state = RunState::load(&runner.state.id).unwrap();
16688            assert!(runner.state.candidates[0].folded);
16689            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16690            assert_eq!(
16691                git::worktree_holding(&repo, &candidate.branch)
16692                    .await
16693                    .unwrap(),
16694                None
16695            );
16696            assert!(!runner.state.candidates[1].folded);
16697            assert!(winner.worktree.exists());
16698            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16699        }
16700    }
16701    #[tokio::test]
16702    async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16703        for sidecar in [false, true] {
16704            let tmp = tempfile::tempdir().unwrap();
16705            let home = tmp.path().join("home");
16706            let repo = tmp.path().join("repo");
16707            std::fs::create_dir(&repo).unwrap();
16708            init_repo(&repo);
16709            let mut state = runner_at(RunStatus::Superseded).state;
16710            state.repo = repo.clone();
16711            state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16712            state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16713            state.driver_exited = true;
16714            state.reserve_under(&home).unwrap();
16715            git::git(
16716                &repo,
16717                &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16718            )
16719            .await
16720            .unwrap();
16721            let wt = state.worktree_root().join("cand-A");
16722            git::worktree_add_branch(&repo, &wt, "candidate", "main")
16723                .await
16724                .unwrap();
16725            std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16726            git::commit_all(&wt, "fixture: independent source")
16727                .await
16728                .unwrap();
16729            let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16730            state.candidates.push(Candidate {
16731                index: 0,
16732                label: 'A',
16733                agent: "fixture".into(),
16734                branch: "candidate".into(),
16735                worktree: wt.clone(),
16736                summary: String::new(),
16737                stat: String::new(),
16738                files: 0,
16739                commits: 0,
16740                empty: false,
16741                failed: None,
16742                verified_noop: None,
16743                duration_ms: 0,
16744                folded: false,
16745            });
16746            let mut task = crate::queue::Task::new(
16747                "task".into(),
16748                "task".into(),
16749                repo.clone(),
16750                crate::queue::Source::Human,
16751            );
16752            task.runs.push(state.id.clone());
16753            crate::queue::Queue::at(home.join("queue"))
16754                .put(&mut task)
16755                .unwrap();
16756            if sidecar {
16757                crate::pr_recovery::retain_sources(&task, &home).unwrap();
16758            }
16759            state.save_under(&home).unwrap();
16760            let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
16761            assert!(
16762                fold_run(&mut state, true, &home)
16763                    .await
16764                    .unwrap_err()
16765                    .to_string()
16766                    .contains(&reason)
16767            );
16768            let mut disk = state.config.disk.clone();
16769            disk.fold_grace_secs = 0;
16770            assert_eq!(
16771                crate::clean::fold_due(
16772                    &home.join("runs"),
16773                    &home,
16774                    &state.worktree_root(),
16775                    &disk,
16776                    Timestamp::now() + std::time::Duration::from_secs(10),
16777                )
16778                .await
16779                .unwrap(),
16780                (0, 0)
16781            );
16782            assert!(wt.exists());
16783            state.status = RunStatus::Blocked;
16784            state.save_under(&home).unwrap();
16785            assert!(
16786                fold_run_overriding_recovery_retention(&mut state, true, &home)
16787                    .await
16788                    .is_err()
16789            );
16790            state.status = RunStatus::Superseded;
16791            state.driver_exited = false;
16792            state.driver_claimed = None;
16793            state.save_under(&home).unwrap();
16794            assert!(
16795                fold_run_overriding_recovery_retention(&mut state, true, &home)
16796                    .await
16797                    .is_err()
16798            );
16799            state.driver_pid = Some(std::process::id());
16800            state.driver_started_at = crate::proc::process_started_at(std::process::id());
16801            state.save_under(&home).unwrap();
16802            assert!(
16803                fold_run_overriding_recovery_retention(&mut state, true, &home)
16804                    .await
16805                    .is_err()
16806            );
16807            state.driver_exited = true;
16808            state.driver_pid = None;
16809            state.driver_started_at = None;
16810            state.save_under(&home).unwrap();
16811            let lease = crate::driver::maintenance(&home, &state.id).unwrap();
16812            assert!(
16813                fold_run_overriding_recovery_retention(&mut state, true, &home)
16814                    .await
16815                    .is_err()
16816            );
16817            drop(lease);
16818            #[cfg(unix)]
16819            {
16820                let output = crate::retention::OutputLease::take(&wt, false).unwrap();
16821                assert!(
16822                    fold_run_overriding_recovery_retention(&mut state, true, &home)
16823                        .await
16824                        .is_err()
16825                );
16826                drop(output);
16827            }
16828            let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
16829            assert!(
16830                fold_run_overriding_recovery_retention(&mut state, true, &home)
16831                    .await
16832                    .is_err()
16833            );
16834            drop(owner);
16835            let checkpoint = git::git(
16836                &wt,
16837                &[
16838                    "rev-parse",
16839                    "--path-format=absolute",
16840                    "--git-path",
16841                    "CHERRY_PICK_HEAD",
16842                ],
16843            )
16844            .await
16845            .unwrap();
16846            std::fs::write(&checkpoint, &head).unwrap();
16847            assert!(
16848                fold_run_overriding_recovery_retention(&mut state, true, &home)
16849                    .await
16850                    .is_err()
16851            );
16852            std::fs::remove_file(checkpoint).unwrap();
16853            std::fs::write(wt.join("dirty"), "keep").unwrap();
16854            assert!(
16855                fold_run_overriding_recovery_retention(&mut state, true, &home)
16856                    .await
16857                    .is_err()
16858            );
16859            assert!(wt.join("dirty").exists());
16860            std::fs::remove_file(wt.join("dirty")).unwrap();
16861            let evidence = home
16862                .join("runs")
16863                .join(&state.id)
16864                .join("artifacts/fold-evidence");
16865            std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
16866            std::fs::write(&evidence, "blocked archive").unwrap();
16867            assert!(
16868                fold_run_overriding_recovery_retention(&mut state, true, &home)
16869                    .await
16870                    .is_err()
16871            );
16872            assert!(wt.exists());
16873            assert!(
16874                !RunState::load_under(&state.id, &home)
16875                    .unwrap()
16876                    .events
16877                    .iter()
16878                    .any(|event| event.message.contains("--override-recovery-retention"))
16879            );
16880            std::fs::remove_file(&evidence).unwrap();
16881            fold_run_overriding_recovery_retention(&mut state, true, &home)
16882                .await
16883                .unwrap();
16884            assert!(!wt.exists());
16885            assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
16886            assert!(
16887                git::worktree_holding(&repo, "candidate")
16888                    .await
16889                    .unwrap()
16890                    .is_none()
16891            );
16892            assert!(
16893                std::fs::read_to_string(evidence.join(format!("{head}.patch")))
16894                    .unwrap()
16895                    .contains("retained implementation")
16896            );
16897            assert!(evidence.join(format!("{head}.index")).is_file());
16898            let saved = RunState::load_under(&state.id, &home).unwrap();
16899            assert_eq!(saved.status, RunStatus::Superseded);
16900            assert!(saved.candidates[0].folded);
16901            assert!(saved.events.iter().any(|event| {
16902                event.message.contains("--override-recovery-retention")
16903                    && event.message.contains(&state.id)
16904                    && event.message.contains(&reason)
16905            }));
16906            assert_eq!(
16907                crate::pr_recovery::explicit_sources_retained(&home, &state.id),
16908                sidecar
16909            );
16910        }
16911    }
16912
16913    #[tokio::test]
16914    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16915        ask_test_home();
16916        let tmp = tempfile::tempdir().unwrap();
16917        let repo = tmp.path().join("repo");
16918        std::fs::create_dir(&repo).unwrap();
16919        init_repo(&repo);
16920        let mut runner = runner_at(RunStatus::Implementing);
16921        runner.state.repo = repo.clone();
16922        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16923        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16924        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16925            let branch = format!("candidate-{label}");
16926            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16927            git::worktree_add_branch(&repo, &wt, &branch, "main")
16928                .await
16929                .unwrap();
16930            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16931            git::git(&wt, &["add", "README.md"]).await.unwrap();
16932            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16933            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16934            runner.state.candidates.push(Candidate {
16935                index,
16936                label,
16937                agent: "alpha".to_owned(),
16938                branch,
16939                worktree: wt,
16940                summary: "completed implementation".to_owned(),
16941                stat: String::new(),
16942                files: 0,
16943                commits: 0,
16944                empty: false,
16945                failed: None,
16946                verified_noop: None,
16947                duration_ms: 42,
16948                folded: false,
16949            });
16950            runner
16951                .state
16952                .candidate_preservation
16953                .insert(label, "awaiting persistence".to_owned());
16954        }
16955        let wt = runner.state.candidates[0].worktree.clone();
16956        let index_lock = git::git(
16957            &wt,
16958            &[
16959                "rev-parse",
16960                "--path-format=absolute",
16961                "--git-path",
16962                "index.lock",
16963            ],
16964        )
16965        .await
16966        .unwrap();
16967        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16968        runner.state.save().unwrap();
16969        assert!(runner.persist_candidates().await.is_err());
16970        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16971        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16972        assert_eq!(
16973            runner.state.candidates[1].commits, 1,
16974            "other results are still collected"
16975        );
16976        assert!(
16977            !runner.state.candidates[0].empty,
16978            "failed rescue is not no change"
16979        );
16980        assert_eq!(
16981            runner.state.candidates[0].summary,
16982            "completed implementation"
16983        );
16984        assert!(runner.after_implement().is_err());
16985        let home = tmp.path().join("fold-home");
16986        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16987        assert!(wt.join("new-file").exists());
16988        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16989        assert!(recovery.join("staged.patch").exists());
16990        assert!(recovery.join("unstaged.patch").exists());
16991        assert_eq!(
16992            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16993            "untracked edits\n"
16994        );
16995        runner.state.begin_pass();
16996        runner.state.end_pass_parked(ParkCause::Upgrade {
16997            from: "1.0".into(),
16998            to: Some("1.1".into()),
16999        });
17000        runner.state.save().unwrap();
17001        runner.state = RunState::load(&runner.state.id).unwrap();
17002        assert!(matches!(
17003            runner.state.passes.last().unwrap().end,
17004            Some(crate::run::PassEnd::Parked {
17005                cause: ParkCause::Upgrade { .. }
17006            })
17007        ));
17008        std::fs::remove_file(index_lock).unwrap();
17009        // No implementer is configured: a persistence retry cannot dispatch a seat.
17010        runner.implement().await.unwrap();
17011        assert!(runner.state.candidate_preservation.is_empty());
17012        assert!(
17013            runner
17014                .state
17015                .candidates
17016                .iter()
17017                .all(|c| c.commits == 1 && !c.empty)
17018        );
17019        assert!(git::is_clean(&wt).await.unwrap());
17020        assert!(
17021            crate::run::read_artifact(&runner.state, "cand-A.patch")
17022                .unwrap()
17023                .contains("later tracked edits")
17024        );
17025    }
17026
17027    #[tokio::test]
17028    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
17029        ask_test_home();
17030        let tmp = tempfile::tempdir().unwrap();
17031        let repo = tmp.path().join("repo");
17032        std::fs::create_dir(&repo).unwrap();
17033        init_repo(&repo);
17034        let wt = tmp.path().join("candidate");
17035        git::worktree_add_branch(&repo, &wt, "candidate", "main")
17036            .await
17037            .unwrap();
17038        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17039        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
17040        assert!(wt.join("uncommitted").exists());
17041        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17042        std::fs::remove_file(wt.join("uncommitted")).unwrap();
17043        let mut runner = runner_at(RunStatus::Implementing);
17044        runner.state.repo = repo.clone();
17045        runner.state.base_commit = "missing-base".to_owned();
17046        runner.state.candidates.push(Candidate {
17047            index: 0,
17048            label: 'A',
17049            agent: "alpha".into(),
17050            branch: "candidate".into(),
17051            worktree: wt.clone(),
17052            summary: "completed".into(),
17053            stat: String::new(),
17054            files: 0,
17055            commits: 0,
17056            empty: false,
17057            failed: None,
17058            verified_noop: None,
17059            duration_ms: 1,
17060            folded: false,
17061        });
17062        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17063        assert!(
17064            fold_run(&mut runner.state, true, &tmp.path().join("home"))
17065                .await
17066                .is_err()
17067        );
17068        assert!(!runner.state.candidates[0].folded);
17069        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17070        std::fs::remove_file(wt.join("uncommitted")).unwrap();
17071        runner
17072            .state
17073            .candidate_preservation
17074            .insert('A', "pending".into());
17075        assert!(runner.persist_candidates().await.is_err());
17076        assert!(!runner.state.candidates[0].empty);
17077        assert!(!runner.state.candidates[0].folded);
17078        assert!(wt.exists());
17079        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
17080        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
17081        std::fs::create_dir_all(&patch_path).unwrap();
17082        assert!(
17083            runner.persist_candidates().await.is_err(),
17084            "artifact failure stays pending"
17085        );
17086        assert!(runner.state.candidate_preservation.contains_key(&'A'));
17087        assert!(!runner.state.candidates[0].empty);
17088        std::fs::remove_dir(patch_path).unwrap();
17089        runner.persist_candidates().await.unwrap();
17090        assert!(
17091            runner.state.candidates[0].empty,
17092            "only successful inspection confirms no change"
17093        );
17094    }
17095
17096    fn init_repo(dir: &Path) {
17097        let run = |args: &[&str]| {
17098            let out = std::process::Command::new("git")
17099                .args(args)
17100                .current_dir(dir)
17101                .quiet()
17102                .output()
17103                .expect("spawn git");
17104            assert!(
17105                out.status.success(),
17106                "git {args:?} failed: {}",
17107                String::from_utf8_lossy(&out.stderr)
17108            );
17109        };
17110        run(&["init", "-b", "main"]);
17111        run(&["config", "user.name", "magi test"]);
17112        run(&["config", "user.email", "magi@example.com"]);
17113        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
17114        run(&["add", "-A"]);
17115        run(&["commit", "-m", "init"]);
17116    }
17117
17118    // `settle_questions` is what closes the ghost the phone showed: a run's
17119    // seat asked something, the run then ended, and nothing was left to
17120    // abandon the question it left `open`. `HOME` is a process-wide
17121    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
17122    // first time it runs in the binary — every test below still reaches the
17123    // same directory whichever call won, and each gets its own run id from
17124    // `RunState::new`, so they never collide there.
17125    fn ask_test_home() {
17126        crate::run::pin_test_home();
17127    }
17128
17129    #[test]
17130    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
17131        ask_test_home();
17132        let mut runner = runner_at(RunStatus::Prep);
17133        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17134        // A blocked temporary output path makes the terminal save fail while
17135        // preserving the already published reservation.
17136        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
17137        let error = guard
17138            .finish(
17139                &mut runner.state,
17140                Err(
17141                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17142                        .into(),
17143                ),
17144            )
17145            .unwrap_err();
17146        assert_eq!(
17147            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17148            std::io::ErrorKind::PermissionDenied
17149        );
17150        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17151        let message = format!("{error:#}");
17152        assert!(message.contains("original refusal"));
17153        assert!(!message.contains("could not save terminal"));
17154        assert_eq!(
17155            error.chain().count(),
17156            2,
17157            "only startup attribution wraps the original error"
17158        );
17159        let saved = RunState::load(&runner.state.id).unwrap();
17160        assert_eq!(saved.status, RunStatus::Blocked);
17161        assert!(!saved.startup.unwrap().complete);
17162    }
17163
17164    #[test]
17165    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17166        ask_test_home();
17167        let mut runner = runner_at(RunStatus::Prep);
17168        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17169        let id = runner.state.id.clone();
17170        drop(guard);
17171        let saved = RunState::load(&id).unwrap();
17172        assert_eq!(saved.status, RunStatus::Blocked);
17173        assert!(!saved.startup.as_ref().unwrap().complete);
17174        assert!(
17175            saved
17176                .startup
17177                .unwrap()
17178                .failure
17179                .unwrap()
17180                .contains("interrupted")
17181        );
17182        assert!(Runner::resume(&id).is_err());
17183    }
17184
17185    #[test]
17186    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17187        ask_test_home();
17188        let mut runner = runner_at(RunStatus::Prep);
17189        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17190        let error = guard
17191            .finish(
17192                &mut runner.state,
17193                Err(
17194                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17195                        .into(),
17196                ),
17197            )
17198            .unwrap_err();
17199        assert_eq!(
17200            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17201            std::io::ErrorKind::PermissionDenied
17202        );
17203        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17204        let mut task = crate::queue::Task::new(
17205            "review".into(),
17206            "criteria".into(),
17207            runner.state.repo.clone(),
17208            crate::queue::Source::Human,
17209        );
17210        task.record_startup_failure(failure.clone());
17211        task.record_startup_failure(failure.clone());
17212        assert_eq!(task.startup_failures.len(), 1);
17213        assert!(task.runs.is_empty());
17214        assert_eq!(task.attempts, 0);
17215    }
17216
17217    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17218        ask_test_home();
17219        let tmp = tempfile::tempdir().unwrap();
17220        let repo = tmp.path().join("repo");
17221        std::fs::create_dir(&repo).unwrap();
17222        init_repo(&repo);
17223        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17224        let origin = tmp.path().join("origin.git");
17225        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17226            .await
17227            .unwrap();
17228        git::git(
17229            &repo,
17230            &["remote", "add", "origin", origin.to_str().unwrap()],
17231        )
17232        .await
17233        .unwrap();
17234        git::git(&repo, &["checkout", "-b", "fix/retained"])
17235            .await
17236            .unwrap();
17237        std::fs::write(repo.join("retained"), "retained change").unwrap();
17238        git::git(&repo, &["add", "."]).await.unwrap();
17239        git::git(&repo, &["commit", "-m", "retained implementation"])
17240            .await
17241            .unwrap();
17242        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17243        git::git(&repo, &["checkout", "main"]).await.unwrap();
17244        std::fs::write(repo.join("base-change"), "new base").unwrap();
17245        git::git(&repo, &["add", "."]).await.unwrap();
17246        git::git(&repo, &["commit", "-m", "base moved"])
17247            .await
17248            .unwrap();
17249        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17250        git::git(&repo, &["checkout", "fix/retained"])
17251            .await
17252            .unwrap();
17253        let mut runner = runner_at(RunStatus::Voting);
17254        runner.state.repo = repo.clone();
17255        runner.state.base_commit = base;
17256        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17257        runner.state.config.disk.min_free_bytes = 0;
17258        runner.state.config.graph.review_rounds = 1;
17259        runner.state.config.verify.e2e = vec!["true".into()];
17260        runner.state.config.verify.gate = vec!["true".into()];
17261        runner.state.candidates = vec![Candidate {
17262            index: 0,
17263            label: 'A',
17264            agent: "fixture".into(),
17265            branch: "fix/retained".into(),
17266            worktree: repo.clone(),
17267            summary: String::new(),
17268            stat: String::new(),
17269            files: 1,
17270            commits: 1,
17271            empty: false,
17272            failed: None,
17273            verified_noop: None,
17274            duration_ms: 0,
17275            folded: false,
17276        }];
17277        runner.state.tally = Some(Tally {
17278            first_choice: BTreeMap::new(),
17279            borda: BTreeMap::new(),
17280            winner: 'A',
17281            rankings: 0,
17282            unanimous_initial: true,
17283            deliberated: false,
17284            changed_votes: 0,
17285            unanimous_final: true,
17286            tie_break: None,
17287            judges: 0,
17288            present: 0,
17289            quorum: 0,
17290            met_quorum: true,
17291            uncontested: Some("review-only".into()),
17292        });
17293        let mut task = crate::queue::Task::new(
17294            "fixture".into(),
17295            "fixture".into(),
17296            repo,
17297            crate::queue::Source::Human,
17298        );
17299        task.link_run(&runner.state.id);
17300        crate::queue::Queue::open().put(&mut task).unwrap();
17301        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17302        runner.state.base_sync = Some(BaseSync {
17303            tip: String::new(),
17304            behind: 1,
17305            attempts: BASE_SYNC_ROUNDS,
17306            conflict: None,
17307            already_in: None,
17308        });
17309        runner.sync_to_base().await.unwrap();
17310        assert_eq!(runner.state.status, RunStatus::Blocked);
17311        (tmp, runner, head, task.id)
17312    }
17313
17314    #[tokio::test]
17315    async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17316        let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17317        runner.state.status = RunStatus::Voting;
17318        runner.state.base_sync = None;
17319        runner.state.base_sync_recovery = Default::default();
17320        let scratch = runner.state.dir().join("base-sync");
17321        std::fs::create_dir_all(&scratch).unwrap();
17322        std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17323        runner.sync_to_base().await.unwrap();
17324        assert!(runner.state.resource_block.is_some());
17325        assert!(!runner.state.base_sync_blocked());
17326        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17327        assert_eq!(runner.state.base_episode_used(), 0);
17328        assert!(runner.state.base_sync_recovery.operations.is_empty());
17329        assert!(runner.state.rebase_fixes.is_empty());
17330        assert_eq!(
17331            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17332            head
17333        );
17334        std::fs::remove_dir_all(&scratch).unwrap();
17335        runner.state.resource_block = None;
17336        runner.sync_to_base().await.unwrap();
17337        assert!(!runner.state.base_sync_blocked());
17338        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17339        assert_eq!(runner.state.base_episode_used(), 1);
17340        assert!(runner.state.base_sync_recovery.operations[0].finished);
17341        assert!(runner.state.rebase_fixes.is_empty());
17342        assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17343        assert!(git::is_clean(&runner.state.repo).await.unwrap());
17344    }
17345
17346    #[cfg(unix)]
17347    #[tokio::test]
17348    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17349        for routed in [false, true] {
17350            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17351            let home = crate::run::home();
17352            let queue = crate::queue::Queue::open();
17353            let task = queue.get(&owner).unwrap();
17354            runner.state.task_instruction = Some(task.instruction.clone());
17355            runner.state.driver_exited = true;
17356            runner.state.save().unwrap();
17357            let questions = crate::ask::Questions::at(home.join("questions"));
17358            let mut q = crate::ask::Question::new(
17359                runner.state.id.clone(),
17360                "base-recovery".into(),
17361                "owner".into(),
17362                "Recover bounded base movement?".into(),
17363                String::new(),
17364                vec!["recover".into()],
17365            );
17366            q.actions.insert(
17367                "recover".into(),
17368                crate::ask::ChoiceAction::RecoverBase {
17369                    run: runner.state.id.clone(),
17370                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17371                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17372                },
17373            );
17374            q.routed_task = Some(owner.clone());
17375            q.answer(crate::ask::Answer::Choice("recover".into()))
17376                .unwrap();
17377            questions.put(&mut q).unwrap();
17378            crate::base_recovery_control::recover(
17379                &queue,
17380                &home,
17381                crate::base_recovery_control::Request {
17382                    run: runner.state.id.clone(),
17383                    head: head.clone(),
17384                    actor: "decisions".into(),
17385                    request: q.id.clone(),
17386                    decision: Some((owner.clone(), q.answered_at.unwrap())),
17387                    expected_stop: Some((
17388                        runner.state.base_sync_recovery.stop.clone().unwrap(),
17389                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
17390                    )),
17391                },
17392            )
17393            .await
17394            .unwrap();
17395            if routed {
17396                questions
17397                    .update(&q.id, |q| {
17398                        q.routed_task = Some("another-owner".into());
17399                        Ok(())
17400                    })
17401                    .unwrap();
17402            } else {
17403                let mut task = queue.get(&owner).unwrap();
17404                task.hold_manual(Some("new owner decision".into()));
17405                // Keep the generation to exercise the separate owner-decision binding.
17406                task.status = crate::queue::TaskStatus::Queued;
17407                queue.put(&mut task).unwrap();
17408            }
17409            runner.state = RunState::load(&runner.state.id).unwrap();
17410            runner.sync_to_base().await.unwrap();
17411            let approval = runner
17412                .state
17413                .base_sync_recovery
17414                .authorizations
17415                .last()
17416                .unwrap();
17417            assert!(approval.consumed);
17418            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17419                "routing"
17420            } else {
17421                "owner decision"
17422            }));
17423            assert_eq!(
17424                runner.state.base_sync.as_ref().unwrap().attempts,
17425                BASE_SYNC_ROUNDS
17426            );
17427            assert_eq!(
17428                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17429                head
17430            );
17431        }
17432    }
17433
17434    #[cfg(unix)]
17435    #[tokio::test]
17436    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17437        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17438        if std::env::var_os(CHILD).is_none() {
17439            let home = tempfile::tempdir().unwrap();
17440            let output = std::process::Command::new(std::env::current_exe().unwrap())
17441                .args([
17442                    "--exact",
17443                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17444                    "--nocapture",
17445                ])
17446                .env(CHILD, "1")
17447                .env("MAGI_HOME", home.path())
17448                .env("MAGI_WORKER_HOME", home.path())
17449                .env("MAGI_WORKER_ID", "abcdef")
17450                .output()
17451                .unwrap();
17452            assert!(
17453                output.status.success()
17454                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17455                "{}\n{}",
17456                String::from_utf8_lossy(&output.stdout),
17457                String::from_utf8_lossy(&output.stderr)
17458            );
17459            return;
17460        }
17461        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17462        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17463        let home = crate::run::home();
17464        let queue = crate::queue::Queue::open();
17465        let mut owner = queue.get(&task).unwrap();
17466        owner.release_operator("test-release", None);
17467        owner.start(runner.state.id.clone());
17468        queue.put(&mut owner).unwrap();
17469        runner.state.driver_exited = true;
17470        runner.state.task_instruction = Some(owner.instruction.clone());
17471        runner.state.save().unwrap();
17472        let _claim = queue.claim(&task).unwrap();
17473        let reserve = |id: &str| {
17474            let directory = home.join("workers/jobs").join(id);
17475            std::fs::create_dir_all(&directory).unwrap();
17476            let job = crate::worker::Job {
17477                id: id.into(),
17478                sequence: 1,
17479                kind: "task".into(),
17480                key: format!("task:{task}"),
17481                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17482                generation: "fixture-generation".into(),
17483                version: None,
17484                status: crate::worker::Status::Running,
17485                result: None,
17486                pid: Some(std::process::id()),
17487                created_at: 0,
17488                finished_at: None,
17489            };
17490            std::fs::write(
17491                directory.join("job.json"),
17492                serde_json::to_vec(&job).unwrap(),
17493            )
17494            .unwrap();
17495            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17496        };
17497        let _own = reserve("abcdef");
17498        let other = reserve("fedcba");
17499        assert!(
17500            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17501                .await
17502                .is_err()
17503        );
17504        assert!(
17505            RunState::load(&runner.state.id)
17506                .unwrap()
17507                .base_sync_recovery
17508                .authorizations
17509                .is_empty()
17510        );
17511        drop(other);
17512        assert!(crate::daemon::is_working_on(
17513            &home,
17514            &runner.state.id,
17515            Timestamp::now()
17516        ));
17517        assert!(
17518            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17519        );
17520        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17521            .await
17522            .unwrap();
17523        assert!(
17524            RunState::load(&runner.state.id)
17525                .unwrap()
17526                .base_recovery_pending()
17527        );
17528    }
17529
17530    #[cfg(unix)]
17531    #[tokio::test]
17532    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17533        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17534        let home = crate::run::home();
17535        runner.state.driver_pid = Some(std::process::id());
17536        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17537        runner.state.driver_exited = false;
17538        runner.state.begin_pass();
17539        runner.state.save().unwrap();
17540        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17541        driver.record(&runner.state).unwrap();
17542        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17543        // Preserve the durable claim as if the process died before Drop saved.
17544        drop(driver);
17545        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17546        assert_eq!(
17547            runner.state.liveness_under(&home, false),
17548            crate::run::Liveness::Dead
17549        );
17550        let queue = crate::queue::Queue::open();
17551        let _claim = queue.claim(&task).unwrap();
17552        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17553            .await
17554            .unwrap();
17555        let disk = RunState::load(&runner.state.id).unwrap();
17556        assert!(!disk.driver_exited);
17557        assert!(disk.base_recovery_pending());
17558    }
17559
17560    #[cfg(unix)]
17561    #[tokio::test]
17562    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17563        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17564        if std::env::var_os(CHILD).is_none() {
17565            use std::os::unix::fs::PermissionsExt;
17566            let tmp = tempfile::tempdir().unwrap();
17567            let gh = tmp.path().join("gh");
17568            std::fs::write(&gh, r#"#!/bin/sh
17569case "$1 $2" in
17570  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17571  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17572  *) exit 2 ;;
17573esac
17574"#).unwrap();
17575            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17576            let mut paths = vec![tmp.path().to_path_buf()];
17577            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17578            let output = std::process::Command::new(std::env::current_exe().unwrap())
17579                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17580                .env(CHILD, "1")
17581                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17582                .env("PATH", std::env::join_paths(paths).unwrap())
17583                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17584                .output().unwrap();
17585            assert!(
17586                output.status.success()
17587                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17588                "{}\n{}",
17589                String::from_utf8_lossy(&output.stdout),
17590                String::from_utf8_lossy(&output.stderr)
17591            );
17592            return;
17593        }
17594        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17595        let repo = runner.state.repo.clone();
17596        git::git(&repo, &["checkout", "main"]).await.unwrap();
17597        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17598        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17599        git::git(&repo, &["checkout", "fix/retained"])
17600            .await
17601            .unwrap();
17602        runner.state.pr = Some(crate::run::PrRecord {
17603            url: "https://example.test/project/pull/1".into(),
17604            number: 1,
17605            state: "open".into(),
17606            checks: "unknown".into(),
17607            round: 0,
17608            rounds: 1,
17609            red_at_merge: Vec::new(),
17610        });
17611        runner.sync_to_base().await.unwrap();
17612        assert_eq!(runner.state.status, RunStatus::Blocked);
17613        assert_eq!(
17614            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17615            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17616        );
17617        assert!(runner.state.base_sync_closure_pending());
17618        assert!(crate::handover::can_resume(&runner.state));
17619        let queue = crate::queue::Queue::open();
17620        assert!(!crate::base_sync_recovery::held_task(
17621            &queue,
17622            &queue.get(&task).unwrap()
17623        ));
17624        assert!(runner.state.authorize_base_sync(&task).is_err());
17625        // Legacy closure stops have no typed stop but must retain the checkpoint.
17626        runner.state.base_sync_recovery.stop = None;
17627        runner.state.status = RunStatus::Voting;
17628        runner.state.gate_ran = true;
17629        runner.state.config.graph.review_rounds = 0;
17630        runner.state.save().unwrap();
17631        assert!(crate::handover::can_resume(&runner.state));
17632        runner.state = RunState::load(&runner.state.id).unwrap();
17633        runner.execute_graph().await.unwrap();
17634        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17635        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17636        assert_eq!(
17637            runner.state.base_sync.as_ref().unwrap().attempts,
17638            BASE_SYNC_ROUNDS
17639        );
17640        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17641        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17642        assert!(runner.state.reviews.is_empty());
17643        assert!(runner.state.gate.is_empty());
17644    }
17645
17646    #[cfg(unix)]
17647    #[tokio::test]
17648    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17649        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17650        runner.state.driver_exited = true;
17651        runner.state.save().unwrap();
17652        let home = crate::run::home();
17653        let queue = crate::queue::Queue::open();
17654        let _claim = queue.claim(&task).unwrap();
17655        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17656        assert!(
17657            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17658                .await
17659                .is_err()
17660        );
17661        drop(lease);
17662        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17663        assert!(
17664            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17665                .await
17666                .is_err()
17667        );
17668        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17669        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17670            .await
17671            .unwrap();
17672        let disk = RunState::load(&runner.state.id).unwrap();
17673        assert!(disk.base_recovery_pending());
17674        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17675        assert!(
17676            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17677                .await
17678                .is_err()
17679        );
17680        assert_eq!(
17681            RunState::load(&runner.state.id)
17682                .unwrap()
17683                .base_sync_recovery
17684                .authorizations
17685                .len(),
17686            1
17687        );
17688        let mut owner = queue.get(&task).unwrap();
17689        owner.runs.push("20260101-000000-ffff".into());
17690        queue.put(&mut owner).unwrap();
17691        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17692            .await
17693            .unwrap_err();
17694        assert!(error.to_string().contains("run task ownership changed"));
17695    }
17696
17697    #[cfg(unix)]
17698    #[tokio::test]
17699    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17700        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17701        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17702        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17703        legacy["status"] = serde_json::json!("voting");
17704        runner.state = serde_json::from_value(legacy).unwrap();
17705        runner.state.save().unwrap();
17706        for _ in 0..3 {
17707            runner.execute_graph().await.unwrap();
17708            assert_eq!(runner.state.status, RunStatus::Blocked);
17709            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17710            assert!(!crate::handover::can_resume(&runner.state));
17711            assert_eq!(
17712                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17713                head
17714            );
17715            assert!(runner.state.reviews.is_empty());
17716            assert!(!runner.state.gate_ran);
17717        }
17718        let notice = crate::notices::run_ended(&runner.state).unwrap();
17719        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17720        runner.state.begin_pass();
17721        runner.state.status = RunStatus::Voting;
17722        runner.deliberate().await.unwrap();
17723        runner.vote().await.unwrap();
17724        runner.finish_after_tally().await.unwrap();
17725        assert_eq!(runner.state.status, RunStatus::Blocked);
17726        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17727        assert_eq!(
17728            crate::notices::run_ended(&runner.state).unwrap().since,
17729            notice.since
17730        );
17731    }
17732
17733    #[cfg(unix)]
17734    #[tokio::test]
17735    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17736        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17737        for detail in [
17738            "fix/retained conflicts with origin/main and did not rebase: conflict",
17739            "rebased fix/retained locally but origin refused the push",
17740            "unknown failure",
17741        ] {
17742            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17743            runner.state.record_base_sync_stop(&head);
17744            assert!(runner.state.authorize_base_sync(&task).is_err());
17745            runner.state.status = RunStatus::Voting;
17746            runner.execute_graph().await.unwrap();
17747            assert_eq!(runner.state.status, RunStatus::Blocked);
17748            assert!(!crate::handover::can_resume(&runner.state));
17749            assert_eq!(
17750                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17751                head
17752            );
17753        }
17754    }
17755
17756    #[cfg(unix)]
17757    #[tokio::test]
17758    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17759        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17760        runner.state.authorize_base_sync(&task).unwrap();
17761        assert!(runner.state.consume_base_recovery(&head));
17762        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17763        runner.state.reserve_base_operation(&head, &tip);
17764        runner.state.save().unwrap();
17765        runner.state = RunState::load(&runner.state.id).unwrap();
17766        assert!(!runner.state.consume_base_recovery(&head));
17767        runner.sync_to_base().await.unwrap();
17768        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17769        assert_eq!(runner.state.base_episode_used(), 1);
17770        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17771        assert!(
17772            runner
17773                .state
17774                .base_sync_recovery
17775                .operations
17776                .last()
17777                .unwrap()
17778                .finished
17779        );
17780        assert!(!runner.state.base_sync_blocked());
17781        runner.sync_to_base().await.unwrap();
17782        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17783    }
17784
17785    #[cfg(unix)]
17786    #[tokio::test]
17787    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17788        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17789        // Both historical diagnostic spellings retain an exhaustion identity.
17790        let original = runner
17791            .state
17792            .base_sync
17793            .as_ref()
17794            .unwrap()
17795            .conflict
17796            .clone()
17797            .unwrap();
17798        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17799            original
17800                .replace("synchronization(s)", "rebase(s)")
17801                .replace("synchronizing", "rebasing"),
17802        );
17803        runner.state.base_sync_recovery = Default::default();
17804        runner.state.record_base_sync_stop(&head);
17805        assert_eq!(
17806            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17807            crate::base_sync_recovery::StopKind::Exhausted
17808        );
17809        runner.state.authorize_base_sync(&task).unwrap();
17810        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17811        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17812            .as_object_mut()
17813            .unwrap();
17814        auth.remove("interval");
17815        auth.remove("scope");
17816        runner.state = serde_json::from_value(legacy).unwrap();
17817        assert!(runner.state.consume_base_recovery(&head));
17818        assert_eq!(runner.state.base_episode_used(), 3);
17819        runner.sync_to_base().await.unwrap();
17820        assert_eq!(runner.state.base_episode_used(), 4);
17821        runner.state.resource_block = Some(crate::run::ResourceBlock {
17822            node: "gate".into(),
17823            kind: "cache".into(),
17824            detail: "legacy wait".into(),
17825            at: Timestamp::now(),
17826            cause: crate::run::ResourceBlockCause::Unrecorded,
17827        });
17828        runner.state.save().unwrap();
17829        assert!(
17830            runner
17831                .state
17832                .base_sync_recovery
17833                .episodes
17834                .last()
17835                .unwrap()
17836                .boundary
17837                .is_none()
17838        );
17839        runner.state.resource_block = None;
17840        advance_base_fixture(&runner).await;
17841        runner.sync_to_base().await.unwrap();
17842        assert!(runner.state.base_sync_blocked());
17843        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17844    }
17845
17846    #[cfg(unix)]
17847    async fn advance_base_fixture(runner: &Runner) {
17848        let repo = &runner.state.repo;
17849        for args in [
17850            vec!["checkout", "main"],
17851            vec!["commit", "--allow-empty", "-m", "base advanced"],
17852            vec!["push", "origin", "main"],
17853            vec!["checkout", "fix/retained"],
17854        ] {
17855            git::git(repo, &args).await.unwrap();
17856        }
17857    }
17858
17859    #[cfg(unix)]
17860    fn fresh_base_interval_fixture(runner: &mut Runner) {
17861        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17862        runner.state.base_sync.as_mut().unwrap().conflict = None;
17863        runner.state.base_sync_recovery = Default::default();
17864        runner.state.status = RunStatus::Voting;
17865    }
17866
17867    #[cfg(unix)]
17868    #[tokio::test]
17869    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17870        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17871        fresh_base_interval_fixture(&mut runner);
17872        runner.state.begin_pass();
17873        runner.sync_to_base().await.unwrap();
17874        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17875        runner.state.resource_block = Some(crate::run::ResourceBlock {
17876            node: "gate".into(),
17877            kind: "build cache".into(),
17878            detail: "busy".into(),
17879            at: Timestamp::now(),
17880            cause: crate::run::ResourceBlockCause::Timeout {
17881                waited_secs: 1,
17882                limit_secs: 1,
17883            },
17884        });
17885        runner.state.save().unwrap();
17886        let queue = crate::queue::Queue::open();
17887        let mut owner = queue.get(&task).unwrap();
17888        runner.state.task_instruction = Some(owner.instruction.clone());
17889        owner.release_operator("fixture-owner", None);
17890        queue.put(&mut owner).unwrap();
17891        let approval = owner.resumption.clone().unwrap();
17892        runner.state.resource_block = None;
17893        runner.state.save().unwrap();
17894        runner.state = RunState::load(&runner.state.id).unwrap();
17895        advance_base_fixture(&runner).await;
17896        runner.sync_to_base().await.unwrap();
17897        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17898        assert_eq!(runner.state.base_episode_used(), 1);
17899        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17900        assert!(
17901            runner.state.base_sync_recovery.episodes[0]
17902                .boundary
17903                .as_ref()
17904                .unwrap()
17905                .consumed
17906        );
17907        assert!(
17908            !runner
17909                .state
17910                .renew_base_episode(&task, &approval, &head, "base")
17911        );
17912        for _ in 0..3 {
17913            owner.release_operator("fixture-owner", None);
17914            queue.put(&mut owner).unwrap();
17915            runner.state = RunState::load(&runner.state.id).unwrap();
17916            runner.state.begin_pass();
17917            runner.sync_to_base().await.unwrap();
17918            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17919            assert_eq!(runner.state.base_episode_used(), 1);
17920        }
17921        runner.state.resource_block = Some(crate::run::ResourceBlock {
17922            node: "gate".into(),
17923            kind: "disk pressure".into(),
17924            detail: "free space required".into(),
17925            at: Timestamp::now(),
17926            cause: crate::run::ResourceBlockCause::Other,
17927        });
17928        runner.state.end_pass_parked(ParkCause::Interrupt {
17929            reason: "disk pressure".into(),
17930        });
17931        runner.state.save().unwrap();
17932        owner.release_operator("fixture-owner-after-disk", None);
17933        queue.put(&mut owner).unwrap();
17934        runner.state.resource_block = None;
17935        advance_base_fixture(&runner).await;
17936        runner.sync_to_base().await.unwrap();
17937        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17938        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17939        assert_eq!(runner.state.base_episode_used(), 1);
17940        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17941    }
17942
17943    #[cfg(unix)]
17944    #[tokio::test]
17945    async fn base_recovery_continuous_advancement_stops_at_four() {
17946        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17947        fresh_base_interval_fixture(&mut runner);
17948        for spent in 1..=BASE_SYNC_ROUNDS {
17949            runner.sync_to_base().await.unwrap();
17950            assert!(!runner.state.base_sync_blocked());
17951            assert_eq!(runner.state.base_episode_used(), spent);
17952            advance_base_fixture(&runner).await;
17953        }
17954        runner.sync_to_base().await.unwrap();
17955        assert_eq!(
17956            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17957            crate::base_sync_recovery::StopKind::Exhausted
17958        );
17959        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17960        assert!(stop.unverified_sync);
17961        let action = runner.state.base_sync_next_action();
17962        assert!(action.contains("never reached its gate"));
17963        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17964        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17965        runner.state = RunState::load(&runner.state.id).unwrap();
17966        runner.sync_to_base().await.unwrap();
17967        assert_eq!(runner.state.base_episode_used(), 4);
17968        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17969    }
17970
17971    #[cfg(unix)]
17972    #[tokio::test]
17973    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17974        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17975        runner.state.ensure_base_episode();
17976        // Verification progress alone is not evidence of a successful sync.
17977        runner.state.record_base_progress(&head);
17978        runner.state.base_sync_recovery.stop = None;
17979        runner.state.record_base_sync_stop(&head);
17980        assert!(
17981            !runner
17982                .state
17983                .base_sync_recovery
17984                .stop
17985                .as_ref()
17986                .unwrap()
17987                .unverified_sync
17988        );
17989
17990        runner.state.finish_base_operation(Some(&head));
17991        // A gate for another head cannot verify this synchronization.
17992        runner.state.gate_ran = true;
17993        runner.state.gate_head = Some("older-head".into());
17994        runner.state.base_sync_recovery.stop = None;
17995        runner.state.record_base_sync_stop(&head);
17996        runner.state.save().unwrap();
17997        runner.state = RunState::load(&runner.state.id).unwrap();
17998        assert!(
17999            runner
18000                .state
18001                .base_sync_recovery
18002                .stop
18003                .as_ref()
18004                .unwrap()
18005                .unverified_sync
18006        );
18007
18008        runner.state.gate_head = Some(head.clone());
18009        runner.state.base_sync_recovery.stop = None;
18010        runner.state.record_base_sync_stop(&head);
18011        assert!(
18012            !runner
18013                .state
18014                .base_sync_recovery
18015                .stop
18016                .as_ref()
18017                .unwrap()
18018                .unverified_sync
18019        );
18020        assert!(
18021            !runner
18022                .state
18023                .base_sync_next_action()
18024                .contains("never reached its gate")
18025        );
18026
18027        // Historical records cannot establish the missing synchronization evidence.
18028        let mut legacy = serde_json::to_value(&runner.state).unwrap();
18029        legacy["base_sync_recovery"]["stop"]
18030            .as_object_mut()
18031            .unwrap()
18032            .remove("unverified_sync");
18033        legacy["base_sync_recovery"]["episodes"][0]
18034            .as_object_mut()
18035            .unwrap()
18036            .remove("last_sync_head");
18037        runner.state = serde_json::from_value(legacy).unwrap();
18038        assert!(
18039            !runner
18040                .state
18041                .base_sync_recovery
18042                .stop
18043                .as_ref()
18044                .unwrap()
18045                .unverified_sync
18046        );
18047        assert!(
18048            runner.state.base_sync_recovery.episodes[0]
18049                .last_sync_head
18050                .is_none()
18051        );
18052    }
18053
18054    #[cfg(unix)]
18055    #[tokio::test]
18056    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
18057        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18058        runner.state.authorize_base_sync(&task).unwrap();
18059        assert!(
18060            runner
18061                .state
18062                .base_sync_next_action()
18063                .contains("Approval recorded")
18064        );
18065        assert!(
18066            runner
18067                .state
18068                .base_sync_next_action()
18069                .contains("4/4 in episode 0, 4 lifetime")
18070        );
18071        assert!(runner.state.consume_base_recovery(&head));
18072        assert!(
18073            runner
18074                .state
18075                .base_sync_next_action()
18076                .contains("0/4 in episode 1, 4 lifetime")
18077        );
18078    }
18079
18080    #[cfg(unix)]
18081    #[tokio::test]
18082    async fn base_recovery_changed_head_invalidates_without_spending() {
18083        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
18084        runner.state.authorize_base_sync(&task).unwrap();
18085        git::git(
18086            &runner.state.repo,
18087            &["commit", "--allow-empty", "-m", "external head change"],
18088        )
18089        .await
18090        .unwrap();
18091        runner.sync_to_base().await.unwrap();
18092        let a = runner
18093            .state
18094            .base_sync_recovery
18095            .authorizations
18096            .last()
18097            .unwrap();
18098        assert!(a.consumed);
18099        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
18100        runner.state = RunState::load(&runner.state.id).unwrap();
18101        assert!(
18102            runner
18103                .state
18104                .base_sync_next_action()
18105                .contains("Approval invalidated")
18106        );
18107        assert!(!runner.state.base_recovery_pending());
18108        assert!(runner.state.base_sync_blocked());
18109        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18110    }
18111
18112    #[cfg(unix)]
18113    #[tokio::test]
18114    async fn base_recovery_stale_fetched_base_invalidates_approval() {
18115        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18116        runner.state.driver_exited = true;
18117        runner.state.save().unwrap();
18118        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18119            .await
18120            .unwrap();
18121        runner.state = RunState::load(&runner.state.id).unwrap();
18122        advance_base_fixture(&runner).await;
18123        runner.sync_to_base().await.unwrap();
18124        let a = runner
18125            .state
18126            .base_sync_recovery
18127            .authorizations
18128            .last()
18129            .unwrap();
18130        assert!(
18131            a.consumed
18132                && a.invalidated
18133                    .as_deref()
18134                    .unwrap()
18135                    .contains("fetched base changed")
18136        );
18137        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18138        assert_eq!(
18139            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18140            head
18141        );
18142        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18143            .await
18144            .unwrap();
18145        runner.state = RunState::load(&runner.state.id).unwrap();
18146        runner.sync_to_base().await.unwrap();
18147        assert!(!runner.state.base_sync_blocked());
18148        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18149    }
18150
18151    #[cfg(unix)]
18152    #[tokio::test]
18153    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18154        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18155        runner.state.authorize_base_sync(&task).unwrap();
18156        assert!(runner.state.consume_base_recovery(&head));
18157        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18158        runner.state.reserve_base_operation(&head, &tip);
18159        runner.state.save().unwrap();
18160        let scratch = runner.state.dir().join("base-sync");
18161        assert_eq!(
18162            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18163                .await
18164                .unwrap(),
18165            crate::rebase::Rebased::Applied
18166        );
18167        runner.state = RunState::load(&runner.state.id).unwrap();
18168        runner.sync_to_base().await.unwrap();
18169        assert!(
18170            runner
18171                .state
18172                .base_sync_recovery
18173                .operations
18174                .last()
18175                .unwrap()
18176                .finished
18177        );
18178        assert_eq!(runner.state.base_episode_used(), 1);
18179        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18180        assert!(!runner.state.base_sync_blocked());
18181    }
18182
18183    #[cfg(unix)]
18184    #[tokio::test]
18185    async fn base_recovery_crash_preserves_untracked_files() {
18186        for ignored in [false, true] {
18187            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18188            runner.state.authorize_base_sync(&task).unwrap();
18189            assert!(runner.state.consume_base_recovery(&head));
18190            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18191            runner.state.reserve_base_operation(&head, &tip);
18192            runner.state.save().unwrap();
18193            let scratch = runner.state.dir().join("base-sync");
18194            assert_eq!(
18195                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18196                    .await
18197                    .unwrap(),
18198                crate::rebase::Rebased::Applied
18199            );
18200            let repo = &runner.state.repo;
18201            let result = git::rev_parse(repo, "HEAD").await.unwrap();
18202            assert_ne!(result, head);
18203            if ignored {
18204                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18205                    .await
18206                    .unwrap();
18207                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18208            }
18209            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18210            assert!(
18211                git::git_raw(repo, &["diff", "--quiet", &head])
18212                    .await
18213                    .unwrap()
18214                    .ok()
18215            );
18216            // Resume after the branch moved, before its checkout was reset.
18217            runner.state = RunState::load(&runner.state.id).unwrap();
18218            runner.sync_to_base().await.unwrap();
18219            assert!(runner.state.base_sync_blocked());
18220            assert!(!runner.state.base_sync_recovery.operations[0].finished);
18221            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18222            assert_eq!(
18223                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18224                result
18225            );
18226            assert_eq!(
18227                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18228                "retained work\n"
18229            );
18230            assert!(
18231                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18232                    .await
18233                    .unwrap()
18234                    .ok()
18235            );
18236        }
18237    }
18238
18239    #[cfg(unix)]
18240    #[tokio::test]
18241    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18242        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18243        runner.state.authorize_base_sync(&task).unwrap();
18244        assert!(runner.state.consume_base_recovery(&head));
18245        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18246        runner.state.reserve_base_operation(&head, &tip);
18247        runner.state.save().unwrap();
18248        let scratch = runner.state.dir().join("base-sync");
18249        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18250            .await
18251            .unwrap();
18252        git::sync_to_head(&runner.state.repo).await.unwrap();
18253        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18254        git::git(&runner.state.repo, &["add", "foreign"])
18255            .await
18256            .unwrap();
18257        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18258            .await
18259            .unwrap();
18260        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18261        runner.state = RunState::load(&runner.state.id).unwrap();
18262        runner.sync_to_base().await.unwrap();
18263        assert!(runner.state.base_sync_blocked());
18264        assert!(
18265            !runner
18266                .state
18267                .base_sync_recovery
18268                .operations
18269                .last()
18270                .unwrap()
18271                .finished
18272        );
18273        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18274        assert_eq!(
18275            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18276            foreign_head
18277        );
18278    }
18279
18280    #[cfg(unix)]
18281    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18282    {
18283        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18284        fresh_base_interval_fixture(&mut runner);
18285        let repo = &runner.state.repo;
18286        git::git(repo, &["checkout", "main"]).await.unwrap();
18287        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18288        git::git(repo, &["add", "retained"]).await.unwrap();
18289        git::git(repo, &["commit", "-m", "upstream retained change"])
18290            .await
18291            .unwrap();
18292        git::git(repo, &["push", "origin", "main"]).await.unwrap();
18293        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18294        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18295        let scratch = runner.state.dir().join("base-sync");
18296        assert!(matches!(
18297            git::rebase_start(repo, &scratch, "fix/retained", &tip)
18298                .await
18299                .unwrap(),
18300            git::RebaseStart::Conflicted(_)
18301        ));
18302        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18303            agent: "fixture".into(),
18304            paths: vec!["retained".into()],
18305            from: Some(head.clone()),
18306            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18307            index_outside: None,
18308            finished: false,
18309            error: None,
18310            launcher: None,
18311            infra: None,
18312            outage: None,
18313        });
18314        (tmp, runner, head, tip, scratch)
18315    }
18316
18317    #[cfg(unix)]
18318    #[tokio::test]
18319    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18320        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18321        // A resource stop from before reservations existed, with resolved
18322        // contents left for the launcher's ordinary staging continuation.
18323        std::fs::write(
18324            scratch.join("retained"),
18325            "upstream and retained resolution\n",
18326        )
18327        .unwrap();
18328        runner.state.resource_block = Some(crate::run::ResourceBlock {
18329            node: "rebase".into(),
18330            kind: "repository coordination".into(),
18331            detail: "busy".into(),
18332            at: Timestamp::now(),
18333            cause: crate::run::ResourceBlockCause::Other,
18334        });
18335        runner.state.save().unwrap();
18336        // Advance the remote without moving either checkout participating
18337        // in the standing rebase.
18338        let newer = git::git(
18339            &runner.state.repo,
18340            &[
18341                "commit-tree",
18342                "main^{tree}",
18343                "-p",
18344                "main",
18345                "-m",
18346                "base advanced during wait",
18347            ],
18348        )
18349        .await
18350        .unwrap();
18351        git::git(
18352            &runner.state.repo,
18353            &[
18354                "push",
18355                "origin",
18356                &format!("{}:refs/heads/main", newer.trim()),
18357            ],
18358        )
18359        .await
18360        .unwrap();
18361        runner.state = RunState::load(&runner.state.id).unwrap();
18362        runner.state.resource_block = None;
18363        runner.sync_to_base().await.unwrap();
18364        assert!(!runner.state.base_sync_blocked());
18365        let op = &runner.state.base_sync_recovery.operations[0];
18366        assert_eq!(op.head, head);
18367        assert_eq!(op.tip, tip);
18368        assert!(op.finished);
18369        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18370        assert_eq!(runner.state.rebase_fixes.len(), 1);
18371        let latest = git::rev_parse(&runner.state.repo, "origin/main")
18372            .await
18373            .unwrap();
18374        assert_ne!(latest, tip);
18375        runner.state = RunState::load(&runner.state.id).unwrap();
18376        runner.sync_to_base().await.unwrap();
18377        assert!(!runner.state.base_sync_blocked());
18378        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18379        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18380        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18381    }
18382
18383    #[cfg(unix)]
18384    #[tokio::test]
18385    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18386        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18387        runner.state.reserve_base_operation(&head, &tip);
18388        runner.state.save().unwrap();
18389        std::fs::write(
18390            scratch.join("retained"),
18391            "upstream and retained resolution\n",
18392        )
18393        .unwrap();
18394        git::git(&scratch, &["add", "retained"]).await.unwrap();
18395        git::git(
18396            &scratch,
18397            &["-c", "core.editor=true", "rebase", "--continue"],
18398        )
18399        .await
18400        .unwrap();
18401        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18402        assert!(
18403            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18404                .await
18405                .unwrap()
18406                .is_empty()
18407        );
18408        assert_eq!(
18409            crate::rebase::finish(
18410                &mut runner.state,
18411                &scratch,
18412                "fix/retained",
18413                &head,
18414                &tip,
18415                &["retained".into()],
18416                ""
18417            )
18418            .await
18419            .unwrap(),
18420            crate::rebase::Rebased::Applied
18421        );
18422        assert!(!scratch.exists());
18423        // Simulate losing the caller before finish_base_operation/save.
18424        runner.state = RunState::load(&runner.state.id).unwrap();
18425        let op = &runner.state.base_sync_recovery.operations[0];
18426        assert!(!op.finished);
18427        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18428        runner.sync_to_base().await.unwrap();
18429        assert!(!runner.state.base_sync_blocked());
18430        assert!(runner.state.base_sync_recovery.operations[0].finished);
18431        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18432        assert_eq!(runner.state.rebase_fixes.len(), 1);
18433        let winner = runner.state.winner().unwrap().clone();
18434        let wait = runner.repository_wait();
18435        assert!(
18436            verify_current_head(&mut runner.state, &winner, &wait)
18437                .await
18438                .unwrap()
18439        );
18440        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18441    }
18442
18443    #[cfg(unix)]
18444    #[tokio::test]
18445    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18446        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18447            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18448            runner.state.reserve_base_operation(&head, &tip);
18449            runner.state.save().unwrap();
18450            std::fs::write(
18451                scratch.join("retained"),
18452                "upstream and retained resolution\n",
18453            )
18454            .unwrap();
18455            git::git(&scratch, &["add", "retained"]).await.unwrap();
18456            git::git(
18457                &scratch,
18458                &["-c", "core.editor=true", "rebase", "--continue"],
18459            )
18460            .await
18461            .unwrap();
18462            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18463            assert!(!git::rebase_in_progress(&scratch).await);
18464            assert!(
18465                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18466                    .await
18467                    .unwrap()
18468                    .is_empty()
18469            );
18470            match damage {
18471                Some("record") => {
18472                    let path =
18473                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18474                            .await
18475                            .unwrap();
18476                    std::fs::write(scratch.join(path), "").unwrap();
18477                }
18478                Some("foreign") => {
18479                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18480                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
18481                    git::git(&scratch, &["commit", "-m", "foreign work"])
18482                        .await
18483                        .unwrap();
18484                }
18485                Some("dirty") => {
18486                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18487                }
18488                _ => {}
18489            }
18490            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18491            // Crash immediately after Git continuation, before finish runs.
18492            runner.state = RunState::load(&runner.state.id).unwrap();
18493            assert!(
18494                runner.state.base_sync_recovery.operations[0]
18495                    .validated_head
18496                    .is_none()
18497            );
18498            runner.sync_to_base().await.unwrap();
18499            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18500            assert_eq!(runner.state.base_episode_used(), 1);
18501            assert_eq!(runner.state.rebase_fixes.len(), 1);
18502            assert_eq!(
18503                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18504                retained_head
18505            );
18506            if damage.is_some() {
18507                assert!(runner.state.base_sync_blocked());
18508                assert!(scratch.exists());
18509                assert!(!runner.state.base_sync_recovery.operations[0].finished);
18510                assert!(
18511                    runner.state.base_sync_recovery.operations[0]
18512                        .validated_head
18513                        .is_none()
18514                );
18515            } else {
18516                assert!(!runner.state.base_sync_blocked());
18517                assert!(!scratch.exists());
18518                assert!(runner.state.base_sync_recovery.operations[0].finished);
18519                assert_eq!(
18520                    runner.state.base_sync_recovery.operations[0]
18521                        .validated_head
18522                        .as_deref(),
18523                    Some(result.as_str())
18524                );
18525                let winner = runner.state.winner().unwrap().clone();
18526                let wait = runner.repository_wait();
18527                assert!(
18528                    verify_current_head(&mut runner.state, &winner, &wait)
18529                        .await
18530                        .unwrap()
18531                );
18532                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18533                runner.state = RunState::load(&runner.state.id).unwrap();
18534                runner.sync_to_base().await.unwrap();
18535                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18536            }
18537        }
18538    }
18539
18540    #[cfg(unix)]
18541    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18542        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18543        // This is an ordinary gate stop, not an authorized base-budget recovery.
18544        runner.state.prep_complete = Some(true);
18545        runner.state.base_sync = None;
18546        runner.state.base_sync_recovery = Default::default();
18547        runner.state.config.graph.gate_fix_rounds = 1;
18548        let mut review = review_round(true, 0, 1, 1, false, true);
18549        review.head = head.clone();
18550        review.verified_head = Some(head.clone());
18551        runner.state.reviews = vec![review];
18552        runner.state.gate = vec![CommandOutcome {
18553            command: "old gate".into(),
18554            code: Some(1),
18555            output_tail: "old failure".into(),
18556            duration_ms: 0,
18557            resource_blocked: false,
18558        }];
18559        runner.state.gate_head = Some(head.clone());
18560        runner.state.gate_ran = true;
18561        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18562            agent: "fixture".into(),
18563            failed: runner.state.gate.clone(),
18564            notes: "spent".into(),
18565            committed: false,
18566            error: None,
18567        }];
18568        runner.state.land_approval = Some(crate::run::LandApproval {
18569            question: "old approval".into(),
18570            head: head.clone(),
18571        });
18572        runner.state.status = RunStatus::Blocked;
18573        (tmp, runner, head)
18574    }
18575
18576    #[cfg(unix)]
18577    #[tokio::test]
18578    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18579        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18580            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18581            let phases = tmp.path().join("phases");
18582            runner.state.config.verify.e2e = vec![format!(
18583                "echo e2e >> '{}'; {}",
18584                phases.display(),
18585                if e2e_ok {
18586                    "true"
18587                } else {
18588                    "echo new-e2e-failure; false"
18589                }
18590            )];
18591            runner.state.config.verify.gate = vec![format!(
18592                "echo gate >> '{}'; {}",
18593                phases.display(),
18594                if gate_ok {
18595                    "true"
18596                } else {
18597                    "echo new-gate-failure; false"
18598                }
18599            )];
18600            runner.state.save().unwrap();
18601            runner.state = RunState::load(&runner.state.id).unwrap();
18602            runner.sync_to_base().await.unwrap();
18603            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18604            assert_ne!(head, old);
18605            // Reentry after Git updated but before verification was invalidated.
18606            runner.state = RunState::load(&runner.state.id).unwrap();
18607            runner.execute_graph().await.unwrap();
18608            assert_eq!(runner.state.gate_fixes.len(), 1);
18609            assert_eq!(runner.state.reviews.len(), 1);
18610            assert_eq!(runner.state.reviews[0].head, old);
18611            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18612            assert_eq!(runner.state.verification_history.len(), 1);
18613            let history = &runner.state.verification_history[0];
18614            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18615            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18616            assert!(history.archived_at.is_some());
18617            assert_eq!(history.fixes_spent, Some(1));
18618            assert_eq!(history.gate[0].output_tail, "old failure");
18619            let archive_events: Vec<_> = runner
18620                .state
18621                .events
18622                .iter()
18623                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18624                .collect();
18625            assert_eq!(archive_events.len(), 1);
18626            assert!(archive_events[0].message.contains(&short(&old)));
18627            assert!(archive_events[0].message.contains(&short(&head)));
18628            assert!(
18629                archive_events[0]
18630                    .message
18631                    .contains("1 gate-fix round(s) already spent stay spent")
18632            );
18633            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18634            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18635            if e2e_ok {
18636                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18637                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18638            } else {
18639                assert!(!runner.state.gate_ran);
18640            }
18641            assert_eq!(
18642                runner.state.status,
18643                if e2e_ok && gate_ok {
18644                    RunStatus::Ready
18645                } else {
18646                    RunStatus::Blocked
18647                }
18648            );
18649            if !e2e_ok || !gate_ok {
18650                let cause = runner.state.verification_blocking_cause().unwrap();
18651                assert!(cause.contains(&head));
18652                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18653                // A later successful progress event must not become the cause.
18654                runner.state.event("land", "rebased successfully");
18655                let notice = crate::notices::run_ended(&runner.state).unwrap();
18656                assert!(!format!("{notice:?}").contains("rebased successfully"));
18657                assert!(runner.state.merge.is_none());
18658            }
18659            let before = std::fs::read_to_string(&phases).unwrap();
18660            runner.state.save().unwrap();
18661            runner.state = RunState::load(&runner.state.id).unwrap();
18662            runner.review_loop().await.unwrap();
18663            runner.gate().await.unwrap();
18664            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18665            assert_eq!(runner.state.verification_history.len(), 1);
18666            let history = &runner.state.verification_history[0];
18667            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18668            assert!(history.archived_at.is_some());
18669            assert_eq!(history.fixes_spent, Some(1));
18670        }
18671    }
18672
18673    #[cfg(unix)]
18674    #[tokio::test]
18675    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18676        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18677        let marker = tmp.path().join("gate-ran");
18678        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18679        runner.review_loop().await.unwrap();
18680        runner.gate().await.unwrap();
18681        assert_eq!(runner.state.status, RunStatus::Blocked);
18682        assert!(!marker.exists());
18683        assert!(runner.state.verification_history.is_empty());
18684        runner.state.gate_head = None;
18685        runner.gate().await.unwrap();
18686        assert!(marker.exists());
18687        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18688        assert_eq!(runner.state.verification_history.len(), 1);
18689        assert_eq!(runner.state.gate_fixes.len(), 1);
18690    }
18691
18692    #[cfg(unix)]
18693    #[tokio::test]
18694    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18695        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18696        runner.sync_to_base().await.unwrap();
18697        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18698        runner.state.reconcile_verification_head(&head);
18699        // Both crash-before-command and resource wait are unresolved evidence.
18700        for outcomes in [
18701            Vec::new(),
18702            vec![CommandOutcome {
18703                command: "true".into(),
18704                code: None,
18705                output_tail: "cache unavailable".into(),
18706                duration_ms: 0,
18707                resource_blocked: true,
18708            }],
18709        ] {
18710            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18711                head: head.clone(),
18712                passed: false,
18713                outcomes,
18714            });
18715            runner.state.gate_ran = false;
18716            runner.state.gate.clear();
18717            runner.state.save().unwrap();
18718            runner.state = RunState::load(&runner.state.id).unwrap();
18719            runner.merge().await.unwrap();
18720            assert!(runner.state.merge.is_none());
18721            runner.review_loop().await.unwrap();
18722            runner.gate().await.unwrap();
18723            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18724            assert!(runner.state.gate_allows_progress());
18725            assert_eq!(runner.state.gate_fixes.len(), 1);
18726            assert!(runner.state.merge.is_none());
18727        }
18728        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18729        git::git(
18730            &runner.state.repo,
18731            &["commit", "--allow-empty", "-m", "external update"],
18732        )
18733        .await
18734        .unwrap();
18735        runner.merge().await.unwrap();
18736        assert!(runner.state.merge.is_none());
18737        assert_eq!(runner.state.status, RunStatus::Blocked);
18738    }
18739
18740    #[cfg(unix)]
18741    #[tokio::test]
18742    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18743        for waiting_on_gate in [false, true] {
18744            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18745            runner.sync_to_base().await.unwrap();
18746            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18747            runner.state.reconcile_verification_head(&head);
18748            if waiting_on_gate {
18749                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18750                    head: head.clone(),
18751                    passed: true,
18752                    outcomes: Vec::new(),
18753                });
18754            }
18755            let cache = tmp.path().join("cache");
18756            runner.state.config.verify.e2e =
18757                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18758            runner.state.config.verify.gate = vec!["true".into()];
18759            runner.state.config.graph.timeout_resource_wait = 1;
18760            let owner =
18761                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18762            let guard =
18763                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18764                    crate::cache::AcquireOutcome::Acquired(g) => g,
18765                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18766                };
18767            runner.review_loop().await.unwrap();
18768            runner.gate().await.unwrap();
18769            assert!(!runner.state.gate_ran);
18770            assert_eq!(runner.state.gate_fixes.len(), 1);
18771            assert!(runner.state.resource_block.is_some());
18772            runner.merge().await.unwrap();
18773            assert!(runner.state.merge.is_none());
18774            drop(guard);
18775            runner.state = RunState::load(&runner.state.id).unwrap();
18776            // The driver clears the previous pass's resource stop on resume.
18777            runner.state.resource_block = None;
18778            runner.review_loop().await.unwrap();
18779            runner.gate().await.unwrap();
18780            assert!(runner.state.gate_allows_progress());
18781            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18782            assert_eq!(runner.state.gate_fixes.len(), 1);
18783        }
18784    }
18785
18786    #[cfg(unix)]
18787    #[tokio::test]
18788    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18789        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18790        runner.sync_to_base().await.unwrap();
18791        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18792        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18793        // Verification passes before pre_gate and fails on the mechanically changed head.
18794        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18795        runner.review_loop().await.unwrap();
18796        runner.gate().await.unwrap();
18797        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18798        assert_ne!(before, after);
18799        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18800        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18801        assert!(!runner.state.gate_allows_progress());
18802        runner.merge().await.unwrap();
18803        assert!(runner.state.merge.is_none());
18804        assert!(
18805            runner
18806                .state
18807                .verification_history
18808                .iter()
18809                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18810        );
18811
18812        // An incomplete review is still an independent hold on a changed head.
18813        runner.state.reviews[0].clean = false;
18814        runner.state.reviews[0].answered = 0;
18815        runner.state.config.verify.e2e = vec!["true".into()];
18816        runner.review_loop().await.unwrap();
18817        runner.gate().await.unwrap();
18818        runner.merge().await.unwrap();
18819        assert_eq!(runner.state.status, RunStatus::Blocked);
18820        assert!(runner.state.merge.is_none());
18821        assert!(
18822            runner
18823                .state
18824                .verification_blocking_cause()
18825                .unwrap()
18826                .contains("review blocked")
18827        );
18828    }
18829
18830    #[cfg(unix)]
18831    #[tokio::test]
18832    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18833        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18834        runner.state.gate_head = None;
18835        runner.state.config.verify.gate =
18836            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18837        runner.review_loop().await.unwrap();
18838        runner.gate().await.unwrap();
18839        assert_ne!(
18840            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18841            old
18842        );
18843        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18844        assert!(!runner.state.gate_allows_progress());
18845        assert_eq!(runner.state.status, RunStatus::Blocked);
18846        runner.merge().await.unwrap();
18847        assert!(runner.state.merge.is_none());
18848    }
18849
18850    #[cfg(unix)]
18851    #[tokio::test]
18852    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18853        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18854        let mut old_review = review_round(true, 0, 1, 1, false, true);
18855        old_review.head = head.clone();
18856        old_review.verified_head = Some(head.clone());
18857        runner.state.reviews = vec![old_review];
18858        runner.state.gate = vec![CommandOutcome {
18859            command: "old gate".into(),
18860            code: None,
18861            output_tail: "timed out after 2400s".into(),
18862            duration_ms: 2_400_000,
18863            resource_blocked: false,
18864        }];
18865        runner.state.gate_ran = true;
18866        runner.state.gate_head = Some(head.clone());
18867        runner.state.land_approval = Some(crate::run::LandApproval {
18868            question: "old-head-approval".into(),
18869            head: head.clone(),
18870        });
18871        runner.state.authorize_base_sync(&task).unwrap();
18872        runner.sync_to_base().await.unwrap();
18873        assert!(!runner.state.base_sync_blocked());
18874        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18875        assert_ne!(head, fresh);
18876        assert_eq!(runner.state.reviews.len(), 1);
18877        assert!(!runner.state.gate_ran);
18878        let saved = &runner.state.base_sync_recovery.authorizations[0];
18879        assert_eq!(saved.reviews[0].head, head);
18880        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18881        assert!(saved.gate_ran);
18882        assert!(!saved.gate[0].ok());
18883        runner.review_loop().await.unwrap();
18884        runner.gate().await.unwrap();
18885        assert_eq!(runner.state.reviews[0].head, head);
18886        assert_eq!(
18887            runner.state.reviews[0].verified_head.as_deref(),
18888            Some(head.as_str())
18889        );
18890        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18891        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18892        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18893        assert!(runner.state.gate_ran);
18894        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18895        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18896        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18897        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18898        assert_eq!(
18899            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18900            "retained change"
18901        );
18902    }
18903
18904    #[cfg(unix)]
18905    #[tokio::test]
18906    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18907        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18908        let queue = crate::queue::Queue::open();
18909        let mut owner = queue.get(&task).unwrap();
18910        owner.release_operator("test-resume", None);
18911        queue.put(&mut owner).unwrap();
18912        let generation = owner.resumption.as_ref().unwrap().generation;
18913        runner.state.task_instruction = Some(owner.instruction.clone());
18914        runner.state.driver_exited = true;
18915        runner.state.save().unwrap();
18916        let home = crate::run::home();
18917        let _claim = queue.claim(&task).unwrap();
18918        owner.resumption.as_mut().unwrap().at =
18919            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18920        queue.put(&mut owner).unwrap();
18921        assert!(
18922            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18923                .await
18924                .is_err(),
18925            "an approval from before the stop grants nothing"
18926        );
18927        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18928        queue.put(&mut owner).unwrap();
18929        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18930            .await
18931            .unwrap();
18932        assert!(
18933            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18934                .await
18935                .is_err()
18936        );
18937        runner.state = RunState::load(&runner.state.id).unwrap();
18938        assert_eq!(
18939            runner.state.base_sync_recovery.authorizations[0].generation,
18940            Some(generation)
18941        );
18942        runner.sync_to_base().await.unwrap();
18943        let repo = runner.state.repo.clone();
18944        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18945        git::git(&repo, &["checkout", "main"]).await.unwrap();
18946        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18947        git::git(&repo, &["add", "."]).await.unwrap();
18948        git::git(&repo, &["commit", "-m", "move base again"])
18949            .await
18950            .unwrap();
18951        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18952        git::git(&repo, &["checkout", "fix/retained"])
18953            .await
18954            .unwrap();
18955        runner.sync_to_base().await.unwrap();
18956        assert!(!runner.state.base_sync_blocked());
18957        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18958        assert_eq!(runner.state.base_episode_used(), 2);
18959        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18960        assert_ne!(fresh, head);
18961        assert!(!runner.state.base_recovery_pending());
18962        // Even replaying the generation with a later timestamp grants nothing.
18963        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18964        queue.put(&mut owner).unwrap();
18965        assert!(
18966            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18967                .await
18968                .is_err()
18969        );
18970        runner.state = RunState::load(&runner.state.id).unwrap();
18971        runner.sync_to_base().await.unwrap();
18972        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18973        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18974        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18975    }
18976
18977    #[cfg(unix)]
18978    #[tokio::test]
18979    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18980        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18981        runner.state.base_sync = None;
18982        runner.state.status = RunStatus::Blocked;
18983        let mut second = runner.state.candidates[0].clone();
18984        second.label = 'B';
18985        runner.state.candidates.push(second);
18986        assert!(runner.state.votes.is_empty());
18987        assert!(runner.state.deliberation.is_empty());
18988        runner.deliberate().await.unwrap();
18989        runner.vote().await.unwrap();
18990        assert_eq!(runner.state.status, RunStatus::Blocked);
18991        assert!(runner.state.votes.is_empty());
18992        assert!(runner.state.deliberation.is_empty());
18993    }
18994
18995    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18996    /// reads nothing else off it.
18997    fn runner_at(status: RunStatus) -> Runner {
18998        let mut state = RunState::new(
18999            PathBuf::from("/nonexistent/repo"),
19000            "main".to_owned(),
19001            "deadbeef".to_owned(),
19002            "task".to_owned(),
19003            Config::default(),
19004        );
19005        // These fixtures test execution flow, independent of host disk capacity.
19006        state.config.disk.min_free_bytes = 0;
19007        // These fixtures share one home and save concurrently. Short clock
19008        // IDs can collide, making an ownership test read another test's run.
19009        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
19010        state.id = format!(
19011            "20000101-000000-{:04x}",
19012            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
19013        );
19014        state.status = status;
19015        Runner {
19016            state,
19017            roles: ResolvedRoles {
19018                implementers: Vec::new(),
19019                judges: Vec::new(),
19020                reviewers: Vec::new(),
19021                fixer: None,
19022                conductor: conductor(),
19023                implementer_roster: Vec::new(),
19024                judge_roster: Vec::new(),
19025                reviewer_roster: Vec::new(),
19026            },
19027            sem: Arc::new(Semaphore::new(1)),
19028            pause: Pause::new(),
19029            interrupt: Pause::new(),
19030        }
19031    }
19032
19033    #[tokio::test]
19034    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
19035        crate::run::pin_test_home();
19036        let temp = tempfile::tempdir().unwrap();
19037        let root = temp.path().canonicalize().unwrap();
19038        let repo = root.join("repo");
19039        std::fs::create_dir(&repo).unwrap();
19040        init_repo(&repo);
19041        git::git(&repo, &["config", "core.autocrlf", "false"])
19042            .await
19043            .unwrap();
19044        let mut runner = runner_at(RunStatus::Blocked);
19045        runner.state.repo = repo.clone();
19046        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
19047        runner.state.config.agents = vec![conductor()];
19048        runner.state.config.graph.selected_implementer = Some("conductor".into());
19049        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
19050        let (shell, command, setup_output) = if cfg!(windows) {
19051            (
19052                vec!["cmd".into(), "/C".into()],
19053                "echo ready>seat-ready",
19054                "ready\r\n",
19055            )
19056        } else {
19057            (
19058                vec!["sh".into(), "-c".into()],
19059                "printf ready > seat-ready",
19060                "ready",
19061            )
19062        };
19063        runner.state.config.verify.shell = Some(shell);
19064        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
19065            run: Some(command.into()),
19066            ..Default::default()
19067        }];
19068        runner.state.save().unwrap();
19069        let path = runner.state.worktree_root().join("judge-1");
19070        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
19071            .await
19072            .unwrap();
19073        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
19074        git::git(&path, &["add", "."]).await.unwrap();
19075        let tree = git::git(&path, &["write-tree"]).await.unwrap();
19076        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
19077            .await
19078            .unwrap();
19079        git::git(&path, &["checkout", "--detach", &head])
19080            .await
19081            .unwrap();
19082        let evidence = crate::fold_evidence::snapshot(
19083            &runner.state,
19084            &crate::run::home().canonicalize().unwrap(),
19085            &head,
19086            64 * 1024 * 1024,
19087        )
19088        .await
19089        .unwrap();
19090        let mut contents = BTreeMap::new();
19091        contents.insert(
19092            repo.join("logs/HEAD"),
19093            format!("{head} {head} fixture\n").into_bytes(),
19094        );
19095        let record = serde_json::json!({
19096            "version":1, "run":runner.state.id, "source":path, "head":head,
19097            "evidence":evidence, "admin":{}, "admin_contents":contents,
19098            "history":{head.clone():evidence}, "phase":"evicted"
19099        });
19100        std::fs::write(
19101            runner.state.dir().join("artifacts/eviction-judge-1.json"),
19102            serde_json::to_vec(&record).unwrap(),
19103        )
19104        .unwrap();
19105        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
19106        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
19107            .await
19108            .unwrap();
19109        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
19110        assert!(
19111            !git::git_raw(&repo, &["cat-file", "-e", &head])
19112                .await
19113                .unwrap()
19114                .ok()
19115        );
19116        let mut resumed = Runner::resume(&runner.state.id).unwrap();
19117        let error = resumed.execute().await.unwrap_err();
19118        assert!(
19119            error
19120                .to_string()
19121                .contains("selected implementation is held"),
19122            "{error:#}"
19123        );
19124        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
19125        assert_eq!(
19126            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
19127            setup_output
19128        );
19129        assert_eq!(
19130            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
19131            "unrelated seat"
19132        );
19133        let error = resumed.execute().await.unwrap_err();
19134        assert!(
19135            error
19136                .to_string()
19137                .contains("selected implementation is held"),
19138            "{error:#}"
19139        );
19140        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
19141        git::git(&path, &["add", "."]).await.unwrap();
19142        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19143        let next = git::git(
19144            &path,
19145            &[
19146                "commit-tree",
19147                &next_tree,
19148                "-p",
19149                &head,
19150                "-m",
19151                "next review checkpoint",
19152            ],
19153        )
19154        .await
19155        .unwrap();
19156        git::git(&path, &["checkout", "--detach", &next])
19157            .await
19158            .unwrap();
19159        resumed.state.reviews.push(
19160            serde_json::from_value(serde_json::json!({
19161                "round":1, "head":next, "reviews":[]
19162            }))
19163            .unwrap(),
19164        );
19165        resumed.state.save().unwrap();
19166        let error = resumed.execute().await.unwrap_err();
19167        assert!(
19168            error
19169                .to_string()
19170                .contains("selected implementation is held"),
19171            "{error:#}"
19172        );
19173        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19174    }
19175
19176    #[tokio::test]
19177    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19178        crate::run::pin_test_home();
19179        let mut runner = runner_at(RunStatus::Gating);
19180        runner.state.save().unwrap();
19181        let id = runner.state.id.clone();
19182        let passes = runner.state.passes.len();
19183        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19184        let mut claiming = Box::pin(runner.claim_driver());
19185        std::future::poll_fn(|cx| {
19186            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19187            std::task::Poll::Ready(())
19188        })
19189        .await;
19190        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19191        drop(owner);
19192        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19193            .await
19194            .unwrap()
19195            .unwrap();
19196        assert_eq!(execution.runner.state.passes.len(), passes + 1);
19197        assert!(execution.runner.state.resource_block.is_none());
19198        assert_eq!(
19199            execution.runner.state.liveness(false),
19200            crate::run::Liveness::Live
19201        );
19202        drop(execution);
19203        assert!(RunState::load(&id).unwrap().driver_exited);
19204    }
19205
19206    #[tokio::test]
19207    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19208        crate::run::pin_test_home();
19209        let mut runner = runner_at(RunStatus::Gating);
19210        runner.state.save().unwrap();
19211        // run.json can commit, but the subsequent lease metadata write fails.
19212        // Cleanup must run after releasing the short ownership lock.
19213        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19214        assert!(runner.claim_driver().await.is_err());
19215        assert!(runner.state.driver_exited);
19216        assert_eq!(runner.state.driver_pid, None);
19217        let disk = RunState::load(&runner.state.id).unwrap();
19218        assert!(disk.driver_exited);
19219        assert_eq!(disk.driver_pid, None);
19220        assert_eq!(disk.status, RunStatus::Gating);
19221    }
19222
19223    #[tokio::test]
19224    async fn an_early_graph_error_releases_its_driver() {
19225        crate::run::pin_test_home();
19226        let mut runner = runner_at(RunStatus::Blocked);
19227        runner.state.config.graph.selected_implementer = Some("selected".into());
19228        runner.state.save().unwrap();
19229        assert!(runner.execute().await.is_err());
19230        let disk = RunState::load(&runner.state.id).unwrap();
19231        assert!(disk.driver_exited);
19232        assert_eq!(disk.driver_pid, None);
19233        assert_eq!(disk.status, RunStatus::Blocked);
19234    }
19235
19236    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19237        crate::run::MergeRepair {
19238            question: "repair-question".into(),
19239            head: "old-head".into(),
19240            pr: "https://github.com/o/r/pull/42".into(),
19241            reviews: vec![review_round(false, 1, 1, 1, false, true)],
19242            gate: vec![outcome(Some(0), false)],
19243            gate_head: Some("old-head".into()),
19244            gate_fixes: Vec::new(),
19245            pre_gate: Vec::new(),
19246            phase,
19247            failure: None,
19248        }
19249    }
19250
19251    #[tokio::test]
19252    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19253        crate::run::pin_test_home();
19254        let mut runner = runner_at(RunStatus::Landing);
19255        runner.state.merge_repairs.push(merge_repair_request(
19256            crate::run::MergeRepairPhase::Requested,
19257        ));
19258        runner.state.save().unwrap();
19259        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19260        assert!(
19261            runner
19262                .execute()
19263                .await
19264                .unwrap_err()
19265                .to_string()
19266                .contains("already running")
19267        );
19268        assert_eq!(
19269            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19270            crate::run::MergeRepairPhase::Requested
19271        );
19272        drop(claim);
19273        let mut disk = runner.state.clone();
19274        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19275        disk.save().unwrap();
19276        assert!(
19277            runner
19278                .execute()
19279                .await
19280                .unwrap_err()
19281                .to_string()
19282                .contains("decision changed")
19283        );
19284        assert_eq!(
19285            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19286            crate::run::MergeRepairPhase::Complete
19287        );
19288    }
19289
19290    #[test]
19291    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19292        crate::run::pin_test_home();
19293        let mut runner = runner_at(RunStatus::Landing);
19294        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19295        runner.state.reviews = request.reviews.clone();
19296        runner.state.gate = request.gate.clone();
19297        runner.state.gate_ran = true;
19298        runner.state.land_approval = Some(crate::run::LandApproval {
19299            question: "old".into(),
19300            head: "old-head".into(),
19301        });
19302        runner.state.merge_repairs.push(request);
19303        runner.begin_merge_repair_review(0).unwrap();
19304        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19305        assert!(!runner.state.gate_status().ok());
19306        assert!(runner.state.land_approval.is_none());
19307        let disk = RunState::load(&runner.state.id).unwrap();
19308        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19309        assert!(disk.merge_repairs[0].gate[0].ok());
19310        assert_eq!(
19311            disk.merge_repairs[0].phase,
19312            crate::run::MergeRepairPhase::Reviewing
19313        );
19314        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19315        assert!(disk.reviews.is_empty());
19316    }
19317
19318    #[tokio::test]
19319    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19320        crate::run::pin_test_home();
19321        for phase in [
19322            crate::run::MergeRepairPhase::Fixing,
19323            crate::run::MergeRepairPhase::Requested,
19324        ] {
19325            let mut runner = runner_at(RunStatus::Landing);
19326            runner.state.config.graph.land_rounds = 0;
19327            runner.state.merge_repairs.push(merge_repair_request(phase));
19328            runner.state.save().unwrap();
19329            assert!(!runner.repair_merge_review().await.unwrap());
19330            assert_eq!(runner.state.status, RunStatus::Blocked);
19331            assert_eq!(
19332                runner.state.merge_repairs[0].phase,
19333                crate::run::MergeRepairPhase::Failed
19334            );
19335            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19336            assert!(
19337                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19338                    "interrupted"
19339                } else {
19340                    "budget exhausted"
19341                })
19342            );
19343            assert!(runner.state.merge.is_none());
19344        }
19345    }
19346
19347    /// `park_here` folding in the reason `Pause::park_because` recorded -
19348    /// this is what lets an operator reading a run's events tell an
19349    /// interrupt-driven park from an ordinary shutdown park.
19350    #[test]
19351    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19352        crate::run::pin_test_home();
19353        let mut runner = runner_at(RunStatus::Implementing);
19354        let interrupt = Pause::new();
19355        runner.watch_interrupt(interrupt.clone());
19356
19357        interrupt.park_because("task a1b2 asked to run first");
19358
19359        assert!(runner.park_here().expect("park_here"));
19360        assert!(runner.state.parked);
19361        let last = runner.state.events.last().expect("a park event");
19362        assert_eq!(last.node, "park");
19363        assert!(
19364            last.message.contains("task a1b2 asked to run first"),
19365            "expected the interrupt reason in {:?}",
19366            last.message
19367        );
19368    }
19369
19370    /// The cause is stored on the pass when it parks, and the first cause a
19371    /// shared pause was given is the one that stays.
19372    #[test]
19373    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19374        crate::run::pin_test_home();
19375        let mut runner = runner_at(RunStatus::Implementing);
19376        runner.state.begin_pass();
19377        let shutdown = Pause::new();
19378        runner.on_pause(shutdown.clone());
19379        let upgrade = ParkCause::Upgrade {
19380            from: "0.121.0".to_owned(),
19381            to: Some("0.121.1".to_owned()),
19382        };
19383        shutdown.park_for(upgrade.clone());
19384        shutdown.park_for(ParkCause::Operator);
19385
19386        assert!(runner.park_here().expect("park_here"));
19387        assert_eq!(
19388            runner.state.passes[0].end,
19389            Some(crate::run::PassEnd::Parked { cause: upgrade })
19390        );
19391        let last = runner.state.events.last().expect("a park event");
19392        assert!(last.message.contains("0.121.0"), "{}", last.message);
19393    }
19394
19395    #[tokio::test]
19396    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19397        crate::run::pin_test_home();
19398        let mut runner = runner_at(RunStatus::Reviewing);
19399        runner.state.save().unwrap();
19400        let mut execution = runner.claim_driver().await.expect("first execution");
19401        assert_eq!(execution.runner.state.passes.len(), 1);
19402        execution
19403            .driver
19404            .finish(&mut execution.runner.state)
19405            .await
19406            .unwrap();
19407    }
19408
19409    #[tokio::test]
19410    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19411        crate::run::pin_test_home();
19412        let mut runner = runner_at(RunStatus::Reviewing);
19413        runner.state.config.agents = vec![conductor()];
19414        runner.state.config.graph.timeout_resource_wait = 0;
19415        runner.state.save().unwrap();
19416        let id = runner.state.id.clone();
19417        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19418        assert!(runner.claim_driver().await.is_err());
19419        drop(owner);
19420        drop(runner);
19421        let mut resumed = Runner::resume(&id).unwrap();
19422        assert_eq!(resumed.state.driver_claimed, Some(false));
19423        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19424        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19425        let mut execution = resumed
19426            .claim_driver()
19427            .await
19428            .expect("retry initial claim after reload");
19429        assert_eq!(execution.runner.state.driver_claimed, Some(true));
19430        assert_eq!(execution.runner.state.passes.len(), 1);
19431        execution
19432            .driver
19433            .finish(&mut execution.runner.state)
19434            .await
19435            .unwrap();
19436    }
19437
19438    #[tokio::test]
19439    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19440        crate::run::pin_test_home();
19441        let mut runner = runner_at(RunStatus::Reviewing);
19442        runner.state.config.agents = vec![conductor()];
19443        runner.state.save().unwrap();
19444        let id = runner.state.id.clone();
19445        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19446        let mut resumed = Runner::resume(&id).unwrap();
19447        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19448        assert!(resumed.claim_driver().await.is_err());
19449        assert!(RunState::load(&id).unwrap().passes.is_empty());
19450        drop(lease);
19451        let mut execution = resumed.claim_driver().await.unwrap();
19452        execution
19453            .driver
19454            .finish(&mut execution.runner.state)
19455            .await
19456            .unwrap();
19457    }
19458
19459    #[tokio::test]
19460    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19461        crate::run::pin_test_home();
19462        let mut runner = runner_at(RunStatus::Reviewing);
19463        let state = &mut runner.state;
19464        state.config.agents = vec![conductor()];
19465        state.driver_claimed = None;
19466        state.driver_pid = Some(std::process::id());
19467        state.driver_started_at = None;
19468        state.begin_pass();
19469        state.save().unwrap();
19470        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19471        std::fs::write(
19472            state.dir().join("driver.json"),
19473            serde_json::to_vec(&serde_json::json!({
19474                "pid": state.driver_pid, "started_at": null,
19475                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19476            }))
19477            .unwrap(),
19478        )
19479        .unwrap();
19480        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19481        drop(lease);
19482        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19483        let mut resumed = Runner::resume(&state.id).unwrap();
19484        assert!(!resumed.state.driver_exited);
19485        let mut execution = resumed
19486            .claim_driver()
19487            .await
19488            .expect("lease proves previous execution ended");
19489        assert_eq!(execution.runner.state.passes.len(), 2);
19490        execution
19491            .driver
19492            .finish(&mut execution.runner.state)
19493            .await
19494            .unwrap();
19495    }
19496
19497    #[tokio::test]
19498    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19499        crate::run::pin_test_home();
19500        let mut runner = runner_at(RunStatus::Reviewing);
19501        let mut json = serde_json::to_value(&runner.state).unwrap();
19502        json.as_object_mut().unwrap().remove("driver_claimed");
19503        runner.state = serde_json::from_value(json).unwrap();
19504        assert_eq!(runner.state.driver_claimed, None);
19505        runner.state.save().unwrap();
19506        let mut execution = runner
19507            .claim_driver()
19508            .await
19509            .expect("the execution lease, not a missing marker, excludes a second driver");
19510        assert_eq!(execution.runner.state.passes.len(), 1);
19511        execution
19512            .driver
19513            .finish(&mut execution.runner.state)
19514            .await
19515            .unwrap();
19516    }
19517
19518    #[tokio::test]
19519    async fn driver_admission_refuses_a_proven_live_driver() {
19520        crate::run::pin_test_home();
19521        let mut runner = runner_at(RunStatus::Reviewing);
19522        let pid = std::process::id();
19523        runner.state.driver_pid = Some(pid);
19524        runner.state.driver_started_at = crate::proc::process_started_at(pid);
19525        assert!(runner.state.driver_started_at.is_some());
19526        runner.state.save().unwrap();
19527        let error = runner
19528            .claim_driver()
19529            .await
19530            .err()
19531            .expect("a live driver must refuse");
19532        assert!(error.to_string().contains("live driver"), "{error:#}");
19533        let saved = RunState::load(&runner.state.id).unwrap();
19534        assert_eq!(saved.driver_pid, Some(pid));
19535        assert_eq!(saved.status, RunStatus::Reviewing);
19536        assert!(saved.passes.is_empty());
19537    }
19538
19539    #[test]
19540    fn parking_resource_halt_persists_the_pass_once() {
19541        crate::run::pin_test_home();
19542        let mut runner = runner_at(RunStatus::Reviewing);
19543        runner.state.begin_pass();
19544        runner.state.resource_block = Some(crate::run::ResourceBlock {
19545            node: "fix".into(),
19546            kind: "shared build cache".into(),
19547            detail: "cancelled".into(),
19548            at: Timestamp::now(),
19549            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19550        });
19551        let cause = ParkCause::Upgrade {
19552            from: "old".into(),
19553            to: Some("new".into()),
19554        };
19555        runner.pause.park_for(cause.clone());
19556        // Both direct fixer exits and graph resource boundaries must complete
19557        // parking, rather than returning a stale Reviewing record.
19558        runner.stop_for_resource().unwrap();
19559        assert!(runner.resource_halt().unwrap());
19560        assert!(runner.park_here().unwrap());
19561        let saved = RunState::load(&runner.state.id).unwrap();
19562        assert!(saved.parked);
19563        assert_eq!(saved.status, RunStatus::Reviewing);
19564        assert!(saved.passes[0].ended_at.is_some());
19565        assert_eq!(
19566            saved.passes[0].end,
19567            Some(crate::run::PassEnd::Parked {
19568                cause: cause.clone()
19569            })
19570        );
19571        assert_eq!(
19572            saved.resource_block.unwrap().cause,
19573            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19574        );
19575        assert_eq!(
19576            saved
19577                .events
19578                .iter()
19579                .filter(|event| event.node == "park")
19580                .count(),
19581            1
19582        );
19583    }
19584
19585    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19586    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19587    /// cleared) must not make a *different* run - one only watching its own,
19588    /// unshared interrupt `Pause` - see itself as parked. If a future change
19589    /// ever collapsed these back into one handle, the interrupt scheduler
19590    /// would park every run for the rest of the daemon's life, not just the
19591    /// one it meant to interrupt.
19592    #[test]
19593    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19594        crate::run::pin_test_home();
19595        let mut runner = runner_at(RunStatus::Implementing);
19596        let shutdown = Pause::new();
19597        runner.on_pause(shutdown.clone());
19598        let interrupt = Pause::new();
19599        runner.watch_interrupt(interrupt.clone());
19600
19601        // Nobody has asked for anything yet.
19602        assert!(!runner.park_here().expect("park_here"));
19603        assert!(!runner.state.parked);
19604
19605        // Only the interrupt handle fires; the shutdown handle stays clear.
19606        interrupt.park_because("test");
19607        assert!(!shutdown.parked());
19608        assert!(runner.park_here().expect("park_here"));
19609    }
19610
19611    /// The property every prior attempt at this feature failed to pin down:
19612    /// asking a run to park while one of its nodes has a real, in-flight
19613    /// async operation running (an agent call, in production) must not cut
19614    /// that operation short. `park_here` is only ever consulted *between*
19615    /// `execute`'s node calls - see its own doc - so nothing inside a node
19616    /// can observe a park request until the node itself returns. This proves
19617    /// that structurally, with real `tokio` concurrency and a channel
19618    /// handshake (never a sleep, which would only prove "usually", not
19619    /// "cannot"): the "node" below reports that it has genuinely started,
19620    /// and only then is the park requested; the node still has to be told to
19621    /// finish before `park_here` is ever called, exactly mirroring every
19622    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19623    /// in `execute`.
19624    #[tokio::test]
19625    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19626        crate::run::pin_test_home();
19627        let mut runner = runner_at(RunStatus::Implementing);
19628        let interrupt = Pause::new();
19629        runner.watch_interrupt(interrupt.clone());
19630
19631        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19632        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19633
19634        // Stands in for one node's in-flight agent call: it proves it has
19635        // genuinely started, then blocks - exactly as a spawned CLI process
19636        // does - until told to finish.
19637        let node = async move {
19638            started_tx.send(()).expect("send started");
19639            finish_rx.await.expect("recv finish");
19640            "node finished"
19641        };
19642
19643        let interrupter = async move {
19644            started_rx.await.expect("recv started");
19645            // The call is now genuinely in flight. Ask it to park.
19646            interrupt.park_because("higher-priority task waiting");
19647            // Nothing the node does can observe this yet - there is no
19648            // check inside it, by construction - so let the executor run
19649            // anything pending and then let the node finish on its own.
19650            tokio::task::yield_now().await;
19651            finish_tx.send(()).expect("send finish");
19652        };
19653
19654        let (node_result, ()) = tokio::join!(node, interrupter);
19655        assert_eq!(
19656            node_result, "node finished",
19657            "the in-flight call ran to completion"
19658        );
19659
19660        // Only now, at the boundary the real `execute` would check right
19661        // after this node, does the park take effect.
19662        assert!(runner.park_here().expect("park_here"));
19663        assert!(runner.state.parked);
19664    }
19665
19666    /// A run parked mid-competition carries every field it had accumulated
19667    /// through the exact same disk round-trip an ordinary resume uses -
19668    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19669    /// Nothing about parking for an interrupt is a special case of that path;
19670    /// this is what proves it rather than assuming it.
19671    #[test]
19672    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19673        crate::run::pin_test_home();
19674        let mut runner = runner_at(RunStatus::Judging);
19675        // `Runner::resume` re-resolves roles from the saved config, which
19676        // refuses an empty roster - give it the same minimal one `conductor`
19677        // itself uses.
19678        runner.state.config.agents = vec![conductor()];
19679        runner.state.candidates = vec![Candidate {
19680            index: 0,
19681            label: 'A',
19682            agent: "alpha".to_owned(),
19683            branch: "magi/x/A".to_owned(),
19684            worktree: PathBuf::from("/nonexistent/worktree"),
19685            summary: "did the thing".to_owned(),
19686            stat: "1 file changed".to_owned(),
19687            files: 1,
19688            commits: 1,
19689            empty: false,
19690            failed: None,
19691            verified_noop: None,
19692            duration_ms: 1234,
19693            folded: false,
19694        }];
19695        let run_id = runner.state.id.clone();
19696
19697        let interrupt = Pause::new();
19698        runner.watch_interrupt(interrupt.clone());
19699        interrupt.park_because("task c3d4 asked to run first");
19700        assert!(runner.park_here().expect("park_here"));
19701
19702        let resumed = Runner::resume(&run_id).expect("resume");
19703        assert_eq!(resumed.state.candidates.len(), 1);
19704        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19705        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19706        assert_eq!(resumed.state.status, runner.state.status);
19707        assert!(
19708            resumed.state.parked,
19709            "still parked until `execute` actually walks the graph again"
19710        );
19711        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19712    }
19713
19714    /// A fresh open question on `run`, stored and handed back for assertions.
19715    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19716        let mut q = ask::Question::new(
19717            run.to_owned(),
19718            "implement".to_owned(),
19719            "impl-A".to_owned(),
19720            "Which storage backend should the cache use?".to_owned(),
19721            String::new(),
19722            vec!["SQLite".to_owned(), "Redis".to_owned()],
19723        );
19724        store.put(&mut q).unwrap();
19725        q
19726    }
19727
19728    #[test]
19729    fn a_failed_runs_open_question_is_abandoned() {
19730        ask_test_home();
19731        let store = ask::Questions::open();
19732        let mut runner = runner_at(RunStatus::Failed);
19733        let run = runner.state.id.clone();
19734        let q = ask_open_question(&store, &run);
19735
19736        runner.settle_questions();
19737
19738        let back = store.get(&q.id).unwrap();
19739        assert!(
19740            !back.status.open(),
19741            "the seat that asked died with the run; nobody is left to read an answer"
19742        );
19743        assert!(
19744            back.detail.contains(&run) && back.detail.contains("failed"),
19745            "the reason names what the run became, not just that it is gone: {}",
19746            back.detail
19747        );
19748    }
19749
19750    #[test]
19751    fn a_merged_runs_open_question_is_abandoned_too() {
19752        ask_test_home();
19753        let store = ask::Questions::open();
19754        // A run that finishes cleanly still leaves nobody to read an answer -
19755        // this is not only a failure-path cleanup.
19756        for status in [RunStatus::Merged, RunStatus::Ready] {
19757            let mut runner = runner_at(status);
19758            let run = runner.state.id.clone();
19759            let q = ask_open_question(&store, &run);
19760
19761            runner.settle_questions();
19762
19763            let back = store.get(&q.id).unwrap();
19764            assert!(
19765                !back.status.open(),
19766                "{status:?} run's question must not outlive the run"
19767            );
19768        }
19769    }
19770
19771    #[test]
19772    fn a_still_resumable_runs_open_question_is_left_alone() {
19773        ask_test_home();
19774        let store = ask::Questions::open();
19775        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19776        // review round and the seat sessions are all still on disk — so a
19777        // question asked mid-round may yet get a real answer from a real
19778        // resume. Sweeping it here would be exactly the failure mode this
19779        // whole feature exists to avoid on the other side.
19780        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19781            let mut runner = runner_at(status);
19782            let run = runner.state.id.clone();
19783            let q = ask_open_question(&store, &run);
19784
19785            runner.settle_questions();
19786
19787            let back = store.get(&q.id).unwrap();
19788            assert!(
19789                back.status.open(),
19790                "{status:?} is still alive; the question must still be waiting"
19791            );
19792        }
19793    }
19794
19795    #[test]
19796    fn settle_questions_never_touches_an_already_answered_question() {
19797        ask_test_home();
19798        let store = ask::Questions::open();
19799        let mut runner = runner_at(RunStatus::Failed);
19800        let run = runner.state.id.clone();
19801        let mut q = ask_open_question(&store, &run);
19802        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19803            .unwrap();
19804        store.put(&mut q).unwrap();
19805
19806        // Called twice, the way a crash-recovered daemon reclaim and the
19807        // graph's own cleanup both can for the same run — `abandon_for_run`
19808        // only ever touches what is still open, so this must be inert both
19809        // times, not merely the second.
19810        runner.settle_questions();
19811        runner.settle_questions();
19812
19813        let back = store.get(&q.id).unwrap();
19814        assert_eq!(
19815            back.status,
19816            ask::QuestionStatus::Answered,
19817            "a real answer is a decision on record, never overwritten by a sweep"
19818        );
19819    }
19820
19821    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19822    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19823    /// without merging, whose winner is still the operator's answer to read.
19824    /// Nothing previously called `fold_run` itself with a real `tally`, so
19825    /// this is the first test to pin down the one distinction the whole
19826    /// automatic-fold feature depends on: the winner's worktree and branch
19827    /// must survive, everything else sharing the run's worktree bay - a
19828    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19829    /// second sweep treats every non-winner directory under the bay alike -
19830    /// must not.
19831    #[tokio::test]
19832    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19833        crate::run::pin_test_home();
19834        let tmp = tempfile::tempdir().expect("tempdir");
19835        let repo = tmp.path().join("repo");
19836        std::fs::create_dir_all(&repo).unwrap();
19837        init_repo(&repo);
19838
19839        let mut config = Config::default();
19840        config.graph.worktree_root = Some(tmp.path().join("wt"));
19841
19842        let mut state = RunState::new(
19843            repo.clone(),
19844            "main".to_owned(),
19845            "deadbeef".to_owned(),
19846            "task".to_owned(),
19847            config,
19848        );
19849        let root = state.worktree_root();
19850        let wt_a = root.join("cand-A");
19851        let wt_b = root.join("cand-B");
19852        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19853            .await
19854            .expect("worktree A");
19855        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19856            .await
19857            .expect("worktree B");
19858
19859        state.candidates = vec![
19860            Candidate {
19861                index: 0,
19862                label: 'A',
19863                agent: "alpha".to_owned(),
19864                branch: "magi/x/A".to_owned(),
19865                worktree: wt_a.clone(),
19866                summary: String::new(),
19867                stat: String::new(),
19868                files: 0,
19869                commits: 0,
19870                empty: false,
19871                failed: None,
19872                verified_noop: None,
19873                duration_ms: 0,
19874                folded: false,
19875            },
19876            Candidate {
19877                index: 1,
19878                label: 'B',
19879                agent: "beta".to_owned(),
19880                branch: "magi/x/B".to_owned(),
19881                worktree: wt_b.clone(),
19882                summary: String::new(),
19883                stat: String::new(),
19884                files: 0,
19885                commits: 0,
19886                empty: false,
19887                failed: None,
19888                verified_noop: None,
19889                duration_ms: 0,
19890                folded: false,
19891            },
19892        ];
19893        state.tally = Some(Tally {
19894            first_choice: BTreeMap::from([('A', 1)]),
19895            borda: BTreeMap::new(),
19896            winner: 'A',
19897            rankings: 1,
19898            unanimous_initial: true,
19899            deliberated: false,
19900            changed_votes: 0,
19901            unanimous_final: true,
19902            tie_break: None,
19903            judges: 1,
19904            present: 1,
19905            quorum: 1,
19906            met_quorum: true,
19907            uncontested: None,
19908        });
19909        state.status = RunStatus::Ready;
19910
19911        fold_run(&mut state, false, &crate::run::home())
19912            .await
19913            .expect("fold_run");
19914
19915        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19916        assert!(
19917            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19918            "the unmerged winner's branch survives"
19919        );
19920        assert!(
19921            !state.candidates[0].folded,
19922            "the winner is not marked folded"
19923        );
19924
19925        assert!(!wt_b.exists(), "the loser's worktree is removed");
19926        assert!(
19927            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19928            "the loser's branch is removed"
19929        );
19930        assert!(state.candidates[1].folded, "the loser is marked folded");
19931    }
19932
19933    #[tokio::test]
19934    #[cfg(unix)]
19935    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19936        use std::os::fd::AsRawFd as _;
19937        use std::time::Duration;
19938
19939        let tmp = tempfile::tempdir().unwrap();
19940        let repo = tmp.path().join("repo");
19941        std::fs::create_dir_all(&repo).unwrap();
19942        init_repo(&repo);
19943        let common = repo.join(".git");
19944        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19945        let home = tmp.path().join("home");
19946        let mut config = Config::default();
19947        config.coordination = crate::config::test_coordination();
19948        config.graph.worktree_root = Some(tmp.path().join("wt"));
19949        config.graph.timeout_resource_wait = 1;
19950        let mut state = RunState::new(
19951            repo.clone(),
19952            "main".into(),
19953            "deadbeef".into(),
19954            "task".into(),
19955            config,
19956        );
19957        let root = state.worktree_root();
19958        let candidate = root.join("cand-A");
19959        let reviewer = root.join("review-1");
19960        let branch = state.branch_for('A');
19961        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19962            .await
19963            .unwrap();
19964        git::worktree_add_detached(&repo, &reviewer, "main")
19965            .await
19966            .unwrap();
19967        git::acquire_worktree_config(&repo).await.unwrap();
19968        state.enabled_worktree_config = true;
19969        state.status = RunStatus::Ready;
19970        state.candidates.push(Candidate {
19971            index: 0,
19972            label: 'A',
19973            agent: "implement".into(),
19974            branch: branch.clone(),
19975            worktree: candidate.clone(),
19976            summary: String::new(),
19977            stat: String::new(),
19978            files: 0,
19979            commits: 0,
19980            empty: false,
19981            failed: None,
19982            verified_noop: None,
19983            duration_ms: 0,
19984            folded: false,
19985        });
19986        state.save_under(&home).unwrap();
19987        let config_before = std::fs::read(common.join("config")).unwrap();
19988        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19989            .await
19990            .unwrap();
19991        // A separate OS handle bypasses the launcher's local gate, exactly as
19992        // an external updater does, and must still prevent all folding work.
19993        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19994        assert_eq!(
19995            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19996            0
19997        );
19998        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19999        assert!(
20000            tokio::time::timeout(Duration::from_millis(200), &mut fold)
20001                .await
20002                .is_err()
20003        );
20004        assert!(candidate.exists());
20005        assert!(reviewer.exists());
20006        assert!(git::branch_exists(&repo, &branch).await.unwrap());
20007        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20008        assert_eq!(
20009            git::git(&repo, &["worktree", "list", "--porcelain"])
20010                .await
20011                .unwrap(),
20012            worktrees_before
20013        );
20014        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
20015            .await
20016            .unwrap()
20017            .unwrap_err();
20018        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
20019        drop(fold);
20020        assert!(candidate.exists() && reviewer.exists());
20021        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
20022        assert_eq!(
20023            state.resource_block.as_ref().unwrap().kind,
20024            "repository coordination"
20025        );
20026        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20027        drop(external);
20028        let mut fold = Box::pin(fold_run(&mut state, true, &home));
20029        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
20030            .await
20031            .unwrap()
20032            .unwrap();
20033        drop(fold);
20034        assert!(removed.contains(&branch));
20035        assert!(!root.exists());
20036        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
20037        assert!(state.candidates[0].folded);
20038        assert!(!state.enabled_worktree_config);
20039        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
20040        let enabled = git::git_raw(
20041            &repo,
20042            &["config", "--local", "--get", "extensions.worktreeConfig"],
20043        )
20044        .await
20045        .unwrap();
20046        assert!(!enabled.ok(), "the last config reference was released");
20047    }
20048
20049    /// A branch handed to a later run is that run's (and its pull request's):
20050    /// folding the run that released it must not delete it.
20051    #[tokio::test]
20052    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
20053        let tmp = tempfile::tempdir().expect("tempdir");
20054        let repo = tmp.path().join("repo");
20055        std::fs::create_dir_all(&repo).unwrap();
20056        init_repo(&repo);
20057        let home = tmp.path().join("home");
20058
20059        let mut config = Config::default();
20060        config.graph.worktree_root = Some(tmp.path().join("wt"));
20061        let mut state = RunState::new(
20062            repo.clone(),
20063            "main".to_owned(),
20064            "deadbeef".to_owned(),
20065            "task".to_owned(),
20066            config,
20067        );
20068        // The worktree is already gone (released); the branch survives.
20069        git::git(&repo, &["branch", "magi/x/A", "main"])
20070            .await
20071            .expect("branch");
20072        state.candidates = vec![Candidate {
20073            index: 0,
20074            label: 'A',
20075            agent: "alpha".to_owned(),
20076            branch: "magi/x/A".to_owned(),
20077            worktree: state.worktree_root().join("cand-A"),
20078            summary: String::new(),
20079            stat: String::new(),
20080            files: 0,
20081            commits: 0,
20082            empty: false,
20083            failed: None,
20084            verified_noop: None,
20085            duration_ms: 0,
20086            folded: true,
20087        }];
20088        state.released_to = Some("20260901-000000-new1".to_owned());
20089        state.released_branches = vec!["magi/x/A".to_owned()];
20090
20091        fold_run(&mut state, true, &home).await.expect("fold_run");
20092        state.status = RunStatus::Merged;
20093        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
20094        state.merge = Some(crate::run::MergeOutcome {
20095            mode: MergeMode::Pr,
20096            ok: true,
20097            detail: "confirmed merged".into(),
20098            empty: false,
20099        });
20100        fold_run(&mut state, true, &home)
20101            .await
20102            .expect("merged fold");
20103
20104        assert!(
20105            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
20106            "the handed-over branch survives a fold"
20107        );
20108    }
20109
20110    /// A winner with nothing ahead of the base is caught before `gh` is ever
20111    /// asked for a pull request, and the message carries what the task's
20112    /// references resolved to.
20113    #[tokio::test]
20114    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
20115        let tmp = tempfile::tempdir().expect("tempdir");
20116        let repo = tmp.path().join("repo");
20117        std::fs::create_dir_all(&repo).unwrap();
20118        init_repo(&repo);
20119        let run = |args: &[&str]| {
20120            let out = std::process::Command::new("git")
20121                .quiet()
20122                .args(args)
20123                .current_dir(&repo)
20124                .output()
20125                .expect("spawn git");
20126            assert!(out.status.success(), "git {args:?}");
20127        };
20128        run(&["branch", "magi/x/A"]);
20129        run(&["checkout", "-q", "-b", "magi/x/B"]);
20130        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
20131        run(&["add", "-A"]);
20132        run(&["commit", "-q", "-m", "work"]);
20133        run(&["checkout", "-q", "main"]);
20134
20135        // A pull request is compared against the remote's base, so the
20136        // fixture needs one. Before it exists nothing can be read, and the
20137        // answer must be "not empty".
20138        let probe = RunState::new(
20139            repo.clone(),
20140            "main".to_owned(),
20141            "deadbeef".to_owned(),
20142            "task".to_owned(),
20143            Config::default(),
20144        );
20145        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20146        let bare = tmp.path().join("origin.git");
20147        let out = std::process::Command::new("git")
20148            .quiet()
20149            .args(["init", "-q", "--bare"])
20150            .arg(&bare)
20151            .output()
20152            .expect("spawn git");
20153        assert!(out.status.success(), "git init --bare");
20154        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20155        run(&["push", "-q", "origin", "main"]);
20156
20157        let mut state = RunState::new(
20158            repo.clone(),
20159            "main".to_owned(),
20160            "deadbeef".to_owned(),
20161            "task".to_owned(),
20162            Config::default(),
20163        );
20164        state.seeds = vec![refs::Seed {
20165            token: "magi/27b2/A".to_owned(),
20166            kind: refs::SeedKind::Unresolved,
20167            sha: String::new(),
20168            branch: true,
20169            detail: "no branch or commit named magi/27b2/A".to_owned(),
20170        }];
20171
20172        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20173        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20174        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20175        let detail = empty_candidate_detail(&state, "main");
20176        assert!(detail.starts_with("empty candidate"), "{detail}");
20177        assert!(!detail.contains("magi/27b2/A"), "{detail}");
20178        assert!(detail.contains("--start-from"), "{detail}");
20179    }
20180
20181    /// `status == Ready` used to be read as "this is the harmless
20182    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
20183    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
20184    /// run's PR was closed without merging — and reentering `merge` with
20185    /// `mode` still `Pr` does not know the difference, so it pushed and
20186    /// opened a second pull request. `mode == Local` reproduces the same
20187    /// blind spot without a network call: reentry must not attempt another
20188    /// git merge once this node has already recorded an outcome.
20189    #[tokio::test]
20190    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20191        let tmp = tempfile::tempdir().expect("tempdir");
20192        let repo = tmp.path().join("repo");
20193        std::fs::create_dir_all(&repo).unwrap();
20194        init_repo(&repo);
20195
20196        let mut config = Config::default();
20197        config.merge.mode = MergeMode::Local;
20198
20199        let mut state = RunState::new(
20200            repo.clone(),
20201            "main".to_owned(),
20202            "deadbeef".to_owned(),
20203            "task".to_owned(),
20204            config,
20205        );
20206        state.candidates = vec![Candidate {
20207            index: 0,
20208            label: 'A',
20209            agent: "alpha".to_owned(),
20210            branch: "does-not-exist".to_owned(),
20211            worktree: repo.clone(),
20212            summary: String::new(),
20213            stat: String::new(),
20214            files: 0,
20215            commits: 0,
20216            empty: false,
20217            failed: None,
20218            verified_noop: None,
20219            duration_ms: 0,
20220            folded: false,
20221        }];
20222        state.tally = Some(Tally {
20223            first_choice: BTreeMap::from([('A', 1)]),
20224            borda: BTreeMap::new(),
20225            winner: 'A',
20226            rankings: 1,
20227            unanimous_initial: true,
20228            deliberated: false,
20229            changed_votes: 0,
20230            unanimous_final: true,
20231            tie_break: None,
20232            judges: 0,
20233            present: 0,
20234            quorum: 0,
20235            met_quorum: true,
20236            uncontested: Some("only candidate A produced a change".to_owned()),
20237        });
20238        state.reviews = vec![ReviewRound {
20239            round: 1,
20240            head: "deadbeef".to_owned(),
20241            verified_head: None,
20242            verified_at: None,
20243            reviews: Vec::new(),
20244            e2e: Vec::new(),
20245            fix: None,
20246            blocking: 0,
20247            answered: 0,
20248            expected: 0,
20249            clean: true,
20250            verify_retried: false,
20251            e2e_deferred: false,
20252            e2e_defer_reason: None,
20253            progressed: false,
20254            vote_split: false,
20255            reconsideration: Vec::new(),
20256            verdict: None,
20257        }];
20258        state.gate = vec![CommandOutcome {
20259            command: "test".to_owned(),
20260            code: Some(0),
20261            output_tail: String::new(),
20262            duration_ms: 0,
20263            resource_blocked: false,
20264        }];
20265        state.gate_ran = true;
20266        // Reached its conclusion already — e.g. `land` closing the PR without
20267        // merging it, which (like the honest `MergeMode::None` path) leaves
20268        // `status` at `Ready`. The recorded outcome is what actually marks
20269        // this node done.
20270        state.status = RunStatus::Ready;
20271        state.merge = Some(MergeOutcome {
20272            mode: MergeMode::Local,
20273            ok: false,
20274            detail: "already concluded".to_owned(),
20275            empty: false,
20276        });
20277
20278        let mut runner = Runner {
20279            state,
20280            roles: ResolvedRoles {
20281                implementers: Vec::new(),
20282                judges: Vec::new(),
20283                reviewers: Vec::new(),
20284                fixer: None,
20285                conductor: conductor(),
20286                implementer_roster: Vec::new(),
20287                judge_roster: Vec::new(),
20288                reviewer_roster: Vec::new(),
20289            },
20290            sem: Arc::new(Semaphore::new(1)),
20291            pause: Pause::new(),
20292            interrupt: Pause::new(),
20293        };
20294
20295        runner.merge().await.expect("merge");
20296
20297        assert_eq!(
20298            runner.state.status,
20299            RunStatus::Ready,
20300            "a concluded run's status must not change on reentry"
20301        );
20302        assert_eq!(
20303            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20304            Some("already concluded"),
20305            "merge must not run again once the node already recorded an outcome"
20306        );
20307    }
20308
20309    /// `gate` leaves `state.gate_ran` false both before it has ever run and
20310    /// when its last attempt was resource-blocked (the shared build cache
20311    /// could not be acquired or confirmed fresh in time - see
20312    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
20313    /// `Vec` this also leaves behind used to read as "nothing failed" and let
20314    /// a run merge a tree the gate never actually checked - exactly the case
20315    /// a contended cache produces on every retry until it clears. `merge`
20316    /// must refuse until `gate` has actually recorded an attempt.
20317    #[tokio::test]
20318    async fn merge_refuses_a_gate_that_has_not_actually_run() {
20319        let tmp = tempfile::tempdir().expect("tempdir");
20320        let repo = tmp.path().join("repo");
20321        std::fs::create_dir_all(&repo).unwrap();
20322        init_repo(&repo);
20323
20324        let mut config = Config::default();
20325        config.merge.mode = MergeMode::Local;
20326
20327        let mut state = RunState::new(
20328            repo.clone(),
20329            "main".to_owned(),
20330            "deadbeef".to_owned(),
20331            "task".to_owned(),
20332            config,
20333        );
20334        state.candidates = vec![Candidate {
20335            index: 0,
20336            label: 'A',
20337            agent: "alpha".to_owned(),
20338            branch: "does-not-exist".to_owned(),
20339            worktree: repo.clone(),
20340            summary: String::new(),
20341            stat: String::new(),
20342            files: 0,
20343            commits: 0,
20344            empty: false,
20345            failed: None,
20346            verified_noop: None,
20347            duration_ms: 0,
20348            folded: false,
20349        }];
20350        state.tally = Some(Tally {
20351            first_choice: BTreeMap::from([('A', 1)]),
20352            borda: BTreeMap::new(),
20353            winner: 'A',
20354            rankings: 1,
20355            unanimous_initial: true,
20356            deliberated: false,
20357            changed_votes: 0,
20358            unanimous_final: true,
20359            tie_break: None,
20360            judges: 0,
20361            present: 0,
20362            quorum: 0,
20363            met_quorum: true,
20364            uncontested: Some("only candidate A produced a change".to_owned()),
20365        });
20366        state.reviews = vec![ReviewRound {
20367            round: 1,
20368            head: "deadbeef".to_owned(),
20369            verified_head: None,
20370            verified_at: None,
20371            reviews: Vec::new(),
20372            e2e: Vec::new(),
20373            fix: None,
20374            blocking: 0,
20375            answered: 0,
20376            expected: 0,
20377            clean: true,
20378            verify_retried: false,
20379            e2e_deferred: false,
20380            e2e_defer_reason: None,
20381            progressed: false,
20382            vote_split: false,
20383            reconsideration: Vec::new(),
20384            verdict: None,
20385        }];
20386        // The point: `gate` has not recorded anything yet.
20387        state.gate = Vec::new();
20388        state.gate_ran = false;
20389        state.status = RunStatus::Gating;
20390
20391        let mut runner = Runner {
20392            state,
20393            roles: ResolvedRoles {
20394                implementers: Vec::new(),
20395                judges: Vec::new(),
20396                reviewers: Vec::new(),
20397                fixer: None,
20398                conductor: conductor(),
20399                implementer_roster: Vec::new(),
20400                judge_roster: Vec::new(),
20401                reviewer_roster: Vec::new(),
20402            },
20403            sem: Arc::new(Semaphore::new(1)),
20404            pause: Pause::new(),
20405            interrupt: Pause::new(),
20406        };
20407
20408        runner.merge().await.expect("merge");
20409
20410        assert!(
20411            runner.state.merge.is_none(),
20412            "an empty gate must never be read as a passing one: {:?}",
20413            runner.state.merge
20414        );
20415    }
20416
20417    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
20418    /// commands configured and `merge.mode` is `none` (a review-only run).
20419    /// `gate` must still record a real attempt — zero commands, vacuously
20420    /// passed — rather than leaving `state.gate` empty in a way `merge`
20421    /// cannot tell apart from "never ran"; otherwise the run reaches
20422    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
20423    #[tokio::test]
20424    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20425        let tmp = tempfile::tempdir().expect("tempdir");
20426        let repo = tmp.path().join("repo");
20427        std::fs::create_dir_all(&repo).unwrap();
20428        init_repo(&repo);
20429
20430        // Default config: `verify.gate` empty, `merge.mode` is `none`.
20431        let config = Config::default();
20432
20433        let mut state = RunState::new(
20434            repo.clone(),
20435            "main".to_owned(),
20436            "deadbeef".to_owned(),
20437            "task".to_owned(),
20438            config,
20439        );
20440        state.candidates = vec![Candidate {
20441            index: 0,
20442            label: 'A',
20443            agent: "alpha".to_owned(),
20444            branch: "does-not-exist".to_owned(),
20445            worktree: repo.clone(),
20446            summary: String::new(),
20447            stat: String::new(),
20448            files: 0,
20449            commits: 0,
20450            empty: false,
20451            failed: None,
20452            verified_noop: None,
20453            duration_ms: 0,
20454            folded: false,
20455        }];
20456        state.tally = Some(Tally {
20457            first_choice: BTreeMap::from([('A', 1)]),
20458            borda: BTreeMap::new(),
20459            winner: 'A',
20460            rankings: 1,
20461            unanimous_initial: true,
20462            deliberated: false,
20463            changed_votes: 0,
20464            unanimous_final: true,
20465            tie_break: None,
20466            judges: 0,
20467            present: 0,
20468            quorum: 0,
20469            met_quorum: true,
20470            uncontested: Some("only candidate A produced a change".to_owned()),
20471        });
20472        state.reviews = vec![ReviewRound {
20473            round: 1,
20474            head: "deadbeef".to_owned(),
20475            verified_head: None,
20476            verified_at: None,
20477            reviews: Vec::new(),
20478            e2e: Vec::new(),
20479            fix: None,
20480            blocking: 0,
20481            answered: 0,
20482            expected: 0,
20483            clean: true,
20484            verify_retried: false,
20485            e2e_deferred: false,
20486            e2e_defer_reason: None,
20487            progressed: false,
20488            vote_split: false,
20489            reconsideration: Vec::new(),
20490            verdict: None,
20491        }];
20492
20493        let mut runner = Runner {
20494            state,
20495            roles: ResolvedRoles {
20496                implementers: Vec::new(),
20497                judges: Vec::new(),
20498                reviewers: Vec::new(),
20499                fixer: None,
20500                conductor: conductor(),
20501                implementer_roster: Vec::new(),
20502                judge_roster: Vec::new(),
20503                reviewer_roster: Vec::new(),
20504            },
20505            sem: Arc::new(Semaphore::new(1)),
20506            pause: Pause::new(),
20507            interrupt: Pause::new(),
20508        };
20509
20510        runner.gate().await.expect("gate");
20511        assert!(
20512            runner.state.gate_ran,
20513            "zero configured commands is still a real attempt, not an unrun gate"
20514        );
20515        assert!(runner.state.gate.is_empty());
20516        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20517        assert_ne!(
20518            runner.state.status,
20519            RunStatus::Blocked,
20520            "a gate with nothing to check must not read as failed"
20521        );
20522
20523        runner.merge().await.expect("merge");
20524        assert_eq!(
20525            runner.state.status,
20526            RunStatus::Ready,
20527            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20528        );
20529    }
20530
20531    /// `Config::cache_dir` is derived from `verify.e2e` as well as
20532    /// `verify.gate` (so the e2e leg and the final gate never build against
20533    /// different directories). With zero `verify.gate` commands but a
20534    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
20535    /// that lease before discovering it had nothing to run - so a repo with
20536    /// no gate commands could come back `resource_blocked` (and therefore
20537    /// still `gate_ran == false`) on nothing but an unrelated run holding the
20538    /// cache, exactly the contention this run's own zero commands could
20539    /// never have touched. `gate` must recognise there is nothing to check
20540    /// before it ever asks for the lease.
20541    #[tokio::test]
20542    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20543        crate::run::pin_test_home();
20544        let home = crate::run::home();
20545
20546        let tmp = tempfile::tempdir().expect("tempdir");
20547        let repo = tmp.path().join("repo");
20548        std::fs::create_dir_all(&repo).unwrap();
20549        init_repo(&repo);
20550        // Unique to this test, so holding its lease cannot collide with
20551        // another test sharing the same process-wide `home`.
20552        let cache_dir = tmp.path().join("target");
20553
20554        let mut config = Config::default();
20555        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20556        // `verify.gate` stays empty (the default). Any accidental cache
20557        // acquisition must report contention immediately, regardless of how
20558        // long unrelated Git subprocesses take under parallel test load.
20559        config.graph.timeout_verify = Some(2);
20560        config.graph.timeout_resource_wait = 0;
20561
20562        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20563        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20564            .expect("no io error acquiring directly")
20565        {
20566            crate::cache::AcquireOutcome::Acquired(g) => g,
20567            crate::cache::AcquireOutcome::Busy(b) => {
20568                panic!("expected the direct acquire to win the lease first: {b:?}")
20569            }
20570        };
20571
20572        let mut state = RunState::new(
20573            repo.clone(),
20574            "main".to_owned(),
20575            "deadbeef".to_owned(),
20576            "task".to_owned(),
20577            config,
20578        );
20579        state.candidates = vec![Candidate {
20580            index: 0,
20581            label: 'A',
20582            agent: "alpha".to_owned(),
20583            branch: "does-not-exist".to_owned(),
20584            worktree: repo.clone(),
20585            summary: String::new(),
20586            stat: String::new(),
20587            files: 0,
20588            commits: 0,
20589            empty: false,
20590            failed: None,
20591            verified_noop: None,
20592            duration_ms: 0,
20593            folded: false,
20594        }];
20595        state.tally = Some(Tally {
20596            first_choice: BTreeMap::from([('A', 1)]),
20597            borda: BTreeMap::new(),
20598            winner: 'A',
20599            rankings: 1,
20600            unanimous_initial: true,
20601            deliberated: false,
20602            changed_votes: 0,
20603            unanimous_final: true,
20604            tie_break: None,
20605            judges: 0,
20606            present: 0,
20607            quorum: 0,
20608            met_quorum: true,
20609            uncontested: Some("only candidate A produced a change".to_owned()),
20610        });
20611        state.reviews = vec![ReviewRound {
20612            round: 1,
20613            head: "deadbeef".to_owned(),
20614            verified_head: None,
20615            verified_at: None,
20616            reviews: Vec::new(),
20617            e2e: Vec::new(),
20618            fix: None,
20619            blocking: 0,
20620            answered: 0,
20621            expected: 0,
20622            clean: true,
20623            verify_retried: false,
20624            e2e_deferred: false,
20625            e2e_defer_reason: None,
20626            progressed: false,
20627            vote_split: false,
20628            reconsideration: Vec::new(),
20629            verdict: None,
20630        }];
20631
20632        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20633        state.reviews[0].e2e = vec![CommandOutcome {
20634            command: state.config.verify.e2e[0].clone(),
20635            code: Some(0),
20636            output_tail: String::new(),
20637            duration_ms: 0,
20638            resource_blocked: false,
20639        }];
20640        let mut runner = Runner {
20641            state,
20642            roles: ResolvedRoles {
20643                implementers: Vec::new(),
20644                judges: Vec::new(),
20645                reviewers: Vec::new(),
20646                fixer: None,
20647                conductor: conductor(),
20648                implementer_roster: Vec::new(),
20649                judge_roster: Vec::new(),
20650                reviewer_roster: Vec::new(),
20651            },
20652            sem: Arc::new(Semaphore::new(1)),
20653            pause: Pause::new(),
20654            interrupt: Pause::new(),
20655        };
20656
20657        runner.gate().await.expect("gate");
20658        assert!(runner.state.resource_block.is_none());
20659        assert!(
20660            !runner
20661                .state
20662                .events
20663                .iter()
20664                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20665            "a gate with nothing to run must never request the held cache lease"
20666        );
20667        assert!(
20668            runner.state.gate_ran,
20669            "zero commands is still a real, immediate attempt"
20670        );
20671        assert!(runner.state.gate.is_empty());
20672        assert_ne!(
20673            runner.state.status,
20674            RunStatus::Blocked,
20675            "must not read as resource-blocked on a lease it never asked for"
20676        );
20677    }
20678
20679    /// A separate process carries a live worker's environment throughout all
20680    /// verification entry points; the parallel harness never changes its env.
20681    #[cfg(unix)]
20682    #[tokio::test]
20683    async fn verification_is_standalone_inside_a_live_worker() {
20684        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20685        if std::env::var_os(CHILD).is_none() {
20686            let home = tempfile::tempdir().unwrap();
20687            let generation = crate::worker::pin_current(home.path()).unwrap();
20688            let output = std::process::Command::new(std::env::current_exe().unwrap())
20689                .args([
20690                    "--exact",
20691                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20692                    "--nocapture",
20693                ])
20694                .env(CHILD, "1")
20695                .env("MAGI_HOME", home.path())
20696                .env("MAGI_WORKER_HOME", home.path())
20697                .env("MAGI_WORKER_ID", "abcdef")
20698                .env("MAGI_WORKER_GENERATION", &generation.id)
20699                .env("MAGI_WORKER_EXE", &generation.executable)
20700                .env("MAGI_RUN", "verification-parent")
20701                .env("MAGI_NODE", "gate")
20702                .env("MAGI_SEAT", "worker-fixture")
20703                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20704                .env("MAGI_NO_AUTOUPDATE", "0")
20705                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20706                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20707                .env(
20708                    "VERIFY_EXPECT_TOOLCHAIN",
20709                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20710                )
20711                .env(
20712                    "VERIFY_EXPECT_CACHE",
20713                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20714                )
20715                .output()
20716                .unwrap();
20717            assert!(
20718                output.status.success()
20719                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20720                "{}\n{}",
20721                String::from_utf8_lossy(&output.stdout),
20722                String::from_utf8_lossy(&output.stderr)
20723            );
20724            return;
20725        }
20726        let home = crate::run::home();
20727        let directory = home.join("workers/jobs/abcdef");
20728        std::fs::create_dir_all(&directory).unwrap();
20729        let job = crate::worker::Job {
20730            id: "abcdef".into(),
20731            sequence: 1,
20732            kind: "task".into(),
20733            key: "task:verification".into(),
20734            payload: serde_json::json!({}),
20735            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20736            version: Some(env!("CARGO_PKG_VERSION").into()),
20737            status: crate::worker::Status::Running,
20738            result: None,
20739            pid: Some(std::process::id()),
20740            created_at: 0,
20741            finished_at: None,
20742        };
20743        std::fs::write(
20744            directory.join("job.json"),
20745            serde_json::to_vec(&job).unwrap(),
20746        )
20747        .unwrap();
20748        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20749        let assert_worker = || {
20750            assert_eq!(
20751                crate::worker::current_job(&home).unwrap().unwrap().id,
20752                job.id
20753            );
20754            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20755            assert_eq!(
20756                crate::worker::helper_executable(&home).unwrap(),
20757                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20758            );
20759            assert!(
20760                crate::updater::Checker::new(&crate::config::Update {
20761                    mode: crate::config::UpdateMode::Notify,
20762                    interval: None,
20763                    auto_restart: false,
20764                })
20765                .is_none(),
20766                "the driver remains pinned, even while tests are standalone"
20767            );
20768        };
20769        assert_worker();
20770        let repo = home.join("repo");
20771        std::fs::create_dir_all(&repo).unwrap();
20772        init_repo(&repo);
20773        // Keep storage, seat/control context, containment and tool lookup.
20774        // No inherited values (or credentials) are printed on success.
20775        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20776            test -z "${MAGI_WORKER_GENERATION+x}" &&
20777            test -z "${MAGI_WORKER_HOME+x}" &&
20778            test -z "${MAGI_WORKER_EXE+x}" &&
20779            test "$MAGI_HOME/repo" -ef . &&
20780            test "$MAGI_RUN" = verification-parent &&
20781            test "$MAGI_NODE" = gate &&
20782            test "$MAGI_SEAT" = worker-fixture &&
20783            test -n "$MAGI_PROCESS_OWNER" &&
20784            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20785            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20786            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20787            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20788        // Negative control: an ordinary child inherits the contamination.
20789        assert!(
20790            !std::process::Command::new("sh")
20791                .args(["-c", probe])
20792                .current_dir(&repo)
20793                .status()
20794                .unwrap()
20795                .success()
20796        );
20797        let unrelated_home = tempfile::tempdir().unwrap();
20798        assert!(
20799            crate::worker::current_job(unrelated_home.path()).is_err(),
20800            "an inherited worker id cannot be resolved in a unit fixture's home"
20801        );
20802        let mut runner = runner_at(RunStatus::Gating);
20803        runner.state.repo = repo.clone();
20804        runner.state.config.disk.min_free_bytes = 0;
20805        runner.state.config.coordination = crate::config::test_coordination();
20806        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20807        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20808        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20809        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20810        let winner = Candidate {
20811            index: 0,
20812            label: 'A',
20813            agent: "fixture".into(),
20814            branch: "main".into(),
20815            worktree: repo.clone(),
20816            summary: String::new(),
20817            stat: String::new(),
20818            files: 0,
20819            commits: 0,
20820            empty: false,
20821            failed: None,
20822            verified_noop: None,
20823            duration_ms: 0,
20824            folded: false,
20825        };
20826        let (e2e, retried, pids) = run_e2e_with_retry(
20827            &mut runner.state,
20828            &["sh".into(), "-c".into()],
20829            &[format!("{probe} && printf initial")],
20830            &repo,
20831            Duration::from_secs(10),
20832            "initial",
20833            None,
20834        )
20835        .await;
20836        assert!(!retried && pids.is_empty());
20837        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20838        assert_eq!(e2e[0].output_tail, "initial");
20839        let gate = runner.run_gate(&winner).await.unwrap();
20840        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20841        // An interrupted repair must recheck e2e from saved state. Landing
20842        // reuses that state and must run both pre-gate and gate again.
20843        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20844            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20845            passed: false,
20846            outcomes: Vec::new(),
20847        });
20848        runner.state.save().unwrap();
20849        let mut resumed = RunState::load(&runner.state.id).unwrap();
20850        let wait = crate::repo_lock::WaitPolicy::default();
20851        assert!(matches!(
20852            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20853                .await
20854                .unwrap(),
20855            GateFix::Retry
20856        ));
20857        let saved = RunState::load(&resumed.id).unwrap();
20858        let repair = saved.gate_repair_e2e.unwrap();
20859        assert!(repair.passed);
20860        assert_eq!(repair.outcomes.len(), 1);
20861        assert!(repair.outcomes[0].ok());
20862        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20863            .await
20864            .unwrap();
20865        assert_eq!(landing.len(), 1);
20866        assert!(landing[0].ok(), "{landing:?}");
20867        assert_eq!(
20868            std::fs::read_to_string(repo.join("phases")).unwrap(),
20869            "pre\ngate\ne2e\npre\ngate\n"
20870        );
20871        assert_worker();
20872    }
20873
20874    #[cfg(unix)]
20875    #[tokio::test]
20876    async fn verification_keeps_the_panic_and_the_full_output() {
20877        crate::run::pin_test_home();
20878        let tmp = tempfile::tempdir().unwrap();
20879        let mut config = Config::default();
20880        config.disk.min_free_bytes = 0;
20881        let mut state = RunState::new(
20882            tmp.path().to_owned(),
20883            "main".into(),
20884            "base".into(),
20885            "verification evidence".into(),
20886            config,
20887        );
20888        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20889                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20890                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20891        let mut names = Vec::new();
20892        for _ in 0..2 {
20893            let (outcomes, _) = run_commands(
20894                &mut state,
20895                "gate",
20896                "gate",
20897                0,
20898                &["sh".into(), "-c".into()],
20899                &[script.into()],
20900                tmp.path(),
20901                Duration::from_secs(60),
20902            )
20903            .await;
20904            let tail = &outcomes[0].output_tail;
20905            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20906            assert!(tail.contains("boom assertion"));
20907            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20908            let line = tail.lines().next().unwrap();
20909            names.push(line.to_owned());
20910        }
20911        assert_ne!(names[0], names[1], "the second run must not overwrite");
20912        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20913        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20914        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20915
20916        // Storage failures are evidence too; preserve the command's verdict.
20917        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20918        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20919        let (outcomes, _) = run_commands(
20920            &mut state,
20921            "gate",
20922            "gate",
20923            0,
20924            &["sh".into(), "-c".into()],
20925            &["echo failure-output; exit 7".into()],
20926            tmp.path(),
20927            Duration::from_secs(10),
20928        )
20929        .await;
20930        assert_eq!(outcomes[0].code, Some(7));
20931        assert!(state.events.iter().any(|e| {
20932            e.node == "gate" && e.message.contains("could not save the full gate output:")
20933        }));
20934    }
20935
20936    #[cfg(unix)]
20937    #[tokio::test]
20938    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20939        crate::run::pin_test_home();
20940        let tmp = tempfile::tempdir().unwrap();
20941        let mut config = Config::default();
20942        config.disk.min_free_bytes = 0;
20943        let mut state = RunState::new(
20944            tmp.path().to_owned(),
20945            "main".into(),
20946            "base".into(),
20947            "verification evidence".into(),
20948            config,
20949        );
20950        let (outcomes, pids) = run_commands(
20951            &mut state,
20952            "gate",
20953            "gate",
20954            1,
20955            &["sh".into(), "-c".into()],
20956            &[
20957                "printf before-timeout; exec sleep 30".into(),
20958                "kill -TERM $$".into(),
20959                "exit 7".into(),
20960                "printf success".into(),
20961            ],
20962            tmp.path(),
20963            Duration::from_secs(1),
20964        )
20965        .await;
20966        wait_for_timed_out_children_to_die(&pids).await;
20967        assert_eq!(
20968            pids.len(),
20969            1,
20970            "unleased timeout retains its death-check PID"
20971        );
20972        assert_eq!(outcomes.len(), 4);
20973        assert_eq!(outcomes[0].code, None);
20974        assert!(outcomes[0].output_tail.contains("before-timeout"));
20975        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20976        assert_eq!(outcomes[1].code, None);
20977        assert!(outcomes[1].output_tail.contains("command terminated:"));
20978        assert!(!outcomes[1].output_tail.contains("timed out"));
20979        assert_eq!(outcomes[2].code, Some(7));
20980        assert!(!outcomes[2].output_tail.contains("timed out"));
20981        assert_eq!(outcomes[3].code, Some(0));
20982        assert_eq!(outcomes[3].output_tail, "success");
20983        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20984    }
20985
20986    /// The addendum's second gap: a `verify.gate` command running for real
20987    /// wall-clock time had nothing at all to show for it in `active` before
20988    /// `run_commands` learned to record it — a run could sit in `Gating` for
20989    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20990    /// what was actually happening. Proven with a genuinely still-running
20991    /// command, not just a before/after check on the final state: a poller
20992    /// task reads the same `run.json` `gate()` is writing, the same way the
20993    /// phone or `magi show` would, while the shell command is still blocked
20994    /// on its own release marker.
20995    #[tokio::test]
20996    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20997        crate::run::pin_test_home();
20998
20999        let tmp = tempfile::tempdir().expect("tempdir");
21000        let repo = tmp.path().join("repo");
21001        std::fs::create_dir_all(&repo).unwrap();
21002        init_repo(&repo);
21003
21004        let mut config = Config::default();
21005        config.disk.min_free_bytes = 0;
21006        config.verify.gate = vec![
21007            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
21008             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
21009                .to_owned(),
21010        ];
21011
21012        let mut state = RunState::new(
21013            repo.clone(),
21014            "main".to_owned(),
21015            "deadbeef".to_owned(),
21016            "task".to_owned(),
21017            config,
21018        );
21019        let run_id = state.id.clone();
21020        state.candidates = vec![Candidate {
21021            index: 0,
21022            label: 'A',
21023            agent: "alpha".to_owned(),
21024            branch: "does-not-exist".to_owned(),
21025            worktree: repo.clone(),
21026            summary: String::new(),
21027            stat: String::new(),
21028            files: 0,
21029            commits: 0,
21030            empty: false,
21031            failed: None,
21032            verified_noop: None,
21033            duration_ms: 0,
21034            folded: false,
21035        }];
21036        state.tally = Some(Tally {
21037            first_choice: BTreeMap::from([('A', 1)]),
21038            borda: BTreeMap::new(),
21039            winner: 'A',
21040            rankings: 1,
21041            unanimous_initial: true,
21042            deliberated: false,
21043            changed_votes: 0,
21044            unanimous_final: true,
21045            tie_break: None,
21046            judges: 0,
21047            present: 0,
21048            quorum: 0,
21049            met_quorum: true,
21050            uncontested: Some("only candidate A produced a change".to_owned()),
21051        });
21052        state.reviews = vec![ReviewRound {
21053            round: 1,
21054            head: "deadbeef".to_owned(),
21055            verified_head: None,
21056            verified_at: None,
21057            reviews: Vec::new(),
21058            e2e: Vec::new(),
21059            fix: None,
21060            blocking: 0,
21061            answered: 0,
21062            expected: 0,
21063            clean: true,
21064            verify_retried: false,
21065            e2e_deferred: false,
21066            e2e_defer_reason: None,
21067            progressed: false,
21068            vote_split: false,
21069            reconsideration: Vec::new(),
21070            verdict: None,
21071        }];
21072
21073        let mut runner = Runner {
21074            state,
21075            roles: ResolvedRoles {
21076                implementers: Vec::new(),
21077                judges: Vec::new(),
21078                reviewers: Vec::new(),
21079                fixer: None,
21080                conductor: conductor(),
21081                implementer_roster: Vec::new(),
21082                judge_roster: Vec::new(),
21083                reviewer_roster: Vec::new(),
21084            },
21085            sem: Arc::new(Semaphore::new(1)),
21086            pause: Pause::new(),
21087            interrupt: Pause::new(),
21088        };
21089
21090        let started_marker = repo.join("started.marker");
21091        let release_marker = repo.join("release.marker");
21092        let poller = tokio::spawn(async move {
21093            // Bounded so a regression that never records the task entry
21094            // fails this test in seconds instead of hanging the suite —
21095            // the same shape `a_park_requested_while_a_seat_is_mid_call_
21096            // does_not_cut_it_short` uses for the same reason.
21097            for _ in 0..100 {
21098                if started_marker.exists()
21099                    && let Ok(s) = crate::run::RunState::load(&run_id)
21100                    && let Some(a) = s.active.get("gate")
21101                {
21102                    std::fs::write(&release_marker, b"go").expect("release marker");
21103                    return Some(a.clone());
21104                }
21105                tokio::time::sleep(Duration::from_millis(50)).await;
21106            }
21107            None
21108        });
21109
21110        runner.gate().await.expect("gate");
21111        let captured = poller.await.expect("poller task");
21112        let captured = captured.expect(
21113            "the poller never saw a `gate` task entry in run.json while the command was \
21114             still blocked on its own release marker",
21115        );
21116
21117        assert_eq!(captured.task.as_deref(), Some("gate"));
21118        assert_eq!(captured.node, "gate");
21119        assert_eq!(captured.index, Some(1));
21120        assert_eq!(captured.total, Some(1));
21121        assert!(
21122            captured
21123                .command
21124                .as_deref()
21125                .is_some_and(|c| c.contains("started.marker")),
21126            "{captured:?}"
21127        );
21128
21129        assert!(
21130            runner.state.active.is_empty(),
21131            "the entry must be cleared once the command actually finished: {:?}",
21132            runner.state.active
21133        );
21134        assert!(runner.state.gate_ran);
21135        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
21136    }
21137
21138    /// The hand-off over a blocking finding a reviewer rejected on leaves a
21139    /// record for `land`; one with only a Minor, or no reject, leaves none.
21140    #[tokio::test]
21141    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
21142        use crate::verdict::{Finding, ReviewVote, Severity};
21143        crate::run::pin_test_home();
21144        let tmp = tempfile::tempdir().expect("tempdir");
21145        let repo = tmp.path().join("repo");
21146        std::fs::create_dir_all(&repo).unwrap();
21147        init_repo(&repo);
21148
21149        for (severity, vote, expect) in [
21150            (Severity::Major, ReviewVote::Reject, true),
21151            (Severity::Minor, ReviewVote::Reject, false),
21152            (Severity::Major, ReviewVote::Approve, false),
21153        ] {
21154            let mut round = review_round(false, 1, 1, 1, false, true);
21155            round.reviews = vec![ReviewRecord {
21156                reviewer: 1,
21157                agent: "alpha".to_owned(),
21158                summary: String::new(),
21159                findings: vec![Finding {
21160                    id: "R1-1-1".to_owned(),
21161                    severity,
21162                    file: None,
21163                    line: None,
21164                    title: "t".to_owned(),
21165                    detail: String::new(),
21166                }],
21167                vote: Some(vote),
21168                failed: None,
21169                duration_ms: 0,
21170                attempts: 0,
21171            }];
21172            let mut state = RunState::new(
21173                repo.clone(),
21174                "main".to_owned(),
21175                "deadbeef".to_owned(),
21176                "task".to_owned(),
21177                Config::default(),
21178            );
21179            state.reviews = vec![round];
21180            let mut runner = Runner {
21181                state,
21182                roles: ResolvedRoles {
21183                    implementers: Vec::new(),
21184                    judges: Vec::new(),
21185                    reviewers: Vec::new(),
21186                    fixer: None,
21187                    conductor: conductor(),
21188                    implementer_roster: Vec::new(),
21189                    judge_roster: Vec::new(),
21190                    reviewer_roster: Vec::new(),
21191                },
21192                sem: Arc::new(Semaphore::new(1)),
21193                pause: Pause::new(),
21194                interrupt: Pause::new(),
21195            };
21196            let shell = runner.state.config.shell();
21197            runner
21198                .stop_reviewing("round budget spent", &shell, &repo)
21199                .await
21200                .expect("stop_reviewing");
21201            assert_eq!(runner.state.status, RunStatus::Gating);
21202            assert_eq!(
21203                runner.state.contested_handoff.is_some(),
21204                expect,
21205                "{severity:?} + {vote:?}"
21206            );
21207        }
21208    }
21209
21210    /// The shape the incident this whole fix responds to actually had: the
21211    /// round budget spent, the last round's own e2e blocked on the shared
21212    /// build cache (held here by a live pid — this test process — exactly
21213    /// `cache`'s own unit tests' pattern for "another owner, still alive"
21214    /// without forking a process). `stop_reviewing` must retry it — not
21215    /// silently leave the round looking untouched (the catch-up-only half of
21216    /// the bug), and not read the contention as a red `e2e` and block the
21217    /// run on it (the other half). Called directly, the same way
21218    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
21219    /// above exercises `gate`, so this never needs a real cargo build to
21220    /// reach: the lease is never released, so `with_cache_lease` never gets
21221    /// past acquiring it into anything that would need a real workspace.
21222    #[tokio::test]
21223    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21224        crate::run::pin_test_home();
21225        let home = crate::run::home();
21226
21227        let tmp = tempfile::tempdir().expect("tempdir");
21228        let repo = tmp.path().join("repo");
21229        std::fs::create_dir_all(&repo).unwrap();
21230        init_repo(&repo);
21231        let head = crate::git::rev_parse(&repo, "HEAD")
21232            .await
21233            .expect("rev-parse");
21234        // Unique to this test, so holding its lease cannot collide with
21235        // another test sharing the same process-wide `home`.
21236        let cache_dir = tmp.path().join("target");
21237
21238        let mut config = Config::default();
21239        config.verify.e2e = vec![format!(
21240            "CARGO_TARGET_DIR='{}' test -f README.md",
21241            cache_dir.display()
21242        )];
21243        config.graph.review_rounds = 1;
21244        // Bounded so a regression that does start waiting fails the test in
21245        // seconds, not hangs it.
21246        config.graph.timeout_verify = Some(2);
21247        config.graph.timeout_resource_wait = 2;
21248
21249        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21250        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21251            .expect("no io error acquiring directly")
21252        {
21253            crate::cache::AcquireOutcome::Acquired(g) => g,
21254            crate::cache::AcquireOutcome::Busy(b) => {
21255                panic!("expected the direct acquire to win the lease first: {b:?}")
21256            }
21257        };
21258
21259        let mut state = RunState::new(
21260            repo.clone(),
21261            "main".to_owned(),
21262            head.clone(),
21263            "task".to_owned(),
21264            config,
21265        );
21266        state.candidates = vec![Candidate {
21267            index: 0,
21268            label: 'A',
21269            agent: "alpha".to_owned(),
21270            branch: "does-not-exist".to_owned(),
21271            worktree: repo.clone(),
21272            summary: String::new(),
21273            stat: String::new(),
21274            files: 0,
21275            commits: 0,
21276            empty: false,
21277            failed: None,
21278            verified_noop: None,
21279            duration_ms: 0,
21280            folded: false,
21281        }];
21282        state.tally = Some(Tally {
21283            first_choice: BTreeMap::from([('A', 1)]),
21284            borda: BTreeMap::new(),
21285            winner: 'A',
21286            rankings: 1,
21287            unanimous_initial: true,
21288            deliberated: false,
21289            changed_votes: 0,
21290            unanimous_final: true,
21291            tie_break: None,
21292            judges: 0,
21293            present: 0,
21294            quorum: 0,
21295            met_quorum: true,
21296            uncontested: Some("only candidate A produced a change".to_owned()),
21297        });
21298        // The round budget's last round, deferred: `needs_catchup_run`'s
21299        // other trigger. `stop_reviewing`'s retry machinery must treat this
21300        // exactly like a resource-blocked attempt once it actually runs.
21301        state.reviews = vec![ReviewRound {
21302            round: 1,
21303            head: head.clone(),
21304            verified_head: None,
21305            verified_at: None,
21306            reviews: Vec::new(),
21307            e2e: Vec::new(),
21308            fix: None,
21309            blocking: 1,
21310            answered: 1,
21311            expected: 1,
21312            clean: false,
21313            verify_retried: false,
21314            e2e_deferred: true,
21315            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21316            progressed: false,
21317            vote_split: false,
21318            reconsideration: Vec::new(),
21319            verdict: None,
21320        }];
21321
21322        let mut runner = Runner {
21323            state,
21324            roles: ResolvedRoles {
21325                implementers: Vec::new(),
21326                judges: Vec::new(),
21327                reviewers: Vec::new(),
21328                fixer: None,
21329                conductor: conductor(),
21330                implementer_roster: Vec::new(),
21331                judge_roster: Vec::new(),
21332                reviewer_roster: Vec::new(),
21333            },
21334            sem: Arc::new(Semaphore::new(1)),
21335            pause: Pause::new(),
21336            interrupt: Pause::new(),
21337        };
21338
21339        let shell = runner.state.config.shell();
21340        runner
21341            .stop_reviewing("round budget spent", &shell, &repo)
21342            .await
21343            .expect("stop_reviewing");
21344
21345        let last = runner.state.reviews.last().expect("round record");
21346        assert_eq!(
21347            last.e2e_status(),
21348            E2eStatus::ResourceBlocked,
21349            "the shared cache is still held; the attempt must read as blocked, not deferred or \
21350             failed: {last:?}"
21351        );
21352        assert_eq!(
21353            last.verified_head.as_deref(),
21354            Some(head.as_str()),
21355            "which commit this attempt targeted is known even though nothing finished checking \
21356             it"
21357        );
21358        let first_attempt_at = last
21359            .verified_at
21360            .expect("when this attempt ran is known too");
21361        assert_ne!(
21362            runner.state.status,
21363            RunStatus::Blocked,
21364            "contention is evidence about the machine, not the patch — it must not settle the \
21365             run as blocked: {:?}",
21366            runner.state.status
21367        );
21368        assert!(
21369            !runner
21370                .state
21371                .events
21372                .iter()
21373                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21374            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21375            runner.state.events
21376        );
21377
21378        // The cache is still held: a later reentry must retry the same
21379        // round's verification again — not leave it looking exactly as
21380        // untouched as the first blocked attempt, which is indistinguishable
21381        // from never having tried again at all.
21382        runner
21383            .stop_reviewing("round budget spent", &shell, &repo)
21384            .await
21385            .expect("stop_reviewing retry");
21386        assert_eq!(
21387            runner.state.reviews.len(),
21388            1,
21389            "no new round was started: {:?}",
21390            runner.state.reviews
21391        );
21392        let last = runner.state.reviews.last().expect("round record");
21393        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21394        assert!(
21395            last.verified_at.expect("still known") > first_attempt_at,
21396            "a second reentry must be a fresh attempt, not a stale copy of the first"
21397        );
21398        assert_ne!(runner.state.status, RunStatus::Blocked);
21399
21400        held.release();
21401    }
21402
21403    #[cfg(unix)]
21404    #[tokio::test]
21405    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21406        ask_test_home();
21407        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21408            let tmp = tempfile::tempdir().unwrap();
21409            let repo = tmp.path().join("repo");
21410            std::fs::create_dir(&repo).unwrap();
21411            init_repo(&repo);
21412            let run_git = |args: &[&str]| {
21413                let out = std::process::Command::new("git")
21414                    .args(args)
21415                    .current_dir(&repo)
21416                    .output()
21417                    .unwrap();
21418                assert!(
21419                    out.status.success(),
21420                    "{args:?}: {}",
21421                    String::from_utf8_lossy(&out.stderr)
21422                );
21423            };
21424            let origin = tmp.path().join("origin.git");
21425            run_git(&["init", "--bare", origin.to_str().unwrap()]);
21426            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21427            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21428            run_git(&["add", "spec.rs"]);
21429            run_git(&["commit", "-m", "initial spec"]);
21430            run_git(&["push", "origin", "main"]);
21431            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21432            run_git(&["checkout", "-b", "fix/review"]);
21433            std::fs::write(
21434                repo.join("check.rs"),
21435                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21436            )
21437            .unwrap();
21438            run_git(&["add", "check.rs"]);
21439            run_git(&["commit", "-m", "use spec"]);
21440            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21441            run_git(&["checkout", "main"]);
21442            std::fs::write(
21443                repo.join("spec.rs"),
21444                "struct AgentSpec { max_turn_secs: u64 }\n",
21445            )
21446            .unwrap();
21447            run_git(&["add", "spec.rs"]);
21448            run_git(&["commit", "-m", "extend spec"]);
21449            run_git(&["push", "origin", "main"]);
21450            run_git(&["checkout", "fix/review"]);
21451
21452            let mut runner = runner_at(RunStatus::Reviewing);
21453            runner.state.repo = repo.clone();
21454            runner.state.base_commit = base;
21455            runner.state.pr_head = Some("fix/review".into());
21456            runner.state.config.graph.review_rounds = max_rounds;
21457            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21458            runner.state.config.graph.sessions = false;
21459            runner.state.config.disk.min_free_bytes = 0;
21460            runner.state.config.verify.e2e =
21461                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21462            runner.state.candidates = vec![Candidate {
21463                index: 0,
21464                label: 'A',
21465                agent: "fixture".into(),
21466                branch: "fix/review".into(),
21467                worktree: repo.clone(),
21468                summary: String::new(),
21469                stat: String::new(),
21470                files: 1,
21471                commits: 1,
21472                empty: false,
21473                failed: None,
21474                verified_noop: None,
21475                duration_ms: 0,
21476                folded: false,
21477            }];
21478            runner.state.tally = Some(Tally {
21479                first_choice: BTreeMap::new(),
21480                borda: BTreeMap::new(),
21481                winner: 'A',
21482                rankings: 0,
21483                unanimous_initial: true,
21484                deliberated: false,
21485                changed_votes: 0,
21486                unanimous_final: true,
21487                tie_break: None,
21488                judges: 0,
21489                present: 0,
21490                quorum: 0,
21491                met_quorum: true,
21492                uncontested: Some("review-only".into()),
21493            });
21494            let mut last = review_round(false, 0, 1, 1, false, false);
21495            last.head = reviewed.clone();
21496            last.e2e_deferred = true;
21497            last.e2e[0].resource_blocked = true;
21498            last.e2e[0].output_tail = "waiting for the shared build cache".into();
21499            if max_rounds == 1 {
21500                last.e2e.clear();
21501            }
21502            runner.state.reviews = vec![last];
21503            let calls = tmp.path().join("fix-calls");
21504            let mut fixer = spec("repair");
21505            fixer
21506                .env
21507                .insert("FIX_CALLS".into(), calls.display().to_string());
21508            fixer.command = vec!["sh".into(), "-c".into(), r#"
21509                echo fix >> "$FIX_CALLS"
21510                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21511                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21512                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21513                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21514            "#.into()];
21515            let mut reviewer = spec("reviewer");
21516            reviewer.command = vec![
21517                "sh".into(),
21518                "-c".into(),
21519                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21520                    .into(),
21521            ];
21522            runner.roles.fixer = Some(vec![fixer]);
21523            runner.roles.reviewers = vec![reviewer];
21524
21525            runner.sync_to_base().await.unwrap();
21526            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21527            assert_ne!(rebased, reviewed);
21528            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21529            if reentry {
21530                // Simulate a crash after catch-up evidence was saved but before
21531                // dispatching the fixer. No dedicated pending-repair field exists.
21532                let shell = runner.state.config.shell();
21533                runner
21534                    .stop_unresolved_review("deferred verification", &shell, &repo)
21535                    .await
21536                    .unwrap();
21537                assert!(runner.catchup_repairable());
21538                assert!(!calls.exists());
21539                runner.state = RunState::load(&runner.state.id).unwrap();
21540                assert!(runner.catchup_repairable());
21541            }
21542            runner.review_loop().await.unwrap();
21543            let first = &runner.state.reviews[0];
21544            assert_eq!(first.head, reviewed);
21545            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21546            assert!(
21547                first.verify_retried,
21548                "a reproduced build failure gets exactly one retry: {:?}",
21549                first.e2e
21550            );
21551            assert!(first.e2e[0].output_tail.contains("E0063"));
21552            assert!(!runner.catchup_repairable());
21553            if max_rounds == 2 {
21554                assert_eq!(
21555                    runner.state.status,
21556                    RunStatus::Gating,
21557                    "{}",
21558                    crate::report::run(&runner.state)
21559                );
21560                assert_eq!(runner.state.reviews.len(), 2);
21561                assert!(first.fix.as_ref().unwrap().committed);
21562                assert!(runner.state.reviews[1].clean);
21563                assert_eq!(
21564                    runner
21565                        .state
21566                        .reviews
21567                        .iter()
21568                        .filter(|r| r.fix.is_some())
21569                        .count(),
21570                    1
21571                );
21572                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21573                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21574            } else {
21575                assert_eq!(runner.state.status, RunStatus::Blocked);
21576                assert!(first.fix.is_none());
21577                assert!(!calls.exists());
21578                let recovery = crate::recovery::assess(
21579                    "prior",
21580                    Some(&runner.state),
21581                    &runner.landing_base(),
21582                    crate::recovery::Relation::Diverged,
21583                );
21584                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21585                let reasons = recovery.reasons.join("\n");
21586                assert!(
21587                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21588                    "{reasons}"
21589                );
21590                assert!(
21591                    reasons.contains("base sync to")
21592                        && reasons.contains("additional review is required"),
21593                    "{reasons}"
21594                );
21595            }
21596        }
21597    }
21598
21599    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21600    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21601    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21602    /// own top-of-function fast path (`review_conclusion`) correctly reads
21603    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21604    /// own `for` range is empty once the round budget is spent, so
21605    /// `review_loop` must retry the check itself rather than silently doing
21606    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21607    /// above exercises directly, but through `review_loop`'s own entry point
21608    /// this time, proving the wiring between the two rather than just the
21609    /// retry logic in isolation.
21610    #[tokio::test]
21611    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21612        crate::run::pin_test_home();
21613        let home = crate::run::home();
21614
21615        let tmp = tempfile::tempdir().expect("tempdir");
21616        let repo = tmp.path().join("repo");
21617        std::fs::create_dir_all(&repo).unwrap();
21618        init_repo(&repo);
21619        let head = crate::git::rev_parse(&repo, "HEAD")
21620            .await
21621            .expect("rev-parse");
21622        let cache_dir = tmp.path().join("target");
21623
21624        let mut config = Config::default();
21625        config.verify.e2e = vec![format!(
21626            "CARGO_TARGET_DIR='{}' test -f README.md",
21627            cache_dir.display()
21628        )];
21629        config.graph.review_rounds = 1;
21630        config.graph.timeout_verify = Some(2);
21631        config.graph.timeout_resource_wait = 2;
21632
21633        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21634        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21635            .expect("no io error acquiring directly")
21636        {
21637            crate::cache::AcquireOutcome::Acquired(g) => g,
21638            crate::cache::AcquireOutcome::Busy(b) => {
21639                panic!("expected the direct acquire to win the lease first: {b:?}")
21640            }
21641        };
21642
21643        let mut state = RunState::new(
21644            repo.clone(),
21645            "main".to_owned(),
21646            head.clone(),
21647            "task".to_owned(),
21648            config,
21649        );
21650        state.candidates = vec![Candidate {
21651            index: 0,
21652            label: 'A',
21653            agent: "alpha".to_owned(),
21654            branch: "does-not-exist".to_owned(),
21655            worktree: repo.clone(),
21656            summary: String::new(),
21657            stat: String::new(),
21658            files: 0,
21659            commits: 0,
21660            empty: false,
21661            failed: None,
21662            verified_noop: None,
21663            duration_ms: 0,
21664            folded: false,
21665        }];
21666        state.tally = Some(Tally {
21667            first_choice: BTreeMap::from([('A', 1)]),
21668            borda: BTreeMap::new(),
21669            winner: 'A',
21670            rankings: 1,
21671            unanimous_initial: true,
21672            deliberated: false,
21673            changed_votes: 0,
21674            unanimous_final: true,
21675            tie_break: None,
21676            judges: 0,
21677            present: 0,
21678            quorum: 0,
21679            met_quorum: true,
21680            uncontested: Some("only candidate A produced a change".to_owned()),
21681        });
21682        // The exact shape a prior process's `stop_reviewing` would have left
21683        // on disk: the round budget's last round, a real attempt already
21684        // made and already resource-blocked.
21685        state.reviews = vec![ReviewRound {
21686            round: 1,
21687            head: head.clone(),
21688            verified_head: Some(head.clone()),
21689            verified_at: Some(jiff::Timestamp::now()),
21690            reviews: Vec::new(),
21691            e2e: vec![CommandOutcome {
21692                command: format!(
21693                    "CARGO_TARGET_DIR='{}' test -f README.md",
21694                    cache_dir.display()
21695                ),
21696                code: None,
21697                output_tail: "waiting for the shared build cache".to_owned(),
21698                duration_ms: 0,
21699                resource_blocked: true,
21700            }],
21701            fix: None,
21702            blocking: 1,
21703            answered: 1,
21704            expected: 1,
21705            clean: false,
21706            verify_retried: false,
21707            e2e_deferred: false,
21708            e2e_defer_reason: None,
21709            progressed: false,
21710            vote_split: false,
21711            reconsideration: Vec::new(),
21712            verdict: None,
21713        }];
21714
21715        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21716        let mut runner = Runner {
21717            state,
21718            roles: ResolvedRoles {
21719                implementers: Vec::new(),
21720                judges: Vec::new(),
21721                reviewers: Vec::new(),
21722                fixer: None,
21723                conductor: conductor(),
21724                implementer_roster: Vec::new(),
21725                judge_roster: Vec::new(),
21726                reviewer_roster: Vec::new(),
21727            },
21728            sem: Arc::new(Semaphore::new(1)),
21729            pause: Pause::new(),
21730            interrupt: Pause::new(),
21731        };
21732
21733        // The lease is still held throughout, so this reentry's own retry is
21734        // also contended — proving `review_loop` actually tried again (not
21735        // that it happened to succeed) is what the timestamp comparison
21736        // below is for.
21737        runner.review_loop().await.expect("review_loop");
21738
21739        assert_eq!(
21740            runner.state.reviews.len(),
21741            1,
21742            "no new round was started on top of the unresolved one: {:?}",
21743            runner.state.reviews
21744        );
21745        let last = &runner.state.reviews[0];
21746        assert_eq!(
21747            last.e2e_status(),
21748            E2eStatus::ResourceBlocked,
21749            "still contended: {last:?}"
21750        );
21751        assert!(
21752            last.verified_at.expect("still known") > first_attempt_at,
21753            "review_loop must have actually retried the check, not left it exactly as found"
21754        );
21755        assert_ne!(
21756            runner.state.status,
21757            RunStatus::Blocked,
21758            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21759            runner.state.status
21760        );
21761
21762        held.release();
21763    }
21764
21765    #[tokio::test]
21766    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21767        crate::run::pin_test_home();
21768        for status in [RunStatus::Blocked, RunStatus::Gating] {
21769            let mut runner = runner_at(status);
21770            runner.state.config.graph.land = true;
21771            runner.state.gate_ran = true;
21772            runner.state.merge = Some(MergeOutcome {
21773                mode: MergeMode::Pr,
21774                ok: false,
21775                detail: "the fixer produced no commit".into(),
21776                empty: false,
21777            });
21778            let url = "https://example.invalid/o/r/pull/1";
21779            runner.state.pr = Some(crate::run::PrRecord {
21780                url: url.into(),
21781                number: 1,
21782                state: "open".into(),
21783                checks: "red".into(),
21784                round: 1,
21785                rounds: 1,
21786                red_at_merge: Vec::new(),
21787            });
21788            runner.state.save().unwrap();
21789            // The nonexistent repository makes observation fail before a forge
21790            // command can run. We still must reach land, rather than silently
21791            // return from the graph with its old gate and merge records.
21792            runner.execute_graph().await.unwrap();
21793            assert_eq!(runner.state.status, RunStatus::Blocked);
21794            assert!(runner.state.land_sync_pending);
21795            assert!(runner.state.events.iter().any(|event|
21796                event.node == "land" && event.message == format!("watching {url}")));
21797            assert_eq!(
21798                runner.state.merge.as_ref().unwrap().detail,
21799                "the fixer produced no commit"
21800            );
21801        }
21802    }
21803
21804    #[tokio::test]
21805    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21806        crate::run::pin_test_home();
21807        let tmp = tempfile::tempdir().expect("tempdir");
21808        let repo = tmp.path().join("repo");
21809        std::fs::create_dir_all(&repo).unwrap();
21810        init_repo(&repo);
21811
21812        let mut config = Config::default();
21813        config.merge.mode = MergeMode::Pr;
21814        config.graph.land = true;
21815        config.graph.land_approval = false;
21816
21817        let mut state = RunState::new(
21818            repo.clone(),
21819            "main".to_owned(),
21820            "deadbeef".to_owned(),
21821            "task".to_owned(),
21822            config,
21823        );
21824        state.candidates = vec![Candidate {
21825            index: 0,
21826            label: 'A',
21827            agent: "alpha".to_owned(),
21828            branch: "does-not-exist".to_owned(),
21829            worktree: repo.clone(),
21830            summary: String::new(),
21831            stat: String::new(),
21832            files: 0,
21833            commits: 0,
21834            empty: false,
21835            failed: None,
21836            verified_noop: None,
21837            duration_ms: 0,
21838            folded: false,
21839        }];
21840        state.tally = Some(Tally {
21841            first_choice: BTreeMap::from([('A', 1)]),
21842            borda: BTreeMap::new(),
21843            winner: 'A',
21844            rankings: 1,
21845            unanimous_initial: true,
21846            deliberated: false,
21847            changed_votes: 0,
21848            unanimous_final: true,
21849            tie_break: None,
21850            judges: 0,
21851            present: 0,
21852            quorum: 0,
21853            met_quorum: true,
21854            uncontested: Some("only candidate A produced a change".to_owned()),
21855        });
21856        state.reviews = vec![ReviewRound {
21857            round: 1,
21858            head: "deadbeef".to_owned(),
21859            verified_head: None,
21860            verified_at: None,
21861            reviews: Vec::new(),
21862            e2e: Vec::new(),
21863            fix: None,
21864            blocking: 0,
21865            answered: 0,
21866            expected: 0,
21867            clean: true,
21868            verify_retried: false,
21869            e2e_deferred: false,
21870            e2e_defer_reason: None,
21871            progressed: false,
21872            vote_split: false,
21873            reconsideration: Vec::new(),
21874            verdict: None,
21875        }];
21876        state.gate = vec![CommandOutcome {
21877            command: "test".to_owned(),
21878            code: Some(0),
21879            output_tail: String::new(),
21880            duration_ms: 0,
21881            resource_blocked: false,
21882        }];
21883        state.gate_ran = true;
21884        // A first pass through `merge` already pushed and opened this pull
21885        // request; `status` is `Landing` because a previous call into `land`
21886        // parked or was interrupted before it reached a terminal outcome.
21887        state.status = RunStatus::Landing;
21888        state.merge = Some(MergeOutcome {
21889            mode: MergeMode::Pr,
21890            ok: true,
21891            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21892            empty: false,
21893        });
21894
21895        // The Landing-resume shortcut calls `run_land` directly rather than
21896        // through `merge`, which is exactly the call site that used to skip
21897        // `settle_questions` - see the fixture below.
21898        ask_test_home();
21899        // A real resume loads a persisted run; driver ownership is checked
21900        // against that record before the Landing shortcut is entered.
21901        state.save().expect("save landing run");
21902        let store = ask::Questions::open();
21903        let q = ask_open_question(&store, &state.id);
21904
21905        let mut runner = Runner {
21906            state,
21907            roles: ResolvedRoles {
21908                implementers: Vec::new(),
21909                judges: Vec::new(),
21910                reviewers: Vec::new(),
21911                fixer: None,
21912                conductor: conductor(),
21913                implementer_roster: Vec::new(),
21914                judge_roster: Vec::new(),
21915                reviewer_roster: Vec::new(),
21916            },
21917            sem: Arc::new(Semaphore::new(1)),
21918            pause: Pause::new(),
21919            interrupt: Pause::new(),
21920        };
21921
21922        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21923        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21924        // exactly because `review_loop` would otherwise clobber the marker
21925        // first.
21926        runner.execute().await.expect("execute");
21927
21928        assert_eq!(
21929            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21930            Some("https://example.invalid/x/y/pull/1"),
21931            "reentry must not push again or open a second pull request over the \
21932             one `land` is already watching"
21933        );
21934        assert_ne!(
21935            runner.state.status,
21936            RunStatus::Landing,
21937            "land could not actually reach the fake pull request, so it must \
21938             have given up rather than left the run silently parked forever"
21939        );
21940        // `land` could not reach the fake pull request, so it gave up into
21941        // `Blocked` - still resumable, so the question must not have been
21942        // swept just because this branch now also calls `settle_questions`.
21943        assert_eq!(runner.state.status, RunStatus::Blocked);
21944        assert!(
21945            store.get(&q.id).unwrap().status.open(),
21946            "Blocked is still alive; settle_questions must have been a no-op here"
21947        );
21948    }
21949
21950    fn state_with_round(round: ReviewRound) -> RunState {
21951        let mut s = RunState::new(
21952            PathBuf::from("/repo"),
21953            "main".to_owned(),
21954            "abc1234".to_owned(),
21955            "add retries".to_owned(),
21956            Config::default(),
21957        );
21958        s.reviews = vec![round];
21959        s
21960    }
21961
21962    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21963        crate::verdict::Finding {
21964            id: id.to_owned(),
21965            severity,
21966            file: None,
21967            line: None,
21968            title: title.to_owned(),
21969            detail: String::new(),
21970        }
21971    }
21972
21973    #[test]
21974    fn pr_body_names_open_findings_and_declined_ones() {
21975        let round = ReviewRound {
21976            round: 2,
21977            head: "deadbee".to_owned(),
21978            verified_head: None,
21979            verified_at: None,
21980            reviews: vec![ReviewRecord {
21981                attempts: 0,
21982                reviewer: 1,
21983                agent: "alpha".to_owned(),
21984                summary: String::new(),
21985                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21986                vote: None,
21987                failed: None,
21988                duration_ms: 0,
21989            }],
21990            e2e: vec![CommandOutcome {
21991                command: "cargo test".to_owned(),
21992                code: Some(0),
21993                output_tail: String::new(),
21994                duration_ms: 0,
21995                resource_blocked: false,
21996            }],
21997            verify_retried: false,
21998            e2e_deferred: false,
21999            e2e_defer_reason: None,
22000            fix: Some(FixRecord {
22001                agent: "alpha".to_owned(),
22002                addressed: Vec::new(),
22003                rejected: vec![crate::verdict::Rejection {
22004                    id: "R1-1-1".to_owned(),
22005                    why: "not reachable from any caller".to_owned(),
22006                }],
22007                notes: String::new(),
22008                committed: true,
22009                failed: None,
22010                duration_ms: 0,
22011                continuation: None,
22012            }),
22013            blocking: 0,
22014            answered: 1,
22015            expected: 1,
22016            clean: false,
22017            progressed: true,
22018            vote_split: false,
22019            reconsideration: Vec::new(),
22020            verdict: None,
22021        };
22022        let state = state_with_round(round);
22023        let body = pr_message(&state, 'A').body;
22024
22025        assert!(body.contains("add retries"), "the task must still be there");
22026        assert!(body.contains("R2-1-1"), "{body}");
22027        assert!(!body.contains("unused import"), "{body}");
22028        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
22029        assert!(
22030            !body.contains("not reachable from any caller"),
22031            "the reason it was declined: {body}"
22032        );
22033    }
22034
22035    #[test]
22036    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
22037        let round = ReviewRound {
22038            round: 1,
22039            head: "deadbee".to_owned(),
22040            verified_head: None,
22041            verified_at: None,
22042            reviews: vec![ReviewRecord {
22043                attempts: 0,
22044                reviewer: 1,
22045                agent: "alpha".to_owned(),
22046                summary: String::new(),
22047                findings: Vec::new(),
22048                vote: None,
22049                failed: None,
22050                duration_ms: 0,
22051            }],
22052            e2e: Vec::new(),
22053            verify_retried: false,
22054            e2e_deferred: false,
22055            e2e_defer_reason: None,
22056            fix: None,
22057            blocking: 0,
22058            answered: 1,
22059            expected: 1,
22060            clean: true,
22061            progressed: false,
22062            vote_split: false,
22063            reconsideration: Vec::new(),
22064            verdict: None,
22065        };
22066        let state = state_with_round(round);
22067        let body = pr_message(&state, 'A').body;
22068        assert!(!body.contains("Open review findings"), "{body}");
22069        assert!(!body.contains("Declined"), "{body}");
22070    }
22071
22072    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
22073        let mut state = RunState::new(
22074            PathBuf::from("/repo"),
22075            "main".to_owned(),
22076            "abc1234".to_owned(),
22077            instruction.to_owned(),
22078            Config::default(),
22079        );
22080        // These fixtures test execution flow, independent of host disk capacity.
22081        state.config.disk.min_free_bytes = 0;
22082        state.candidates.push(Candidate {
22083            index: 0,
22084            label: 'A',
22085            agent: "alpha".to_owned(),
22086            branch: "magi/x/A".to_owned(),
22087            worktree: PathBuf::from("/wt"),
22088            summary: summary.to_owned(),
22089            stat: String::new(),
22090            files: 1,
22091            commits: 1,
22092            empty: false,
22093            failed: None,
22094            verified_noop: None,
22095            folded: false,
22096            duration_ms: 0,
22097        });
22098        state
22099    }
22100
22101    fn review_state(subjects: &[&str]) -> RunState {
22102        let mut state = state_with_summary(
22103            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
22104            "",
22105        );
22106        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22107        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
22108        state
22109    }
22110
22111    /// A branch rebased onto a main that moved past the recorded
22112    /// `base_commit` is titled from its own first commit, never main's.
22113    #[tokio::test]
22114    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
22115        ask_test_home();
22116        let tmp = tempfile::tempdir().unwrap();
22117        let repo = tmp.path().join("repo");
22118        std::fs::create_dir_all(&repo).unwrap();
22119        init_repo(&repo);
22120        let origin = tmp.path().join("origin.git");
22121        let g = |dir: &Path, args: &[&str]| {
22122            let out = std::process::Command::new("git")
22123                .args(args)
22124                .current_dir(dir)
22125                .quiet()
22126                .output()
22127                .expect("spawn git");
22128            assert!(
22129                out.status.success(),
22130                "git {args:?}: {}",
22131                String::from_utf8_lossy(&out.stderr)
22132            );
22133        };
22134        g(
22135            tmp.path(),
22136            &[
22137                "clone",
22138                "--bare",
22139                "-q",
22140                repo.to_str().unwrap(),
22141                origin.to_str().unwrap(),
22142            ],
22143        );
22144        g(
22145            &repo,
22146            &["remote", "add", "origin", origin.to_str().unwrap()],
22147        );
22148        let c1 = git::rev_parse(&repo, "main").await.unwrap();
22149
22150        // Main moves on; the branch is built on top of the new main.
22151        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22152        g(&repo, &["add", "-A"]);
22153        g(
22154            &repo,
22155            &["commit", "-q", "-m", "chore(deps): update a crate"],
22156        );
22157        g(&repo, &["push", "-q", "origin", "main"]);
22158        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22159        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22160        g(&repo, &["add", "-A"]);
22161        g(
22162            &repo,
22163            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22164        );
22165        g(&repo, &["checkout", "-q", "main"]);
22166
22167        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22168        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22169        // Without a readable tracking ref the recorded base's merge base is used.
22170        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22171        assert_eq!(fallback, c1);
22172
22173        let mut state = review_state(&[
22174            "chore(deps): update a crate",
22175            "fix(daemon): apply a chosen action",
22176        ]);
22177        state.repo = repo.clone();
22178        state.base_branch = "main".to_owned();
22179        state.base_commit = c1;
22180        refresh_reviewed_commits(&mut state, "feat/own").await;
22181        assert_eq!(
22182            state.reviewed_commits,
22183            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22184        );
22185        assert_eq!(
22186            review_title(&state).as_deref(),
22187            Some("fix(daemon): apply a chosen action")
22188        );
22189        assert_eq!(
22190            leaked_subjects(&state, "feat/own").await,
22191            Some(vec!["chore(deps): update a crate".to_owned()])
22192        );
22193        // A stale tracking ref is still used when the fetch fails, but the
22194        // leak list is withheld.
22195        state.config.merge.remote = "nowhere".to_owned();
22196        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22197    }
22198
22199    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
22200    /// with `origin/main` one commit behind it, and a summarizing agent that
22201    /// prints whatever `reply` holds.
22202    struct SummaryFixture {
22203        _tmp: tempfile::TempDir,
22204        repo: PathBuf,
22205        reply: PathBuf,
22206        runner: Runner,
22207    }
22208
22209    fn summary_fixture() -> SummaryFixture {
22210        ask_test_home();
22211        let tmp = tempfile::tempdir().unwrap();
22212        let repo = tmp.path().join("repo");
22213        std::fs::create_dir_all(&repo).unwrap();
22214        init_repo(&repo);
22215        let origin = tmp.path().join("origin.git");
22216        let g = |args: &[&str]| {
22217            let out = std::process::Command::new("git")
22218                .args(args)
22219                .current_dir(&repo)
22220                .quiet()
22221                .output()
22222                .expect("spawn git");
22223            assert!(
22224                out.status.success(),
22225                "git {args:?}: {}",
22226                String::from_utf8_lossy(&out.stderr)
22227            );
22228        };
22229        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22230        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22231        g(&["push", "-q", "origin", "main"]);
22232        g(&["checkout", "-q", "-b", "feat/own"]);
22233        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22234        g(&["add", "-A"]);
22235        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22236        let reply = tmp.path().join("reply.txt");
22237        std::fs::write(
22238            &reply,
22239            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22240        )
22241        .unwrap();
22242        let mut runner = runner_at(RunStatus::Gating);
22243        let mut state = review_state(&["feat(own): add the own file"]);
22244        state.repo = repo.clone();
22245        state.base_branch = "main".to_owned();
22246        state.candidates[0].branch = "feat/own".to_owned();
22247        state.candidates[0].worktree = repo.clone();
22248        let mut author = spec("alpha");
22249        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22250        author
22251            .env
22252            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22253        state.config.agents = vec![author];
22254        runner.state = state;
22255        SummaryFixture {
22256            _tmp: tmp,
22257            repo,
22258            reply,
22259            runner,
22260        }
22261    }
22262
22263    async fn posted(f: &mut SummaryFixture) -> PrMessage {
22264        let winner = f.runner.state.candidates[0].clone();
22265        f.runner
22266            .guarded_pr_message(&winner, None, true)
22267            .await
22268            .unwrap()
22269            .expect("an English description never parks")
22270    }
22271
22272    fn clean_round_with_minor_findings() -> ReviewRound {
22273        let mut round = passing_round(2, None);
22274        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22275        round.reviews = vec![ReviewRecord {
22276            attempts: 0,
22277            reviewer: 1,
22278            agent: "alpha".into(),
22279            summary: "内部レビュー".into(),
22280            findings: vec![
22281                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22282                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22283            ],
22284            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22285            failed: None,
22286            duration_ms: 0,
22287        }];
22288        round
22289    }
22290
22291    #[tokio::test]
22292    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22293        for review_only in [false, true] {
22294            let mut f = summary_fixture();
22295            if !review_only {
22296                f.runner.state.candidates[0].agent = "alpha".into();
22297            }
22298            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22299            assert!(f.runner.state.reviews[0].clean);
22300            assert!(f.runner.state.reviews[0].fix.is_none());
22301            assert!(f.runner.state.open_findings().is_empty());
22302            assert_eq!(
22303                crate::pr_outcome::required_finding_ids(&f.runner.state),
22304                ["R2-1-1", "R2-1-2"]
22305            );
22306            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22307            let evidence = crate::pr_outcome::summary_evidence(
22308                &f.runner.state,
22309                crate::pr_outcome::FinalHead::At(&head),
22310            );
22311            assert!(
22312                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22313            );
22314            // The ordinary reply omits both IDs: no cached description is adopted.
22315            let omitted = posted(&mut f).await;
22316            assert!(omitted.body.contains("No description"));
22317            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22318            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22319            assert!(f.runner.state.change_summary.is_none());
22320            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22321            let described = posted(&mut f).await;
22322            assert!(described.body.contains("improve documentation for callers"));
22323            assert!(described.body.contains("remaining edge case"));
22324            assert!(described.body.contains("## Open review findings"));
22325            assert!(!described.body.contains("未修正"));
22326        }
22327    }
22328
22329    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22330        let mut f = summary_fixture();
22331        if !review_only {
22332            f.runner.state.candidates[0].agent = "alpha".into();
22333        }
22334        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22335        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22336        fixed
22337            .fix
22338            .as_mut()
22339            .unwrap()
22340            .rejected
22341            .push(crate::verdict::Rejection {
22342                id: "R1-1-1".into(),
22343                why: "呼び出されないコードのため辞退".into(),
22344            });
22345        let mut clean = clean_round_with_minor_findings();
22346        clean.verified_head = Some(head.clone());
22347        f.runner.state.reviews = vec![fixed, clean];
22348        ok_gate(&mut f.runner.state, Some(&head));
22349        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22350        f.runner.state.config.agents.push(scripted("beta", reply));
22351        f.runner.state.config.roles.language_judge =
22352            Some(crate::config::AgentChoice::One("beta".into()));
22353        f
22354    }
22355
22356    #[tokio::test]
22357    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22358        for review_only in [false, true] {
22359            let mut f = rewrite_fixture_with_findings(review_only,
22360                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22361            let m = posted(&mut f).await;
22362            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22363            let expected =
22364                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22365            assert!(m.body.contains(&expected));
22366            assert!(m.body.contains("## Review rounds"));
22367            assert!(
22368                m.body
22369                    .contains("Gate (`verify.gate`): passed on the final commit")
22370            );
22371            assert!(
22372                m.body
22373                    .contains("Tests (`verify.e2e`): passed on the final commit")
22374            );
22375            let c = f.runner.state.change_summary.as_ref().unwrap();
22376            assert!(
22377                m.body
22378                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22379            );
22380            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22381            assert!(crate::pr_outcome::covers_required_findings(
22382                &f.runner.state,
22383                &m.body
22384            ));
22385            assert!(m.body.contains("documentation remains incomplete"));
22386            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22387            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22388            assert!(f.runner.state.github_text.is_none());
22389        }
22390    }
22391
22392    #[tokio::test]
22393    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22394        for review_only in [false, true] {
22395            for reply in [
22396                "TITLE: fix: public description\nA short English description.",
22397                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22398            ] {
22399                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22400                let winner = f.runner.state.candidates[0].clone();
22401                let posted = f
22402                    .runner
22403                    .guarded_pr_message(&winner, None, true)
22404                    .await
22405                    .unwrap();
22406                assert!(posted.is_none());
22407                assert!(f.runner.state.parked);
22408                let withheld = f.runner.state.github_text.as_ref().unwrap();
22409                assert!(
22410                    withheld.rewrite.is_none(),
22411                    "an incomplete rewrite must not be selectable"
22412                );
22413                let q = ask::Questions::open()
22414                    .get(withheld.question.as_ref().unwrap())
22415                    .unwrap();
22416                assert!(
22417                    !q.choices
22418                        .contains(&crate::github_text::USE_REWRITE.to_owned())
22419                );
22420                assert!(
22421                    q.detail.contains("R1-1-1")
22422                        && q.detail.contains("R2-1-1")
22423                        && q.detail.contains("R2-1-2")
22424                );
22425            }
22426        }
22427    }
22428
22429    #[tokio::test]
22430    async fn japanese_reports_stay_local_in_both_public_run_modes() {
22431        for review_only in [false, true] {
22432            let mut f = summary_fixture();
22433            if !review_only {
22434                f.runner.state.candidates[0].agent = "alpha".into();
22435            }
22436            f.runner.state.instruction = "内部タスクの説明".into();
22437            f.runner.state.candidates[0].summary = "内部の候補報告".into();
22438            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22439            f.runner.state.reviews = vec![
22440                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22441                round_with_notes(2, Some("二回目の内部修正報告です。")),
22442                passing_round(3, Some(&head)),
22443            ];
22444            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22445            ok_gate(&mut f.runner.state, Some(&head));
22446            let captured = f.repo.join("public-prompt.txt");
22447            let author = &mut f.runner.state.config.agents[0];
22448            author.command = vec![
22449                "sh".into(),
22450                "-c".into(),
22451                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22452            ];
22453            author.env.insert(
22454                "PROMPT_FILE".into(),
22455                captured.to_string_lossy().into_owned(),
22456            );
22457            let m = posted(&mut f).await;
22458            assert!(m.body.contains("Adds `own.txt`"));
22459            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22460            assert!(m.body.contains("R1-1-1"));
22461            assert!(
22462                m.body
22463                    .contains("Gate (`verify.gate`): passed on the final commit")
22464            );
22465            assert!(
22466                m.body
22467                    .contains("Tests (`verify.e2e`): passed on the final commit")
22468            );
22469            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22470            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22471            let prompt = std::fs::read_to_string(captured).unwrap();
22472            assert!(prompt.contains(&format!(
22473                "git diff {} {head}",
22474                f.runner.state.change_summary.as_ref().unwrap().base
22475            )));
22476            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22477            assert_eq!(
22478                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22479                "二回目の内部修正報告です。"
22480            );
22481        }
22482    }
22483
22484    #[tokio::test]
22485    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22486        for review_only in [false, true] {
22487            let mut f = summary_fixture();
22488            if !review_only {
22489                f.runner.state.candidates[0].agent = "alpha".into();
22490            }
22491            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22492            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22493            std::fs::write(&f.reply, "no usable description").unwrap();
22494            let failed = posted(&mut f).await;
22495            assert!(
22496                failed
22497                    .body
22498                    .contains("No description of what this change does could be generated")
22499            );
22500            assert!(failed.body.is_ascii());
22501            assert!(f.runner.state.github_text.is_none());
22502            assert!(f.runner.state.change_summary.is_none());
22503            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22504            let evidence = crate::pr_outcome::summary_evidence(
22505                &f.runner.state,
22506                crate::pr_outcome::FinalHead::At(&head),
22507            );
22508            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22509            assert!(
22510                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22511                    .exists()
22512            );
22513            f.runner.state =
22514                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22515            std::fs::write(
22516                &f.reply,
22517                "TITLE: feat(own): add the own file\nAdds the committed own file.",
22518            )
22519            .unwrap();
22520            let retried = posted(&mut f).await;
22521            assert!(retried.body.contains("Adds the committed own file."));
22522            assert_eq!(
22523                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22524                "日本語の修正報告"
22525            );
22526        }
22527    }
22528
22529    #[tokio::test]
22530    async fn same_head_verification_and_findings_invalidate_public_prose() {
22531        let mut f = summary_fixture();
22532        posted(&mut f).await;
22533        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22534        let original = f
22535            .runner
22536            .state
22537            .change_summary
22538            .as_ref()
22539            .unwrap()
22540            .evidence
22541            .clone();
22542        ok_gate(&mut f.runner.state, Some(&head));
22543        std::fs::write(
22544            &f.reply,
22545            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22546        )
22547        .unwrap();
22548        let verified = posted(&mut f).await;
22549        assert!(verified.body.contains("now verified"));
22550        assert_ne!(
22551            f.runner.state.change_summary.as_ref().unwrap().evidence,
22552            original
22553        );
22554        let mut round = round_with_notes(1, Some("内部報告"));
22555        round.reviews.push(ReviewRecord {
22556            attempts: 0,
22557            reviewer: 1,
22558            agent: "alpha".into(),
22559            summary: "内部レビュー".into(),
22560            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22561            vote: None,
22562            failed: None,
22563            duration_ms: 0,
22564        });
22565        round
22566            .fix
22567            .as_mut()
22568            .unwrap()
22569            .rejected
22570            .push(crate::verdict::Rejection {
22571                id: "R1-1-1".into(),
22572                why: "呼び出し元から到達できない".into(),
22573            });
22574        f.runner.state.reviews = vec![round];
22575        // A description omitting the required ID is unusable, even with a title.
22576        let omitted = posted(&mut f).await;
22577        assert!(omitted.body.contains("No description"));
22578        assert!(omitted.body.contains("R1-1-1"));
22579        assert!(!omitted.body.contains("到達"));
22580        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22581        let described = posted(&mut f).await;
22582        assert!(described.body.contains("unreachable from callers"));
22583        assert!(described.body.contains("R1-1-1"));
22584        assert!(!described.body.contains("到達"));
22585        let evidence = f
22586            .runner
22587            .state
22588            .change_summary
22589            .as_ref()
22590            .unwrap()
22591            .evidence
22592            .clone();
22593        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22594        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22595        assert!(posted(&mut f).await.body.contains("rationale updated"));
22596        assert_ne!(
22597            f.runner.state.change_summary.as_ref().unwrap().evidence,
22598            evidence
22599        );
22600        // Old serialized summaries have no evidence fingerprint and must regenerate.
22601        f.runner
22602            .state
22603            .change_summary
22604            .as_mut()
22605            .unwrap()
22606            .evidence
22607            .clear();
22608        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22609        assert!(posted(&mut f).await.body.contains("current specification"));
22610    }
22611
22612    #[tokio::test]
22613    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22614        let mut f = summary_fixture();
22615        let m = posted(&mut f).await;
22616        assert_eq!(m.title, "feat(own): add the own file");
22617        assert!(
22618            m.body
22619                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22620            "{}",
22621            m.body
22622        );
22623        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22624        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22625        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22626        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22627        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22628        let kept = f.runner.state.change_summary.as_ref().unwrap();
22629        assert_eq!(
22630            (kept.head.as_str(), kept.base.as_str()),
22631            (head.as_str(), base.as_str())
22632        );
22633        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22634    }
22635
22636    #[tokio::test]
22637    async fn review_only_summary_receives_the_configured_github_language() {
22638        let mut f = summary_fixture();
22639        f.runner.state.config.graph.github_language = "ja".into();
22640        let captured = f.repo.join("summary-prompt.txt");
22641        let author = &mut f.runner.state.config.agents[0];
22642        author.command = vec![
22643            "sh".into(),
22644            "-c".into(),
22645            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22646        ];
22647        author.env.insert(
22648            "PROMPT_FILE".into(),
22649            captured.to_string_lossy().into_owned(),
22650        );
22651        std::fs::write(
22652            &f.reply,
22653            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22654        )
22655        .unwrap();
22656        let m = posted(&mut f).await;
22657        let prompt = std::fs::read_to_string(captured).unwrap();
22658        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22659        assert!(!prompt.contains("in English"));
22660        assert_eq!(m.title, "feat(own): ファイルを追加");
22661        assert!(m.body.contains("own.txt を追加します。"));
22662    }
22663
22664    #[tokio::test]
22665    async fn review_fixes_come_after_the_whole_change_summary() {
22666        let mut f = summary_fixture();
22667        f.runner.state.reviews = vec![
22668            round_with_notes(1, Some("handled the PATH case")),
22669            round_with_notes(2, None),
22670        ];
22671        let m = posted(&mut f).await;
22672        let summary = m.body.find("## Summary").expect("summary");
22673        let fixes = m.body.find("## Review rounds").expect("review rounds");
22674        assert!(summary < fixes, "{}", m.body);
22675        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22676        assert!(m.body.contains("## Final verification"), "{}", m.body);
22677    }
22678
22679    #[tokio::test]
22680    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22681        let mut f = summary_fixture();
22682        let first = posted(&mut f).await;
22683        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22684        // The same head reuses the record: a different reply is not read.
22685        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22686        let again = posted(&mut f).await;
22687        assert_eq!((again.title, again.body), (first.title, first.body));
22688
22689        let g = |args: &[&str]| {
22690            let out = std::process::Command::new("git")
22691                .args(args)
22692                .current_dir(&f.repo)
22693                .quiet()
22694                .output()
22695                .expect("spawn git");
22696            assert!(out.status.success(), "git {args:?}");
22697        };
22698        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22699        g(&["add", "-A"]);
22700        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22701        let second = posted(&mut f).await;
22702        let kept = f.runner.state.change_summary.clone().unwrap();
22703        assert_ne!(kept.head, first_head);
22704        assert!(second.body.contains("Not read yet."), "{}", second.body);
22705        assert!(!second.body.contains("so the first summary is read"));
22706        assert_eq!(second.title, "feat(own): other");
22707    }
22708
22709    #[tokio::test]
22710    async fn an_unavailable_summary_is_stated_and_never_invented() {
22711        let mut f = summary_fixture();
22712        std::fs::write(&f.reply, "no title line here").unwrap();
22713        let m = posted(&mut f).await;
22714        assert!(
22715            m.body
22716                .contains("No description of what this change does could be generated"),
22717            "{}",
22718            m.body
22719        );
22720        assert!(m.body.contains("internal reports remain in local records"));
22721        assert!(f.runner.state.change_summary.is_none());
22722        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22723        // A preview asks nobody and claims nothing.
22724        let winner = f.runner.state.candidates[0].clone();
22725        let preview = f
22726            .runner
22727            .guarded_pr_message(&winner, None, false)
22728            .await
22729            .unwrap()
22730            .unwrap();
22731        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22732    }
22733
22734    #[test]
22735    fn pr_message_review_single_commit_uses_its_subject() {
22736        let state = review_state(&["feat(nats): per-role user"]);
22737        let m = pr_message(&state, 'A');
22738        assert_eq!(m.title, "feat(nats): per-role user");
22739        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22740        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22741    }
22742
22743    #[test]
22744    fn pr_message_review_multi_commit_takes_the_oldest() {
22745        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22746        let m = pr_message(&state, 'A');
22747        assert_eq!(m.title, "feat: the change");
22748        for s in ["feat: the change", "fix: typo", "fix: again"] {
22749            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22750        }
22751    }
22752
22753    #[test]
22754    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22755        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22756            let mut state = review_state(&[first, "fix: later fixup"]);
22757            state.candidates[0].branch = "機能/ログイン".to_owned();
22758            let m = pr_message(&state, 'A');
22759            assert!(
22760                m.title.starts_with("chore: land candidate A of run"),
22761                "{}",
22762                m.title
22763            );
22764        }
22765    }
22766
22767    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22768        BranchFacts {
22769            commits: commits
22770                .iter()
22771                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22772                .collect(),
22773            stat: stat.to_owned(),
22774        }
22775    }
22776
22777    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22778        let mut r = review_round(true, 0, 1, 1, true, true);
22779        r.round = round;
22780        r.fix = notes.map(|n| FixRecord {
22781            agent: "fixer".to_owned(),
22782            addressed: Vec::new(),
22783            rejected: Vec::new(),
22784            notes: n.to_owned(),
22785            committed: true,
22786            failed: None,
22787            duration_ms: 0,
22788            continuation: None,
22789        });
22790        r
22791    }
22792
22793    #[test]
22794    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22795        let state = review_state(&["ignored"]);
22796        let f = facts(
22797            &[
22798                (
22799                    "fix(login): resolve PATH on macOS",
22800                    "Login shells skip rc files.",
22801                ),
22802                ("fix: address review", ""),
22803            ],
22804            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22805        );
22806        let m = pr_message_with(&state, 'A', Some(&f));
22807        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22808        assert!(
22809            m.body
22810                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22811            "{}",
22812            m.body
22813        );
22814        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22815        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22816        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22817        for banned in [
22818            "Review the work already",
22819            "no task statement",
22820            "Original task",
22821        ] {
22822            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22823        }
22824        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22825    }
22826
22827    #[test]
22828    fn pr_message_review_truncates_a_huge_first_commit_body() {
22829        let state = review_state(&["ignored"]);
22830        let f = facts(
22831            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22832            "s",
22833        );
22834        let m = pr_message_with(&state, 'A', Some(&f));
22835        assert!(m.body.len() < 30_000, "{}", m.body.len());
22836        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22837        assert!(
22838            m.body.contains("- ... 1 more commit(s)"),
22839            "{}",
22840            m.body.len()
22841        );
22842        assert!(m.body.ends_with("magi:candidate-a\n"));
22843    }
22844
22845    #[test]
22846    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22847        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22848        assert_eq!(m.title, "feat: x");
22849        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22850        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22851        assert!(!m.body.contains("no task statement"), "{}", m.body);
22852    }
22853
22854    #[test]
22855    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22856        let mut state = review_state(&["日本語の件名"]);
22857        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22858        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22859        state.candidates[0].branch = "機能/ログイン".to_owned();
22860        assert!(
22861            pr_message(&state, 'A')
22862                .title
22863                .starts_with("chore: land candidate A")
22864        );
22865    }
22866
22867    #[test]
22868    fn pr_message_review_fixes_survive_a_clean_final_round() {
22869        let mut state = review_state(&["feat: x"]);
22870        state.reviews = vec![
22871            round_with_notes(1, Some("handled the PATH case")),
22872            round_with_notes(2, None),
22873        ];
22874        let body = pr_message(&state, 'A').body;
22875        assert!(
22876            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22877            "{body}"
22878        );
22879        assert!(body.contains("### Round 1"), "{body}");
22880
22881        state.reviews = vec![
22882            round_with_notes(1, Some("first fix")),
22883            round_with_notes(2, Some("")),
22884            round_with_notes(3, Some("second fix")),
22885            round_with_notes(4, None),
22886        ];
22887        let body = pr_message(&state, 'A').body;
22888        assert!(
22889            body.contains("### Round 1") && !body.contains("first fix"),
22890            "{body}"
22891        );
22892        assert!(
22893            body.contains("### Round 3") && !body.contains("second fix"),
22894            "{body}"
22895        );
22896        assert!(body.contains("### Round 2"), "{body}");
22897    }
22898
22899    const FINAL: &str = "abcdef1234567890";
22900
22901    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22902        let mut r = round_with_notes(round, None);
22903        r.verified_head = verified.map(str::to_owned);
22904        r
22905    }
22906
22907    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22908        state.gate = vec![CommandOutcome {
22909            command: "gate".to_owned(),
22910            code: Some(0),
22911            output_tail: String::new(),
22912            duration_ms: 0,
22913            resource_blocked: false,
22914        }];
22915        state.gate_ran = true;
22916        state.gate_head = head.map(str::to_owned);
22917    }
22918
22919    fn body_at(state: &RunState) -> String {
22920        pr_message_raw(
22921            state,
22922            'A',
22923            None,
22924            crate::pr_outcome::FinalHead::At(FINAL),
22925            ChangeNote::NotAsked,
22926        )
22927        .body
22928    }
22929
22930    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22931
22932    #[test]
22933    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22934        let mut state = review_state(&["feat: x"]);
22935        state.reviews = vec![
22936            round_with_notes(1, Some(SANDBOXED)),
22937            passing_round(2, Some(FINAL)),
22938        ];
22939        ok_gate(&mut state, Some(FINAL));
22940        let body = body_at(&state);
22941        assert!(!body.contains("Handled the PATH case."), "{body}");
22942        assert!(!body.contains("could not commit"), "{body}");
22943        assert!(!body.contains("could not run"), "{body}");
22944        assert!(
22945            body.contains("The fix is committed on the branch"),
22946            "{body}"
22947        );
22948        assert!(!body.contains("sandbox"), "{body}");
22949        assert!(
22950            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22951            "{body}"
22952        );
22953        assert!(
22954            body.contains("Gate (`verify.gate`): passed on the final commit"),
22955            "{body}"
22956        );
22957        // The raw report is untouched in the record.
22958        assert!(
22959            state.reviews[0]
22960                .fix
22961                .as_ref()
22962                .unwrap()
22963                .notes
22964                .contains("could not commit")
22965        );
22966    }
22967
22968    #[test]
22969    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22970        let mut state = review_state(&["feat: x"]);
22971        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22972        r1.fix.as_mut().unwrap().committed = false;
22973        let mut r2 = passing_round(2, Some(FINAL));
22974        r2.e2e[0].code = Some(1);
22975        state.reviews = vec![r1, r2];
22976        let body = body_at(&state);
22977        assert!(!body.contains("could not commit"), "{body}");
22978        assert!(!body.contains("could not run"), "{body}");
22979        assert!(body.contains("This round produced no commit"), "{body}");
22980        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22981        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22982        assert!(!body.contains("passed"), "{body}");
22983    }
22984
22985    #[test]
22986    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22987        let mut state = review_state(&["feat: x"]);
22988        state.reviews = vec![
22989            round_with_notes(1, Some(SANDBOXED)),
22990            passing_round(2, Some("0123456789abcdef")),
22991        ];
22992        ok_gate(&mut state, Some("0123456789abcdef"));
22993        let body = body_at(&state);
22994        assert!(!body.contains("HTTP tests could not run"), "{body}");
22995        // The commit claim is still settled by the structured record.
22996        assert!(!body.contains("could not commit"), "{body}");
22997        assert!(
22998            body.contains("passed on an earlier commit (`0123456`)"),
22999            "{body}"
23000        );
23001        assert!(!body.contains("passed on the final commit"), "{body}");
23002
23003        // A gate recorded before `gate_head` existed is "not recorded".
23004        ok_gate(&mut state, None);
23005        let body = body_at(&state);
23006        assert!(
23007            body.contains("passed on a commit that was not recorded"),
23008            "{body}"
23009        );
23010    }
23011
23012    #[test]
23013    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
23014        let state = state_with_summary(
23015            "add retries to the client",
23016            "TITLE: feat: retries\n\nDid it.",
23017        );
23018        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
23019        assert_eq!(m.title, "feat: retries");
23020        assert!(
23021            m.body.contains("<summary>Original task</summary>"),
23022            "{}",
23023            m.body
23024        );
23025        assert!(!m.body.contains("Commits under review"), "{}", m.body);
23026        assert!(
23027            m.body
23028                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
23029            "{}",
23030            m.body
23031        );
23032    }
23033
23034    #[test]
23035    fn pr_message_review_bounds_a_long_english_subject() {
23036        let long = format!("feat: {}", "word ".repeat(100));
23037        let m = pr_message(&review_state(&[&long]), 'A');
23038        assert!(m.title.starts_with("feat: word"), "{}", m.title);
23039        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23040    }
23041
23042    #[test]
23043    fn pr_message_implementation_run_is_unchanged_by_review_support() {
23044        let state = state_with_summary("add retries\n\ndetails", "- did some things");
23045        let m = pr_message(&state, 'A');
23046        assert_eq!(m.title, "add retries");
23047        assert!(m.body.contains("<summary>Original task</summary>"));
23048        assert!(!m.body.contains("Commits under review"));
23049        assert_eq!(landing_subject_source(&state), state.instruction);
23050    }
23051
23052    #[test]
23053    fn review_run_squash_subject_is_the_change_not_the_prompt() {
23054        let state = review_state(&["feat: the change", "fix: typo"]);
23055        let source = landing_subject_source(&state);
23056        assert_eq!(land::merge_subject("", &source), "feat: the change");
23057        assert_eq!(
23058            land::merge_subject("magi: candidate A (uncommitted work)", &source),
23059            "feat: the change"
23060        );
23061        // An operator's rename still wins.
23062        assert_eq!(
23063            land::merge_subject("feat: renamed by hand", &source),
23064            "feat: renamed by hand"
23065        );
23066        let blank = review_state(&["日本語"]);
23067        assert!(
23068            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
23069        );
23070    }
23071
23072    #[test]
23073    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
23074        let state = review_state(&["feat: the change"]);
23075        let source = landing_subject_source(&state);
23076        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
23077        assert_eq!(
23078            land::merge_subject(landing_title(&state, old), &source),
23079            "feat: the change"
23080        );
23081        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
23082        let task = state_with_summary("add retries", "");
23083        assert_eq!(landing_title(&task, old), old);
23084    }
23085
23086    #[test]
23087    fn pr_message_describes_the_change_not_the_task() {
23088        let state = state_with_summary(
23089            "今回やってほしいこと: results projector を直す",
23090            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
23091        );
23092        let m = pr_message(&state, 'A');
23093        assert_eq!(m.title, "fix(web): batch the runs list reads");
23094        assert!(
23095            m.body.starts_with("## Summary\n\n- reads run.json once"),
23096            "{}",
23097            m.body
23098        );
23099        assert!(!m.body.contains("TITLE:"), "{}", m.body);
23100        let task_at = m.body.find("今回やってほしいこと").unwrap();
23101        let details_at = m.body.find("<details>").unwrap();
23102        assert!(
23103            details_at < task_at,
23104            "the task lives inside <details>: {}",
23105            m.body
23106        );
23107        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
23108        assert!(m.body.contains("magi:candidate-a"));
23109    }
23110
23111    #[test]
23112    fn pr_message_falls_back_to_the_task_without_a_title_line() {
23113        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
23114        let m = pr_message(&state, 'A');
23115        assert_eq!(m.title, "add retries");
23116        assert!(
23117            m.body.contains("## Summary\n\n- did some things"),
23118            "{}",
23119            m.body
23120        );
23121
23122        let none = RunState::new(
23123            PathBuf::from("/repo"),
23124            "main".to_owned(),
23125            "abc1234".to_owned(),
23126            "add retries".to_owned(),
23127            Config::default(),
23128        );
23129        let m = pr_message(&none, 'A');
23130        assert_eq!(m.title, "add retries");
23131        assert!(!m.body.contains("## Summary"), "{}", m.body);
23132    }
23133
23134    #[test]
23135    fn pr_message_refuses_the_candidate_commit_subject() {
23136        for bad in [
23137            "TITLE: magi: candidate A (uncommitted work)",
23138            "TITLE: chore: stuff (uncommitted work)",
23139            "TITLE:   ",
23140        ] {
23141            let state = state_with_summary("add retries", bad);
23142            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23143        }
23144    }
23145
23146    #[test]
23147    fn pr_message_bounds_a_very_long_task_and_title() {
23148        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23149        let state = state_with_summary(&long, "- nothing");
23150        let m = pr_message(&state, 'A');
23151        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23152        assert!(!m.title.contains('\n'));
23153
23154        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23155        let m = pr_message(&state, 'A');
23156        assert!(m.title.starts_with("feat: "));
23157        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23158        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23159    }
23160
23161    fn long_title_of(instruction: &str) -> String {
23162        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23163    }
23164
23165    #[test]
23166    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23167        let first = "Make the landing path keep a readable title for long tasks";
23168        let line = format!(
23169            "{first}. {}",
23170            "And then keep going with more words ".repeat(20)
23171        );
23172        let t = long_title_of(&line);
23173        assert_eq!(t, first);
23174        assert!(!t.starts_with("chore: land"));
23175    }
23176
23177    #[test]
23178    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23179        let line = "word ".repeat(200);
23180        let t = long_title_of(&line);
23181        assert!(t.ends_with("word..."), "{t}");
23182        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23183    }
23184
23185    #[test]
23186    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23187        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23188            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23189        }
23190    }
23191
23192    #[test]
23193    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23194        let mut state = state_with_summary("再試行を追加する", "- nothing");
23195        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23196        state.config.graph.github_language = "ja".into();
23197        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23198    }
23199
23200    #[test]
23201    fn pr_message_title_limit_is_exact() {
23202        let at = "a".repeat(PR_TITLE_MAX);
23203        assert_eq!(long_title_of(&at), at);
23204        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23205        assert!(over.ends_with("..."), "{over}");
23206        assert_eq!(over.chars().count(), PR_TITLE_MAX);
23207    }
23208
23209    #[test]
23210    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23211        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23212        let t = long_title_of(&line);
23213        assert!(t.ends_with("..."), "{t}");
23214        assert!(t.is_ascii(), "{t}");
23215    }
23216
23217    #[test]
23218    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23219        let line = format!(
23220            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23221            "plus more filler words ".repeat(20)
23222        );
23223        let t = long_title_of(&line);
23224        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23225    }
23226
23227    #[test]
23228    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23229        let line = format!("\n\n# {}", "title words ".repeat(40));
23230        let state = state_with_summary(&line, "- nothing");
23231        let m = pr_message(&state, 'A');
23232        assert!(m.title.starts_with("title words"), "{}", m.title);
23233        assert_eq!(
23234            land::merge_subject(&m.title, &landing_subject_source(&state)),
23235            m.title
23236        );
23237        // An operator's rename wins untouched.
23238        assert_eq!(
23239            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23240            "feat: renamed by hand"
23241        );
23242    }
23243
23244    #[tokio::test]
23245    async fn github_text_rewrite_is_bounded_and_falls_back() {
23246        crate::run::pin_test_home();
23247        for (reply, accepted) in [
23248            (
23249                "TITLE: fix: retries\nAdd retries for failed requests.",
23250                true,
23251            ),
23252            (
23253                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23254                false,
23255            ),
23256            (
23257                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23258                true,
23259            ),
23260            (
23261                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23262                true,
23263            ),
23264            (
23265                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23266                true,
23267            ),
23268            (
23269                "TITLE: fix: retries\n日本語の説明です。 token=secret",
23270                false,
23271            ),
23272            ("TITLE: fix: retries\n", false),
23273        ] {
23274            let dir = tempfile::tempdir().unwrap();
23275            let mut runner = runner_at(RunStatus::Gating);
23276            runner.state = state_with_summary(
23277                "add retries",
23278                "TITLE: fix: retries\n日本語の説明を書きます。",
23279            );
23280            runner.state.repo = dir.path().to_owned();
23281            runner.state.candidates[0].worktree = dir.path().to_owned();
23282            let mut author = spec("alpha");
23283            author.command = vec![
23284                "sh".into(),
23285                "-c".into(),
23286                "printf '%s' \"$REWRITE_REPLY\"".into(),
23287            ];
23288            author.env.insert("REWRITE_REPLY".into(), reply.into());
23289            runner.state.config.agents = vec![author];
23290            seed_public_summary(&mut runner);
23291            let winner = runner.state.candidates[0].clone();
23292            let first = runner
23293                .guarded_pr_message(&winner, None, true)
23294                .await
23295                .unwrap();
23296            let message = if accepted {
23297                first.expect("a passing rewrite never asks")
23298            } else {
23299                // Withheld: the run parks on a question and posts nothing yet.
23300                assert!(first.is_none());
23301                assert!(runner.state.parked);
23302                let store = ask::Questions::open();
23303                let asked: Vec<_> = store
23304                    .list()
23305                    .into_iter()
23306                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23307                    .collect();
23308                assert_eq!(asked.len(), 1);
23309                assert!(!asked[0].detail.contains("secret"));
23310                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23311                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23312                assert_eq!(
23313                    asked[0]
23314                        .choices
23315                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
23316                    rewrite.is_some()
23317                );
23318                // Silence: the question lapses and the neutral text is used,
23319                // without a second rewrite call or a second question.
23320                store
23321                    .update(&asked[0].id, |q| {
23322                        q.abandon("test");
23323                        Ok(())
23324                    })
23325                    .unwrap();
23326                let again = runner
23327                    .guarded_pr_message(&winner, None, true)
23328                    .await
23329                    .unwrap()
23330                    .expect("an abandoned question falls back");
23331                let n = store
23332                    .list()
23333                    .into_iter()
23334                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23335                    .count();
23336                assert_eq!(n, 1, "asked once per rejected text");
23337                again
23338            };
23339            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23340            assert!(!message.body.contains("secret"));
23341            assert!(!message.body.contains("prefix"));
23342            assert!(!message.title.contains("secret"));
23343            assert_eq!(
23344                message.body.contains("Add retries for failed requests."),
23345                accepted
23346            );
23347            assert_eq!(
23348                runner.state.seats["github-text-rewrite"].turns, 1,
23349                "only one rewrite invocation"
23350            );
23351            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23352            if accepted {
23353                assert!(runner.state.github_text.is_none());
23354                assert!(!runner.state.parked);
23355                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23356                assert!(message.body.contains("## Final verification"));
23357                assert!(message.body.contains("Evidence: final diff"));
23358                assert!(
23359                    message
23360                        .body
23361                        .contains(&format!("magi:run/{}", runner.state.id))
23362                );
23363            }
23364            if !accepted {
23365                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23366                assert!(
23367                    message
23368                        .body
23369                        .contains(&format!("magi:run/{}", runner.state.id))
23370                );
23371            }
23372        }
23373    }
23374
23375    fn scripted(id: &str, reply: &str) -> AgentSpec {
23376        let mut a = spec(id);
23377        a.command = vec![
23378            "sh".into(),
23379            "-c".into(),
23380            "printf '%s' \"$REWRITE_REPLY\"".into(),
23381        ];
23382        a.env.insert("REWRITE_REPLY".into(), reply.into());
23383        a
23384    }
23385
23386    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23387    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23388
23389    /// Gate tests inject generated public output, not internal reports.
23390    fn seed_public_summary(runner: &mut Runner) {
23391        let repo = &runner.state.repo;
23392        let g = |args: &[&str]| {
23393            let out = std::process::Command::new("git")
23394                .args(args)
23395                .current_dir(repo)
23396                .quiet()
23397                .output()
23398                .unwrap();
23399            assert!(out.status.success(), "git {args:?}");
23400            String::from_utf8_lossy(&out.stdout).trim().to_owned()
23401        };
23402        g(&["init", "-q", "-b", "main"]);
23403        g(&[
23404            "-c",
23405            "user.name=Test",
23406            "-c",
23407            "user.email=test@example.invalid",
23408            "commit",
23409            "-q",
23410            "--allow-empty",
23411            "-m",
23412            "feat: fixture",
23413        ]);
23414        let head = g(&["rev-parse", "HEAD"]);
23415        g(&["update-ref", "refs/remotes/origin/main", &head]);
23416        let evidence = crate::pr_outcome::summary_evidence(
23417            &runner.state,
23418            crate::pr_outcome::FinalHead::At(&head),
23419        );
23420        let generated = &runner.state.candidates[0].summary;
23421        runner.state.change_summary = Some(crate::run::ChangeSummary {
23422            base: head.clone(),
23423            head,
23424            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23425            title: summary_title(generated).unwrap(),
23426            body: summary_without_title(generated),
23427        });
23428    }
23429
23430    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23431        let mut runner = runner_at(RunStatus::Gating);
23432        runner.state = state_with_summary("add retries", JA_SUMMARY);
23433        if review_only {
23434            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23435        }
23436        runner.state.repo = dir.to_owned();
23437        runner.state.candidates[0].worktree = dir.to_owned();
23438        runner.state.config.agents = agents;
23439        seed_public_summary(&mut runner);
23440        runner
23441    }
23442
23443    fn gate_questions(runner: &Runner) -> usize {
23444        ask::Questions::open()
23445            .list()
23446            .into_iter()
23447            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23448            .count()
23449    }
23450
23451    #[tokio::test]
23452    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23453        crate::run::pin_test_home();
23454        let dir = tempfile::tempdir().unwrap();
23455        let mut runner = rewrite_runner(
23456            dir.path(),
23457            true,
23458            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23459        );
23460        runner.state.config.roles.language_judge =
23461            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23462        let winner = runner.state.candidates[0].clone();
23463        let msg = runner
23464            .guarded_pr_message(&winner, None, true)
23465            .await
23466            .unwrap()
23467            .expect("an English rewrite posts without asking");
23468        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23469        assert!(msg.body.contains("<details>kept</details>"));
23470        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23471        assert_eq!(gate_questions(&runner), 0);
23472        assert!(runner.state.github_text.is_none());
23473        assert!(!runner.state.seats.contains_key("impl-A"));
23474        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23475    }
23476
23477    #[tokio::test]
23478    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23479        crate::run::pin_test_home();
23480        let dir = tempfile::tempdir().unwrap();
23481        let mut runner = rewrite_runner(
23482            dir.path(),
23483            false,
23484            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23485        );
23486        runner.state.config.roles.language_judge =
23487            Some(crate::config::AgentChoice::One("beta".into()));
23488        runner.state.seats.insert(
23489            "impl-A".into(),
23490            SeatState::new("impl-A", "alpha", runner.state.seed),
23491        );
23492        let winner = runner.state.candidates[0].clone();
23493        let msg = runner
23494            .guarded_pr_message(&winner, None, true)
23495            .await
23496            .unwrap()
23497            .expect("rewritten");
23498        assert!(msg.body.contains("<details>kept</details>"));
23499        assert_eq!(runner.state.seats["impl-A"].turns, 0);
23500        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23501        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23502    }
23503
23504    #[tokio::test]
23505    async fn rewrite_reuses_a_resumable_implementer_session() {
23506        crate::run::pin_test_home();
23507        let dir = tempfile::tempdir().unwrap();
23508        let mut runner = rewrite_runner(
23509            dir.path(),
23510            false,
23511            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23512        );
23513        runner.state.config.roles.language_judge =
23514            Some(crate::config::AgentChoice::One("beta".into()));
23515        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23516        seat.turns = 1;
23517        runner.state.seats.insert("impl-A".into(), seat);
23518        let winner = runner.state.candidates[0].clone();
23519        runner
23520            .guarded_pr_message(&winner, None, true)
23521            .await
23522            .unwrap()
23523            .expect("rewritten");
23524        assert_eq!(runner.state.seats["impl-A"].turns, 2);
23525        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23526    }
23527
23528    #[tokio::test]
23529    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23530        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
23531        // session, was out of quota, and nothing else was asked.
23532        crate::run::pin_test_home();
23533        let dir = tempfile::tempdir().unwrap();
23534        let mut exhausted = spec("alpha");
23535        exhausted.command = vec!["false".into()];
23536        let mut runner = rewrite_runner(
23537            dir.path(),
23538            false,
23539            vec![exhausted, scripted("beta", EN_REWRITE)],
23540        );
23541        // The implementer's own agent is in the role too: it is not asked twice.
23542        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23543            "alpha".into(),
23544            "beta".into(),
23545        ]));
23546        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23547        seat.turns = 1;
23548        runner.state.seats.insert("impl-A".into(), seat);
23549        let winner = runner.state.candidates[0].clone();
23550        let msg = runner
23551            .guarded_pr_message(&winner, None, true)
23552            .await
23553            .unwrap()
23554            .expect("the role rewrites after the implementer fails");
23555        assert!(msg.body.contains("`src/net.rs`"));
23556        assert_eq!(
23557            runner.state.seats["impl-A"].turns, 2,
23558            "the implementer was tried once"
23559        );
23560        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23561        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23562    }
23563
23564    #[tokio::test]
23565    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23566        crate::run::pin_test_home();
23567        let dir = tempfile::tempdir().unwrap();
23568        let mut broken = spec("bad");
23569        broken.command = vec!["false".into()];
23570        let mut runner = rewrite_runner(
23571            dir.path(),
23572            true,
23573            vec![
23574                broken,
23575                scripted("noformat", "Add retries."),
23576                scripted("good", EN_REWRITE),
23577            ],
23578        );
23579        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23580            "bad".into(),
23581            "noformat".into(),
23582            "good".into(),
23583        ]));
23584        let winner = runner.state.candidates[0].clone();
23585        let msg = runner
23586            .guarded_pr_message(&winner, None, true)
23587            .await
23588            .unwrap()
23589            .expect("the third agent rewrites");
23590        assert!(msg.body.contains("`src/net.rs`"));
23591        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23592        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23593    }
23594
23595    #[tokio::test]
23596    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23597        crate::run::pin_test_home();
23598        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23599        for (agents, role, saved) in [
23600            (
23601                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23602                Some("beta"),
23603                true,
23604            ),
23605            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23606            (Vec::new(), None, false),
23607        ] {
23608            let dir = tempfile::tempdir().unwrap();
23609            let mut runner = rewrite_runner(dir.path(), true, agents);
23610            runner.state.config.roles.language_judge =
23611                role.map(|r| crate::config::AgentChoice::One(r.into()));
23612            let winner = runner.state.candidates[0].clone();
23613            let first = runner
23614                .guarded_pr_message(&winner, None, true)
23615                .await
23616                .unwrap();
23617            assert!(first.is_none(), "withheld text parks the run");
23618            assert!(runner.state.parked);
23619            assert_eq!(gate_questions(&runner), 1);
23620            let g = runner.state.github_text.as_ref().unwrap();
23621            assert_eq!(g.rewrite.is_some(), saved);
23622            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23623        }
23624    }
23625
23626    #[tokio::test]
23627    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23628        crate::run::pin_test_home();
23629        let dir = tempfile::tempdir().unwrap();
23630        let mut runner = runner_at(RunStatus::Gating);
23631        runner.state =
23632            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23633        runner.state.repo = dir.path().to_owned();
23634        runner.state.candidates[0].worktree = dir.path().to_owned();
23635        let mut author = spec("alpha");
23636        author.command = vec![
23637            "sh".into(),
23638            "-c".into(),
23639            "printf '%s' \"$REWRITE_REPLY\"".into(),
23640        ];
23641        author.env.insert(
23642            "REWRITE_REPLY".into(),
23643            format!(
23644                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23645                "English sentence. ".repeat(600)
23646            ),
23647        );
23648        runner.state.config.agents = vec![author];
23649        seed_public_summary(&mut runner);
23650        let winner = runner.state.candidates[0].clone();
23651        assert!(
23652            runner
23653                .guarded_pr_message(&winner, None, true)
23654                .await
23655                .unwrap()
23656                .is_none()
23657        );
23658        let record = runner.state.github_text.clone().unwrap();
23659        let rewrite = record.rewrite.clone().unwrap();
23660        assert_eq!(rewrite.failed, ["body-language"]);
23661        assert!(
23662            rewrite
23663                .body
23664                .ends_with(&format!("magi:run/{}", runner.state.id))
23665        );
23666        let store = ask::Questions::open();
23667        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23668        assert!(
23669            q.choices
23670                .contains(&crate::github_text::USE_REWRITE.to_owned())
23671        );
23672        for label in [
23673            "ORIGINAL",
23674            "FALLBACK",
23675            "REWRITE",
23676            "Rules still failed: body-language",
23677            "truncated",
23678        ] {
23679            assert!(q.detail.contains(label), "{label}");
23680        }
23681        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23682            &runner.state,
23683            &crate::github_text::artifact_name(&record.fingerprint),
23684        ))
23685        .unwrap();
23686        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23687        // Restart from serialized state, then retry after a bad owner title.
23688        runner.state =
23689            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23690        store
23691            .update(&q.id, |q| {
23692                q.say("まだ日本語")?;
23693                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23694            })
23695            .unwrap();
23696        assert!(
23697            runner
23698                .guarded_pr_message(&winner, None, true)
23699                .await
23700                .unwrap()
23701                .is_none()
23702        );
23703        let retry = runner.state.github_text.as_ref().unwrap();
23704        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23705        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23706        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23707        assert!(
23708            q.posting_comparison
23709                .as_ref()
23710                .unwrap()
23711                .matches_detail(&q.detail)
23712        );
23713        store
23714            .update(&q.id, |q| {
23715                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23716            })
23717            .unwrap();
23718        let message = runner
23719            .guarded_pr_message(&winner, None, true)
23720            .await
23721            .unwrap()
23722            .unwrap();
23723        assert_eq!(message.title, rewrite.title);
23724        assert_eq!(message.body, rewrite.body);
23725        assert_eq!(
23726            runner.state.github_text.as_ref().unwrap().choice,
23727            Some(GithubTextChoice::Rewrite)
23728        );
23729        let again = runner
23730            .guarded_pr_message(&winner, None, true)
23731            .await
23732            .unwrap()
23733            .unwrap();
23734        assert_eq!(again.title, message.title);
23735        assert_eq!(again.body, message.body);
23736        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23737        assert!(
23738            !runner
23739                .state
23740                .events
23741                .iter()
23742                .any(|e| e.message.contains(&rewrite.title)
23743                    || e.message.contains("English sentence."))
23744        );
23745    }
23746
23747    #[tokio::test]
23748    async fn historical_posting_reply_never_expires_into_fallback() {
23749        crate::run::pin_test_home();
23750        let dir = tempfile::tempdir().unwrap();
23751        let mut runner = runner_at(RunStatus::Gating);
23752        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23753        runner.state.repo = dir.path().to_owned();
23754        runner.state.candidates[0].worktree = dir.path().to_owned();
23755        seed_public_summary(&mut runner);
23756        let winner = runner.state.candidates[0].clone();
23757        assert!(
23758            runner
23759                .guarded_pr_message(&winner, None, true)
23760                .await
23761                .unwrap()
23762                .is_none()
23763        );
23764        let id = runner
23765            .state
23766            .github_text
23767            .as_ref()
23768            .unwrap()
23769            .question
23770            .clone()
23771            .unwrap();
23772        let store = ask::Questions::open();
23773        let mut q = store.get(&id).unwrap();
23774        q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23775        q.answer(ask::Answer::Choice(
23776            crate::github_text::POST_ORIGINAL.into(),
23777        ))
23778        .unwrap();
23779        q.answer_delivered = true;
23780        store.put(&mut q).unwrap();
23781        let source = std::fs::read(store.path_of(&id)).unwrap();
23782        let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23783        std::fs::create_dir_all(&migration).unwrap();
23784        std::fs::write(
23785            migration.join("state.json"),
23786            serde_json::to_vec(&serde_json::json!({
23787                "schema": 1, "sources": [id], "pending": [id]
23788            }))
23789            .unwrap(),
23790        )
23791        .unwrap();
23792        for _ in 0..2 {
23793            assert!(
23794                runner
23795                    .guarded_pr_message(&winner, None, true)
23796                    .await
23797                    .unwrap()
23798                    .is_none()
23799            );
23800            assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23801            assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23802        }
23803        let home = crate::run::home();
23804        crate::task_chat::sweep(
23805            &crate::queue::Queue::at(home.join("queue")),
23806            &crate::talk::Talks::at(home.join("talks")),
23807            &store,
23808            &home,
23809        );
23810        let outbox = crate::task_inquiry::Outbox::at(&home);
23811        outbox
23812            .update_decision(&id, |d| {
23813                d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23814                d.source_answer_at = q.answered_at;
23815                Ok(true)
23816            })
23817            .unwrap();
23818        outbox
23819            .settle(
23820                &id,
23821                "owner-chat",
23822                "Owner confirms no publication remains necessary",
23823                None,
23824            )
23825            .unwrap();
23826        assert!(
23827            runner
23828                .guarded_pr_message(&winner, None, true)
23829                .await
23830                .unwrap()
23831                .is_none()
23832        );
23833        assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23834        assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23835    }
23836
23837    #[tokio::test]
23838    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23839        crate::run::pin_test_home();
23840        let dir = tempfile::tempdir().unwrap();
23841        let mut runner = runner_at(RunStatus::Gating);
23842        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23843        runner.state.repo = dir.path().to_owned();
23844        runner.state.candidates[0].worktree = dir.path().to_owned();
23845        seed_public_summary(&mut runner);
23846        let winner = runner.state.candidates[0].clone();
23847        assert!(
23848            runner
23849                .guarded_pr_message(&winner, None, true)
23850                .await
23851                .unwrap()
23852                .is_none()
23853        );
23854        let record = runner.state.github_text.as_ref().unwrap();
23855        assert!(record.rewrite.is_none());
23856        // Records written before the additive field deserialize with no rewrite.
23857        let mut old = serde_json::to_value(record).unwrap();
23858        old.as_object_mut().unwrap().remove("rewrite");
23859        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23860        assert!(old.rewrite.is_none());
23861        let id = record.question.clone().unwrap();
23862        ask::Questions::open()
23863            .update(&id, |q| {
23864                q.choices.push(crate::github_text::USE_REWRITE.into());
23865                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23866            })
23867            .unwrap();
23868        let message = runner
23869            .guarded_pr_message(&winner, None, true)
23870            .await
23871            .unwrap()
23872            .unwrap();
23873        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23874        assert_eq!(
23875            runner.state.github_text.as_ref().unwrap().choice,
23876            Some(GithubTextChoice::Fallback)
23877        );
23878    }
23879
23880    #[tokio::test]
23881    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23882        crate::run::pin_test_home();
23883        let dir = tempfile::tempdir().unwrap();
23884        let mut runner = runner_at(RunStatus::Gating);
23885        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23886        runner.state.repo = dir.path().to_owned();
23887        runner.state.candidates[0].worktree = dir.path().to_owned();
23888        seed_public_summary(&mut runner);
23889        let winner = runner.state.candidates[0].clone();
23890        assert!(
23891            runner
23892                .guarded_pr_message(&winner, None, true)
23893                .await
23894                .unwrap()
23895                .is_none()
23896        );
23897        let store = ask::Questions::open();
23898        let find = |run: &str| {
23899            store
23900                .list()
23901                .into_iter()
23902                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23903                .unwrap()
23904        };
23905        let run = runner.state.id.clone();
23906        let q = find(&run);
23907        let fp = runner
23908            .state
23909            .github_text
23910            .as_ref()
23911            .unwrap()
23912            .fingerprint
23913            .clone();
23914        let name = crate::github_text::artifact_name(&fp);
23915        let path = crate::run::artifact_path(&runner.state, &name);
23916        assert!(q.detail.contains(&path.display().to_string()));
23917        assert!(q.detail.contains("title-language: title"));
23918        let saved = std::fs::read_to_string(&path).expect("artifact written");
23919        assert!(saved.contains("日本語のタイトル"));
23920        assert!(q.cwd.is_none());
23921        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23922        // A bad replacement is refused and asked about once more.
23923        let say = |id: &str, text: &str, choice: &str| {
23924            store
23925                .update(id, |q| {
23926                    q.thread.push(ask::Turn {
23927                        who: ask::Who::Operator,
23928                        body: text.to_owned(),
23929                        at: jiff::Timestamp::now(),
23930                        note: None,
23931                    });
23932                    q.answer(ask::Answer::Choice(choice.to_owned()))
23933                })
23934                .unwrap();
23935        };
23936        say(&q.id, "まだ日本語", "use my text");
23937        assert!(
23938            runner
23939                .guarded_pr_message(&winner, None, true)
23940                .await
23941                .unwrap()
23942                .is_none()
23943        );
23944        let second = find(&run);
23945        assert_ne!(second.id, q.id);
23946        assert!(second.detail.contains("did not pass"));
23947        // The retry still compares the original text; the rejected
23948        // replacement is named by category only.
23949        assert!(second.detail.contains("日本語のタイトル"));
23950        assert!(!second.detail.contains("まだ日本語"));
23951        assert!(second.detail.contains(&path.display().to_string()));
23952        say(&second.id, "fix: retry failed requests", "use my text");
23953        let message = runner
23954            .guarded_pr_message(&winner, None, true)
23955            .await
23956            .unwrap()
23957            .expect("a vetted replacement is posted");
23958        assert_eq!(message.title, "fix: retry failed requests");
23959        assert_eq!(
23960            runner
23961                .state
23962                .github_text
23963                .as_ref()
23964                .unwrap()
23965                .chosen_title
23966                .as_deref(),
23967            Some("fix: retry failed requests")
23968        );
23969        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23970        // Resumed again: the saved decision is applied, nothing is re-asked.
23971        let again = runner
23972            .guarded_pr_message(&winner, None, true)
23973            .await
23974            .unwrap()
23975            .unwrap();
23976        assert_eq!(again.title, message.title);
23977        assert_eq!(
23978            store
23979                .list()
23980                .into_iter()
23981                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23982                .count(),
23983            2
23984        );
23985    }
23986
23987    #[test]
23988    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23989        // What magi itself writes stays English under any configured language,
23990        // so a future localisation of these headings fails here. (The agents'
23991        // own text is also checked by the posting gate.)
23992        let mut state = state_with_summary(
23993            "add retries",
23994            "TITLE: fix(web): batch reads\n- reads run.json once",
23995        );
23996        state.config.graph.language = "ja".to_owned();
23997        let m = pr_message(&state, 'A');
23998        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23999        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
24000
24001        // The task is the operator's own text: it goes in untouched, and the
24002        // fallback title (no summary) may be in its language too.
24003        let task = "今回やってほしいこと: results projector を直す";
24004        let mut state = state_with_summary(task, "- no title line");
24005        state.config.graph.language = "ja".to_owned();
24006        let m = pr_message(&state, 'A');
24007        assert_eq!(
24008            m.title,
24009            format!("chore: land candidate A of run {}", state.id)
24010        );
24011        assert!(
24012            m.body.contains(&format!(
24013                "<summary>Original task</summary>\n\n{task}\n\n</details>"
24014            )),
24015            "{}",
24016            m.body
24017        );
24018    }
24019
24020    #[test]
24021    fn pr_message_scrubs_home_paths_and_addresses() {
24022        let state = state_with_summary(
24023            "fix it in /Users/someone/src/x",
24024            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
24025        );
24026        let m = pr_message(&state, 'A');
24027        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
24028            assert!(!m.body.contains(leak), "{}", m.body);
24029        }
24030        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
24031    }
24032
24033    #[test]
24034    fn pr_message_survives_a_task_that_closes_details() {
24035        let state = state_with_summary("a </details> b", "TITLE: fix: x");
24036        let m = pr_message(&state, 'A');
24037        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
24038    }
24039
24040    #[test]
24041    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
24042        let cmd = manual_merge_command(
24043            MergeStyle::Squash,
24044            Path::new("/repo"),
24045            "b",
24046            "fix: \"quoted\" $(x) `y`\n\nbody",
24047        );
24048        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
24049    }
24050
24051    #[test]
24052    fn manual_merge_command_matches_the_configured_style() {
24053        let repo = Path::new("/repo");
24054        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
24055
24056        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
24057        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
24058
24059        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
24060        assert_eq!(
24061            squash,
24062            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
24063             \"Merge magi run 0832 (candidate A)\""
24064        );
24065
24066        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
24067        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
24068    }
24069
24070    #[test]
24071    fn a_nudge_gets_a_quarter_of_the_budget() {
24072        // The judge and implement budgets magi ships with.
24073        assert_eq!(retry_budget(secs(1200), true), secs(300));
24074        assert_eq!(retry_budget(secs(3600), true), secs(900));
24075    }
24076
24077    #[test]
24078    fn a_resent_prompt_keeps_the_whole_budget() {
24079        // The seat kept no context, so the retry is the original job again and
24080        // shortening it would only guarantee a second failure.
24081        assert_eq!(retry_budget(secs(1200), false), secs(1200));
24082        assert_eq!(retry_budget(secs(60), false), secs(60));
24083    }
24084
24085    #[test]
24086    fn the_floor_never_exceeds_the_original_budget() {
24087        // A short configured timeout must not be *raised* by the floor: the
24088        // operator asked for a bound, and a retry may not outlast the attempt
24089        // it is retrying.
24090        assert_eq!(retry_budget(secs(60), true), secs(60));
24091        assert_eq!(retry_budget(secs(480), true), secs(120));
24092        assert_eq!(retry_budget(secs(0), true), secs(0));
24093    }
24094
24095    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
24096        agent::CommandEvidence {
24097            id: "item1".to_owned(),
24098            description: "cargo test".to_owned(),
24099            exit_code,
24100            result_summary: String::new(),
24101            source: "codex".to_owned(),
24102        }
24103    }
24104
24105    #[test]
24106    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
24107        // No evidence is not the same fact as unconfirmed evidence: a
24108        // backend with no adapter, or a reply that ran no commands at all,
24109        // must not be misread as carrying a dangling job.
24110        assert!(!has_unconfirmed_command(&[]));
24111    }
24112
24113    #[test]
24114    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
24115        // Deliberately not a check on the exit code's *value*: a fixer
24116        // legitimately runs something that fails mid-iteration before it
24117        // succeeds, and that must never by itself reopen a valid report.
24118        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
24119        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
24120        assert!(!has_unconfirmed_command(&[
24121            evidence(Some(0)),
24122            evidence(Some(101))
24123        ]));
24124    }
24125
24126    #[test]
24127    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
24128        assert!(has_unconfirmed_command(&[
24129            evidence(Some(0)),
24130            evidence(None)
24131        ]));
24132    }
24133
24134    #[test]
24135    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
24136        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24137        assert_eq!(
24138            verified_noop_claim(true, &[], text).as_deref(),
24139            Some("already fixed by b32cfc4, on main.")
24140        );
24141    }
24142
24143    #[test]
24144    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24145        // A timeout or a bad exit code reads as the ordinary loss it is,
24146        // whatever the reply's own prose claims.
24147        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24148        assert!(verified_noop_claim(false, &[], text).is_none());
24149    }
24150
24151    #[test]
24152    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24153        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24154        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24155        // A confirmed command alongside the marker is fine.
24156        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24157    }
24158
24159    #[test]
24160    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24161        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24162    }
24163
24164    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
24165    /// pair, in order, labelled A, B, C, ...
24166    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24167        runner.state.candidates = shape
24168            .iter()
24169            .enumerate()
24170            .map(|(i, &(empty, verified))| Candidate {
24171                index: i,
24172                label: (b'A' + i as u8) as char,
24173                agent: "sonnet".to_owned(),
24174                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24175                worktree: PathBuf::from(format!("/wt/{i}")),
24176                summary: String::new(),
24177                stat: String::new(),
24178                files: 0,
24179                commits: 0,
24180                empty,
24181                failed: None,
24182                verified_noop: verified.map(str::to_owned),
24183                duration_ms: 0,
24184                folded: false,
24185            })
24186            .collect();
24187    }
24188
24189    #[test]
24190    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24191        ask_test_home();
24192        let mut runner = runner_at(RunStatus::Implementing);
24193        set_candidates(
24194            &mut runner,
24195            &[
24196                (true, Some("already on main at b32cfc4")),
24197                (true, Some("same fix, see the existing test")),
24198            ],
24199        );
24200
24201        runner
24202            .after_implement()
24203            .expect("a verified no-op is not an error");
24204
24205        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24206    }
24207
24208    #[test]
24209    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24210        ask_test_home();
24211        let mut runner = runner_at(RunStatus::Implementing);
24212        // Candidate A declares a verified no-op; candidate B simply wrote
24213        // nothing and said nothing about why. One candidate's claim is not
24214        // the whole run's agreement.
24215        set_candidates(
24216            &mut runner,
24217            &[(true, Some("already on main at b32cfc4")), (true, None)],
24218        );
24219
24220        let err = runner
24221            .after_implement()
24222            .expect_err("an unverified empty candidate must still fail the run");
24223
24224        assert!(
24225            err.to_string().contains("no candidate produced a change"),
24226            "{err}"
24227        );
24228        assert_eq!(runner.state.status, RunStatus::Failed);
24229    }
24230
24231    #[test]
24232    fn after_implement_still_fails_an_ordinary_all_empty_run() {
24233        ask_test_home();
24234        let mut runner = runner_at(RunStatus::Implementing);
24235        set_candidates(&mut runner, &[(true, None), (true, None)]);
24236
24237        let err = runner
24238            .after_implement()
24239            .expect_err("no candidate declared anything; this is an ordinary failure");
24240
24241        assert!(
24242            err.to_string().contains("no candidate produced a change"),
24243            "{err}"
24244        );
24245        assert_eq!(runner.state.status, RunStatus::Failed);
24246    }
24247}
24248
24249/// A graft rollback whose restore could not be confirmed (see
24250/// [`Runner::graft`]).
24251#[derive(Debug)]
24252struct GraftRestoreFailed(String);
24253
24254impl std::fmt::Display for GraftRestoreFailed {
24255    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24256        write!(
24257            f,
24258            "could not restore the winner to its pre-graft commit: {}",
24259            self.0
24260        )
24261    }
24262}
24263
24264impl std::error::Error for GraftRestoreFailed {}
24265
24266/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
24267/// not the agent: they never draw on an agent's own timeout, and running out of
24268/// them never reads as an agent failure.
24269#[cfg(test)]
24270mod resource_wait_tests {
24271    use super::*;
24272    use std::collections::BTreeMap;
24273
24274    fn sh(id: &str, script: &str) -> AgentSpec {
24275        AgentSpec {
24276            id: id.to_owned(),
24277            kind: crate::config::AgentKind::Command,
24278            model: None,
24279            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24280            extra_args: Vec::new(),
24281            env: BTreeMap::new(),
24282            prompt_delivery: None,
24283            max_turn_secs: None,
24284        }
24285    }
24286
24287    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24288        SeatJob {
24289            seat: SeatState::new("impl-A", &spec.id, 1),
24290            spec,
24291            cwd: dir.to_path_buf(),
24292            prompt: "do it".to_owned(),
24293            timeout,
24294            allow_write: true,
24295            sessions: false,
24296            artifacts: dir.join("art"),
24297            stem: "impl-A".to_owned(),
24298            handover: None,
24299        }
24300    }
24301
24302    fn state(resource_wait_secs: u64) -> RunState {
24303        let mut config = Config::default();
24304        config.graph.timeout_resource_wait = resource_wait_secs;
24305        RunState::new(
24306            PathBuf::from("/repo"),
24307            "main".to_owned(),
24308            "abc1234".to_owned(),
24309            "task".to_owned(),
24310            config,
24311        )
24312    }
24313
24314    fn hold(cache: &Path) -> crate::cache::Guard {
24315        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24316        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24317            crate::cache::AcquireOutcome::Acquired(g) => g,
24318            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24319        }
24320    }
24321
24322    async fn run(
24323        jobs: Vec<SeatJob>,
24324        sem: Arc<Semaphore>,
24325        cache: Option<&Path>,
24326        st: &mut RunState,
24327    ) -> Vec<(usize, SeatState, AgentOutcome)> {
24328        let prompts = Prompts::default();
24329        let ctx = WaveCtx {
24330            run: "r",
24331            node: "implement",
24332            prompts: &prompts,
24333            cache,
24334            round: None,
24335            carry_seats: false,
24336            pauses: None,
24337        };
24338        wave(jobs, sem, &ctx, st, 0).await
24339    }
24340
24341    #[tokio::test]
24342    async fn verification_cache_wait_preserves_the_command_budget() {
24343        crate::run::pin_test_home();
24344        let tmp = tempfile::tempdir().unwrap();
24345        let cache = tmp.path().join("target");
24346        let guard = hold(&cache);
24347        let release = tokio::spawn(async move {
24348            tokio::time::sleep(Duration::from_millis(100)).await;
24349            guard.release();
24350        });
24351        let mut st = state(10);
24352        let budget = Duration::from_millis(50);
24353        let wait = repository_wait(&st, None);
24354        let (outcomes, _) = with_cache_lease(
24355            &mut st,
24356            Some(&cache),
24357            "gate",
24358            "gate",
24359            tmp.path(),
24360            "abc1234",
24361            budget,
24362            "test gate",
24363            &wait,
24364            |_, available| async move {
24365                assert_eq!(
24366                    available, budget,
24367                    "contention must not shorten verification"
24368                );
24369                (Vec::new(), false, Vec::new())
24370            },
24371        )
24372        .await
24373        .unwrap();
24374        release.await.unwrap();
24375        assert!(outcomes.is_empty(), "{outcomes:?}");
24376        assert!(
24377            st.events
24378                .iter()
24379                .any(|event| event.message.contains("waiting for"))
24380        );
24381        assert!(st.resource_wait.is_none());
24382        // The verification body no longer owns the lease after returning.
24383        hold(&cache).release();
24384    }
24385
24386    #[tokio::test]
24387    async fn verification_cache_wait_expires_without_starting_commands() {
24388        crate::run::pin_test_home();
24389        let tmp = tempfile::tempdir().unwrap();
24390        let cache = tmp.path().join("target");
24391        let _held = hold(&cache);
24392        let mut st = state(0);
24393        let wait = repository_wait(&st, None);
24394        let (outcomes, _) = with_cache_lease(
24395            &mut st,
24396            Some(&cache),
24397            "gate",
24398            "gate",
24399            tmp.path(),
24400            "abc1234",
24401            Duration::from_secs(60),
24402            "test gate",
24403            &wait,
24404            |_, _| async { panic!("commands must not start without the cache lease") },
24405        )
24406        .await
24407        .unwrap();
24408        assert_eq!(outcomes.len(), 1);
24409        assert!(outcomes[0].resource_blocked);
24410        assert!(st.resource_wait.is_none());
24411        assert!(matches!(
24412            st.resource_block.unwrap().cause,
24413            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24414        ));
24415    }
24416
24417    #[tokio::test]
24418    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24419        crate::run::pin_test_home();
24420        let tmp = tempfile::tempdir().unwrap();
24421        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24422        let cache = tmp.path().join("cache");
24423        let mut state = state(21600);
24424        let wait = repository_wait(&state, None);
24425        let (outcomes, _) = with_cache_lease(
24426            &mut state,
24427            Some(&cache),
24428            "gate",
24429            "gate",
24430            tmp.path(),
24431            "head",
24432            Duration::from_secs(60),
24433            "gate",
24434            &wait,
24435            |_, _| async { panic!("freshness failure must not start verification") },
24436        )
24437        .await
24438        .unwrap();
24439        assert_eq!(outcomes.len(), 1);
24440        assert!(outcomes[0].resource_blocked);
24441        let block = state.resource_block.unwrap();
24442        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24443        assert!(!block.description().contains("timed out"));
24444    }
24445
24446    #[tokio::test]
24447    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24448        crate::run::pin_test_home();
24449        let tmp = tempfile::tempdir().unwrap();
24450        let cache = tmp.path().join("cache");
24451        let _held = hold(&cache);
24452        let mut state = state(21600);
24453        let pause = Pause::new();
24454        let interrupt = Pause::new();
24455        let cause = ParkCause::Operator;
24456        pause.park_for(cause.clone());
24457        let prompts = Prompts::default();
24458        let ctx = WaveCtx {
24459            run: "test",
24460            node: "fix",
24461            prompts: &prompts,
24462            cache: Some(&cache),
24463            round: None,
24464            carry_seats: false,
24465            pauses: Some((&pause, &interrupt)),
24466        };
24467        let result = wave(
24468            vec![job(
24469                sh("writer", "exit 99"),
24470                tmp.path(),
24471                Duration::from_secs(1),
24472            )],
24473            Arc::new(Semaphore::new(1)),
24474            &ctx,
24475            &mut state,
24476            0,
24477        )
24478        .await;
24479        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24480        assert_eq!(result[0].1.turns, 0);
24481        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24482        assert_eq!(
24483            state.resource_block.unwrap().cause,
24484            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24485        );
24486        assert!(state.active.is_empty());
24487    }
24488
24489    #[tokio::test]
24490    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24491        crate::run::pin_test_home();
24492        let tmp = tempfile::tempdir().unwrap();
24493        let cache = tmp.path().join("target-a");
24494        let guard = hold(&cache);
24495        // Held for longer than the agent's whole 1s budget.
24496        tokio::spawn(async move {
24497            tokio::time::sleep(Duration::from_millis(1500)).await;
24498            guard.release();
24499        });
24500        let mut st = state(60);
24501        let results = run(
24502            vec![job(
24503                sh("alpha", "echo ok"),
24504                tmp.path(),
24505                Duration::from_secs(1),
24506            )],
24507            Arc::new(Semaphore::new(1)),
24508            Some(&cache),
24509            &mut st,
24510        )
24511        .await;
24512        assert!(
24513            matches!(results[0].2, AgentOutcome::Ok(_)),
24514            "the wait must not be charged to the agent"
24515        );
24516        assert!(st.resource_block.is_none());
24517        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24518        assert!(st.resource_wait.is_none() && st.active.is_empty());
24519    }
24520
24521    #[tokio::test]
24522    #[cfg(unix)]
24523    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24524        use std::os::fd::AsRawFd as _;
24525        crate::run::pin_test_home();
24526        let tmp = tempfile::tempdir().unwrap();
24527        crate::git::git(tmp.path(), &["init"]).await.unwrap();
24528        let common = tmp.path().join(".git");
24529        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24530        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24531        assert_eq!(
24532            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24533            0
24534        );
24535        let mut st = state(1);
24536        st.config.coordination = crate::config::test_coordination();
24537        let mut reader = job(
24538            sh("reader", "echo must-not-run"),
24539            tmp.path(),
24540            Duration::from_secs(30),
24541        );
24542        reader.allow_write = false;
24543        let results = run(
24544            vec![reader.clone()],
24545            Arc::new(Semaphore::new(1)),
24546            None,
24547            &mut st,
24548        )
24549        .await;
24550        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24551        assert_eq!(results[0].1.turns, 0);
24552        assert!(FailClass::of(&results[0].2).is_none());
24553        assert_eq!(
24554            st.resource_block.as_ref().unwrap().kind,
24555            "repository coordination"
24556        );
24557        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24558        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24559        drop(external);
24560        let editor = crate::repo_lock::SeatLease::for_checkout(
24561            tmp.path(),
24562            true,
24563            crate::config::test_coordination().as_ref(),
24564        )
24565        .await
24566        .unwrap();
24567        let pause = Pause::new();
24568        let interrupt = Pause::new();
24569        let prompts = Prompts::default();
24570        let ctx = WaveCtx {
24571            run: "r",
24572            node: "review",
24573            prompts: &prompts,
24574            cache: None,
24575            round: None,
24576            carry_seats: false,
24577            pauses: Some((&pause, &interrupt)),
24578        };
24579        let (results, ()) = tokio::join!(
24580            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24581            async {
24582                tokio::time::sleep(Duration::from_millis(50)).await;
24583                pause.park();
24584            }
24585        );
24586        assert!(
24587            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24588        );
24589        assert_eq!(results[0].1.turns, 0);
24590        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24591        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24592        drop(editor);
24593    }
24594
24595    #[tokio::test]
24596    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24597        crate::run::pin_test_home();
24598        let tmp = tempfile::tempdir().unwrap();
24599        let cache = tmp.path().join("target-b");
24600        let _held = hold(&cache);
24601        let mut st = state(1);
24602        let results = run(
24603            vec![job(
24604                sh("alpha", "echo ok"),
24605                tmp.path(),
24606                Duration::from_secs(1),
24607            )],
24608            Arc::new(Semaphore::new(1)),
24609            Some(&cache),
24610            &mut st,
24611        )
24612        .await;
24613        let (_, seat, out) = &results[0];
24614        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24615        assert!(wave_blocked(&results));
24616        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24617        assert_eq!(seat.turns, 0, "nothing was asked");
24618        assert!(
24619            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24620            "no agent was spawned"
24621        );
24622        assert!(FailClass::of(out).is_none(), "not a failure class");
24623        let block = st.resource_block.as_ref().expect("recorded for the resume");
24624        assert_eq!(block.kind, "shared build cache");
24625        assert!(matches!(
24626            block.cause,
24627            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24628        ));
24629        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24630        assert!(st.active.is_empty());
24631    }
24632
24633    #[tokio::test]
24634    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24635        crate::run::pin_test_home();
24636        let tmp = tempfile::tempdir().unwrap();
24637        let mut st = state(1);
24638        let results = run(
24639            vec![job(
24640                sh("alpha", "echo ok"),
24641                tmp.path(),
24642                Duration::from_secs(1),
24643            )],
24644            Arc::new(Semaphore::new(0)),
24645            None,
24646            &mut st,
24647        )
24648        .await;
24649        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24650        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24651        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24652    }
24653
24654    #[tokio::test]
24655    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24656        crate::run::pin_test_home();
24657        let tmp = tempfile::tempdir().unwrap();
24658        let mut st = state(60);
24659        let results = run(
24660            vec![job(
24661                sh("alpha", "sleep 30"),
24662                tmp.path(),
24663                Duration::from_secs(1),
24664            )],
24665            Arc::new(Semaphore::new(1)),
24666            None,
24667            &mut st,
24668        )
24669        .await;
24670        let out = &results[0].2;
24671        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24672        let class = FailClass::of(out).expect("a failure");
24673        assert_eq!(class, FailClass::Timeout);
24674        assert!(should_hand_over(None, &class), "existing handover policy");
24675        assert!(st.resource_block.is_none());
24676    }
24677}