Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        anyhow::ensure!(
1443            state
1444                .base_sync_recovery
1445                .stop
1446                .as_ref()
1447                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449        );
1450        let fetched =
1451            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452        anyhow::ensure!(
1453            fetched.ok(),
1454            "cannot authorize base recovery without a fetched base"
1455        );
1456        let approved_tip = git::rev_parse(
1457            &state.repo,
1458            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459        )
1460        .await?;
1461        if let Some(control) = control.as_deref_mut() {
1462            let queue = crate::queue::Queue::at(home.join("queue"));
1463            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464            if let Some(prior) = state
1465                .base_sync_recovery
1466                .authorizations
1467                .iter()
1468                .find(|a| a.request.as_deref() == Some(&control.key()))
1469            {
1470                anyhow::ensure!(
1471                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472                    "recovery request was consumed or invalidated; it cannot queue again"
1473                );
1474                return control.enqueue(&queue);
1475            }
1476        }
1477        if resumption {
1478            anyhow::ensure!(
1479                crate::daemon::compatible_run_instruction(&state, &queued),
1480                "resumption instruction does not match the retained run"
1481            );
1482            let approval = queued
1483                .resumption
1484                .as_ref()
1485                .context("no operator resumption")?;
1486            anyhow::ensure!(
1487                approval.instruction == queued.instruction,
1488                "resumption approval does not match the current task instruction"
1489            );
1490            state.authorize_base_sync_resumption(task, approval)?;
1491        } else {
1492            state.authorize_base_sync(task)?;
1493        }
1494        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495        state
1496            .base_sync_recovery
1497            .authorizations
1498            .last_mut()
1499            .unwrap()
1500            .scope = Some(crate::base_sync_recovery::Scope {
1501            tip: approved_tip,
1502            instruction: queued.instruction.clone(),
1503            episode,
1504        });
1505        if let Some(control) = control.as_deref_mut() {
1506            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507            authorization.request = Some(control.key());
1508            authorization.generation = control.generation();
1509            authorization.decision = control.decision();
1510            authorization.question = control
1511                .request
1512                .decision
1513                .as_ref()
1514                .map(|_| control.request.request.clone());
1515        }
1516        state.event(
1517            "base-recovery",
1518            "owner authorized a bounded base synchronization interval; fresh verification required",
1519        );
1520        state.save_under(home)?;
1521        if let Some(control) = control {
1522            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523        }
1524        Ok(())
1525    }
1526
1527    /// Reopen an existing run.
1528    pub fn resume(id: &str) -> Result<Self> {
1529        let state = RunState::load(id)?;
1530        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531        if state.startup.as_ref().is_some_and(|s| !s.complete)
1532            || (state.status == RunStatus::Prep
1533                && state.instruction.trim().is_empty()
1534                && state.candidates.is_empty()
1535                && state.prep_complete == Some(false))
1536        {
1537            bail!(
1538                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539            );
1540        }
1541        crate::pr_head::ensure_owner(&state)?;
1542        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543            bail!(
1544                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545            );
1546        }
1547        if let Some(to) = &state.released_to {
1548            bail!(
1549                "run {} cannot be resumed: its worktree was released to run {}",
1550                state.short(),
1551                crate::run::short_of(to)
1552            );
1553        }
1554        let roles = state.config.resolve_roles()?;
1555        if let Some(id) = &state.config.graph.selected_implementer
1556            && state.candidates.iter().any(|c| c.agent != *id)
1557        {
1558            bail!(
1559                "replacement implementer pin does not match the saved candidates; task must stay held"
1560            );
1561        }
1562        let max_parallel = state.config.graph.max_parallel.max(1);
1563        Ok(Self {
1564            state,
1565            roles,
1566            sem: Arc::new(Semaphore::new(max_parallel)),
1567            pause: Pause::new(),
1568            interrupt: Pause::new(),
1569        })
1570    }
1571
1572    /// Walk the graph to a terminal state, skipping nodes already recorded.
1573    ///
1574    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1575    /// the phone - ends here, so this is the one place a run that ended
1576    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1577    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1578    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579        // Allocate before polling: boxing inside an async wrapper would still
1580        // retain its construction frame while the nested graph is polled.
1581        Box::pin(self.execute_inner())
1582    }
1583
1584    async fn execute_inner(&mut self) -> Result<()> {
1585        anyhow::ensure!(
1586            self.state.startup.as_ref().is_none_or(|s| s.complete),
1587            "startup incomplete; inspect retained evidence before explicit recovery"
1588        );
1589        if let Some(selection) = &self.state.pr_recovery
1590            && (selection.source_proof.is_some() || selection.source_run.is_some())
1591        {
1592            Box::pin(selection.verify_source(&self.state.repo)).await?;
1593        }
1594        // Queue claims cover daemon work; this also serializes manual drivers
1595        // consuming a repair answer, including the new review and decision.
1596        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597            || self.state.land_approval.is_some()
1598            || self.state.merge_decision.is_some()
1599            || !self.state.merge_repairs.is_empty()
1600            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601        {
1602            let claim = FixClaim::acquire(&self.state.dir())?;
1603            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604            let disk = RunState::load(&self.state.id)?;
1605            let checkpoint = |s: &RunState| {
1606                (
1607                    s.merge_repairs.len(),
1608                    s.merge_repairs
1609                        .last()
1610                        .map(|r| (r.question.clone(), r.phase)),
1611                    s.land_approval
1612                        .as_ref()
1613                        .map(|a| (a.question.clone(), a.head.clone())),
1614                )
1615            };
1616            if checkpoint(&disk) != checkpoint(&self.state) {
1617                bail!(
1618                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619                );
1620            }
1621            Some(claim)
1622        } else {
1623            None
1624        };
1625        let mut execution = self.claim_driver().await?;
1626        let runner = &mut *execution.runner;
1627        let wait = runner.repository_wait();
1628        // Keep the graph's large future off the callers' stack, including
1629        // standalone task attempts running on a default-sized test thread.
1630        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631        let result = match result {
1632            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633                record_repository_block(&mut runner.state, &e);
1634                runner.park_here()?;
1635                Ok(())
1636            }
1637            other => other,
1638        };
1639        execution.driver.finish(&mut runner.state).await?;
1640        let ended = if result.is_err() {
1641            Some({
1642                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643                if let Err(error) = &result {
1644                    notice.message = format!(
1645                        "Run {} stopped with an error: {}",
1646                        runner.state.id,
1647                        error.to_string().lines().next().unwrap_or("unknown error")
1648                    );
1649                }
1650                notice
1651            })
1652        } else {
1653            crate::notices::run_ended(&runner.state)
1654        };
1655        if let Some(notice) = ended {
1656            crate::notices::raise(notice);
1657        }
1658        result
1659    }
1660
1661    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663    }
1664
1665    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666        let wait = self.repository_wait();
1667        coordinate_operation(&mut self.state, cwd, &wait).await
1668    }
1669
1670    /// Validate ownership and persist the driver claim in one critical section.
1671    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672        let owner = match crate::pr_head::owner_lock_wait(
1673            &crate::run::home(),
1674            &self.state.id,
1675            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676        )
1677        .await
1678        {
1679            Ok(owner) => owner,
1680            Err(error) => {
1681                let detail = format!("worktree maintenance or ownership update: {error:#}");
1682                // Admission did not own run.json: retain this resource outcome
1683                // in memory for queue settlement without overwriting a concurrent
1684                // retirement or publication transaction.
1685                self.state.resource_block = Some(crate::run::ResourceBlock {
1686                    cause: crate::run::ResourceBlockCause::Other,
1687                    node: "start".into(),
1688                    kind: "worktree maintenance".into(),
1689                    detail: detail.clone(),
1690                    at: Timestamp::now(),
1691                });
1692                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693            }
1694        };
1695        let disk = RunState::load(&self.state.id)?;
1696        crate::pr_head::ensure_owner(&disk)?;
1697        let claimed =
1698            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699        // Only proven-live ownership refuses here. Unknown records (legacy runs
1700        // with no claim marker or process identity) fall through to the
1701        // exclusive execution lease taken just below, which is what actually
1702        // keeps a second driver out.
1703        anyhow::ensure!(
1704            disk.liveness(claimed) != crate::run::Liveness::Live,
1705            "run already has a live driver; wait for its execution to end"
1706        );
1707        if let Some(to) = &disk.released_to {
1708            bail!(
1709                "run {} cannot continue: its worktree was released to run {}",
1710                self.state.short(),
1711                crate::run::short_of(to)
1712            );
1713        }
1714        anyhow::ensure!(
1715            serde_json::to_value(&disk.base_sync_recovery)?
1716                == serde_json::to_value(&self.state.base_sync_recovery)?,
1717            "base recovery changed since this run was loaded; reload the saved run"
1718        );
1719        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720        let mut execution = DriverExecution {
1721            runner: self,
1722            driver,
1723        };
1724        // Declare after the execution guard: on any claim error this short
1725        // lock must drop before that guard reloads and releases ownership.
1726        let _owner = owner;
1727        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728            .await?;
1729        let state = &mut execution.runner.state;
1730        // Moving again, so it is no longer parked. Set before the walk rather
1731        // than in `resume`, so every way of re-entering the graph clears it
1732        // and a card cannot claim a run is waiting to be resumed while the
1733        // agents are already working.
1734        state.parked = false;
1735        state.resource_wait = None;
1736        state.resource_block = None;
1737        // Any seat this state still lists as answering belongs to whatever
1738        // process last drove this run — this one included, if it crashed
1739        // mid-wave. Cleared and flushed immediately, before anything else
1740        // runs, so a resume can never show a seat as live when nothing is
1741        // asking it anything yet; the node that actually dispatches the next
1742        // wave repopulates it.
1743        state.clear_active();
1744        // Recorded in the same spot, and flushed together with the clear
1745        // above: this is the pid a reader checks (`RunState::liveness`) when
1746        // no daemon claim exists to answer "is a process still driving this
1747        // run" — a plain `magi run` / `magi review` typed into a terminal
1748        // claims nothing there. Always overwritten, never only-if-absent, so
1749        // a resumed run's stale pid from a previous, possibly-dead process
1750        // can never survive into this one's own report. Unlike
1751        // `clear_active`, this changes on every single `execute()` call, so
1752        // the save below is now unconditional rather than only-if-cleared.
1753        //
1754        // `driver_started_at` is recorded in the same breath, from this same
1755        // pid, so `liveness` can tell a live pid that is genuinely still us
1756        // apart from one the OS has since handed to an unrelated process —
1757        // see that field's own doc for why the pid alone is not enough.
1758        let pid = std::process::id();
1759        state.driver_pid = Some(pid);
1760        state.driver_started_at = crate::proc::process_started_at(pid);
1761        state.driver_exited = false;
1762        state.driver_claimed = Some(true);
1763        state.begin_pass();
1764        execution.driver.bind(state);
1765        state.save()?;
1766        execution.driver.record(state)?;
1767        Ok(execution)
1768    }
1769
1770    /// No conflict checkpoint proves dispatch was never reached. End that
1771    /// consumed generation so an owner can explicitly authorize a fresh one.
1772    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774        if let Err(error) = &result
1775            && !error.is::<crate::repo_lock::ResourceBlocked>()
1776            && self
1777                .state
1778                .repair_grants
1779                .last()
1780                .is_some_and(|g| g.conflict.is_none())
1781        {
1782            self.state.repair_grants.last_mut().unwrap().phase =
1783                crate::repair_budget::Phase::Complete;
1784            self.state.status = RunStatus::Blocked;
1785            self.state.save()?;
1786            bail!(
1787                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788                self.state.passes.last().map_or(0, |p| p.index)
1789            );
1790        }
1791        result
1792    }
1793
1794    /// A grant owns only these repair attempts. Confirmation reviews cannot
1795    /// create another fixer allowance, and every operation is restart-checkpointed.
1796    async fn execute_granted_repair(&mut self) -> Result<()> {
1797        use crate::repair_budget::{Kind, Phase};
1798        let index = self.state.repair_grants.len() - 1;
1799        let winner = self
1800            .state
1801            .winner()
1802            .cloned()
1803            .context("repair winner missing")?;
1804        loop {
1805            let grant = self.state.repair_grants[index].clone();
1806            match grant.phase {
1807                Phase::Pending => {
1808                    if grant.spent >= grant.receipt.request.rounds {
1809                        self.state.repair_grants[index].phase = Phase::Complete;
1810                        self.state.status = RunStatus::Blocked;
1811                        self.state.save()?;
1812                        return Ok(());
1813                    }
1814                    // Reserve before dispatch. A crash can consume an attempt,
1815                    // but can never duplicate it or reset saved fixer history.
1816                    self.state.repair_grants[index].spent += 1;
1817                    self.state.repair_grants[index].phase = Phase::Fixing;
1818                    self.state.save()?;
1819                    match grant.receipt.request.kind {
1820                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821                        Kind::Gate => {
1822                            let failed = self.state.gate.clone();
1823                            let _ = self.gate_fix_round(&winner, &failed).await?;
1824                        }
1825                        Kind::Review => {
1826                            let mut record = self
1827                                .state
1828                                .reviews
1829                                .last()
1830                                .cloned()
1831                                .context("review failure missing")?;
1832                            let round = record.round;
1833                            let step = self.run_fix_round(&winner, &mut record).await?;
1834                            self.state.reviews[round - 1] = record;
1835                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1836                                self.state.repair_grants[index].phase = Phase::Complete;
1837                                self.state.status = RunStatus::Blocked;
1838                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839                                self.state.save()?;
1840                                return Ok(());
1841                            }
1842                            if matches!(step, FixStep::Resource) {
1843                                self.stop_for_resource()?;
1844                                return Ok(());
1845                            }
1846                            self.state.save()?;
1847                        }
1848                    }
1849                    if self.resource_halt()? {
1850                        return Ok(());
1851                    }
1852                }
1853                Phase::Fixing => {
1854                    if grant.receipt.request.kind == Kind::BaseConflict {
1855                        self.repair_base_conflict(false).await?;
1856                    }
1857                    // Interrupted fixers may have left source changes or an
1858                    // unpushed commit. Preserve them before fresh assessment.
1859                    let _coordination = self.coordinate(&winner.worktree).await?;
1860                    let rescued = git::rescue_commit(
1861                        &winner.worktree,
1862                        "magi: preserve interrupted granted repair",
1863                    )
1864                    .await?;
1865                    self.state.note_withheld("repair-grant", &rescued.withheld);
1866                    // Fresh checks must supersede even a same-head failure.
1867                    // Archive rather than erase failure evidence or counters.
1868                    self.state
1869                        .verification_history
1870                        .push(crate::run::VerificationEvidence {
1871                            superseded_by: None,
1872                            archived_at: Some(Timestamp::now()),
1873                            fixes_spent: Some(self.state.gate_fixes.len()),
1874                            gate_head: self.state.gate_head.take(),
1875                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1876                            gate: std::mem::take(&mut self.state.gate),
1877                            e2e: self.state.head_e2e.take(),
1878                        });
1879                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880                        self.state
1881                            .verification_history
1882                            .push(crate::run::VerificationEvidence {
1883                                superseded_by: None,
1884                                archived_at: Some(Timestamp::now()),
1885                                fixes_spent: Some(self.state.gate_fixes.len()),
1886                                gate_head: None,
1887                                gate_ran: false,
1888                                gate: Vec::new(),
1889                                e2e: Some(e2e),
1890                            });
1891                    }
1892                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1893                    self.state.repair_grants[index].head =
1894                        git::rev_parse(&winner.worktree, "HEAD").await?;
1895                    self.state.repair_grants[index].phase = Phase::Reviewing;
1896                    if grant.receipt.request.kind == Kind::BaseConflict {
1897                        self.state.base_sync_recovery.stop = None;
1898                        if let Some(sync) = &mut self.state.base_sync {
1899                            sync.tip = grant
1900                                .receipt
1901                                .request
1902                                .tip
1903                                .clone()
1904                                .context("repair base tip missing")?;
1905                            sync.conflict = None;
1906                            sync.behind = 0;
1907                        }
1908                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909                    }
1910                    self.state.save()?;
1911                }
1912                Phase::Reviewing => {
1913                    self.review_loop_pass().await?;
1914                    if self.resource_halt()? {
1915                        return Ok(());
1916                    }
1917                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918                    let clean = self.state.reviews.len() > grant.review_start
1919                        && self.state.reviews.last().is_some_and(|r| {
1920                            r.head == head
1921                                && r.clean
1922                                && r.expected > 0
1923                                && r.answered == r.expected
1924                                && matches!(
1925                                    r.e2e_status(),
1926                                    E2eStatus::Passed | E2eStatus::NotConfigured
1927                                )
1928                        });
1929                    if !clean {
1930                        // A confirmation review carries no additional fix budget.
1931                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932                            == Kind::Review
1933                            && grant.spent < grant.receipt.request.rounds
1934                            && self.state.reviews.last().is_some_and(|r| {
1935                                r.fix.is_none()
1936                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937                            }) {
1938                            Phase::Pending
1939                        } else {
1940                            Phase::Complete
1941                        };
1942                        self.state.status = RunStatus::Blocked;
1943                        self.state.event(
1944                            "repair-grant",
1945                            "fresh review did not clear the repaired head; no publication",
1946                        );
1947                        self.state.save()?;
1948                        if self.state.repair_grants[index].phase == Phase::Pending {
1949                            continue;
1950                        }
1951                        return Ok(());
1952                    }
1953                    self.state.repair_grants[index].phase = Phase::Gating;
1954                    self.state.save()?;
1955                }
1956                Phase::Gating => {
1957                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958                    self.state.reconcile_verification_head(&head);
1959                    let wait = self.repository_wait();
1960                    if self
1961                        .state
1962                        .reviews
1963                        .last()
1964                        .is_none_or(|r| r.head != head || !r.clean)
1965                    {
1966                        self.state.status = RunStatus::Blocked;
1967                        self.state.repair_grants[index].phase = Phase::Complete;
1968                        self.state.event(
1969                            "repair-grant",
1970                            "head changed since fresh review; repair held",
1971                        );
1972                        self.state.save()?;
1973                        return Ok(());
1974                    }
1975                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976                        return Ok(());
1977                    }
1978                    let outcomes = self.run_gate(&winner).await?;
1979                    if verify_inconclusive(&outcomes) {
1980                        return Ok(());
1981                    }
1982                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983                    if after != head {
1984                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1985                        self.state.repair_grants[index].head = after;
1986                        self.state.repair_grants[index].phase = Phase::Reviewing;
1987                        self.state.status = RunStatus::Blocked;
1988                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989                        self.state.save()?;
1990                        return Ok(());
1991                    }
1992                    let passed = outcomes.iter().all(CommandOutcome::ok);
1993                    // Follow-up grants owe the same guarded checkout delivery
1994                    // as the conflict repair that introduced the new head.
1995                    if passed
1996                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997                    {
1998                        return Ok(());
1999                    }
2000                    self.state.repair_grants[index].succeeded = passed;
2001                    self.state.gate = outcomes;
2002                    self.state.gate_ran = true;
2003                    self.state.status = if passed {
2004                        RunStatus::Gating
2005                    } else {
2006                        RunStatus::Blocked
2007                    };
2008                    self.state.repair_grants[index].phase = if !passed
2009                        && grant.receipt.request.kind == Kind::Gate
2010                        && grant.spent < grant.receipt.request.rounds
2011                    {
2012                        Phase::Pending
2013                    } else {
2014                        Phase::Complete
2015                    };
2016                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2017                        // Persist publication debt with the successful checkpoint.
2018                        self.state.pr_publication_pending = true;
2019                    } else if passed {
2020                        self.state.status = RunStatus::Ready;
2021                    }
2022                    self.state.save()?;
2023                    if !passed {
2024                        continue;
2025                    }
2026                    // Existing PR ownership and push lease stay in place. The
2027                    // normal publisher owes the new reviewed/gated SHA, and
2028                    // land's fresh owner approval remains mandatory.
2029                    if self.state.config.merge.mode != MergeMode::Pr {
2030                        self.state.status = RunStatus::Ready;
2031                        self.state.event(
2032                            "repair-grant",
2033                            "repair verified locally; grant carries no merge authorization",
2034                        );
2035                        self.state.save()?;
2036                        return Ok(());
2037                    }
2038                    return self.merge().await;
2039                }
2040                Phase::Complete => return Ok(()),
2041            }
2042        }
2043    }
2044
2045    async fn execute_graph(&mut self) -> Result<()> {
2046        // Even a terminal run may reenter through an explicit recovery action.
2047        // Restore before prep shortcuts, saved conversations or graph nodes use
2048        // a disposable path; GC may already have removed its original objects.
2049        let archive_home = crate::run::home().canonicalize()?;
2050        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051            let repo = self.state.repo.clone();
2052            let _coordination = self.coordinate(&repo).await?;
2053            ensure_setup_config(&mut self.state, &repo).await?;
2054            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055        }
2056        self.state.configure_recovery_resume()?;
2057        if self
2058            .state
2059            .repair_grants
2060            .last()
2061            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062        {
2063            return self.execute_granted_repair().await;
2064        }
2065        if self
2066            .state
2067            .repair_grants
2068            .last()
2069            .is_some_and(|g| !g.succeeded)
2070        {
2071            self.state.status = RunStatus::Blocked;
2072            self.state.save()?;
2073            return Ok(());
2074        }
2075        // Preservation may invalidate an old empty classification. Resolve it
2076        // before any resume shortcut can reuse a verdict or publish a branch.
2077        self.persist_candidates().await?;
2078        // A saved base stop outranks stage markers, including legacy Voting.
2079        // Recovery never reenters implementation, deliberation or voting.
2080        if self.state.base_sync_blocked() {
2081            self.sync_to_base().await?;
2082            if !self.state.base_sync_blocked()
2083                && self.state.status != RunStatus::AlreadyInBase
2084                && !self.resource_halt()?
2085            {
2086                self.finish_after_tally().await?;
2087            }
2088            return Ok(());
2089        }
2090        if self
2091            .state
2092            .merge_decision
2093            .as_ref()
2094            .is_some_and(|decision| !decision.closed)
2095            && self.state.status != RunStatus::Landing
2096        {
2097            if crate::merge_decision::resumable_hold(&self.state) {
2098                self.state.merge = Some(MergeOutcome {
2099                    mode: MergeMode::Pr,
2100                    ok: false,
2101                    detail: self
2102                        .state
2103                        .merge_decision
2104                        .as_ref()
2105                        .unwrap()
2106                        .target
2107                        .pr
2108                        .clone(),
2109                    empty: false,
2110                });
2111                self.run_land().await?;
2112                self.settle_questions();
2113                return Ok(());
2114            }
2115            bail!(
2116                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117            );
2118        }
2119        if self.state.config.graph.selected_implementer.is_some()
2120            && self.state.status == RunStatus::Blocked
2121            && self.state.tally.is_none()
2122            && self.state.candidate_preservation.is_empty()
2123            && self.state.interrupted_implementations.is_empty()
2124        {
2125            bail!(
2126                "selected implementation is held; explicitly release the owning task to retry it"
2127            );
2128        }
2129
2130        if self.state.merge_repairs.last().is_some_and(|r| {
2131            matches!(
2132                r.phase,
2133                crate::run::MergeRepairPhase::Requested
2134                    | crate::run::MergeRepairPhase::Fixing
2135                    | crate::run::MergeRepairPhase::Reviewing
2136            )
2137        }) {
2138            if self.repair_merge_review().await? {
2139                self.run_land().await?;
2140            }
2141            self.settle_questions();
2142            return Ok(());
2143        }
2144        // Publication already passed review and gate. Resume only that node:
2145        // syncing to a newer base here could change the verified commit.
2146        if self.state.pr_publication_pending {
2147            self.merge().await?;
2148            if self.resource_halt()? {
2149                return Ok(());
2150            }
2151            self.state.save()?;
2152            self.settle_questions();
2153            return Ok(());
2154        }
2155        // A run that already lost its quorum never resumes into the verdict
2156        // machinery: `deliberate` and `vote` would otherwise clobber the
2157        // stalled marker back to Voting and the run would keep going past a
2158        // verdict that is no longer trustworthy. Everything already recorded is
2159        // kept, so the run stays resumable (or foldable) for a human to pick up.
2160        //
2161        // On --resume the run gets one chance to repair itself: the seats a
2162        // rate limit took out are re-asked. If their quota has since reset and
2163        // the quorum is restored, the run picks up and finishes; otherwise it
2164        // stays stale and still-resumable for a later retry. If it does not
2165        // recover, the returned status stays `Stalled` and nothing was
2166        // clobbered (the recovery only mutates entries for the lost seats).
2167        if self.state.status == RunStatus::Stalled {
2168            if self.recover_stall().await? {
2169                self.finish_after_tally().await?;
2170            } else {
2171                // Still below quorum: persist the marker and stay resumable.
2172                self.state.save()?;
2173            }
2174            return Ok(());
2175        }
2176        // A run parked inside `land` - watching CI, mid fix-round, or
2177        // waiting on the owner's merge approval - resumes directly into it,
2178        // never back through `prep`. Everything before `merge` already
2179        // concluded; that is the only way `status` reaches `Landing` in the
2180        // first place. Re-walking `review_loop` first would also be actively
2181        // wrong: its own status recomputation (see its doc) treats any
2182        // clean round as reason to set `status` to `Gating`, which would
2183        // clobber this marker before `merge` ever ran, and this run would
2184        // never find its way back into `land` at all.
2185        if self.state.status == RunStatus::Landing
2186            || crate::handover::landing_checkpoint(&self.state)
2187        {
2188            self.state.land_sync_pending = true;
2189            self.state.save()?;
2190            self.run_land().await?;
2191            // `run_land` may have settled the run right here - CI came back
2192            // green and the PR merged, say - without ever passing back
2193            // through `merge`'s own trailing call. Whatever it left `status`
2194            // as is what this has to read.
2195            self.settle_questions();
2196            return Ok(());
2197        }
2198        // A run parked at `merge` on the owner's word about a withheld PR text
2199        // resumes straight into `merge`. Re-walking the tail would sync to the
2200        // base again (possibly rebasing, possibly spending the fixer) after the
2201        // gate had passed, and then push a tree nobody reviewed or gated.
2202        if self.state.github_text.is_some()
2203            && self.state.merge.is_none()
2204            && self.state.status == RunStatus::Gating
2205        {
2206            self.merge().await?;
2207            if self.resource_halt()? {
2208                return Ok(());
2209            }
2210            self.state.save()?;
2211            return Ok(());
2212        }
2213        if let Err(e) = self.prep().await {
2214            if e.is::<crate::repo_lock::ResourceBlocked>() {
2215                return Err(e);
2216            }
2217            self.state.status = RunStatus::Blocked;
2218            self.state
2219                .prep_failure
2220                .get_or_insert_with(|| format!("{e:#}"));
2221            self.state
2222                .event("prep", format!("preparation stopped: {e:#}"));
2223            self.state.save()?;
2224            return Err(e);
2225        }
2226        if self.park_here()? {
2227            return Ok(());
2228        }
2229        self.advise().await?;
2230        if self.park_here()? {
2231            return Ok(());
2232        }
2233        self.implement().await?;
2234        if self.park_here()? {
2235            return Ok(());
2236        }
2237        // `after_implement` already saved the state and settled any open
2238        // questions when it set this; nothing later in the graph has
2239        // anything to judge.
2240        if self.state.status == RunStatus::VerifiedNoop {
2241            return Ok(());
2242        }
2243        self.judge().await?;
2244        if self.park_here()? {
2245            return Ok(());
2246        }
2247        self.deliberate().await?;
2248        if self.park_here()? {
2249            return Ok(());
2250        }
2251        self.vote().await?;
2252        if self.park_here()? {
2253            return Ok(());
2254        }
2255        self.tally()?;
2256        // A verdict that lost its quorum is not trustworthy: do not review,
2257        // gate, or merge on it. Everything already done is kept, so the run
2258        // stays resumable (or foldable); the human can replace the agent that
2259        // ran out of quota and pick it up.
2260        if self.state.status == RunStatus::Stalled {
2261            // Persist the stalled marker now — the normal end-of-execute save
2262            // below is below this early return, and without it a resumed run
2263            // would reload a pre-tally status and keep going.
2264            self.state.save()?;
2265            return Ok(());
2266        }
2267        self.finish_after_tally().await?;
2268        Ok(())
2269    }
2270
2271    /// A wave could not start its agents because a shared resource stayed busy
2272    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2273    /// resumable. Unstarted seats have no result; completed peers retain
2274    /// their saved candidate state.
2275    fn stop_for_resource(&mut self) -> Result<()> {
2276        // Resource acquisition observes cancellation inside a node. Complete
2277        // the same durable park as every other node boundary before returning.
2278        if self.pause.parked() || self.interrupt.parked() {
2279            self.park_here()?;
2280            return Ok(());
2281        }
2282        if self.state.resource_block.is_some() {
2283            self.state.status = RunStatus::Blocked;
2284        }
2285        self.state.save()
2286    }
2287
2288    /// Stop walking the graph if a node ended on a resource block.
2289    fn resource_halt(&mut self) -> Result<bool> {
2290        if self.state.resource_block.is_none() {
2291            return Ok(false);
2292        }
2293        self.stop_for_resource()?;
2294        Ok(true)
2295    }
2296
2297    /// Park here if asked to, recording it in the run's own timeline.
2298    ///
2299    /// Returns whether the caller should stop walking the graph. The state is
2300    /// saved either way by the node that just finished; this adds the event so
2301    /// the operator's card says why a run that is neither finished nor moving
2302    /// is sitting where it is.
2303    fn park_here(&mut self) -> Result<bool> {
2304        let parking = self.pause.parked() || self.interrupt.parked();
2305        if !parking && self.state.resource_block.is_some() {
2306            self.stop_for_resource()?;
2307            return Ok(true);
2308        }
2309        // Either handle asking is enough - see `Pause`'s own doc for why
2310        // they are never the same one. `interrupt` is checked second so a
2311        // reason it carries is preferred in the message below over a plain
2312        // shutdown park racing it at the same boundary.
2313        if !self.pause.parked() && !self.interrupt.parked() {
2314            return Ok(false);
2315        }
2316        if self.state.parked {
2317            return Ok(true);
2318        }
2319        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320        if let Some(block) = &mut self.state.resource_block
2321            && matches!(
2322                block.cause,
2323                crate::run::ResourceBlockCause::Cancelled { .. }
2324            )
2325        {
2326            block.cause = crate::run::ResourceBlockCause::Cancelled {
2327                park: cause.clone(),
2328            };
2329        }
2330        let why = match cause.as_ref().map(ParkCause::label) {
2331            Some(reason) => format!(
2332                "parked after `{}` ({reason}) — resume to carry on from here",
2333                self.state.status.as_str()
2334            ),
2335            None => format!(
2336                "parked after `{}` — resume to carry on from here",
2337                self.state.status.as_str()
2338            ),
2339        };
2340        self.state.event("park", why);
2341        self.state
2342            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343        self.state.parked = true;
2344        self.state.save()?;
2345        Ok(true)
2346    }
2347
2348    /// Only a new explicit operator approval may retry failed preparation.
2349    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350        if self.state.prep_failure.is_none() {
2351            return Ok(());
2352        }
2353        let failed_at = self
2354            .state
2355            .events
2356            .iter()
2357            .rev()
2358            .find(|event| event.node == "prep")
2359            .map_or(self.state.updated_at, |event| event.at);
2360        anyhow::ensure!(
2361            approval_at > failed_at,
2362            "preparation retry requires a new explicit release after the failure"
2363        );
2364        self.state.prep_failure = None;
2365        self.state.prep_complete = Some(false);
2366        self.state
2367            .event("prep-retry", "operator approved retry of preparation");
2368        Ok(())
2369    }
2370
2371    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2372    pub fn on_pause(&mut self, pause: Pause) {
2373        self.pause = pause;
2374    }
2375
2376    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2377    /// scheduler asking this one run - and no other - to park so a task
2378    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2379    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2380    /// handle.
2381    pub fn watch_interrupt(&mut self, pause: Pause) {
2382        self.interrupt = pause;
2383    }
2384
2385    /// Abandon this run's own open questions, once `status` has actually
2386    /// settled rather than merely paused.
2387    ///
2388    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2389    /// either back up with the candidates, the review round and the seat
2390    /// sessions already on disk, so a question an implementer asked mid-round
2391    /// may still get a real answer read by a real resume. Only the statuses
2392    /// `resumable` excludes are actually final: the run merged, it reached
2393    /// `Ready` with nothing left to do, it failed outright with no
2394    /// established point to continue from, or every candidate agreed, with
2395    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2396    /// every one of those the seat that asked is gone for good, exactly like
2397    /// the run being deleted under `magi run rm` - so the same cleanup
2398    /// applies, worded for what actually happened instead of "the run was
2399    /// deleted".
2400    ///
2401    /// Best-effort and silent on success: called from every place `status`
2402    /// can land on one of those three, including ones a resumed run revisits,
2403    /// so it must cost nothing when there was nothing open to begin with.
2404    fn settle_questions(&mut self) {
2405        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407        }
2408    }
2409
2410    /// The tail of the graph after a trustworthy tally: fold losers, review,
2411    /// gate, merge, and persist.
2412    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413        Box::pin(self.finish_after_tally_inner())
2414    }
2415
2416    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417        // Before the losers are folded: their branches are what the graft
2418        // reads from.
2419        self.graft().await?;
2420        if self.resource_halt()? {
2421            return Ok(());
2422        }
2423        self.fold_losers().await?;
2424        // Before review starts, and again right before the gate: a run's
2425        // review rounds can themselves take long enough for the base to move
2426        // a second time, and the gate is the one node whose "green" gets
2427        // acted on.
2428        self.sync_to_base().await?;
2429        if self.hold_base_sync().await? || self.resource_halt()? {
2430            return Ok(());
2431        }
2432        if self.state.status == RunStatus::AlreadyInBase {
2433            return Ok(());
2434        }
2435        self.review_loop().await?;
2436        if self.resource_halt()? {
2437            return Ok(());
2438        }
2439        self.sync_to_base().await?;
2440        if self.hold_base_sync().await? || self.resource_halt()? {
2441            return Ok(());
2442        }
2443        if self.state.status == RunStatus::AlreadyInBase {
2444            return Ok(());
2445        }
2446        self.gate().await?;
2447        self.merge().await?;
2448        if self.resource_halt()? {
2449            return Ok(());
2450        }
2451        self.state.save()?;
2452        Ok(())
2453    }
2454
2455    // ---------------------------------------------------------------- prep
2456
2457    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2458    /// run with the step and its output in the event; it is never swallowed.
2459    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2460    /// resume starts over instead of finding candidates and missing seats.
2461    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462        let result = match ensure_setup_config(&mut self.state, repo).await {
2463            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464            Err(e) => Err(e),
2465        };
2466        if let Err(e) = result {
2467            git::worktree_remove_clean(repo, wt).await.ok();
2468            if in_prep {
2469                self.rollback_prep(repo).await;
2470            }
2471            self.state.status = RunStatus::Blocked;
2472            self.state.event(
2473                "setup",
2474                format!("worktree setup failed in {}: {e:#}", wt.display()),
2475            );
2476            self.state.save()?;
2477            return Err(e);
2478        }
2479        Ok(())
2480    }
2481
2482    /// Undo everything `prep` created, so a resume re-runs it whole.
2483    async fn rollback_prep(&mut self, repo: &Path) {
2484        let root = self.state.worktree_root();
2485        for c in std::mem::take(&mut self.state.candidates) {
2486            if matches!(
2487                git::worktree_remove_clean(repo, &c.worktree).await,
2488                Ok(true)
2489            ) {
2490                git::branch_delete(repo, &c.branch).await.ok();
2491            } else {
2492                self.state.candidates.push(c);
2493            }
2494        }
2495        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496        for j in 1..=seats {
2497            remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2498                .await
2499                .ok();
2500        }
2501        for k in 1..=self.state.config.graph.advisors {
2502            remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2503                .await
2504                .ok();
2505        }
2506    }
2507
2508    async fn prep(&mut self) -> Result<()> {
2509        if let Some(reason) = &self.state.prep_failure {
2510            bail!(
2511                "candidate preparation previously failed: {reason}; correct the source before retrying"
2512            );
2513        }
2514        if self
2515            .state
2516            .prep_complete
2517            .unwrap_or(!self.state.candidates.is_empty())
2518        {
2519            return Ok(());
2520        }
2521        let repo = self.state.repo.clone();
2522        // Includes rollback, config, refs, checkout creation and setup. No
2523        // repository mutation may precede the real shared lock acquisition.
2524        let _coordination = self.coordinate(&repo).await?;
2525        let base = self.state.base_commit.clone();
2526        let plan = self.state.sources.plan(&repo, &base).await?;
2527        self.rollback_prep(&repo).await;
2528        if !self.state.candidates.is_empty() {
2529            bail!(
2530                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531            );
2532        }
2533        self.state.prep_complete = Some(false);
2534        self.state.status = RunStatus::Prep;
2535        self.state.save()?;
2536        let repo = self.state.repo.clone();
2537        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538        let root = self.state.worktree_root();
2539        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541        // The hook is the write-time half of the blindness contract; the
2542        // presentation filter in `blind` is the half that cannot be bypassed.
2543        let hooks_dir = self.state.dir().join("hooks");
2544        if self.state.config.blind.commit_msg_hook {
2545            std::fs::create_dir_all(&hooks_dir)
2546                .with_context(|| format!("create {}", hooks_dir.display()))?;
2547            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548            let path = hooks_dir.join("commit-msg");
2549            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550            make_executable(&path)?;
2551            // Ref-counted rather than a plain idempotent set: with more than
2552            // one run able to be in flight in the same repository at once
2553            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2554            // true?" check cannot tell "another run of mine still needs
2555            // this" from "nobody does", and the run that happens to finish
2556            // first would disable the hook out from under a sibling still
2557            // relying on it.
2558            git::acquire_worktree_config(&repo).await?;
2559            self.state.enabled_worktree_config = true;
2560        }
2561
2562        // `[worktree] setup` hides its products per worktree, which needs the
2563        // same per-worktree config the hook does. Held until fold, like it.
2564        ensure_setup_config(&mut self.state, &repo).await?;
2565
2566        for (index, (spec, label)) in self
2567            .roles
2568            .implementers
2569            .clone()
2570            .into_iter()
2571            .zip(labels)
2572            .enumerate()
2573        {
2574            let branch = self.state.branch_for(label);
2575            let worktree = root.join(format!("cand-{label}"));
2576            // Persist intent before Git can leave a partially created tree.
2577            // Candidate registration follows setup, so it cannot inventory
2578            // crashes or refusals during creation itself.
2579            self.state.event(
2580                "prep-intent",
2581                serde_json::json!({
2582                    "branch": branch,
2583                    "worktree": worktree,
2584                    "start": start,
2585                    "picks": plan.picks,
2586                })
2587                .to_string(),
2588            );
2589            self.state.save()?;
2590            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591            if self.state.config.blind.commit_msg_hook {
2592                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593            }
2594            git::local_exclude(&worktree, "/.magi/").await?;
2595            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596                // Nothing of this prep may survive: a resume must start over,
2597                // not find half the candidates and skip the rest.
2598                git::worktree_remove_clean(&repo, &worktree).await.ok();
2599                git::branch_delete(&repo, &branch).await.ok();
2600                self.rollback_prep(&repo).await;
2601                self.state.status = RunStatus::Blocked;
2602                self.state
2603                    .event("prep", format!("worktree setup failed: {e:#}"));
2604                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605                self.state.save()?;
2606                return Err(e);
2607            }
2608            self.state.candidates.push(Candidate {
2609                index,
2610                label,
2611                agent: spec.id.clone(),
2612                branch,
2613                worktree,
2614                summary: String::new(),
2615                stat: String::new(),
2616                files: 0,
2617                commits: 0,
2618                empty: false,
2619                failed: None,
2620                verified_noop: None,
2621                duration_ms: 0,
2622                folded: false,
2623            });
2624            self.state.save()?;
2625            for pick in &plan.picks {
2626                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627                {
2628                    self.state.status = RunStatus::Blocked;
2629                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630                    self.state
2631                        .event("prep", format!("cannot apply referenced commit: {e}"));
2632                    self.state.save()?;
2633                    return Err(e);
2634                }
2635            }
2636        }
2637
2638        for j in 1..=self.roles.judges.len() {
2639            let wt = root.join(format!("judge-{j}"));
2640            if !wt.exists() {
2641                git::worktree_add_detached(&repo, &wt, &base).await?;
2642                self.setup_seat_worktree(&repo, &wt, true).await?;
2643            }
2644        }
2645
2646        // Disposable, detached checkouts for the design-deliberation stage's
2647        // advisor seats — the same shape as the judges' above, at the same
2648        // base commit, since advisors also only ever read. Sized off the
2649        // configured count directly rather than a resolved roster: unlike
2650        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2651        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2652        // `prep` has no `ResolvedRoles` field to read a count from here.
2653        if self.state.config.graph.advise {
2654            for k in 1..=self.state.config.graph.advisors {
2655                let wt = root.join(format!("advisor-{k}"));
2656                if !wt.exists() {
2657                    git::worktree_add_detached(&repo, &wt, &base).await?;
2658                    self.setup_seat_worktree(&repo, &wt, true).await?;
2659                }
2660            }
2661        }
2662
2663        // A judge cannot tell it is looking at its own patch — the seats keep
2664        // separate conversations — but a panel that shares agents with the
2665        // field is less independent than it looks, and that is worth saying out
2666        // loud once per run rather than leaving it in the config.
2667        let authors: Vec<&str> = self
2668            .roles
2669            .implementers
2670            .iter()
2671            .map(|a| a.id.as_str())
2672            .collect();
2673        let overlap: Vec<String> = self
2674            .roles
2675            .judges
2676            .iter()
2677            .enumerate()
2678            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680            .collect();
2681        if !overlap.is_empty() {
2682            let note = format!(
2683                "{} also authored a candidate; blind, but the panel is less \
2684                 independent than {} distinct agents would be",
2685                overlap.join(", "),
2686                self.roles.judges.len()
2687            );
2688            self.state.event("prep", note);
2689        }
2690
2691        self.state.event(
2692            "prep",
2693            format!(
2694                "{} candidates, {} judges, base {} ({})",
2695                self.state.candidates.len(),
2696                self.roles.judges.len(),
2697                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698                self.state.base_branch
2699            ),
2700        );
2701        self.state.prep_complete = Some(true);
2702        self.state.status = RunStatus::Implementing;
2703        self.state.save()?;
2704        Ok(())
2705    }
2706
2707    // -------------------------------------------------------------- advise
2708
2709    /// The design-deliberation stage: independent, read-only advisor seats
2710    /// each sketch a design before any implementer touches the repository,
2711    /// and (when at least one produced a usable proposal) a synthesis seat
2712    /// blends them into a brief `implement` carries in every candidate's
2713    /// prompt.
2714    ///
2715    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2716    /// advisors` is the proposal count. Everything here is best-effort and
2717    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2718    /// that cannot reach quota, or a synthesis seat that produced nothing
2719    /// usable all leave `implement` exactly as it was before this stage
2720    /// existed — the task instruction alone — rather than failing the whole
2721    /// competition over an enrichment stage. Every outcome is still recorded
2722    /// as an event, so a run that got nothing from this stage says why.
2723    ///
2724    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2725    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2726    /// resumed run whose stage failed would re-run it, and re-spend the
2727    /// agent calls, on every reentry before `implement`.
2728    ///
2729    /// Also skipped once any candidate shows implementation progress — the
2730    /// exact predicate `implement` itself uses to decide a candidate is no
2731    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2732    /// is not enough: a run created by an older binary that predates this
2733    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2734    /// an already-`Implementing`-or-later run under this build would
2735    /// otherwise walk straight back through `prep` (a no-op once candidates
2736    /// exist) into this node and spawn every advisor seat against worktrees
2737    /// `prep` never recreated — after implementation has already started,
2738    /// which is exactly the invariant this stage exists to guarantee.
2739    async fn advise(&mut self) -> Result<()> {
2740        let implement_untouched = self
2741            .state
2742            .candidates
2743            .iter()
2744            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745        if !self.state.config.graph.advise || self.state.advise_attempted {
2746            return Ok(());
2747        }
2748        if !implement_untouched {
2749            self.state.event(
2750                "advise",
2751                "skipping the design-deliberation stage: at least one \
2752                 candidate already shows implementation progress, so this \
2753                 run is past the point the stage exists to run before"
2754                    .to_owned(),
2755            );
2756            self.state.advise_attempted = true;
2757            self.state.save()?;
2758            return Ok(());
2759        }
2760        let run_id = self.state.id.clone();
2761        let prompts = self.state.config.prompts.clone();
2762        let instruction = self.state.instruction.clone();
2763        let language = self.state.config.graph.language.clone();
2764        let root = self.state.worktree_root();
2765        let n = self.state.config.graph.advisors;
2766        let where_recorded = self.state.dir().join("run.json");
2767
2768        let seats = match self.state.config.advisors() {
2769            Ok(seats) if !seats.is_empty() => seats,
2770            Ok(_) => {
2771                self.state.event(
2772                    "advise",
2773                    format!(
2774                        "[graph] advisors is 0; skipping the design-deliberation \
2775                         stage and continuing without a synthesis brief (see {})",
2776                        where_recorded.display()
2777                    ),
2778                );
2779                self.state.advise_attempted = true;
2780                self.state.save()?;
2781                return Ok(());
2782            }
2783            Err(e) => {
2784                self.state.event(
2785                    "advise",
2786                    format!(
2787                        "could not resolve advisor seats ({e:#}); continuing \
2788                         without a design-deliberation brief (see {})",
2789                        where_recorded.display()
2790                    ),
2791                );
2792                self.state.advise_attempted = true;
2793                self.state.save()?;
2794                return Ok(());
2795            }
2796        };
2797
2798        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799        let artifacts = agent::artifacts_dir(&self.state.dir());
2800        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802        let mut jobs = Vec::new();
2803        for (i, spec) in seats.iter().cloned().enumerate() {
2804            let seat_key = format!("advisor-{}", i + 1);
2805            let seat = self.seat(&seat_key, &spec.id);
2806            jobs.push(SeatJob {
2807                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808                spec,
2809                seat,
2810                cwd: worktrees[i % worktrees.len()].clone(),
2811                timeout,
2812                allow_write: false,
2813                sessions: false,
2814                artifacts: artifacts.clone(),
2815                stem: seat_key,
2816                handover: None,
2817            });
2818        }
2819
2820        self.state.event(
2821            "advise",
2822            format!(
2823                "{} advisor seat(s) sketching a design in parallel",
2824                jobs.len()
2825            ),
2826        );
2827        let mut quota_losses = Vec::new();
2828        let cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            carry_seats: false,
2831            pauses: Some((&self.pause, &self.interrupt)),
2832            run: &run_id,
2833            node: "advise",
2834            prompts: &prompts,
2835            cache: cache.as_deref(),
2836            round: None,
2837        };
2838        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839        let results = ask_json_wave::<Proposal>(
2840            jobs,
2841            Arc::clone(&self.sem),
2842            self.state.config.graph.retries,
2843            &advisor_roster,
2844            &ctx,
2845            &mut quota_losses,
2846            &mut self.state,
2847            &|p: &Proposal| p.validate(),
2848        )
2849        .await;
2850        if self.state.resource_block.is_some() {
2851            return self.stop_for_resource();
2852        }
2853        self.state.quota.extend(quota_losses);
2854
2855        let mut records = Vec::with_capacity(results.len());
2856        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857            let agent_id = seat.agent.clone();
2858            self.state.seats.insert(seat.key.clone(), seat);
2859            match res {
2860                Ok((proposal, out)) => {
2861                    self.state
2862                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2863                    records.push(advise::AdvisorRecord::proposed(
2864                        i + 1,
2865                        agent_id,
2866                        proposal,
2867                        out.duration_ms,
2868                    ));
2869                }
2870                Err(e) => {
2871                    self.state.event(
2872                        "advise",
2873                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874                    );
2875                    records.push(advise::AdvisorRecord::failed(
2876                        i + 1,
2877                        agent_id,
2878                        e.to_string(),
2879                    ));
2880                }
2881            }
2882        }
2883
2884        let mut advice = advise::Advice {
2885            records,
2886            synthesis: None,
2887        };
2888        if advice.proposals().is_empty() {
2889            self.state.event(
2890                "advise",
2891                "no advisor produced a usable proposal; continuing without a \
2892                 synthesis brief"
2893                    .to_owned(),
2894            );
2895        } else {
2896            match self
2897                .synthesize_brief(
2898                    &advice,
2899                    &instruction,
2900                    &language,
2901                    &worktrees[0],
2902                    &artifacts,
2903                    &run_id,
2904                    &prompts,
2905                    cache.as_deref(),
2906                )
2907                .await
2908            {
2909                Ok(Some(text)) => {
2910                    self.state.event(
2911                        "advise",
2912                        "synthesized a design brief for the implementer".to_owned(),
2913                    );
2914                    advice.synthesis = Some(text);
2915                }
2916                Ok(None) => {
2917                    self.state.event(
2918                        "advise",
2919                        "the synthesis seat produced nothing usable; continuing \
2920                         without a design brief"
2921                            .to_owned(),
2922                    );
2923                }
2924                Err(e) => {
2925                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2926                        return Err(e);
2927                    }
2928                    self.state.event(
2929                        "advise",
2930                        format!("could not synthesize a design brief: {e:#}"),
2931                    );
2932                }
2933            }
2934        }
2935        advise::apply_reflection(&mut advice);
2936
2937        self.state.advice = Some(advice);
2938        self.state.advise_attempted = true;
2939        self.state.save()?;
2940        Ok(())
2941    }
2942
2943    /// The synthesis seat: reads every advisor's proposal and blends them
2944    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2945    /// out of [`Runner::advise`] only for readability — it is not called
2946    /// anywhere else.
2947    ///
2948    /// Picked the same way [`crate::talk`]'s standing conversation and
2949    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2950    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2951    /// order (a claude seat, else the first runnable agent in roster order)
2952    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2953    /// in [`crate::config`] for why a dedicated field exists here at all.
2954    #[allow(clippy::too_many_arguments)]
2955    async fn synthesize_brief(
2956        &mut self,
2957        advice: &advise::Advice,
2958        instruction: &str,
2959        language: &str,
2960        cwd: &Path,
2961        artifacts: &Path,
2962        run_id: &str,
2963        prompts: &Prompts,
2964        cache: Option<&Path>,
2965    ) -> Result<Option<String>> {
2966        let chain = agent::pick_chain(
2967            &self.state.config.agents,
2968            self.state.config.roles.synthesizer.as_ref(),
2969            &agent::installed,
2970            "synthesizer",
2971        )?;
2972        let proposals = advice.proposals();
2973        let mut prompt = prompt::with_overlay(
2974            prompt::synthesize_brief(instruction, &proposals, language),
2975            prompts.overlay("advise"),
2976        );
2977        if cache.is_some() {
2978            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2979            // below — see `prompt::build_cache_note`'s doc for why telling a
2980            // read-only seat to build through the shared cache is exactly how
2981            // a sandbox's write refusal gets misread as a defect.
2982            prompt.push('\n');
2983            prompt.push_str(&prompt::build_cache_note("advise", false));
2984        }
2985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986        // Each id is tried once, in order; a quota hit, error or unusable
2987        // answer moves to the next. The seat is single-turn (`sessions:
2988        // false`) and the prompt is the whole context, so a fallback agent
2989        // needs nothing carried over.
2990        let mut last = None;
2991        for (n, spec) in chain.iter().enumerate() {
2992            if n > 0 {
2993                self.state
2994                    .event("advise", format!("synthesis falling back to {}", spec.id));
2995            }
2996            let mut seat = self.seat("advise-synthesis", &spec.id);
2997            let wait = self.repository_wait();
2998            let outcome = wait
2999                .scope(agent::invoke(
3000                    spec,
3001                    &mut seat,
3002                    &Invocation {
3003                        coordination: self.state.config.coordination.as_ref(),
3004                        fetched_base: None,
3005                        cwd,
3006                        prompt: &prompt,
3007                        timeout,
3008                        allow_write: false,
3009                        sessions: false,
3010                        artifacts,
3011                        stem: &if n == 0 {
3012                            "advise-synthesis".to_owned()
3013                        } else {
3014                            format!("advise-synthesis-{}", spec.id)
3015                        },
3016                        run: run_id,
3017                        node: "advise",
3018                        cache_dir: None,
3019                        attachments: &[],
3020                        writable: &[],
3021                    },
3022                ))
3023                .await;
3024            if outcome.is_ok() {
3025                self.state.seats.insert(seat.key.clone(), seat);
3026            }
3027            let advance = agent::chain_advances(&outcome);
3028            last = Some(outcome);
3029            if !advance {
3030                break;
3031            }
3032        }
3033        // Exhausted: the last attempt's result is what a single failed seat
3034        // would have produced.
3035        let out = last.expect("a chain holds at least one agent")?;
3036        if !out.usable() {
3037            return Ok(None);
3038        }
3039        let text =
3040            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041        Ok((!text.trim().is_empty()).then_some(text))
3042    }
3043
3044    // ----------------------------------------------------------- implement
3045
3046    async fn implement(&mut self) -> Result<()> {
3047        self.persist_candidates().await?;
3048        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3049        // agent files with `magi task add` name the run that paid for it. The
3050        // prompt overlay is cloned alongside it because the waves borrow it
3051        // while `self` is mutably borrowed by the node's own bookkeeping.
3052        let run_id = self.state.id.clone();
3053        let prompts = self.state.config.prompts.clone();
3054        let todo: Vec<usize> = self
3055            .state
3056            .candidates
3057            .iter()
3058            .enumerate()
3059            .filter(|(_, c)| {
3060                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061                    && c.failed.is_none()
3062                    && !c.empty
3063            })
3064            .map(|(i, _)| i)
3065            .collect();
3066        if todo.is_empty() {
3067            return self.after_implement();
3068        }
3069        self.state.status = RunStatus::Implementing;
3070
3071        let language = self.state.config.graph.language.clone();
3072        let github_language = self.state.config.graph.github_language.clone();
3073        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074        let sessions = self.state.config.graph.sessions;
3075        let artifacts = agent::artifacts_dir(&self.state.dir());
3076        // The design-deliberation stage's blended brief, when `advise` found
3077        // one — carried into every implementer's prompt the same way
3078        // regardless of which candidate it is.
3079        let brief = self
3080            .state
3081            .advice
3082            .as_ref()
3083            .and_then(|a| a.synthesis.as_deref())
3084            .map(str::to_owned);
3085        let attachments = self.state.attachments.clone();
3086
3087        let mut jobs = Vec::new();
3088        for &i in &todo {
3089            let (index, label, worktree) = {
3090                let c = &self.state.candidates[i];
3091                (c.index, c.label, c.worktree.clone())
3092            };
3093            let spec = self.roles.implementers[index].clone();
3094            let seat_key = format!("impl-{label}");
3095            let seat = self.seat(&seat_key, &spec.id);
3096            let instruction = seeded_instruction(&self.state);
3097            jobs.push(SeatJob {
3098                spec,
3099                seat,
3100                prompt: prompt::implement_in(
3101                    &instruction,
3102                    &worktree.to_string_lossy(),
3103                    &language,
3104                    &github_language,
3105                    brief.as_deref(),
3106                    &attachments,
3107                ),
3108                cwd: worktree,
3109                timeout,
3110                allow_write: true,
3111                sessions,
3112                artifacts: artifacts.clone(),
3113                stem: format!("impl-{label}"),
3114                handover: None,
3115            });
3116        }
3117
3118        self.state.event(
3119            "implement",
3120            format!("{} candidates in parallel", jobs.len()),
3121        );
3122        // Kept so a seat whose CLI hung up can be asked again from the same
3123        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3124        // can update a seat's own entry once a fallback agent takes it over —
3125        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3126        // whichever agent actually answered, not the one that quota'd out.
3127        let mut sent = jobs.clone();
3128        let cache = self.state.config.cache_dir();
3129        let ctx = WaveCtx {
3130            carry_seats: false,
3131            pauses: Some((&self.pause, &self.interrupt)),
3132            run: &run_id,
3133            node: "implement",
3134            prompts: &prompts,
3135            cache: cache.as_deref(),
3136            round: None,
3137        };
3138        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139        if !wave_blocked(&results) {
3140            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141                .await;
3142            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143                .await?;
3144            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145                .await;
3146        }
3147        // Do not finalize failed candidates when the entire wave needs machine
3148        // recovery: leaving them pending resumes this node with its work intact.
3149        if !results.is_empty()
3150            && results
3151                .iter()
3152                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153        {
3154            let losses: Vec<_> = results
3155                .iter()
3156                .map(|(_, seat, out)| {
3157                    let AgentOutcome::Quota(o) = out else {
3158                        unreachable!()
3159                    };
3160                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161                })
3162                .collect();
3163            self.state.quota.extend(losses.clone());
3164            for (_, seat, _) in results {
3165                self.state.seats.insert(seat.key.clone(), seat);
3166            }
3167            for &i in &todo {
3168                let label = self.state.candidates[i].label;
3169                self.state.interrupted_implementations.insert(label);
3170                self.state
3171                    .candidate_preservation
3172                    .insert(label, "machine outage awaiting persistence".into());
3173            }
3174            // Preserve partial edits too, while keeping every interrupted seat
3175            // pending even when its tree already contains an inherited patch.
3176            block_machine_outage(&mut self.state, "implement", &losses);
3177            self.state.save()?;
3178            self.persist_candidates().await?;
3179            for &i in &todo {
3180                self.state
3181                    .interrupted_implementations
3182                    .insert(self.state.candidates[i].label);
3183            }
3184            return self.stop_for_resource();
3185        }
3186        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189            && results
3190                .iter()
3191                .any(|(_, _, out)| FailClass::of(out).is_some());
3192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194                // Executed turns interrupted by disk pressure already carry
3195                // a preservation barrier from wave. Admission-only waits do
3196                // not. Keep the interrupted seat's session for recovery too.
3197                if self
3198                    .state
3199                    .candidate_preservation
3200                    .contains_key(&self.state.candidates[i].label)
3201                {
3202                    self.state.seats.insert(seat.key.clone(), seat);
3203                }
3204                continue;
3205            }
3206            let seat_key = seat.key.clone();
3207            // A quota fallback (`resume_seat_handovers`) may have handed this
3208            // seat to a different agent than the one `prep` recorded on the
3209            // candidate; the stats tables and any later fixer-defaults-to-
3210            // winner's-author lookup must credit whoever actually answered —
3211            // unless every fallback also quota'd out, in which case nobody
3212            // actually answered and crediting the last agent tried would
3213            // erase every earlier agent's own quota loss from the stats
3214            // tables instead of just this one seat's.
3215            let agent = seat.agent.clone();
3216            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217            self.state.seats.insert(seat.key.clone(), seat);
3218            let label = self.state.candidates[i].label;
3219            self.state.interrupted_implementations.remove(&label);
3220
3221            let (summary, duration, failed, verified_claim) = match out {
3222                AgentOutcome::Ok(o) => {
3223                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224                    let failed = (!o.usable()).then(|| {
3225                        if o.timed_out {
3226                            "agent timed out".to_owned()
3227                        } else {
3228                            format!(
3229                                "agent exited with {}",
3230                                crate::notices::exit_status(o.exit_code)
3231                            )
3232                        }
3233                    });
3234                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235                    (text, o.duration_ms, failed, verified_claim)
3236                }
3237                // Left un-resumed by `resume_undelivered` (a dirty tree
3238                // already rescues the work, or there was no session left to
3239                // resume into) — reported like the ordinary failure it is,
3240                // never as if `o.text` (the CLI's raw error JSON) were an
3241                // answer.
3242                AgentOutcome::Dropped(o) => {
3243                    let why = o
3244                        .dropped
3245                        .as_ref()
3246                        .map(|d| d.why.as_str())
3247                        .unwrap_or("the CLI ended the stream without delivering its answer");
3248                    (
3249                        String::new(),
3250                        o.duration_ms,
3251                        Some(format!("the CLI dropped the stream ({why})")),
3252                        None,
3253                    )
3254                }
3255                AgentOutcome::Quota(o) => {
3256                    self.state.quota.push(QuotaLoss::from_report(
3257                        seat_key,
3258                        "implement",
3259                        o.quota.as_ref(),
3260                    ));
3261                    (
3262                        String::new(),
3263                        o.duration_ms,
3264                        Some("rate limited (quota); produced no change".to_owned()),
3265                        None,
3266                    )
3267                }
3268                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269                    (String::new(), 0, Some(e), None)
3270                }
3271            };
3272
3273            // Save every completed turn before any fallible Git operation. A failed
3274            // rescue must not erase the reply or prevent collection of other seats.
3275            let c = &mut self.state.candidates[i];
3276            if !exhausted_the_fallback_chain {
3277                c.agent = agent;
3278            }
3279            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280            c.duration_ms = duration;
3281            c.failed = failed;
3282            c.verified_noop = verified_claim;
3283            self.state
3284                .candidate_preservation
3285                .insert(label, "awaiting persistence".to_owned());
3286        }
3287        self.state.save()?;
3288        self.persist_candidates().await?;
3289
3290        if resource_blocked {
3291            return self.stop_for_resource();
3292        }
3293        if selected_failed {
3294            self.state.status = RunStatus::Blocked;
3295            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296            self.state.save()?;
3297            bail!(
3298                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299            );
3300        }
3301        self.after_implement()
3302    }
3303
3304    /// Retry only persistence after a completed implementation turn.
3305    async fn persist_candidates(&mut self) -> Result<()> {
3306        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307        for label in labels {
3308            let i = self
3309                .state
3310                .candidates
3311                .iter()
3312                .position(|c| c.label == label)
3313                .context("pending candidate is missing")?;
3314            let c = self.state.candidates[i].clone();
3315            let interrupted = self.state.interrupted_implementations.contains(&label);
3316            let result = async {
3317                let wait = crate::repo_lock::WaitPolicy::new(
3318                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319                    || false,
3320                );
3321                let _coordination =
3322                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323                // Best effort evidence requires no index writes. A successful snapshot
3324                // never authorizes removal of a tree whose rescue failed.
3325                if let Err(e) = git::snapshot_uncommitted(
3326                    &c.worktree,
3327                    &self
3328                        .state
3329                        .dir()
3330                        .join("artifacts")
3331                        .join(format!("candidate-{label}-recovery")),
3332                )
3333                .await
3334                {
3335                    self.state.event(
3336                        "preserve",
3337                        format!("candidate {label} snapshot incomplete: {e:#}"),
3338                    );
3339                }
3340                let rescued = git::rescue_commit(
3341                    &c.worktree,
3342                    &format!("magi: candidate {label} (uncommitted work)"),
3343                )
3344                .await?;
3345                self.state.note_withheld("implement", &rescued.withheld);
3346                let base = &self.state.base_commit;
3347                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352                if c.empty && commits > 0 && !patch.trim().is_empty()
3353                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355                        || self.state.tally.is_some())
3356                {
3357                    // Keep the old classification and pending marker so every
3358                    // retry stops too. Retain decisions as recovery evidence;
3359                    // only a supported recovery operation may replace them.
3360                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361                }
3362                let c = &mut self.state.candidates[i];
3363                c.commits = commits;
3364                c.stat = stat;
3365                c.files = files;
3366                let no_changes = commits == 0 || patch.trim().is_empty();
3367                c.empty = no_changes && !interrupted;
3368                if interrupted && no_changes {
3369                    c.failed = None;
3370                    c.verified_noop = None;
3371                }
3372                if !c.empty {
3373                    c.failed = None;
3374                    c.verified_noop = None;
3375                }
3376                self.state.event(
3377                    "preserve",
3378                    format!("candidate {label}: {files} files, {commits} commits"),
3379                );
3380                Ok::<_, anyhow::Error>(())
3381            }
3382            .await;
3383            match result {
3384                Ok(()) => {
3385                    self.state.candidate_preservation.remove(&label);
3386                    if !interrupted || self.state.candidates[i].files > 0 {
3387                        self.state.interrupted_implementations.remove(&label);
3388                    }
3389                }
3390                Err(e) => {
3391                    let reason = format!("candidate {label} preservation failed: {e:#}");
3392                    self.state.event("preserve", &reason);
3393                    self.state.candidate_preservation.insert(label, reason);
3394                }
3395            }
3396            if self.state.candidate_preservation.is_empty()
3397                && self.state.status == RunStatus::Blocked
3398                && self.state.candidates.iter().all(|c| c.failed.is_none())
3399            {
3400                // Persist the recovered phase with the cleared pending marker;
3401                // a crash between saves must not turn this into a selected-seat
3402                // failure hold on the next resume.
3403                self.state.status = RunStatus::Implementing;
3404                self.state.event(
3405                    "preserve",
3406                    "candidate persistence recovered; implementation can continue",
3407                );
3408            }
3409            self.state.save()?;
3410        }
3411        if !self.state.candidate_preservation.is_empty() {
3412            self.state.status = RunStatus::Blocked;
3413            self.state.save()?;
3414            bail!(
3415                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416            );
3417        }
3418        Ok(())
3419    }
3420
3421    /// Ask again, once, for work a CLI did and then failed to hand over.
3422    ///
3423    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3424    /// status with an empty response and a usage report showing output tokens,
3425    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3426    /// seven minutes and 14,267 output tokens that arrived as an empty
3427    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3428    ///
3429    /// Two conditions, and both matter:
3430    ///
3431    /// - **Only when the tree is untouched.** Often the agent has already
3432    ///   written its files and only the closing message was lost; the rescue
3433    ///   commit below picks that up and there is nothing to ask for. Re-asking
3434    ///   then would pay for a second implementation of work already on disk.
3435    /// - **Once.** A CLI that drops one stream can drop the next, and this
3436    ///   node is the most expensive in the graph.
3437    ///
3438    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3439    /// dropped reply still carried its `conversation_id`, so the seat is asked
3440    /// to finish what it was doing rather than sent the whole task again. It
3441    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3442    /// node's - for the same reason a re-ranked judge does: restating finished
3443    /// work is not the work.
3444    ///
3445    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3446    /// same way until it resets, while an abandoned conversation is still
3447    /// there to be picked up.
3448    async fn resume_undelivered(
3449        &mut self,
3450        results: &mut [(usize, SeatState, AgentOutcome)],
3451        sent: &[SeatJob],
3452        prompts: &Prompts,
3453        run_id: &str,
3454    ) {
3455        for (wi, seat, out) in results.iter_mut() {
3456            let Some(dropped) = (match &*out {
3457                AgentOutcome::Dropped(o) => o.dropped.clone(),
3458                _ => None,
3459            }) else {
3460                continue;
3461            };
3462            let Some(job) = sent.get(*wi) else { continue };
3463            // Already on disk? Then only the closing message was lost.
3464            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465                self.state.event(
3466                    "implement",
3467                    format!(
3468                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469                         work is in the tree",
3470                        seat.key, dropped.output_tokens, dropped.why
3471                    ),
3472                );
3473                continue;
3474            }
3475            // The re-ask only makes sense as a resume: `resume_after_drop`
3476            // says nothing about the task, trusting the seat to still hold it.
3477            // Without a session to resume — sessions disabled, or this CLI's
3478            // drop shape happened not to carry a session id — that prompt
3479            // would open a brand-new conversation with no context at all,
3480            // which is worse than leaving this as the ordinary failure it
3481            // already is.
3482            if !has_context(&job.spec, seat, job.sessions) {
3483                self.state.event(
3484                    "implement",
3485                    format!(
3486                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487                         is no session left to resume",
3488                        seat.key, dropped.output_tokens, dropped.why
3489                    ),
3490                );
3491                continue;
3492            }
3493            self.state.event(
3494                "implement",
3495                format!(
3496                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497                     conversation",
3498                    seat.key, dropped.output_tokens, dropped.why
3499                ),
3500            );
3501            let mut retry = job.clone();
3502            retry.seat = seat.clone();
3503            retry.prompt = prompt::resume_after_drop(&dropped.why);
3504            retry.timeout = retry_budget(job.timeout, true);
3505            retry.stem = format!("{}-resume", job.stem);
3506            let cache = self.state.config.cache_dir();
3507            let ctx = WaveCtx {
3508                carry_seats: false,
3509                pauses: Some((&self.pause, &self.interrupt)),
3510                run: run_id,
3511                node: "implement",
3512                prompts,
3513                cache: cache.as_deref(),
3514                round: None,
3515            };
3516            let (resumed_seat, resumed) =
3517                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518            *seat = resumed_seat;
3519            *out = resumed;
3520        }
3521    }
3522
3523    /// Fall an implement seat through to the next untried agent in the
3524    /// implementer roster when it lost to quota — or, since the handover was
3525    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3526    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3527    /// quota path itself being unchanged — instead of leaving the
3528    /// seat's loss final the moment one agent's account runs dry.
3529    ///
3530    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3531    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3532    /// truncates to the single slot rotation picked, so a solo task whose one
3533    /// implementer hits quota mid-run used to have nothing else to try. This
3534    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3535    /// unrotated roster — which is the only place the *other* candidates in
3536    /// the machine's roster still exist once `implementers` has been cut down
3537    /// to size.
3538    ///
3539    /// Walks forward from just past the seat's own original position in the
3540    /// roster, never wrapping back to the front: a later candidate slot (say
3541    /// `beta`, the roster's second entry) must fall through to the *next*
3542    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3543    /// almost certainly a different candidate's own agent already — and once
3544    /// the roster's tail is exhausted there is nothing left to fall through
3545    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3546    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3547    /// let this retry that id forever. The loop keeps falling through until
3548    /// an attempt lands something other than `Quota` or the roster's tail
3549    /// runs out of untried ids, at which point the seat is left exactly as
3550    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3551    /// `QuotaLoss` recorded, the candidate failed/empty.
3552    ///
3553    /// `sent` is taken mutably and updated with the fallback agent's spec:
3554    /// `resume_unconfirmed_commands`, which runs after this and also reads
3555    /// `sent`, must see whichever agent actually ended up answering the seat
3556    /// — reading the stale, original spec there would check session
3557    /// eligibility against the wrong CLI and could hand a fallback agent's
3558    /// session id to the agent that just lost the seat to quota.
3559    ///
3560    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3561    /// — `self.seat` only reuses state when the agent id is unchanged, so
3562    /// handing it a different id already gets this for free. Reusing the old
3563    /// seat would resume a different CLI's session as if it were a
3564    /// continuation of this one.
3565    ///
3566    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3567    /// a quota loss cuts an agent off mid-turn, so anything already in the
3568    /// tree is unfinished work, not a completed candidate a re-ask would pay
3569    /// for twice. A dirty tree is rescued into a commit first (the same
3570    /// neutral-identity rescue `implement`'s own outcome loop gives every
3571    /// candidate) so the next agent starts clean.
3572    ///
3573    /// The new agent gets the implementer's full prompt and full
3574    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3575    /// it has no session and no context, and is implementing the task from
3576    /// nothing, unlike a resumed drop which is only restating work already
3577    /// done.
3578    ///
3579    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3580    /// `implement` event, never into `self.state.quota` — that is what
3581    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3582    /// go unspent, and a seat that ultimately recovered on its second or
3583    /// third agent is not the stalled panel that check exists to catch. Only
3584    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3585    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3586    /// outcome loop already has — this helper never pushes to it itself.
3587    async fn resume_seat_handovers(
3588        &mut self,
3589        results: &mut [(usize, SeatState, AgentOutcome)],
3590        sent: &mut [SeatJob],
3591        prompts: &Prompts,
3592        run_id: &str,
3593    ) -> Result<()> {
3594        let instruction = seeded_instruction(&self.state);
3595        let language = self.state.config.graph.language.clone();
3596        let github_language = self.state.config.graph.github_language.clone();
3597        let brief = self
3598            .state
3599            .advice
3600            .as_ref()
3601            .and_then(|a| a.synthesis.as_deref())
3602            .map(str::to_owned);
3603        let attachments = self.state.attachments.clone();
3604        for (wi, seat, out) in results.iter_mut() {
3605            // Who holds the other candidate seats of this wave right now
3606            // (earlier handovers already written back to `sent`).
3607            let others: BTreeSet<String> = sent
3608                .iter()
3609                .enumerate()
3610                .filter(|(j, _)| j != wi)
3611                .map(|(_, j)| j.spec.id.clone())
3612                .collect();
3613            let Some(job) = sent.get_mut(*wi) else {
3614                continue;
3615            };
3616            // Where the seat's own original agent sits in the roster — the
3617            // fallback walk starts just past here, never at the front, so a
3618            // later candidate slot's quota loss does not fall back onto an
3619            // earlier slot's own agent.
3620            let start = self
3621                .roles
3622                .implementer_roster
3623                .iter()
3624                .position(|s| s.id == job.spec.id)
3625                .unwrap_or(0);
3626            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627            let mut fallback_attempt = 0usize;
3628            let mut prev: Option<FailClass> = None;
3629            while let Some(cur) = FailClass::of(&*out) {
3630                if !should_hand_over(prev.as_ref(), &cur) {
3631                    break;
3632                }
3633                let Some(next) =
3634                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635                        .cloned()
3636                else {
3637                    break;
3638                };
3639                tried.insert(next.id.clone());
3640                fallback_attempt += 1;
3641
3642                let coordination = self.coordinate(&job.cwd).await?;
3643                if let Ok(r) = git::rescue_commit(
3644                    &job.cwd,
3645                    &format!(
3646                        "magi: candidate {} (uncommitted work before {} fallback)",
3647                        seat.key,
3648                        if cur == FailClass::Quota {
3649                            "quota"
3650                        } else {
3651                            "handover"
3652                        }
3653                    ),
3654                )
3655                .await
3656                {
3657                    self.state.note_withheld("implement", &r.withheld);
3658                }
3659                drop(coordination);
3660
3661                record_handover(
3662                    &mut self.state,
3663                    "implement",
3664                    &seat.key,
3665                    &seat.agent,
3666                    &next.id,
3667                    &cur,
3668                    &fail_reason(&*out),
3669                );
3670                prev = Some(cur.clone());
3671
3672                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674                // Kept in sync on `sent` itself, not just the local retry: a
3675                // later helper (`resume_unconfirmed_commands`) reads `sent`
3676                // after this one returns and must see whichever agent is now
3677                // occupying the seat, not the one that just quota'd out —
3678                // otherwise it would judge session/continuation eligibility
3679                // by the wrong CLI and could resend a fallback's session id
3680                // to the agent that lost it the seat in the first place.
3681                job.spec = next.clone();
3682                let mut retry = job.clone();
3683                retry.seat = new_seat;
3684                retry.prompt = prompt::implement_in(
3685                    &instruction,
3686                    &job.cwd.to_string_lossy(),
3687                    &language,
3688                    &github_language,
3689                    brief.as_deref(),
3690                    &attachments,
3691                );
3692                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693                let cache = self.state.config.cache_dir();
3694                let ctx = WaveCtx {
3695                    carry_seats: false,
3696                    pauses: Some((&self.pause, &self.interrupt)),
3697                    run: run_id,
3698                    node: "implement",
3699                    prompts,
3700                    cache: cache.as_deref(),
3701                    round: None,
3702                };
3703                let (fallback_seat, fallback_out) = run_one(
3704                    retry,
3705                    Arc::clone(&self.sem),
3706                    &ctx,
3707                    &mut self.state,
3708                    fallback_attempt,
3709                )
3710                .await;
3711                *seat = fallback_seat;
3712                *out = fallback_out;
3713            }
3714        }
3715        Ok(())
3716    }
3717
3718    /// Ask an implement seat's own CLI to confirm what it started, once, when
3719    /// its reply reported a command whose completion status it never
3720    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3721    /// that does and does not mean.
3722    ///
3723    /// The completion contract this task asks for, extended to `implement`
3724    /// with the same signal `continue_fix_report` reads for the fixer,
3725    /// rather than a keyword search over the reply or a hard requirement on
3726    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3727    /// a candidate's CLI turn ended cleanly while a test run it had started
3728    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3729    /// named in it at all is untouched by this: `commands` is empty, so
3730    /// there is nothing to be unconfirmed.
3731    ///
3732    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3733    /// this is not about recovering edits that might already be on disk, it
3734    /// is about a result the seat itself never vouched for, which resuming
3735    /// asks for regardless of what the tree already holds. Bounded to one
3736    /// attempt for the same reason `resume_undelivered` is — this is the
3737    /// most expensive node in the graph — and a seat that still cannot
3738    /// confirm on that attempt is left as whatever its (possibly still
3739    /// unconfirmed) reply says; this does not invent a new "failed" reason
3740    /// for a candidate that otherwise produced a real, committed change.
3741    async fn resume_unconfirmed_commands(
3742        &mut self,
3743        results: &mut [(usize, SeatState, AgentOutcome)],
3744        sent: &[SeatJob],
3745        prompts: &Prompts,
3746        run_id: &str,
3747    ) {
3748        for (wi, seat, out) in results.iter_mut() {
3749            let AgentOutcome::Ok(o) = &*out else {
3750                continue;
3751            };
3752            if !has_unconfirmed_command(&o.commands) {
3753                continue;
3754            }
3755            let Some(job) = sent.get(*wi) else { continue };
3756            if !has_context(&job.spec, seat, job.sessions) {
3757                self.state.event(
3758                    "implement",
3759                    format!(
3760                        "{}: the reply named a command whose own CLI never confirmed the exit \
3761                         status of, but there is no session left to resume",
3762                        seat.key
3763                    ),
3764                );
3765                continue;
3766            }
3767            self.state.event(
3768                "implement",
3769                format!(
3770                    "{}: the reply named a command whose own CLI never confirmed the exit \
3771                     status of; resuming the conversation",
3772                    seat.key
3773                ),
3774            );
3775            let mut retry = job.clone();
3776            retry.seat = seat.clone();
3777            retry.prompt = prompt::resume_incomplete(
3778                "a command in your last reply had no confirmed exit status",
3779            );
3780            retry.timeout = retry_budget(job.timeout, true);
3781            retry.stem = format!("{}-confirm", job.stem);
3782            let cache = self.state.config.cache_dir();
3783            let ctx = WaveCtx {
3784                carry_seats: false,
3785                pauses: Some((&self.pause, &self.interrupt)),
3786                run: run_id,
3787                node: "implement",
3788                prompts,
3789                cache: cache.as_deref(),
3790                round: None,
3791            };
3792            let (resumed_seat, resumed) =
3793                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794            *seat = resumed_seat;
3795            *out = resumed;
3796        }
3797    }
3798
3799    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3800    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3801    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3802    /// that motivated this.
3803    ///
3804    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3805    /// own nudge loop already covers for judge/review/vote seats, and not a
3806    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3807    /// implement seats: here the CLI turn genuinely finished while the node's
3808    /// own work — the fixer's account of what it did — had not. Gated purely
3809    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3810    /// reply, never on any wording in it, so a fixer whose valid, first-try
3811    /// `FixReport` happens to mention having waited on a background test is
3812    /// never resumed — the `Ok(report)` branch at the call site returns
3813    /// before this is ever invoked.
3814    ///
3815    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3816    /// attempt ([`retry_budget`]), nothing attempted once the session is
3817    /// gone, and a quota hit ends the loop immediately rather than retrying a
3818    /// rate limit that fails the same way again.
3819    async fn continue_fix_report(
3820        &mut self,
3821        mut seat: SeatState,
3822        parse_err: String,
3823        job: &SeatJob,
3824        prompts: &Prompts,
3825        run_id: &str,
3826        round: usize,
3827    ) -> (
3828        SeatState,
3829        Option<FixReport>,
3830        Option<String>,
3831        ContinuationRecord,
3832    ) {
3833        let mut last_err = parse_err;
3834        let mut cumulative_wait_ms = 0u64;
3835        let mut attempts = 0usize;
3836        loop {
3837            if !has_context(&job.spec, &seat, job.sessions) {
3838                self.state.event(
3839                    "fix",
3840                    format!(
3841                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842                         session left to resume into"
3843                    ),
3844                );
3845                let outcome = if attempts == 0 {
3846                    ContinuationOutcome::NoSession
3847                } else {
3848                    ContinuationOutcome::Exhausted
3849                };
3850                return (
3851                    seat,
3852                    None,
3853                    Some(format!("unparsable fix report: {last_err}")),
3854                    ContinuationRecord {
3855                        attempts,
3856                        cumulative_wait_ms,
3857                        outcome,
3858                    },
3859                );
3860            }
3861            if attempts >= MAX_FIX_CONTINUATIONS {
3862                self.state.event(
3863                    "fix",
3864                    format!(
3865                        "round {round}: fixer's reply still had no adoption report after \
3866                         {attempts} continuation(s) ({last_err}); giving up"
3867                    ),
3868                );
3869                return (
3870                    seat,
3871                    None,
3872                    Some(format!(
3873                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3874                    )),
3875                    ContinuationRecord {
3876                        attempts,
3877                        cumulative_wait_ms,
3878                        outcome: ContinuationOutcome::Exhausted,
3879                    },
3880                );
3881            }
3882            attempts += 1;
3883            self.state.event(
3884                "fix",
3885                format!(
3886                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888                ),
3889            );
3890            let mut retry = job.clone();
3891            retry.seat = seat.clone();
3892            retry.prompt = prompt::resume_incomplete(&last_err);
3893            retry.timeout = retry_budget(job.timeout, true);
3894            retry.stem = format!("{}-continue{attempts}", job.stem);
3895            let cache = self.state.config.cache_dir();
3896            let ctx = WaveCtx {
3897                carry_seats: false,
3898                pauses: Some((&self.pause, &self.interrupt)),
3899                run: run_id,
3900                node: "fix",
3901                prompts,
3902                cache: cache.as_deref(),
3903                round: Some(round),
3904            };
3905            let (resumed_seat, resumed_out) = run_one(
3906                retry,
3907                Arc::clone(&self.sem),
3908                &ctx,
3909                &mut self.state,
3910                attempts,
3911            )
3912            .await;
3913            seat = resumed_seat;
3914            match resumed_out {
3915                AgentOutcome::Ok(o) => {
3916                    cumulative_wait_ms += o.duration_ms;
3917                    match verdict::extract_json::<FixReport>(&o.text) {
3918                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919                            self.state.event(
3920                                "fix",
3921                                format!(
3922                                    "round {round}: fixer's adoption report recovered after \
3923                                     {attempts} continuation(s)"
3924                                ),
3925                            );
3926                            return (
3927                                seat,
3928                                Some(report),
3929                                None,
3930                                ContinuationRecord {
3931                                    attempts,
3932                                    cumulative_wait_ms,
3933                                    outcome: ContinuationOutcome::Resumed,
3934                                },
3935                            );
3936                        }
3937                        // The report parsed, but this same reply's own
3938                        // CommandEvidence — the identical record `state.jobs`
3939                        // renders — names a command whose CLI never
3940                        // confirmed an exit status. Read together, that is
3941                        // not a resolved answer: keep nudging rather than
3942                        // accept a report standing next to a command the
3943                        // seat's own CLI cannot vouch for.
3944                        Ok(_) => {
3945                            last_err = "the reply parsed, but it reported a command whose own CLI \
3946                                 never confirmed an exit status"
3947                                .to_owned();
3948                        }
3949                        Err(e) => last_err = e.to_string(),
3950                    }
3951                }
3952                AgentOutcome::Quota(o) => {
3953                    cumulative_wait_ms += o.duration_ms;
3954                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956                    self.state.quota.push(loss);
3957                    self.state.event(
3958                        "fix",
3959                        format!(
3960                            "round {round}: continuation rate limited (quota); not retrying now"
3961                        ),
3962                    );
3963                    return (
3964                        seat,
3965                        None,
3966                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3967                        ContinuationRecord {
3968                            attempts,
3969                            cumulative_wait_ms,
3970                            outcome: ContinuationOutcome::QuotaLost,
3971                        },
3972                    );
3973                }
3974                AgentOutcome::Dropped(o) => {
3975                    cumulative_wait_ms += o.duration_ms;
3976                    let why = o
3977                        .dropped
3978                        .as_ref()
3979                        .map(|d| d.why.as_str())
3980                        .unwrap_or("the CLI ended the stream without delivering its answer");
3981                    last_err = format!("the CLI dropped the stream ({why})");
3982                }
3983                AgentOutcome::Failed(e) => last_err = e,
3984                // The continuation was never started; the report stays lost
3985                // and the caller records that, without any handover.
3986                AgentOutcome::ResourceBlocked(e) => {
3987                    return (
3988                        seat,
3989                        None,
3990                        Some(format!("the continuation was not started: {e}")),
3991                        ContinuationRecord {
3992                            attempts: attempts.saturating_sub(1),
3993                            cumulative_wait_ms,
3994                            outcome: ContinuationOutcome::NoSession,
3995                        },
3996                    );
3997                }
3998            }
3999        }
4000    }
4001
4002    fn after_implement(&mut self) -> Result<()> {
4003        if !self.state.candidate_preservation.is_empty() {
4004            bail!("candidate preservation incomplete; judging is unsafe");
4005        }
4006        // Scan every candidate patch once the set is complete.
4007        if self.state.leaks.is_empty() {
4008            let cfg = self.state.config.blind.clone();
4009            let mut leaks = Vec::new();
4010            for c in &self.state.candidates {
4011                let Some(patch) =
4012                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013                else {
4014                    continue;
4015                };
4016                leaks.extend(blind::scan(
4017                    &format!("candidate {} patch", c.label),
4018                    &patch,
4019                    &cfg.vendor_tokens,
4020                ));
4021            }
4022            if !leaks.is_empty() {
4023                let summary = leaks
4024                    .iter()
4025                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026                    .collect::<Vec<_>>()
4027                    .join(", ");
4028                match cfg.on_leak {
4029                    LeakPolicy::Fail => {
4030                        self.state.status = RunStatus::Failed;
4031                        self.state
4032                            .event("blind", format!("vendor text in a patch: {summary}"));
4033                        self.state.leaks = leaks;
4034                        self.state.save()?;
4035                        self.settle_questions();
4036                        bail!(
4037                            "blind.on_leak = \"fail\" and vendor text reached a \
4038                             judged patch: {summary}"
4039                        );
4040                    }
4041                    LeakPolicy::Redact => self.state.event(
4042                        "blind",
4043                        format!("redacting vendor text for judging: {summary}"),
4044                    ),
4045                    LeakPolicy::Warn => self.state.event(
4046                        "blind",
4047                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048                    ),
4049                }
4050                self.state.leaks = leaks;
4051            }
4052        }
4053
4054        if self.state.viable().is_empty() {
4055            if self.state.all_candidates_verified_noop() {
4056                // Every candidate agreed, with evidence the adoption guard
4057                // accepted, that nothing belongs in this worktree. That is
4058                // not the same fact as a candidate that simply failed to
4059                // write anything, and settling it as an ordinary `Failed`
4060                // (see `SCHEMA`'s doc for schema 10) is what let two of
4061                // task 391f's attempts burn a retry each re-discovering the
4062                // same already-landed fix. Terminal either way, so `judge`
4063                // must never run over an empty candidate set — unlike the
4064                // `Failed` branch below this returns `Ok`, not an error:
4065                // nothing here failed.
4066                self.state.status = RunStatus::VerifiedNoop;
4067                self.state.save()?;
4068                self.settle_questions();
4069                return Ok(());
4070            }
4071            self.state.status = RunStatus::Failed;
4072            self.state.save()?;
4073            self.settle_questions();
4074            bail!("no candidate produced a change; nothing to judge");
4075        }
4076        self.state.status = RunStatus::Judging;
4077        self.state.save()?;
4078        Ok(())
4079    }
4080
4081    // --------------------------------------------------------------- judge
4082
4083    async fn judge(&mut self) -> Result<()> {
4084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4085        // agent files with `magi task add` name the run that paid for it. The
4086        // prompt overlay is cloned alongside it because the waves borrow it
4087        // while `self` is mutably borrowed by the node's own bookkeeping.
4088        let run_id = self.state.id.clone();
4089        let prompts = self.state.config.prompts.clone();
4090        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091            return Ok(());
4092        }
4093        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094        if viable.len() == 1 {
4095            // Recorded so this is a one-time event: `judgements` stays empty
4096            // either way, which without this flag is indistinguishable from
4097            // "not yet judged" on the next reentry — and status is left
4098            // untouched, so a later node's conclusion (e.g. `Blocked` after
4099            // the review budget ran out) survives a resume instead of being
4100            // clobbered back to `Judging` by this node running again.
4101            self.state.judge_skipped = true;
4102            self.state.event(
4103                "judge",
4104                format!(
4105                    "only candidate {} produced a change; judging skipped",
4106                    viable[0].label
4107                ),
4108            );
4109            self.state.save()?;
4110            return Ok(());
4111        }
4112        self.state.status = RunStatus::Judging;
4113
4114        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115        let language = self.state.config.graph.language.clone();
4116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117        let sessions = self.state.config.graph.sessions;
4118        let artifacts = agent::artifacts_dir(&self.state.dir());
4119        let root = self.state.worktree_root();
4120        let base_short = short(&self.state.base_commit);
4121
4122        let mut jobs = Vec::new();
4123        let mut orders = Vec::new();
4124        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128            let seat_key = format!("judge-{}", j + 1);
4129            let seat = self.seat(&seat_key, &spec.id);
4130            jobs.push(SeatJob {
4131                prompt: prompt::judge(
4132                    &self.state.instruction,
4133                    &views,
4134                    self.roles.judges.len(),
4135                    &base_short,
4136                    &language,
4137                ),
4138                spec,
4139                seat,
4140                cwd: root.join(format!("judge-{}", j + 1)),
4141                timeout,
4142                allow_write: false,
4143                sessions,
4144                artifacts: artifacts.clone(),
4145                stem: format!("judge-{}", j + 1),
4146                handover: None,
4147            });
4148        }
4149
4150        self.state.event(
4151            "judge",
4152            format!(
4153                "{} judges ranking {} candidates blind",
4154                jobs.len(),
4155                viable.len()
4156            ),
4157        );
4158        let labels_for_check = labels.clone();
4159        let mut quota_losses = Vec::new();
4160        let cache = self.state.config.cache_dir();
4161        let ctx = WaveCtx {
4162            carry_seats: false,
4163            pauses: Some((&self.pause, &self.interrupt)),
4164            run: &run_id,
4165            node: "judge",
4166            prompts: &prompts,
4167            cache: cache.as_deref(),
4168            round: None,
4169        };
4170        let results = ask_json_wave::<Ranking>(
4171            jobs,
4172            Arc::clone(&self.sem),
4173            self.state.config.graph.retries,
4174            &self.roles.judge_roster,
4175            &ctx,
4176            &mut quota_losses,
4177            &mut self.state,
4178            &move |r: &Ranking| r.validate(&labels_for_check),
4179        )
4180        .await;
4181        if self.state.resource_block.is_some() {
4182            return self.stop_for_resource();
4183        }
4184        self.state.quota.extend(quota_losses);
4185
4186        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187            let agent_id = seat.agent.clone();
4188            self.state.seats.insert(seat.key.clone(), seat);
4189            let mut record = Judgement {
4190                judge: j + 1,
4191                seat: format!("judge-{}", j + 1),
4192                agent: agent_id,
4193                ranking: Vec::new(),
4194                reasons: BTreeMap::new(),
4195                confidence: None,
4196                strengths: BTreeMap::new(),
4197                order: orders[j].clone(),
4198                failed: None,
4199                duration_ms: 0,
4200            };
4201            match res {
4202                Ok((ranking, out)) => {
4203                    record.ranking = ranking.normalized();
4204                    record.reasons = ranking.reasons;
4205                    record.strengths = ranking.strengths;
4206                    record.confidence = ranking.confidence;
4207                    record.duration_ms = out.duration_ms;
4208                    self.state.event(
4209                        "judge",
4210                        format!(
4211                            "judge {} ranked {}",
4212                            j + 1,
4213                            record.ranking.iter().collect::<String>()
4214                        ),
4215                    );
4216                }
4217                Err(e) => {
4218                    record.failed = Some(e.to_string());
4219                    self.state
4220                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221                }
4222            }
4223            self.state.judgements.push(record);
4224            self.state.save()?;
4225        }
4226        Ok(())
4227    }
4228
4229    // ---------------------------------------------------------- deliberate
4230
4231    async fn deliberate(&mut self) -> Result<()> {
4232        if self.hold_base_sync().await? {
4233            return Ok(());
4234        }
4235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4236        // agent files with `magi task add` name the run that paid for it. The
4237        // prompt overlay is cloned alongside it because the waves borrow it
4238        // while `self` is mutably borrowed by the node's own bookkeeping.
4239        let run_id = self.state.id.clone();
4240        let prompts = self.state.config.prompts.clone();
4241        let rounds = self.state.config.graph.deliberate_rounds;
4242        // Resume unfinished rounds after an outage, but never reopen a tally.
4243        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244            return Ok(());
4245        }
4246        let tops: Vec<char> = self
4247            .state
4248            .judgements
4249            .iter()
4250            .filter_map(|j| j.ranking.first().copied())
4251            .collect();
4252        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254                self.state.event(
4255                    "deliberate",
4256                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4257                );
4258            }
4259            self.state.status = RunStatus::Voting;
4260            self.state.save()?;
4261            return Ok(());
4262        }
4263
4264        self.state.status = RunStatus::Deliberating;
4265        self.state.event(
4266            "deliberate",
4267            format!(
4268                "split: first choices were {} — opening {rounds} round(s)",
4269                tops.iter().collect::<String>()
4270            ),
4271        );
4272
4273        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274        let language = self.state.config.graph.language.clone();
4275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276        let sessions = self.state.config.graph.sessions;
4277        let artifacts = agent::artifacts_dir(&self.state.dir());
4278        let root = self.state.worktree_root();
4279        let base_short = short(&self.state.base_commit);
4280
4281        // Judges argue in sequence so that a turn can answer the one before it;
4282        // that is the difference between deliberation and three parallel
4283        // monologues.
4284        for round in self.state.deliberation.len() + 1..=rounds {
4285            let mut turns: Vec<DeliberationTurn> = Vec::new();
4286            let mut round_losses = Vec::new();
4287            let mut asked = 0;
4288            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289                if self.state.judgements[j].failed.is_some() {
4290                    continue;
4291                }
4292                let seat_key = format!("judge-{}", j + 1);
4293                let spec = self.occupant(&seat_key, spec);
4294                let mut seat = self.seat(&seat_key, &spec.id);
4295                let transcript = self.transcript(&turns, j);
4296                let build = |context: Option<&str>| {
4297                    prompt::deliberate(
4298                        &self.state.instruction,
4299                        context,
4300                        &transcript,
4301                        round,
4302                        rounds,
4303                        &language,
4304                    )
4305                };
4306                let block = self.candidate_block(&viable, &base_short);
4307                let full = build(Some(&block));
4308                let text = if has_context(&spec, &seat, sessions) {
4309                    build(None)
4310                } else {
4311                    full.clone()
4312                };
4313                let job = SeatJob {
4314                    spec,
4315                    seat: seat.clone(),
4316                    prompt: text,
4317                    cwd: root.join(format!("judge-{}", j + 1)),
4318                    timeout,
4319                    allow_write: false,
4320                    sessions,
4321                    artifacts: artifacts.clone(),
4322                    stem: format!("delib-{round}-judge-{}", j + 1),
4323                    handover: Some(full),
4324                };
4325                let cache = self.state.config.cache_dir();
4326                let ctx = WaveCtx {
4327                    carry_seats: false,
4328                    pauses: Some((&self.pause, &self.interrupt)),
4329                    run: &run_id,
4330                    node: "deliberate",
4331                    prompts: &prompts,
4332                    cache: cache.as_deref(),
4333                    round: None,
4334                };
4335                // A turn is never nudged (`retries` 0): a failed seat is
4336                // handed to the next roster agent, which gets the full
4337                // context. An empty answer is a turn, not a failure.
4338                let mut losses = Vec::new();
4339                asked += 1;
4340                let mut results = ask_wave_with::<String>(
4341                    vec![job],
4342                    Arc::clone(&self.sem),
4343                    0,
4344                    &self.roles.judge_roster,
4345                    &ctx,
4346                    &mut losses,
4347                    &mut self.state,
4348                    &|text: &str| {
4349                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350                    },
4351                )
4352                .await;
4353                if self.state.resource_block.is_some() {
4354                    return self.stop_for_resource();
4355                }
4356                round_losses.extend(losses.iter().cloned());
4357                self.state.quota.extend(losses);
4358                let (updated, res, _) = results.pop().expect("one job in, one result out");
4359                seat = updated;
4360                let agent_id = seat.agent.clone();
4361                self.state.seats.insert(seat.key.clone(), seat);
4362                let body = match res {
4363                    Ok((body, _)) => body,
4364                    // Skip the seat; a CLI's raw error JSON is never read as
4365                    // this judge's position.
4366                    Err(e) => {
4367                        self.state
4368                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369                        continue;
4370                    }
4371                };
4372                let tentative = verdict::extract_json::<Position>(&body)
4373                    .ok()
4374                    .and_then(|p| p.tentative)
4375                    .and_then(|s| s.trim().chars().next())
4376                    .map(|c| c.to_ascii_uppercase());
4377                self.state.event(
4378                    "deliberate",
4379                    format!(
4380                        "round {round}: judge {} now favours {}",
4381                        j + 1,
4382                        tentative.map_or("—".to_owned(), |c| c.to_string())
4383                    ),
4384                );
4385                turns.push(DeliberationTurn {
4386                    judge: j + 1,
4387                    agent: agent_id,
4388                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4389                    tentative,
4390                });
4391            }
4392            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394                return self.stop_for_resource();
4395            }
4396            self.state
4397                .deliberation
4398                .push(DeliberationRound { round, turns });
4399            self.state.save()?;
4400        }
4401
4402        self.state.status = RunStatus::Voting;
4403        self.state.save()?;
4404        Ok(())
4405    }
4406
4407    // ---------------------------------------------------------------- vote
4408
4409    async fn vote(&mut self) -> Result<()> {
4410        if self.hold_base_sync().await? {
4411            return Ok(());
4412        }
4413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4414        // agent files with `magi task add` name the run that paid for it. The
4415        // prompt overlay is cloned alongside it because the waves borrow it
4416        // while `self` is mutably borrowed by the node's own bookkeeping.
4417        let run_id = self.state.id.clone();
4418        let prompts = self.state.config.prompts.clone();
4419        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420            return Ok(());
4421        }
4422        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423        if viable.len() == 1 {
4424            return Ok(());
4425        }
4426        self.state.status = RunStatus::Voting;
4427
4428        let language = self.state.config.graph.language.clone();
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let root = self.state.worktree_root();
4433        let base_short = short(&self.state.base_commit);
4434        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436        let mut jobs = Vec::new();
4437        let mut seats_at = Vec::new();
4438        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439            if self
4440                .state
4441                .judgements
4442                .get(j)
4443                .is_some_and(|r| r.failed.is_some())
4444            {
4445                continue;
4446            }
4447            let seat_key = format!("judge-{}", j + 1);
4448            let spec = self.occupant(&seat_key, spec);
4449            let seat = self.seat(&seat_key, &spec.id);
4450            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451            let text = if has_context(&spec, &seat, sessions) {
4452                prompt::final_vote(&viable, &language)
4453            } else {
4454                full.clone()
4455            };
4456            jobs.push(SeatJob {
4457                spec,
4458                seat,
4459                prompt: text,
4460                cwd: root.join(format!("judge-{}", j + 1)),
4461                timeout,
4462                allow_write: false,
4463                sessions,
4464                artifacts: artifacts.clone(),
4465                stem: format!("vote-judge-{}", j + 1),
4466                handover: Some(full),
4467            });
4468            seats_at.push(j);
4469        }
4470
4471        self.state.event(
4472            "vote",
4473            format!(
4474                "collecting {} final votes one by one, privately",
4475                jobs.len()
4476            ),
4477        );
4478        let allowed = viable.clone();
4479        let mut quota_losses = Vec::new();
4480        let cache = self.state.config.cache_dir();
4481        let ctx = WaveCtx {
4482            carry_seats: false,
4483            pauses: Some((&self.pause, &self.interrupt)),
4484            run: &run_id,
4485            node: "vote",
4486            prompts: &prompts,
4487            cache: cache.as_deref(),
4488            round: None,
4489        };
4490        let results = ask_json_wave::<FinalVote>(
4491            jobs,
4492            Arc::clone(&self.sem),
4493            self.state.config.graph.retries,
4494            &self.roles.judge_roster,
4495            &ctx,
4496            &mut quota_losses,
4497            &mut self.state,
4498            &move |v: &FinalVote| match v.label() {
4499                Some(c) if allowed.contains(&c) => Ok(()),
4500                other => bail!("vote {other:?} is not one of {allowed:?}"),
4501            },
4502        )
4503        .await;
4504        if self.state.resource_block.is_some() {
4505            return self.stop_for_resource();
4506        }
4507        self.state.quota.extend(quota_losses);
4508
4509        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510            let agent_id = seat.agent.clone();
4511            self.state.seats.insert(seat.key.clone(), seat);
4512            let initial = self
4513                .state
4514                .judgements
4515                .get(j)
4516                .and_then(|r| r.ranking.first().copied());
4517            let mut record = VoteRecord {
4518                judge: j + 1,
4519                agent: agent_id,
4520                vote: None,
4521                reason: String::new(),
4522                changed: false,
4523            };
4524            match res {
4525                Ok((v, _)) => {
4526                    record.vote = v.label();
4527                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529                    self.state.event(
4530                        "vote",
4531                        format!(
4532                            "judge {} voted {}{}",
4533                            j + 1,
4534                            record.vote.unwrap_or('?'),
4535                            if record.changed { " (changed)" } else { "" }
4536                        ),
4537                    );
4538                }
4539                Err(e) => {
4540                    self.state
4541                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542                }
4543            }
4544            self.state.votes.push(record);
4545            self.state.save()?;
4546        }
4547        Ok(())
4548    }
4549
4550    // --------------------------------------------------------------- tally
4551
4552    fn tally(&mut self) -> Result<()> {
4553        if self.state.tally.is_some() {
4554            return Ok(());
4555        }
4556        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557        let tops: Vec<char> = self
4558            .state
4559            .judgements
4560            .iter()
4561            .filter_map(|j| j.ranking.first().copied())
4562            .collect();
4563        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565        // A judge whose private vote failed still counted once, in the initial
4566        // ranking; using it beats discarding a whole seat.
4567        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568        let mut cast: Vec<char> = Vec::new();
4569        for (i, j) in self.state.judgements.iter().enumerate() {
4570            let vote = self
4571                .state
4572                .votes
4573                .iter()
4574                .find(|v| v.judge == i + 1)
4575                .and_then(|v| v.vote)
4576                .or_else(|| j.ranking.first().copied());
4577            if let Some(v) = vote {
4578                *first_choice.entry(v).or_insert(0) += 1;
4579                cast.push(v);
4580            }
4581        }
4582
4583        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584        for j in &self.state.judgements {
4585            let n = j.ranking.len();
4586            for (pos, label) in j.ranking.iter().enumerate() {
4587                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588            }
4589        }
4590
4591        let best = first_choice.values().copied().max().unwrap_or(0);
4592        let mut leaders: Vec<char> = first_choice
4593            .iter()
4594            .filter(|(_, v)| **v == best)
4595            .map(|(k, _)| *k)
4596            .collect();
4597        let mut tie_break = None;
4598        if leaders.len() > 1 {
4599            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600            let borda_leaders: Vec<char> = leaders
4601                .iter()
4602                .copied()
4603                .filter(|l| borda[l] == top_borda)
4604                .collect();
4605            tie_break = Some(if borda_leaders.len() == 1 {
4606                format!(
4607                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608                    leaders.len()
4609                )
4610            } else {
4611                format!(
4612                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4613                    leaders.len()
4614                )
4615            });
4616            leaders = borda_leaders;
4617            leaders.sort_unstable();
4618        }
4619        let winner = *leaders
4620            .first()
4621            .or(viable.first())
4622            .context("no candidate to declare a winner from")?;
4623
4624        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626        let deliberated = !self.state.deliberation.is_empty();
4627
4628        // Whose verdict is this? A rate-limited seat is absent even if it
4629        // ranked before the limit hit, so presence is measured against the
4630        // recorded losses, not just "did a ranking ever appear".
4631        let quota_seats: std::collections::BTreeSet<&str> =
4632            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633        let mut present = 0usize;
4634        for (i, j) in self.state.judgements.iter().enumerate() {
4635            if quota_seats.contains(j.seat.as_str()) {
4636                continue;
4637            }
4638            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639            let voted = self
4640                .state
4641                .votes
4642                .iter()
4643                .any(|v| v.judge == i + 1 && v.vote.is_some());
4644            if ranked || voted {
4645                present += 1;
4646            }
4647        }
4648        // Strict majority of the configured panel. A bare majority is real
4649        // signal we can act on, while a minority verdict must never stand in
4650        // for a healthy one. A one-candidate run needs no panel at all, and
4651        // `judges` stays `0` rather than the roster size a panel that never
4652        // sat would otherwise be credited with.
4653        let needs_quorum = viable.len() > 1;
4654        let judges_total = if needs_quorum {
4655            self.roles.judges.len()
4656        } else {
4657            0
4658        };
4659        let quorum = if needs_quorum {
4660            judges_total / 2 + 1
4661        } else {
4662            0
4663        };
4664        let met_quorum = !needs_quorum || present >= quorum;
4665        let uncontested = (!needs_quorum).then(|| {
4666            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667        });
4668
4669        self.state.event(
4670            "tally",
4671            match &uncontested {
4672                Some(reason) => format!("winner {winner} — {reason}"),
4673                None => format!(
4674                    "winner {winner} — votes {} | initial {} | {} changed | \
4675                     {present}/{judges_total} judges{}",
4676                    first_choice
4677                        .iter()
4678                        .map(|(k, v)| format!("{k}:{v}"))
4679                        .collect::<Vec<_>>()
4680                        .join(" "),
4681                    if unanimous_initial {
4682                        "unanimous"
4683                    } else {
4684                        "split"
4685                    },
4686                    changed_votes,
4687                    if met_quorum {
4688                        String::new()
4689                    } else {
4690                        format!(" — below quorum ({quorum} required)")
4691                    },
4692                ),
4693            },
4694        );
4695        if !met_quorum {
4696            self.state.event(
4697                "stall",
4698                format!(
4699                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700                     the run stops here, resumable"
4701                ),
4702            );
4703        }
4704        self.state.tally = Some(Tally {
4705            first_choice,
4706            borda,
4707            winner,
4708            rankings: tops.len(),
4709            unanimous_initial,
4710            deliberated,
4711            changed_votes,
4712            unanimous_final,
4713            tie_break,
4714            judges: judges_total,
4715            present,
4716            quorum,
4717            met_quorum,
4718            uncontested,
4719        });
4720        self.state.status = if met_quorum {
4721            RunStatus::Reviewing
4722        } else {
4723            RunStatus::Stalled
4724        };
4725        self.state.save()?;
4726        Ok(())
4727    }
4728
4729    // ------------------------------------------------------------- recover
4730
4731    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4732    /// resumed toward completion once the transient cause clears.
4733    ///
4734    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4735    /// it toward the quorum, which is exactly the set of seats whose absence
4736    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4737    /// not depend on which node happened to hit the limit), or an ordinary
4738    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4739    /// seat is never disturbed.
4740    ///
4741    /// A seat that now answers with a usable ranking is "recovered": its
4742    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4743    /// `tally` counts it present again), and its vote re-collected. A seat that
4744    /// still fails keeps its loss and stays absent.
4745    ///
4746    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4747    /// to review/gate/merge), `false` when it is still below quorum (the run
4748    /// stays `Stalled`, still resumable for a later retry).
4749    #[allow(clippy::too_many_lines)]
4750    async fn recover_stall(&mut self) -> Result<bool> {
4751        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4752        // agent files with `magi task add` name the run that paid for it. The
4753        // prompt overlay is cloned alongside it because the waves borrow it
4754        // while `self` is mutably borrowed by the node's own bookkeeping.
4755        let run_id = self.state.id.clone();
4756        let prompts = self.state.config.prompts.clone();
4757        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4758        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4759        // is what keeps a non-quota collapse — or a quota loss recorded at the
4760        // deliberate node — from being a permanent dead-end on `--resume`.
4761        let quota_seats: BTreeSet<&str> =
4762            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763        let absent: Vec<String> = self
4764            .state
4765            .judgements
4766            .iter()
4767            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768            .map(|j| j.seat.clone())
4769            .collect();
4770        if absent.is_empty() {
4771            return Ok(false);
4772        }
4773        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774        if viable.len() <= 1 {
4775            return Ok(false);
4776        }
4777        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778        let language = self.state.config.graph.language.clone();
4779        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780        let sessions = self.state.config.graph.sessions;
4781        let artifacts = agent::artifacts_dir(&self.state.dir());
4782        let root = self.state.worktree_root();
4783        let base_short = short(&self.state.base_commit);
4784        let candidates: Vec<Candidate> = viable.clone();
4785
4786        // Map each absent seat key to its 0-based position in `roles.judges`.
4787        let mut positions: Vec<usize> = absent
4788            .iter()
4789            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790            .collect();
4791        if positions.is_empty() {
4792            return Ok(false);
4793        }
4794        positions.sort_unstable();
4795        positions.dedup();
4796
4797        // Re-rank the lost seats, one blind prompt each.
4798        let mut judge_jobs = Vec::new();
4799        for &j in &positions {
4800            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802            let seat_key = format!("judge-{}", j + 1);
4803            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804            let seat = self.seat(&seat_key, &spec.id);
4805            judge_jobs.push(SeatJob {
4806                spec,
4807                seat,
4808                prompt: prompt::judge(
4809                    &self.state.instruction,
4810                    &views,
4811                    self.roles.judges.len(),
4812                    &base_short,
4813                    &language,
4814                ),
4815                cwd: root.join(seat_key),
4816                timeout,
4817                allow_write: false,
4818                sessions,
4819                artifacts: artifacts.clone(),
4820                stem: format!("judge-{}-recover", j + 1),
4821                handover: None,
4822            });
4823        }
4824
4825        let labels_for_check = labels.clone();
4826        let mut judge_losses = Vec::new();
4827        let retries = self.state.config.graph.retries;
4828        let cache = self.state.config.cache_dir();
4829        let ctx = WaveCtx {
4830            carry_seats: false,
4831            pauses: Some((&self.pause, &self.interrupt)),
4832            run: &run_id,
4833            node: "judge",
4834            prompts: &prompts,
4835            cache: cache.as_deref(),
4836            round: None,
4837        };
4838        let results = ask_json_wave::<Ranking>(
4839            judge_jobs,
4840            Arc::clone(&self.sem),
4841            retries,
4842            &self.roles.judge_roster,
4843            &ctx,
4844            &mut judge_losses,
4845            &mut self.state,
4846            &move |r: &Ranking| r.validate(&labels_for_check),
4847        )
4848        .await;
4849        if self.state.resource_block.is_some() {
4850            self.stop_for_resource()?;
4851            return Ok(false);
4852        }
4853
4854        // Refresh the judgement of every seat that ranked again.
4855        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857            let agent_id = seat.agent.clone();
4858            self.state.seats.insert(seat.key.clone(), seat);
4859            let record = &mut self.state.judgements[j];
4860            match res {
4861                Ok((ranking, out)) => {
4862                    record.agent = agent_id;
4863                    record.ranking = ranking.normalized();
4864                    record.reasons = ranking.reasons;
4865                    record.strengths = ranking.strengths;
4866                    record.confidence = ranking.confidence;
4867                    record.failed = None;
4868                    record.duration_ms = out.duration_ms;
4869                    recovered.insert(j);
4870                    self.state.event(
4871                        "recover",
4872                        format!("judge {} ranked again after the limit", j + 1),
4873                    );
4874                }
4875                Err(e) => {
4876                    self.state
4877                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878                }
4879            }
4880        }
4881
4882        // Re-ask the votes of the seats that recovered a ranking.
4883        let mut vote_jobs = Vec::new();
4884        let mut vote_pos: Vec<usize> = Vec::new();
4885        for &j in &recovered {
4886            let seat_key = format!("judge-{}", j + 1);
4887            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888            let seat = self.seat(&seat_key, &spec.id);
4889            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890            let text = if has_context(&spec, &seat, sessions) {
4891                prompt::final_vote(&labels, &language)
4892            } else {
4893                full.clone()
4894            };
4895            vote_jobs.push(SeatJob {
4896                spec,
4897                seat,
4898                prompt: text,
4899                cwd: root.join(seat_key),
4900                timeout,
4901                allow_write: false,
4902                sessions,
4903                artifacts: artifacts.clone(),
4904                stem: format!("vote-judge-{}-recover", j + 1),
4905                handover: Some(full),
4906            });
4907            vote_pos.push(j);
4908        }
4909        let allowed = labels.clone();
4910        let mut vote_losses = Vec::new();
4911        let vote_retries = self.state.config.graph.retries;
4912        let vote_cache = self.state.config.cache_dir();
4913        let ctx = WaveCtx {
4914            carry_seats: false,
4915            pauses: Some((&self.pause, &self.interrupt)),
4916            run: &run_id,
4917            node: "vote",
4918            prompts: &prompts,
4919            cache: vote_cache.as_deref(),
4920            round: None,
4921        };
4922        let votes = ask_json_wave::<FinalVote>(
4923            vote_jobs,
4924            Arc::clone(&self.sem),
4925            vote_retries,
4926            &self.roles.judge_roster,
4927            &ctx,
4928            &mut vote_losses,
4929            &mut self.state,
4930            &move |v: &FinalVote| match v.label() {
4931                Some(c) if allowed.contains(&c) => Ok(()),
4932                other => bail!("vote {other:?} is not one of {allowed:?}"),
4933            },
4934        )
4935        .await;
4936        if self.state.resource_block.is_some() {
4937            self.stop_for_resource()?;
4938            return Ok(false);
4939        }
4940        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941            let agent_id = seat.agent.clone();
4942            self.state.seats.insert(seat.key.clone(), seat);
4943            match res {
4944                Ok((v, _)) => {
4945                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946                        rec.vote = v.label();
4947                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948                    } else {
4949                        self.state.votes.push(VoteRecord {
4950                            judge: j + 1,
4951                            agent: agent_id,
4952                            vote: v.label(),
4953                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954                            changed: false,
4955                        });
4956                    }
4957                    self.state.event(
4958                        "recover",
4959                        format!("judge {} voted again after the limit", j + 1),
4960                    );
4961                }
4962                Err(e) => {
4963                    self.state
4964                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965                }
4966            }
4967        }
4968
4969        // A seat that ranked again is present even if its re-vote failed —
4970        // `tally` falls back to the initial ranking's first choice — so clear
4971        // its quota loss. Seats that still fail keep theirs and stay absent.
4972        let recovered_keys: BTreeSet<String> = recovered
4973            .iter()
4974            .map(|&j| format!("judge-{}", j + 1))
4975            .collect();
4976        self.state
4977            .quota
4978            .retain(|q| !recovered_keys.contains(&q.seat));
4979        // A seat that hit the limit again is a fresh loss, not the old one:
4980        // replace the stale entry so the history stays one-per-seat and the
4981        // daemon can tell this attempt's loss from a previous session's.
4982        for loss in judge_losses.into_iter().chain(vote_losses) {
4983            if recovered_keys.contains(&loss.seat) {
4984                continue;
4985            }
4986            self.state.quota.retain(|q| q.seat != loss.seat);
4987            self.state.quota.push(loss);
4988        }
4989
4990        // Recompute the verdict from the refreshed panel.
4991        self.state.tally = None;
4992        self.tally()?;
4993        Ok(self
4994            .state
4995            .tally
4996            .as_ref()
4997            .map(|t| t.met_quorum)
4998            .unwrap_or(false))
4999    }
5000
5001    // ---------------------------------------------------------------- graft
5002
5003    /// Port the strengths the judges attributed to the losing candidates onto
5004    /// the winner, once, between the tally and the review loop.
5005    ///
5006    /// Best-effort by construction: it returns nothing and fails nothing. A
5007    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5008    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5009    /// the run carries on as if the phase were off. See [`graft`] for the
5010    /// decisions and `AGENTS.md` for the invariants.
5011    ///
5012    /// The one exception is a rollback that could not be confirmed: the
5013    /// rejected commit may still be the winner's tip, so the run is blocked
5014    /// (resumable; the record stays `Running`, which makes a resume retry the
5015    /// restore) instead of reviewing, gating and merging it.
5016    async fn graft(&mut self) -> Result<()> {
5017        let Err(e) = self.graft_inner().await else {
5018            return Ok(());
5019        };
5020        if e.is::<crate::repo_lock::ResourceBlocked>() {
5021            return Err(e);
5022        }
5023        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024            self.state.status = RunStatus::Blocked;
5025            self.state.event("graft", format!("blocked: {e:#}"));
5026            self.state.save()?;
5027            return Err(e);
5028        }
5029        // An error after the `Running` record was saved may have left the
5030        // fixer's work in the tree: the record must not stay unsettled
5031        // (a later resume would reset past the review fixes), so confirm the
5032        // restore now, or block.
5033        if let Some(rec) = self.state.graft.clone()
5034            && !rec.outcome.settled()
5035        {
5036            if let Some(winner) = self.state.winner().cloned() {
5037                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5039                        return Err(re);
5040                    }
5041                    self.state.status = RunStatus::Blocked;
5042                    self.state.event("graft", format!("blocked: {re:#}"));
5043                    self.state.save()?;
5044                    return Err(re);
5045                }
5046            }
5047            if let Some(g) = self.state.graft.as_mut() {
5048                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049            }
5050        }
5051        self.state.event("graft", format!("skipped: {e:#}"));
5052        if let Err(e) = self.state.save() {
5053            tracing::warn!("save after graft: {e:#}");
5054        }
5055        Ok(())
5056    }
5057
5058    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5059    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5060    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061        let _coordination = self.coordinate(wt).await?;
5062        crate::recovery::restore_repair(wt, pre)
5063            .await
5064            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065    }
5066
5067    /// A skip is a run event, recorded once however often the node is re-entered.
5068    fn graft_skipped(&mut self, why: &str) {
5069        let message = format!("skipped: {why}");
5070        if !self
5071            .state
5072            .events
5073            .iter()
5074            .any(|e| e.node == "graft" && e.message == message)
5075        {
5076            self.state.event("graft", message);
5077        }
5078    }
5079
5080    async fn graft_inner(&mut self) -> Result<()> {
5081        // A round that was started and never finished (crash, kill): the
5082        // fixer may have left anything in the tree, so put the winner back.
5083        if let Some(rec) = self.state.graft.clone()
5084            && !rec.outcome.settled()
5085        {
5086            // Reviews exist only if the graft ended without its record being
5087            // settled (a swallowed error): the tree has moved on since, and
5088            // a reset to `pre` would discard the review fixes.
5089            if self.state.reviews.is_empty()
5090                && let Some(winner) = self.state.winner().cloned()
5091            {
5092                self.graft_restore(&winner.worktree, &rec.pre).await?;
5093            }
5094            if rec.outcome == GraftOutcome::WaitingForRestore {
5095                self.state.graft = None;
5096                self.state.save()?;
5097            } else {
5098                if let Some(g) = self.state.graft.as_mut() {
5099                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100                }
5101                self.state.event(
5102                    "graft",
5103                    "rolled back: the previous round was interrupted; the winner is untouched",
5104                );
5105                self.state.save()?;
5106                return Ok(());
5107            }
5108        }
5109        let plan = match graft::plan(&self.state) {
5110            Ok(plan) => plan,
5111            Err(skip) => {
5112                if skip != graft::Skip::AlreadyDone {
5113                    self.graft_skipped(skip.reason());
5114                    self.state.save()?;
5115                }
5116                return Ok(());
5117            }
5118        };
5119        let Some(winner) = self.state.winner().cloned() else {
5120            return Ok(());
5121        };
5122        let wt = winner.worktree.clone();
5123        if !git::is_clean(&wt).await? {
5124            self.graft_skipped("the winner's worktree has uncommitted changes");
5125            self.state.save()?;
5126            return Ok(());
5127        }
5128        let pre = git::rev_parse(&wt, "HEAD").await?;
5129        // Saved before the fixer is called: a stop from here on is repaired
5130        // by the branch above on the next entry.
5131        self.state.graft = Some(GraftRecord {
5132            pre: pre.clone(),
5133            post: None,
5134            lines: None,
5135            limit: None,
5136            agent: None,
5137            outcome: GraftOutcome::Running,
5138            requested: graft::requested(&plan),
5139            adoption: None,
5140            adoption_error: None,
5141        });
5142        self.state.event(
5143            "graft",
5144            format!(
5145                "porting strengths of candidate(s) {} onto {}",
5146                plan.strengths
5147                    .iter()
5148                    .map(|(l, _, _)| l.to_string())
5149                    .collect::<Vec<_>>()
5150                    .join(", "),
5151                plan.winner
5152            ),
5153        );
5154        self.state.save()?;
5155
5156        let fix_prompt = prompt::graft(
5157            &self.state.instruction,
5158            plan.winner,
5159            &plan.strengths,
5160            &self.state.config.graph.language,
5161        );
5162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163        let sessions = self.state.config.graph.sessions;
5164        let artifacts = agent::artifacts_dir(&self.state.dir());
5165        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166        let (_, seat, out) = self
5167            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168                SeatJob {
5169                    prompt: fix_prompt.clone(),
5170                    spec,
5171                    seat,
5172                    cwd: wt.clone(),
5173                    timeout,
5174                    allow_write: true,
5175                    sessions,
5176                    artifacts: artifacts.clone(),
5177                    stem: "graft".to_owned(),
5178                    handover: None,
5179                }
5180            })
5181            .await;
5182        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183            // Preserve the need to restore earlier fallback edits without
5184            // confusing an unstarted seat with an interrupted invocation.
5185            self.state.graft.as_mut().expect("graft record").outcome =
5186                GraftOutcome::WaitingForRestore;
5187            return self.stop_for_resource();
5188        }
5189        let mut why: Option<String> = None;
5190        match &out {
5191            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192                why = Some("the fixer produced nothing usable".to_owned());
5193            }
5194            AgentOutcome::Ok(_) => {}
5195            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196            AgentOutcome::Quota(o) => {
5197                self.state.quota.push(QuotaLoss::from_report(
5198                    seat.key.clone(),
5199                    "graft",
5200                    o.quota.as_ref(),
5201                ));
5202                why = Some("rate limited (quota)".to_owned());
5203            }
5204            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205                why = Some(format!(
5206                    "the fixer failed: {}",
5207                    e.lines()
5208                        .next()
5209                        .unwrap_or("")
5210                        .chars()
5211                        .take(160)
5212                        .collect::<String>()
5213                ));
5214            }
5215        }
5216        let agent_id = seat.agent.clone();
5217        self.state.seats.insert(seat.key.clone(), seat);
5218
5219        let mut post = None;
5220        let mut lines = None;
5221        let _coordination = self.coordinate(&wt).await?;
5222        if why.is_none() {
5223            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224            if let Ok(r) = &rescued {
5225                self.state.note_withheld("graft", &r.withheld);
5226            }
5227            let head = git::rev_parse(&wt, "HEAD").await?;
5228            // What is measured must be what gets reviewed and gated: a failed
5229            // rescue, or anything still uncommitted (withheld lockfiles
5230            // aside, which are never part of the tree), rejects the graft.
5231            let dirty = match &rescued {
5232                Ok(r) => git::status_porcelain(&wt)
5233                    .await
5234                    .map(|st| {
5235                        st.lines().any(|l| {
5236                            !r.withheld
5237                                .iter()
5238                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239                        })
5240                    })
5241                    .unwrap_or(true),
5242                Err(_) => true,
5243            };
5244            if dirty {
5245                why = Some("the fixer left changes that could not be committed".to_owned());
5246            } else if head == pre {
5247                why = Some("the fixer changed nothing".to_owned());
5248            } else {
5249                lines = git::changed_lines(&wt, &pre, &head).await;
5250                post = Some(head);
5251            }
5252        }
5253        let (adoption, adoption_error) = match &out {
5254            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255            _ => (None, None),
5256        };
5257        let requested = graft::requested(&plan);
5258        let outcome = match why {
5259            None => {
5260                self.state.event(
5261                    "graft",
5262                    format!(
5263                        "applied: {}",
5264                        lines.map_or_else(
5265                            || "changed lines unavailable".to_owned(),
5266                            |n| format!("{n} changed lines")
5267                        )
5268                    ),
5269                );
5270                GraftOutcome::Applied
5271            }
5272            Some(why) => {
5273                // Not settled until the restore is confirmed: the record
5274                // stays `Running`, so a resume retries it.
5275                crate::recovery::restore_repair(&wt, &pre)
5276                    .await
5277                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278                self.state.event(
5279                    "graft",
5280                    format!("rolled back: {why}; the winner is untouched"),
5281                );
5282                GraftOutcome::RolledBack(why)
5283            }
5284        };
5285        self.state.graft = Some(GraftRecord {
5286            pre,
5287            post,
5288            lines,
5289            limit: None,
5290            agent: Some(agent_id),
5291            outcome,
5292            requested,
5293            adoption,
5294            adoption_error,
5295        });
5296        self.state.save()?;
5297        Ok(())
5298    }
5299
5300    // ----------------------------------------------------------------- fold
5301
5302    async fn fold_losers(&mut self) -> Result<()> {
5303        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304            return Ok(());
5305        };
5306        if !self.state.candidate_preservation.is_empty() {
5307            bail!("candidate preservation incomplete; refusing cleanup");
5308        }
5309        let repo = self.state.repo.clone();
5310        let _coordination = self.coordinate(&repo).await?;
5311        let mut folded = Vec::new();
5312        for i in 0..self.state.candidates.len() {
5313            let c = &self.state.candidates[i];
5314            if c.label == winner || c.folded {
5315                continue;
5316            }
5317            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318            remove_preserved_worktree(&repo, &wt).await?;
5319            if git::branch_exists(&repo, &branch).await?
5320                && !git::branch_delete(&repo, &branch).await?
5321            {
5322                bail!("candidate branch removal refused");
5323            }
5324            self.state.candidates[i].folded = true;
5325            self.state.save()?;
5326            folded.push(label.to_string());
5327        }
5328        // The judges are finished; their checkouts are pure cost from here.
5329        let root = self.state.worktree_root();
5330        for j in 1..=self.roles.judges.len() {
5331            let wt = root.join(format!("judge-{j}"));
5332            if wt.exists() {
5333                remove_read_seat_worktree(&mut self.state, &wt).await?;
5334                self.state.save()?;
5335            }
5336        }
5337        // The design-deliberation stage is finished by the time a tally
5338        // exists — same reasoning as the judges above.
5339        if self.state.config.graph.advise {
5340            for k in 1..=self.state.config.graph.advisors {
5341                let wt = root.join(format!("advisor-{k}"));
5342                if wt.exists() {
5343                    remove_read_seat_worktree(&mut self.state, &wt).await?;
5344                    self.state.save()?;
5345                }
5346            }
5347        }
5348        if !folded.is_empty() {
5349            self.state
5350                .event("fold", format!("folded candidates {}", folded.join(", ")));
5351            self.state.save()?;
5352        }
5353        Ok(())
5354    }
5355
5356    // ------------------------------------------------------------ base sync
5357
5358    async fn hold_base_sync(&mut self) -> Result<bool> {
5359        self.hold_base_sync_as(None).await
5360    }
5361
5362    async fn hold_base_sync_kind(
5363        &mut self,
5364        kind: crate::base_sync_recovery::StopKind,
5365    ) -> Result<()> {
5366        self.hold_base_sync_as(Some(kind)).await?;
5367        Ok(())
5368    }
5369
5370    async fn hold_base_sync_as(
5371        &mut self,
5372        kind: Option<crate::base_sync_recovery::StopKind>,
5373    ) -> Result<bool> {
5374        if !self.state.base_sync_blocked() {
5375            self.state.base_sync_recovery.stop = None;
5376            return Ok(false);
5377        }
5378        let head = match self.state.winner() {
5379            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5380                .await
5381                .unwrap_or_default(),
5382            None => String::new(),
5383        };
5384        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5385            && !authorization.consumed
5386            && authorization.stop.head != head
5387        {
5388            authorization.refuse(format!(
5389                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5390                authorization.stop.head
5391            ));
5392        }
5393        self.state.record_base_sync_stop(&head);
5394        if let Some(kind) = kind {
5395            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5396            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5397                stop.kind = kind;
5398            }
5399        }
5400        self.state.save()?;
5401        Ok(true)
5402    }
5403
5404    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5405    /// anything verifies it.
5406    ///
5407    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5408    /// read whatever is checked out in the winner's worktree. Left alone that
5409    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5410    /// it when the run *branched* - and a run takes long enough that the base
5411    /// has usually moved by the time it gets here. A gate that ran there
5412    /// answers "green on the commit this run started from", not "green on
5413    /// what is about to land", and the difference showed up three times in
5414    /// one day as a green run whose merge would have reverted a file another
5415    /// pull request had already landed.
5416    ///
5417    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5418    /// `land::Step::Rebase` calls, rather than a second implementation of the
5419    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5420    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5421    /// doc warns about ("two answers to a question nobody notices until a
5422    /// diff is wrong").
5423    ///
5424    /// Foreign-checkout reviews instead merge the base into the owned shadow
5425    /// worktree, preserving the ancestry required for fast-forward delivery.
5426    /// A conflicting merge is aborted and held for operator resolution.
5427    ///
5428    /// A conflict is not the end of the road: the standing rebase is handed
5429    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5430    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5431    /// land). Once it finishes, review and the gate run as usual on the
5432    /// rebased tree, which is where a breakage the new base caused is caught
5433    /// by the ordinary gate-fix round. magi resolves nothing itself.
5434    ///
5435    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5436    /// the current durable recovery interval, bounds how often the base is
5437    /// synchronized during active recovery (lifetime attempts remain evidence).
5438    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5439    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5440    /// what was tried (rounds spent, paths still conflicted) and the branch
5441    /// and worktree stay exactly as they were - untouched, for a person to
5442    /// look at - which is also what makes re-entering this function
5443    /// afterwards a no-op instead of a second attempt at the same wall. A
5444    /// push failure ends the same way.
5445    async fn sync_to_base(&mut self) -> Result<()> {
5446        if self.state.status == RunStatus::AlreadyInBase {
5447            return Ok(());
5448        }
5449        self.hold_base_sync().await?;
5450        let conflicted = self
5451            .state
5452            .base_sync
5453            .as_ref()
5454            .is_some_and(|s| s.conflict.is_some());
5455        let Some(winner) = self.state.winner().cloned() else {
5456            return Ok(());
5457        };
5458
5459        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5460
5461        let repo = self.state.repo.clone();
5462        let remote = self.state.config.merge.remote.clone();
5463        let base_branch = self.state.base_branch.clone();
5464        let tracking = format!("{remote}/{base_branch}");
5465
5466        let fetched_base =
5467            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5468        // No network, or the remote never had this branch: the run already
5469        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5470        // otherwise), and one that got this far is not blocked by a fetch
5471        // that fails now. It just has no newer tip to compare against.
5472        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5473            return Ok(());
5474        };
5475
5476        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5477        self.state.ensure_base_episode();
5478        let mut pending = self
5479            .state
5480            .base_sync_recovery
5481            .operations
5482            .last()
5483            .filter(|o| !o.finished)
5484            .cloned();
5485        let scratch = self.state.dir().join("base-sync");
5486        if git::rebase_in_progress(&scratch).await {
5487            // Older resource stops predate operation reservations. Import the
5488            // standing Git operation, never retarget it to today's main.
5489            if pending.is_none()
5490                && self.state.base_sync_recovery.operations.is_empty()
5491                && !conflicted
5492                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5493                    .await
5494                    .ok()
5495                    .as_deref()
5496                    == Some(head.as_str())
5497                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5498                && git::is_clean(&winner.worktree).await?
5499            {
5500                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5501                if self.state.base_sync.is_none() {
5502                    self.state.base_sync = Some(BaseSync {
5503                        tip: saved_tip.clone(),
5504                        behind: 0,
5505                        attempts: 0,
5506                        conflict: None,
5507                        already_in: None,
5508                    });
5509                }
5510                self.state.reserve_base_operation(&head, &saved_tip);
5511                self.state.event(
5512                    "base-recovery",
5513                    "imported standing legacy rebase as a spent synchronization reservation",
5514                );
5515                self.state.save()?;
5516                pending = self.state.base_sync_recovery.operations.last().cloned();
5517            }
5518            let expected = pending.as_ref().map(|op| op.tip.as_str());
5519            if expected.is_none()
5520                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5521            {
5522                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5523                if self.state.base_sync.is_none() {
5524                    self.state.base_sync = Some(BaseSync {
5525                        tip: tip.clone(),
5526                        behind: 0,
5527                        attempts: 0,
5528                        conflict: None,
5529                        already_in: None,
5530                    });
5531                }
5532                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5533                self.state.event("land", why);
5534                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535                    .await?;
5536                return Ok(());
5537            }
5538        }
5539        if let Some(op) = &pending {
5540            // A replay may add the pinned base's commits, never foreign work.
5541            let mut replay_proven =
5542                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5543            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5544                let wait = repository_wait(&self.state, None);
5545                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5546                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5547                    Ok(true) => {
5548                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5549                        self.state.save()?;
5550                        git::worktree_remove(&repo, &scratch).await.ok();
5551                        replay_proven = true;
5552                    }
5553                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5554                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5555                }
5556            }
5557            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5558                if self.state.foreign_checkout.is_some() {
5559                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5560                        .await
5561                        .ok()
5562                        .as_deref()
5563                        == Some(op.head.as_str())
5564                        && git::rev_parse(&repo, &format!("{head}^2"))
5565                            .await
5566                            .ok()
5567                            .as_deref()
5568                            == Some(op.tip.as_str());
5569                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5570                    .await
5571                    .is_ok_and(|m| m.is_empty())
5572                    && let Ok(added) =
5573                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5574                {
5575                    replay_proven = true;
5576                    for commit in added {
5577                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5578                            replay_proven = false;
5579                            break;
5580                        }
5581                    }
5582                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5583                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5584                }
5585            }
5586            // Never retarget an operation across a crash, even if main moved.
5587            // sync_to_head resets and cleans with -x, so even ignored untracked
5588            // files must stop recovery independently of the old tracked tree.
5589            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590                || (head != op.head
5591                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5592                        .await?
5593                        .is_empty())
5594                || (!git::is_clean(&winner.worktree).await?
5595                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5596                        .await
5597                        .is_ok_and(|o| o.ok()))
5598                || !replay_proven
5599            {
5600                let why = format!(
5601                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5602                    op.id
5603                );
5604                let sync = self
5605                    .state
5606                    .base_sync
5607                    .as_mut()
5608                    .context("reserved synchronization has no lifetime accounting")?;
5609                sync.conflict = Some(why.clone());
5610                self.state.event("land", why);
5611                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5612                    .await?;
5613                return Ok(());
5614            }
5615            if head != op.head {
5616                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5617            }
5618            tip = op.tip.clone();
5619        }
5620        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5621        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5622        if pending.is_none()
5623            && !conflicted
5624            && fetched_base
5625            && self
5626                .state
5627                .base_sync_recovery
5628                .episodes
5629                .last()
5630                .is_some_and(|e| e.boundary.is_some())
5631            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5632            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5633            && owner.execution_runs().last() == Some(&self.state.id)
5634            && owner.status != crate::queue::TaskStatus::Done
5635            && !owner.fresh_start
5636            && owner.review_branch.is_none()
5637            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5638            && let Some(approval) = owner.resumption.as_ref()
5639            && approval.instruction == owner.instruction
5640            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5641            && git::is_clean(&winner.worktree).await?
5642            && self.state.renew_base_episode(&task, approval, &head, &tip)
5643        {
5644            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5645            self.state.save()?;
5646        }
5647
5648        // Before any rebase, and before a recorded conflict is honoured: a
5649        // branch whose change reached the base under other commit ids has
5650        // nothing to rebase and nothing to conflict with, and a run that
5651        // already stopped on that phantom conflict recovers here on resume.
5652        // `behind == 0` with head == tip is a branch the base has since taken
5653        // in whole, whether or not a conflict was ever recorded: the ancestry
5654        // proof must still run (`classify` ignores a head still on the start
5655        // commit).
5656        if (behind > 0 || conflicted || head == tip)
5657            && self
5658                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5659                .await?
5660        {
5661            return Ok(());
5662        }
5663        if conflicted && self.state.base_recovery_pending() {
5664            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5665            let owner =
5666                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5667            let owner_valid = owner.as_ref().is_ok_and(|t| {
5668                t.execution_runs().last() == Some(&self.state.id)
5669                    && t.status != crate::queue::TaskStatus::Done
5670                    && !t.fresh_start
5671                    && t.review_branch.is_none()
5672            });
5673            let invalidation = if !owner_valid {
5674                Some("latest task ownership changed")
5675            } else if !owner.as_ref().is_ok_and(|t| {
5676                authorization
5677                    .generation
5678                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5679            }) {
5680                Some("owner approval generation changed")
5681            } else if authorization.question.as_ref().is_some_and(|question| {
5682                !crate::base_recovery_control::decision_current(
5683                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5684                    &self.state,
5685                    &authorization.task,
5686                    question,
5687                )
5688            }) {
5689                Some("recovery decision ownership, routing or stop evidence changed")
5690            } else if authorization.request.is_some()
5691                && !owner
5692                    .as_ref()
5693                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5694            {
5695                Some("owner decision changed")
5696            } else if let Some(scope) = &authorization.scope {
5697                if scope.tip != tip {
5698                    Some("fetched base changed")
5699                } else if !self
5700                    .state
5701                    .base_sync_recovery
5702                    .episodes
5703                    .last()
5704                    .is_some_and(|e| e.id == scope.episode)
5705                {
5706                    Some("active synchronization interval changed")
5707                } else if !owner
5708                    .as_ref()
5709                    .is_ok_and(|t| t.instruction == scope.instruction)
5710                {
5711                    Some("task instruction changed")
5712                } else {
5713                    None
5714                }
5715            } else {
5716                None
5717            };
5718            if let Some(reason) = invalidation {
5719                let authorization = self
5720                    .state
5721                    .base_sync_recovery
5722                    .authorizations
5723                    .last_mut()
5724                    .unwrap();
5725                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5726                self.state.save()?;
5727                return Ok(());
5728            }
5729            if !fetched_base
5730                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5731                || !git::is_clean(&winner.worktree).await?
5732                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5733            {
5734                return Ok(());
5735            }
5736        }
5737        let recovery = conflicted && self.state.consume_base_recovery(&head);
5738        if conflicted {
5739            // Consumption (including a refused changed HEAD) is durable before Git.
5740            self.state.save()?;
5741            if !recovery {
5742                return Ok(());
5743            }
5744        }
5745
5746        if behind == 0 {
5747            // A fixer-finished rebase moves the branch ref before the
5748            // winner's worktree is told (`sync_to_head` below). A run that
5749            // died in between resumes here with `behind == 0` and a tree still
5750            // holding the pre-rebase files, which review and the gate would
5751            // then read. That state is exactly: HEAD moved off the tip the
5752            // rebase started from, yet the tree is still identical to that
5753            // tip. A tree with edits of its own differs from it, so nothing
5754            // is thrown away.
5755            if let Some(from) = self
5756                .state
5757                .rebase_fixes
5758                .iter()
5759                .rev()
5760                .find_map(|r| r.from.clone())
5761                && from != head
5762                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5763                    .await
5764                    .is_ok_and(|o| o.ok())
5765            {
5766                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5767            }
5768            // An earlier attempt may have rebased the branch locally and died
5769            // before pushing it (only the fresh-rebase arm below pushes).
5770            // Publish it now, so the plain push at PR time is not refused as
5771            // a non-fast-forward. A run already holding a recorded conflict
5772            // never reaches here; that case is out of scope.
5773            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5774            if let Some(why) = &conflict {
5775                self.state.status = RunStatus::Blocked;
5776                self.state.event("land", why.clone());
5777            }
5778            self.state.base_sync = Some(BaseSync {
5779                tip,
5780                behind: 0,
5781                attempts,
5782                conflict,
5783                already_in: None,
5784            });
5785            if pending.is_some() {
5786                self.state.finish_base_operation(Some(&head));
5787            }
5788            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5789                .await?;
5790            self.state.save()?;
5791            return Ok(());
5792        }
5793
5794        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5795            let why = format!(
5796                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5797                 synchronization(s); synchronizing again would only race it",
5798                winner.branch
5799            );
5800            self.state.status = RunStatus::Blocked;
5801            self.state.base_sync = Some(BaseSync {
5802                tip,
5803                behind,
5804                attempts,
5805                conflict: Some(why.clone()),
5806                already_in: None,
5807            });
5808            self.state.event("land", why);
5809            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5810                .await?;
5811            self.state.save()?;
5812            return Ok(());
5813        }
5814
5815        if self.state.foreign_checkout.is_some() {
5816            // Delivery to the operator's checkout is fast-forward only. A
5817            // rebase here would discard its saved head from our ancestry.
5818            let wait = repository_wait(&self.state, None);
5819            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5820            if self.state.base_sync.is_none() {
5821                self.state.base_sync = Some(BaseSync {
5822                    tip: tip.clone(),
5823                    behind,
5824                    attempts: 0,
5825                    conflict: None,
5826                    already_in: None,
5827                });
5828            }
5829            self.state.reserve_base_operation(&head, &tip);
5830            self.state.save()?;
5831            let merged = git::git_raw(
5832                &winner.worktree,
5833                &[
5834                    "-c",
5835                    "core.hooksPath=/dev/null",
5836                    "merge",
5837                    "--no-ff",
5838                    "--no-autostash",
5839                    "--no-edit",
5840                    "-m",
5841                    &format!("Merge {tracking} before verifying"),
5842                    &tip,
5843                ],
5844            )
5845            .await?;
5846            let conflict = if merged.ok() {
5847                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5848                self.state.event(
5849                    "land",
5850                    format!("merged {tracking} into {} before verifying", winner.branch),
5851                );
5852                None
5853            } else {
5854                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5855                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5856                }
5857                let why = format!(
5858                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5859                    winner.branch,
5860                    merged.stderr.chars().take(600).collect::<String>()
5861                );
5862                self.state.status = RunStatus::Blocked;
5863                self.state.event("land", &why);
5864                Some(why)
5865            };
5866            let synced = conflict.is_none();
5867            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5868            self.state
5869                .finish_base_operation(synced.then_some(fresh.as_str()));
5870            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5871            self.state.base_sync = Some(BaseSync {
5872                tip,
5873                behind: if synced { 0 } else { behind },
5874                attempts,
5875                conflict,
5876                already_in: None,
5877            });
5878            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5879                .await?;
5880            self.state.save()?;
5881            return Ok(());
5882        }
5883
5884        self.state.event(
5885            "land",
5886            format!(
5887                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5888                winner.branch
5889            ),
5890        );
5891        self.state.save()?;
5892
5893        // The remote's copy of the branch, read now and only if the fetch
5894        // really succeeded (a stale tracking ref must never pin a lease). It is
5895        // pushed over after a rebase only when it is a commit this branch
5896        // already contains, by ancestry or by patch (an earlier rebase of ours
5897        // that never reached the remote): anything else is somebody else's work.
5898        let branch_tracking = format!("{remote}/{}", winner.branch);
5899        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5900        let remote_tip = if self.state.task_pr_branch().is_none()
5901            && matches!(&fetched_branch, Ok(o) if o.ok())
5902        {
5903            git::rev_parse(&repo, &branch_tracking).await.ok()
5904        } else {
5905            None
5906        };
5907        // A remote tip this branch does not contain is somebody else's work:
5908        // rebasing would leave a local tip that can never be pushed. Stop
5909        // before touching anything and say so.
5910        if let Some(theirs) = &remote_tip
5911            && !git::is_ancestor(&repo, theirs, &head).await
5912            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5913                .await
5914                .is_ok_and(|missing| missing.is_empty())
5915        {
5916            let why = format!(
5917                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5918                 them",
5919                short(theirs),
5920                winner.branch
5921            );
5922            self.state.status = RunStatus::Blocked;
5923            self.state.base_sync = Some(BaseSync {
5924                tip,
5925                behind,
5926                attempts,
5927                conflict: Some(why.clone()),
5928                already_in: None,
5929            });
5930            self.state.event("land", why);
5931            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5932                .await?;
5933            self.state.save()?;
5934            return Ok(());
5935        }
5936
5937        if self.state.base_sync.is_none() {
5938            self.state.base_sync = Some(BaseSync {
5939                tip: tip.clone(),
5940                behind,
5941                attempts: 0,
5942                conflict: None,
5943                already_in: None,
5944            });
5945        }
5946        let scratch = self.state.dir().join("base-sync");
5947        if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5948        {
5949            if error.is::<crate::repo_lock::ResourceBlocked>() {
5950                return Ok(());
5951            }
5952            return Err(error);
5953        }
5954        self.state.reserve_base_operation(&head, &tip);
5955        self.state.save()?;
5956        let rebased =
5957            match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
5958                .await
5959            {
5960                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5961                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5962                Err(e) => Err(e),
5963            };
5964        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5965        let stop_kind = match &rebased {
5966            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5967            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5968            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5969        };
5970        match rebased {
5971            Ok(None) => {
5972                // The branch ref moved, but a worktree that already had it
5973                // checked out (the winner's) was not told; sync its index and
5974                // files before anything reads them.
5975                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5976                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5977                self.state.finish_base_operation(Some(&fresh));
5978                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5979                let mut conflict = None;
5980                if let Some(pinned) = &remote_tip {
5981                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5982                    match pushed {
5983                        Ok(o) if o.ok() => self.state.event(
5984                            "land",
5985                            format!("pushed rebased {} to {remote}", winner.branch),
5986                        ),
5987                        Ok(o) => {
5988                            conflict = Some(format!(
5989                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5990                                winner.branch,
5991                                short(pinned),
5992                                o.stderr.chars().take(600).collect::<String>()
5993                            ));
5994                        }
5995                        Err(e) => {
5996                            conflict = Some(format!(
5997                                "rebased {} locally but could not push it: {e:#}",
5998                                winner.branch
5999                            ));
6000                        }
6001                    }
6002                }
6003                if let Some(why) = &conflict {
6004                    self.state.status = RunStatus::Blocked;
6005                    self.state.event("land", why.clone());
6006                }
6007                self.state.base_sync = Some(BaseSync {
6008                    tip: tip.clone(),
6009                    behind: 0,
6010                    attempts,
6011                    conflict,
6012                    already_in: None,
6013                });
6014                self.state
6015                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6016            }
6017            Ok(Some(conflict)) => {
6018                self.state.finish_base_operation(None);
6019                let why = format!(
6020                    "{} conflicts with {tracking} and did not rebase: {}",
6021                    winner.branch,
6022                    conflict.chars().take(600).collect::<String>()
6023                );
6024                self.state.status = RunStatus::Blocked;
6025                self.state.base_sync = Some(BaseSync {
6026                    tip,
6027                    behind,
6028                    attempts,
6029                    conflict: Some(why.clone()),
6030                    already_in: None,
6031                });
6032                self.state.event("land", why);
6033            }
6034            // A refused Git write or a busy repository: nothing is wrong with
6035            // the branch. The scratch rebase stays standing and the run
6036            // resumes into it (`base_sync` must not claim a conflict).
6037            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6038                self.state.save()?;
6039                return Ok(());
6040            }
6041            Err(e) => {
6042                self.state.finish_base_operation(None);
6043                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6044                self.state.status = RunStatus::Blocked;
6045                self.state.base_sync = Some(BaseSync {
6046                    tip,
6047                    behind,
6048                    attempts,
6049                    conflict: Some(why.clone()),
6050                    already_in: None,
6051                });
6052                self.state.event("land", why);
6053            }
6054        }
6055        if !self.state.base_sync_blocked() {
6056            self.state.base_sync_recovery.stop = None;
6057        }
6058        self.hold_base_sync_kind(stop_kind).await?;
6059        self.state.save()?;
6060        Ok(())
6061    }
6062
6063    /// Push a branch an earlier attempt rebased locally but never published,
6064    /// pinned to the remote tip read right after a successful fetch. Returns
6065    /// the reason when the run must stop; `None` when there was nothing to do
6066    /// (no remote copy, the same tip, or a remote copy this branch already
6067    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6068    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6069        // Task competitions publish only after the PR text gate, to their
6070        // public head; never resurrect a candidate branch on the remote.
6071        if self.state.task_pr_branch().is_some() {
6072            return None;
6073        }
6074        let repo = self.state.repo.clone();
6075        let remote = self.state.config.merge.remote.clone();
6076        let fetched = git::fetch(&repo, &remote, branch).await;
6077        if !matches!(&fetched, Ok(o) if o.ok()) {
6078            return None;
6079        }
6080        let branch_tracking = format!("{remote}/{branch}");
6081        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6082        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6083            return None;
6084        }
6085        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6086            .await
6087            .is_ok_and(|missing| missing.is_empty())
6088        {
6089            return Some(format!(
6090                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6091                 them",
6092                short(&theirs)
6093            ));
6094        }
6095        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6096            Ok(o) if o.ok() => {
6097                self.state
6098                    .event("land", format!("pushed rebased {branch} to {remote}"));
6099                None
6100            }
6101            Ok(o) => Some(format!(
6102                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6103                 someone may have pushed): {}",
6104                short(&theirs),
6105                o.stderr.chars().take(600).collect::<String>()
6106            )),
6107            Err(e) => Some(format!(
6108                "{branch} is rebased locally but could not be pushed: {e:#}"
6109            )),
6110        }
6111    }
6112
6113    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6114    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6115    /// whether it did.
6116    ///
6117    /// Checked only when the base is ahead of the branch. A failing check is
6118    /// "not proven" - the ordinary rebase path then decides - never a reason to
6119    /// stop the run.
6120    ///
6121    /// The remote copy of the branch is held to the same standard as the local
6122    /// one: if it carries a tip this worktree does not, that tip must itself be
6123    /// proven in the base, or nothing is settled (a pull request would
6124    /// otherwise be closed over commits nobody checked). The pull request is
6125    /// closed *before* the terminal status is saved; if that fails for a
6126    /// reason other than a refusal (no network, a `gh` error) the run is left
6127    /// `Blocked` with the reason as its conflict, which a resume retries -
6128    /// the same recovery a phantom conflict gets.
6129    async fn settle_already_in(
6130        &mut self,
6131        branch: &str,
6132        tip: &str,
6133        head: &str,
6134        attempts: usize,
6135        behind: usize,
6136    ) -> Result<bool> {
6137        let repo = self.state.repo.clone();
6138        let remote = self.state.config.merge.remote.clone();
6139        let start = self.state.base_commit.clone();
6140        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6141            Ok(Some(e)) => e,
6142            Ok(None) => return Ok(false),
6143            Err(e) => {
6144                tracing::warn!("already-in-base check for {branch}: {e:#}");
6145                return Ok(false);
6146            }
6147        };
6148        let mut verified = vec![head.to_owned()];
6149        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6150        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6151        if matches!(&fetched, Ok(o) if o.ok())
6152            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6153            && theirs != head
6154        {
6155            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6156                Ok(Some(_)) => verified.push(theirs),
6157                _ => return Ok(false),
6158            }
6159        }
6160        let base_branch = self.state.base_branch.clone();
6161        let message = format!(
6162            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6163             land",
6164            evidence.names(),
6165            evidence.proof.as_str()
6166        );
6167        let closed =
6168            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6169                .await;
6170        match closed {
6171            Ok(Ok(url)) => self
6172                .state
6173                .event("land", format!("closed {url}: superseded on {base_branch}")),
6174            Ok(Err(why)) => self
6175                .state
6176                .event("land", format!("did not close a pull request: {why}")),
6177            Err(e) => {
6178                let why = format!(
6179                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6180                     not be closed ({e:#}); resume to retry"
6181                );
6182                self.state.status = RunStatus::Blocked;
6183                self.state.base_sync = Some(BaseSync {
6184                    tip: tip.to_owned(),
6185                    behind,
6186                    attempts,
6187                    conflict: Some(why.clone()),
6188                    already_in: None,
6189                });
6190                self.state.event("land", why);
6191                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6192                    .await?;
6193                self.state.save()?;
6194                return Ok(true);
6195            }
6196        }
6197        self.state.status = RunStatus::AlreadyInBase;
6198        self.state.base_sync_recovery.stop = None;
6199        self.state.base_sync = Some(BaseSync {
6200            tip: tip.to_owned(),
6201            behind,
6202            attempts,
6203            conflict: None,
6204            already_in: Some(evidence),
6205        });
6206        self.state.event("land", message);
6207        self.state.save()?;
6208        self.settle_questions();
6209        Ok(true)
6210    }
6211
6212    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6213    /// last landed the winner on, once it has run, else the commit the run
6214    /// branched from.
6215    ///
6216    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6217    /// and `vote` all happen before there is a winner to rebase, so they
6218    /// compare every candidate against the branch point on purpose, and a
6219    /// base that moves after they are already done cannot change an answer
6220    /// they already gave.
6221    fn landing_base(&self) -> String {
6222        self.state
6223            .base_sync
6224            .as_ref()
6225            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6226    }
6227
6228    // ------------------------------------------------------- operator fix
6229
6230    /// Route specific, already-recorded review findings to a fixer for a
6231    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6232    /// entry point.
6233    ///
6234    /// Distinct from `review_loop`'s own fix step in three ways: it never
6235    /// runs a reviewer wave, it never spends review-round budget, and what
6236    /// happened is recorded as an [`OperatorFixRequest`] appended to
6237    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6238    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6239    /// and vote must never be rewritten to look like a manufactured blocking
6240    /// verdict.
6241    ///
6242    /// Only meaningful once review has actually concluded: `Ready` (handed
6243    /// off with findings still open, or simply concluded clean while minor
6244    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6245    /// gate failed). Everything else is refused: a run still in progress
6246    /// should simply be resumed, and a `Merged` run's branch has already
6247    /// landed — reopening *this* run's own record cannot change that, so the
6248    /// answer there is a fresh `magi review <branch>`.
6249    ///
6250    /// A real commit here re-verifies through a fresh, ordinary review-only
6251    /// run on the same branch ([`Self::review`]) rather than reopening this
6252    /// run's own `review_loop`: once any round in this run's history went
6253    /// clean, `review_conclusion` treats that as permanent by design (the
6254    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6255    /// way to force one more genuine reviewer wave out of *this* run without
6256    /// either rewriting history or weakening that guarantee for every other
6257    /// caller. A review-only run costs nothing extra — no implementation, no
6258    /// judging, no vote — and exercises the exact same review → verify →
6259    /// gate → (human) merge path, unmodified.
6260    pub async fn fix_selected(
6261        &mut self,
6262        ids: &[String],
6263        reason: &str,
6264        allow_stale: bool,
6265    ) -> Result<()> {
6266        let reason = reason.trim();
6267        if reason.is_empty() {
6268            bail!("a fix request needs a reason — that is the operator's own record of why");
6269        }
6270        if ids.is_empty() {
6271            bail!("no finding id given");
6272        }
6273        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6274            bail!(
6275                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6276                 has already concluded — can be given a targeted fix. A run still \
6277                 in progress should simply be resumed; a `merged` run's branch has \
6278                 already landed, so its answer is a fresh `magi review <branch>`, \
6279                 not reopening this run's own record",
6280                self.state.id,
6281                self.state.status.as_str()
6282            );
6283        }
6284        // Reassignment shares this claim through its atomic task save. Check
6285        // the canonical run identity inside it, before preflight or any await.
6286        let _claim = FixClaim::acquire(&self.state.dir())?;
6287        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6288        let Some(winner) = self.state.winner().cloned() else {
6289            bail!("run {} has no winning candidate to fix", self.state.id);
6290        };
6291        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6292            bail!(
6293                "branch `{}` no longer exists; this run cannot be extended",
6294                winner.branch
6295            );
6296        }
6297        let home = crate::run::home();
6298        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6299            bail!(
6300                "run {} is currently being worked on by another magi process",
6301                self.state.id
6302            );
6303        }
6304        // Resolve every id before spending anything — an unknown id refuses
6305        // the whole request rather than silently dropping it — and dedup
6306        // while keeping the operator's own order.
6307        let mut seen = BTreeSet::new();
6308        let mut findings = Vec::new();
6309        let mut missing = Vec::new();
6310        for id in ids {
6311            if !seen.insert(id.clone()) {
6312                continue;
6313            }
6314            match self.state.finding(id) {
6315                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6316                    id: f.id.clone(),
6317                    severity: f.severity,
6318                    reviewer_vote: rec.vote,
6319                    round: round.round,
6320                    round_head: round.head.clone(),
6321                    reviewer: rec.reviewer,
6322                    agent: rec.agent.clone(),
6323                    file: f.file.clone(),
6324                    line: f.line,
6325                    title: f.title.clone(),
6326                    detail: f.detail.clone(),
6327                    outcome: OperatorFixOutcome::Pending,
6328                }),
6329                None => missing.push(id.clone()),
6330            }
6331        }
6332        if !missing.is_empty() {
6333            bail!(
6334                "unknown finding id(s): {}; nothing was changed",
6335                missing.join(", ")
6336            );
6337        }
6338
6339        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6340        let stale_details: Vec<(String, String)> = findings
6341            .iter()
6342            .filter(|f| f.round_head != head_at_request)
6343            .map(|f| (f.id.clone(), f.round_head.clone()))
6344            .collect();
6345        let stale = !stale_details.is_empty();
6346        if stale && !allow_stale {
6347            bail!(
6348                "the branch has moved since some finding(s) were raised — {} — now \
6349                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6350                stale_details
6351                    .iter()
6352                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6353                    .collect::<Vec<_>>()
6354                    .join(", "),
6355                short(&head_at_request)
6356            );
6357        }
6358
6359        let request = OperatorFixRequest {
6360            requested_at: Timestamp::now(),
6361            reason: reason.to_owned(),
6362            findings,
6363            head_at_request: head_at_request.clone(),
6364            allow_stale,
6365            stale,
6366            fix: None,
6367            result_head: None,
6368            follow_up_review_run: None,
6369        };
6370        self.state.event(
6371            "fix",
6372            format!(
6373                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6374                request.findings.len(),
6375                request
6376                    .findings
6377                    .iter()
6378                    .map(|f| f.id.as_str())
6379                    .collect::<Vec<_>>()
6380                    .join(", "),
6381            ),
6382        );
6383        // Recorded now, before any worktree work or the fixer call itself —
6384        // and re-saved at each checkpoint below: a crash at any point after
6385        // this (mid fixer call, mid follow-up review) must not lose the fact
6386        // that this was requested, for which findings, and why. Everything
6387        // past this point reads and writes through `request_index` rather
6388        // than a local variable, since `request` itself is moved here.
6389        self.state.operator_fixes.push(request);
6390        self.state.save()?;
6391        let request_index = self.state.operator_fixes.len() - 1;
6392
6393        // A fresh, dedicated worktree for this one call, never the winner's
6394        // own worktree in place: that one may already be gone (folded away),
6395        // and reusing it in place would leave the branch checked out there
6396        // when the follow-up review below tries to check it out again. Freed
6397        // immediately after, either way — but only once confirmed clean:
6398        // `worktree_remove` is a `git worktree remove --force`, which would
6399        // otherwise discard uncommitted work left there by the operator or
6400        // another process before this had a chance to even look at it.
6401        let repo = self.state.repo.clone();
6402        let coordination = self.coordinate(&repo).await?;
6403        if winner.worktree.exists() {
6404            // Lockfiles a rescue commit withheld stay untracked on purpose and
6405            // are already recorded; they are not the operator's work to protect.
6406            let dirty = git::git(
6407                &winner.worktree,
6408                &["status", "--porcelain", "--untracked-files=all"],
6409            )
6410            .await?;
6411            let only_withheld = dirty.lines().all(|l| {
6412                l.strip_prefix("?? ")
6413                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6414            });
6415            if !only_withheld {
6416                bail!(
6417                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6418                     discard them first",
6419                    winner.worktree.display()
6420                );
6421            }
6422            git::worktree_remove(&self.state.repo, &winner.worktree)
6423                .await
6424                .ok();
6425        }
6426        let fix_worktree = self.state.worktree_root().join("operator-fix");
6427        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6428        git::git(
6429            &self.state.repo,
6430            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6431        )
6432        .await
6433        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6434        if !git::is_clean(&fix_worktree).await? {
6435            git::worktree_remove(&self.state.repo, &fix_worktree)
6436                .await
6437                .ok();
6438            bail!(
6439                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6440                winner.branch
6441            );
6442        }
6443        let repo_for_setup = self.state.repo.clone();
6444        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6445            Ok(()) => {
6446                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6447            }
6448            Err(e) => Err(e),
6449        };
6450        if let Err(e) = setup {
6451            git::worktree_remove(&self.state.repo, &fix_worktree)
6452                .await
6453                .ok();
6454            return Err(e);
6455        }
6456
6457        // This explicit operator request retries availability independently
6458        // of execute(); the previous call has already saved and cleaned up.
6459        self.state.resource_block = None;
6460
6461        let run_id = self.state.id.clone();
6462        let prompts = self.state.config.prompts.clone();
6463        let language = self.state.config.graph.language.clone();
6464        let sessions = self.state.config.graph.sessions;
6465        let artifacts = agent::artifacts_dir(&self.state.dir());
6466        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6467            .findings
6468            .iter()
6469            .map(|f| Finding {
6470                id: f.id.clone(),
6471                severity: f.severity,
6472                file: f.file.clone(),
6473                line: f.line,
6474                title: f.title.clone(),
6475                detail: f.detail.clone(),
6476            })
6477            .collect();
6478        let fix_prompt = prompt::operator_fix_in(
6479            &self.state.instruction,
6480            &finding_list,
6481            reason,
6482            &stale_details,
6483            &head_at_request,
6484            &language,
6485            &self.state.config.graph.github_language,
6486        );
6487        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6488        drop(coordination);
6489        let (job, seat, out) = self
6490            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6491                prompt: fix_prompt.clone(),
6492                spec,
6493                seat,
6494                cwd: fix_worktree.clone(),
6495                timeout,
6496                allow_write: true,
6497                sessions,
6498                artifacts: artifacts.clone(),
6499                stem: "operator-fix".to_owned(),
6500                handover: None,
6501            })
6502            .await;
6503        let agent_id = seat.agent.clone();
6504
6505        let mut fix = FixRecord {
6506            agent: agent_id,
6507            addressed: Vec::new(),
6508            rejected: Vec::new(),
6509            notes: String::new(),
6510            committed: false,
6511            failed: None,
6512            duration_ms: 0,
6513            continuation: None,
6514        };
6515        let mut final_seat = seat.clone();
6516        match out {
6517            AgentOutcome::Ok(o) => {
6518                fix.duration_ms = o.duration_ms;
6519                let parsed = verdict::extract_json::<FixReport>(&o.text);
6520                let incomplete_reason = match &parsed {
6521                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6522                        "the reply parsed, but it reported a command whose own CLI \
6523                         never confirmed an exit status"
6524                            .to_owned(),
6525                    ),
6526                    Ok(_) => None,
6527                    Err(e) => Some(e.to_string()),
6528                };
6529                match incomplete_reason {
6530                    None => {
6531                        let report = parsed.expect("checked Ok above");
6532                        fix.addressed = report.addressed;
6533                        fix.rejected = report.rejected;
6534                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6535                    }
6536                    Some(reason) => {
6537                        let (resumed_seat, resolved, failure, cont) = self
6538                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6539                            .await;
6540                        fix.duration_ms += cont.cumulative_wait_ms;
6541                        fix.continuation = Some(cont);
6542                        final_seat = resumed_seat;
6543                        match resolved {
6544                            Some(report) => {
6545                                fix.addressed = report.addressed;
6546                                fix.rejected = report.rejected;
6547                                fix.notes =
6548                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6549                            }
6550                            None => fix.failed = failure,
6551                        }
6552                    }
6553                }
6554            }
6555            AgentOutcome::Dropped(o) => {
6556                fix.duration_ms = o.duration_ms;
6557                let why = o
6558                    .dropped
6559                    .as_ref()
6560                    .map(|d| d.why.as_str())
6561                    .unwrap_or("the CLI ended the stream without delivering its answer");
6562                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6563            }
6564            AgentOutcome::Quota(o) => {
6565                self.state.quota.push(QuotaLoss::from_report(
6566                    final_seat.key.clone(),
6567                    "fix",
6568                    o.quota.as_ref(),
6569                ));
6570                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6571            }
6572            AgentOutcome::Failed(e) => fix.failed = Some(e),
6573            AgentOutcome::ResourceBlocked(e) => {
6574                fix.failed = Some(format!("the fixer was not started: {e}"));
6575            }
6576        }
6577        // Even a deferred call must rescue edits, release the worktree and
6578        // finalize its record below so an explicit `magi fix` can retry.
6579        if fix.continuation.is_none() {
6580            fix.continuation = Some(ContinuationRecord::not_needed());
6581        }
6582        self.state.seats.insert(final_seat.key.clone(), final_seat);
6583
6584        let rescue_message = format!(
6585            "magi: operator-selected fix ({}) (uncommitted work)",
6586            self.state.operator_fixes[request_index]
6587                .findings
6588                .iter()
6589                .map(|f| f.id.as_str())
6590                .collect::<Vec<_>>()
6591                .join(", ")
6592        );
6593        let repo = self.state.repo.clone();
6594        let coordination = self.coordinate(&repo).await?;
6595        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6596            self.state.note_withheld("fix", &r.withheld);
6597        }
6598        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6599        fix.committed = after != head_at_request;
6600        git::worktree_remove(&self.state.repo, &fix_worktree)
6601            .await
6602            .ok();
6603        drop(coordination);
6604
6605        self.state.event(
6606            "fix",
6607            match &fix.failed {
6608                Some(reason) => format!(
6609                    "operator fix: adoption report was lost ({reason}); {}",
6610                    if fix.committed {
6611                        "committed"
6612                    } else {
6613                        "NO new commit"
6614                    }
6615                ),
6616                None => format!(
6617                    "operator fix: {} addressed, {} rejected, {}",
6618                    fix.addressed.len(),
6619                    fix.rejected.len(),
6620                    if fix.committed {
6621                        "committed"
6622                    } else {
6623                        "NO new commit"
6624                    }
6625                ),
6626            },
6627        );
6628
6629        // Every selected finding gets an outcome — never left `Pending` once
6630        // the fixer's own turn is over. A report that never came back at all
6631        // marks every one of them `Unreported`, not silently "not addressed":
6632        // quota, a dropped stream, or an exhausted continuation are gaps in
6633        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6634        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6635        for f in &mut self.state.operator_fixes[request_index].findings {
6636            f.outcome = if fix.failed.is_some() {
6637                OperatorFixOutcome::Unreported
6638            } else if fix.addressed.contains(&f.id) {
6639                OperatorFixOutcome::Addressed
6640            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6641                OperatorFixOutcome::Rejected { why: r.why.clone() }
6642            } else {
6643                OperatorFixOutcome::Unreported
6644            };
6645        }
6646
6647        let committed = fix.committed;
6648        if committed {
6649            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6650        }
6651        self.state.operator_fixes[request_index].fix = Some(fix);
6652        // Saved again now that the fixer's own outcome is final, on top of
6653        // the save right after the request was first pushed above.
6654        self.state.save()?;
6655        if self.state.resource_block.is_some() {
6656            return self.stop_for_resource();
6657        }
6658
6659        if committed {
6660            self.state.event(
6661                "fix",
6662                format!(
6663                    "operator fix committed {}; opening a follow-up review-only run",
6664                    short(&after)
6665                ),
6666            );
6667            // The operator asked for the fix, and the follow-up serves whatever
6668            // task the run it follows served.
6669            let origin =
6670                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6671            match Self::review(
6672                &self.state.repo,
6673                &winner.branch,
6674                self.state.config.clone(),
6675                origin,
6676            )
6677            .await
6678            {
6679                Ok(mut follow_up) => {
6680                    follow_up.state.event(
6681                        "start",
6682                        format!(
6683                            "requested by an operator fix on run {} for finding(s) {}",
6684                            self.state.id,
6685                            self.state.operator_fixes[request_index]
6686                                .findings
6687                                .iter()
6688                                .map(|f| f.id.as_str())
6689                                .collect::<Vec<_>>()
6690                                .join(", "),
6691                        ),
6692                    );
6693                    follow_up.state.save()?;
6694                    let follow_up_id = follow_up.state.id.clone();
6695                    // The follow-up is a run like any other: it belongs to the
6696                    // task of the run it follows, or to one filed for it.
6697                    let adopted = match crate::direct::adopt(
6698                        &follow_up.state,
6699                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6700                    ) {
6701                        Ok(a) => a,
6702                        Err(e) => {
6703                            // An ownerless run must not spend agent calls; it
6704                            // stays saved, and `magi run --resume` adopts it.
6705                            self.state.event(
6706                                "fix",
6707                                format!(
6708                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6709                                ),
6710                            );
6711                            self.state.save()?;
6712                            return Ok(());
6713                        }
6714                    };
6715                    let executed = follow_up.execute().await;
6716                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6717                    if let Some(a) = adopted {
6718                        a.finish(&follow_up.state, executed);
6719                    }
6720                    if let Some(e) = failure {
6721                        self.state.event(
6722                            "fix",
6723                            format!(
6724                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6725                            ),
6726                        );
6727                    }
6728                    self.state.operator_fixes[request_index].follow_up_review_run =
6729                        Some(follow_up_id);
6730                }
6731                Err(e) => {
6732                    self.state.event(
6733                        "fix",
6734                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6735                    );
6736                }
6737            }
6738            self.state.save()?;
6739        }
6740
6741        Ok(())
6742    }
6743
6744    // --------------------------------------------------------------- review
6745
6746    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6747    /// names, in order, each at most once, moving on only when the call
6748    /// advances (an error, a quota hit or nothing usable - the same decision
6749    /// point `agent::chain_advances` is for the other chained roles).
6750    ///
6751    /// Each agent gets its own seat from `Runner::seat`: the same agent
6752    /// continues its conversation (the winner's own implementer seat when it
6753    /// is the winner's author, now that the competition is over), another
6754    /// takes a fresh one so the full prompt is sent again. A handover is
6755    /// recorded under `node`, which is what makes the fallback stick for the
6756    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6757    /// is returned, so an exhausted chain reads exactly like a single failed
6758    /// fixer: one quota loss, the same wording, the same refund. An earlier
6759    /// attempt's edits are left in the tree and judged, with the final
6760    /// attempt's, by what git says afterwards.
6761    ///
6762    /// Returns the job that produced the outcome, for `continue_fix_report`.
6763    async fn ask_fixer(
6764        &mut self,
6765        winner: &Candidate,
6766        node: &'static str,
6767        round: Option<usize>,
6768        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6769    ) -> (SeatJob, SeatState, AgentOutcome) {
6770        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6771        self.ask_fixer_among(attempts, node, round, None, build)
6772            .await
6773    }
6774
6775    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6776    /// worktree and a commit: before the next agent is asked after one that
6777    /// did not satisfy the call, the worktree is put back on that commit, so
6778    /// a failed agent's partial edits are not carried into the next one.
6779    async fn ask_fixer_among(
6780        &mut self,
6781        attempts: Vec<(AgentSpec, String)>,
6782        node: &'static str,
6783        round: Option<usize>,
6784        restore: Option<(&Path, &str)>,
6785        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6786    ) -> (SeatJob, SeatState, AgentOutcome) {
6787        let run_id = self.state.id.clone();
6788        let prompts = self.state.config.prompts.clone();
6789        let cache = self.state.config.cache_dir();
6790        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6791        let mut last = None;
6792        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6793            let seat = self.seat(&key, &spec.id);
6794            let mut job = build(spec.clone(), seat);
6795            if i > 0 {
6796                job.stem = format!("{}-{}", job.stem, spec.id);
6797            }
6798            let ctx = WaveCtx {
6799                carry_seats: false,
6800                pauses: Some((&self.pause, &self.interrupt)),
6801                run: &run_id,
6802                node,
6803                prompts: &prompts,
6804                cache: cache.as_deref(),
6805                round,
6806            };
6807            let (seat, out) =
6808                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6809            // A resource block never advances the chain: the chosen agent
6810            // was not asked, so it must be the one asked again on resume.
6811            let advances = match &out {
6812                AgentOutcome::Ok(o) => agent::output_advances(o),
6813                AgentOutcome::ResourceBlocked(_) => false,
6814                _ => true,
6815            };
6816            if let Some(next) = ids.get(i + 1)
6817                && advances
6818            {
6819                self.state.seats.insert(seat.key.clone(), seat.clone());
6820                let class =
6821                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6822                let reason = match &out {
6823                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6824                    other => fail_reason(other),
6825                };
6826                // A failed agent's edits must not reach the next one, and a
6827                // restore that did not work ends the chain: the call fails
6828                // and the caller's own (confirmed) restore decides.
6829                if let Some((wt, rev)) = restore
6830                    && let Err(e) = self.graft_restore(wt, rev).await
6831                {
6832                    let why =
6833                        format!("could not restore the worktree before the next fixer: {e:#}");
6834                    tracing::warn!("{why}");
6835                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6836                        AgentOutcome::ResourceBlocked(why)
6837                    } else {
6838                        AgentOutcome::Failed(why)
6839                    };
6840                    last = Some((job, seat, out));
6841                    break;
6842                }
6843                record_handover(
6844                    &mut self.state,
6845                    node,
6846                    &seat.key,
6847                    &spec.id,
6848                    next,
6849                    &class,
6850                    &reason,
6851                );
6852                continue;
6853            }
6854            last = Some((job, seat, out));
6855            break;
6856        }
6857        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6858        if let AgentOutcome::Quota(output) = &out
6859            && node != "graft"
6860        {
6861            // Graft is best effort: its caller records the loss and restores
6862            // the pre-graft tree, then proceeds to review and gate.
6863            // Every fallback was tried. Machine outages cannot be repaired by
6864            // spending a fix round; use each caller's resumable resource path.
6865            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6866            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6867            self.state.quota.push(loss);
6868            self.state.seats.insert(seat.key.clone(), seat.clone());
6869            self.state.status = RunStatus::Blocked;
6870            return (
6871                job,
6872                seat,
6873                AgentOutcome::ResourceBlocked(
6874                    "fixer unavailable; restore agent availability and resume".into(),
6875                ),
6876            );
6877        }
6878        (job, seat, out)
6879    }
6880
6881    /// The one repair a verification-only failure earns (see
6882    /// [`crate::recovery`]), before the first review round of a run that took
6883    /// a branch over.
6884    ///
6885    /// Saved as started *before* the fixer is called, so a resume never
6886    /// repairs twice; a start found on re-entry is rolled back to the commit
6887    /// it began from. A failed or unusable fixer leaves the tree as it was and
6888    /// the review goes on over it, so the first round's own verification
6889    /// reports the failure again. The failure is never carried into a round as
6890    /// a result: verification is re-run on whatever commit this leaves.
6891    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6892        use crate::recovery::{Mode, Repair};
6893        if !self.state.reviews.is_empty() {
6894            return Ok(());
6895        }
6896        let Some(mut rec) = self.state.recovery.clone() else {
6897            return Ok(());
6898        };
6899        if rec.mode != Mode::VerifyRepair {
6900            return Ok(());
6901        }
6902        let wt = winner.worktree.clone();
6903        let mut head = git::rev_parse(&wt, "HEAD").await?;
6904        match rec.repair.clone() {
6905            Repair::Done { .. } => return Ok(()),
6906            Repair::WaitingForRestore { before } => {
6907                self.graft_restore(&wt, &before).await?;
6908                head = before;
6909                rec.repair = Repair::NotStarted;
6910                self.state.recovery = Some(rec.clone());
6911                self.state.save()?;
6912            }
6913            Repair::Started { before } => {
6914                // Interrupted: the unfinished edits are not trusted.
6915                // An unconfirmed rollback blocks the run with `Started` kept,
6916                // so a resume retries it; the tree is never reviewed as is.
6917                if let Err(e) = self.graft_restore(&wt, &before).await {
6918                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6919                        return Err(e);
6920                    }
6921                    return self
6922                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6923                        .await;
6924                }
6925                rec.repair = Repair::Done {
6926                    before: before.clone(),
6927                    after: before,
6928                    committed: false,
6929                    failed: Some("interrupted; rolled back".to_owned()),
6930                };
6931                self.state
6932                    .event("recovery", "the repair was interrupted and rolled back");
6933                self.state.recovery = Some(rec);
6934                self.state.save()?;
6935                return Ok(());
6936            }
6937            Repair::NotStarted => {}
6938        }
6939        if crate::recovery::recheck(&mut rec, &head) {
6940            self.state.event("recovery", rec.summary());
6941            self.state.recovery = Some(rec);
6942            self.state.save()?;
6943            return Ok(());
6944        }
6945        rec.repair = Repair::Started {
6946            before: head.clone(),
6947        };
6948        self.state.recovery = Some(rec.clone());
6949        self.state.save()?;
6950
6951        let language = self.state.config.graph.language.clone();
6952        let sessions = self.state.config.graph.sessions;
6953        let artifacts = agent::artifacts_dir(&self.state.dir());
6954        let fix_prompt = prompt::recovery_fix(
6955            &self.state.instruction,
6956            &rec,
6957            self.state.config.graph.review_rounds,
6958            &language,
6959        );
6960        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6961        self.state.event(
6962            "recovery",
6963            format!(
6964                "repair: the fixer starts from the failed verification on {}",
6965                short(&head)
6966            ),
6967        );
6968        let (_job, seat, out) = self
6969            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6970                prompt: fix_prompt.clone(),
6971                spec,
6972                seat,
6973                cwd: wt.clone(),
6974                timeout,
6975                allow_write: true,
6976                sessions,
6977                artifacts: artifacts.clone(),
6978                stem: "fix-recovery".to_owned(),
6979                handover: None,
6980            })
6981            .await;
6982        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6983            rec.repair = Repair::WaitingForRestore { before: head };
6984            self.state.recovery = Some(rec);
6985            return self.stop_for_resource();
6986        }
6987        let failed = match &out {
6988            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6989            AgentOutcome::Quota(o) => {
6990                self.state.quota.push(QuotaLoss::from_report(
6991                    seat.key.clone(),
6992                    "fix",
6993                    o.quota.as_ref(),
6994                ));
6995                Some("rate limited (quota); the fixer could not run".to_owned())
6996            }
6997            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6998            AgentOutcome::Failed(e) => Some(e.clone()),
6999        };
7000        self.state.seats.insert(seat.key.clone(), seat);
7001        let coordination = self.coordinate(&wt).await?;
7002        let failed = match failed {
7003            // The agent's partial edits are not kept.
7004            Some(why) => {
7005                if let Err(e) = git::restore_to(&wt, &head).await {
7006                    return self
7007                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7008                        .await;
7009                }
7010                Some(why)
7011            }
7012            None => {
7013                if let Ok(r) = git::rescue_commit(
7014                    &wt,
7015                    "magi: recovery repair of the failed verification (uncommitted work)",
7016                )
7017                .await
7018                {
7019                    self.state.note_withheld("fix", &r.withheld);
7020                }
7021                None
7022            }
7023        };
7024        let after = git::rev_parse(&wt, "HEAD").await?;
7025        drop(coordination);
7026        self.state.event(
7027            "recovery",
7028            match &failed {
7029                Some(why) => format!("repair: nothing kept ({why})"),
7030                None if after != head => {
7031                    format!("repair: committed {}", short(&after))
7032                }
7033                None => "repair: the fixer made no commit".to_owned(),
7034            },
7035        );
7036        rec.repair = Repair::Done {
7037            before: head.clone(),
7038            committed: after != head,
7039            after,
7040            failed,
7041        };
7042        self.state.recovery = Some(rec);
7043        self.state.save()?;
7044        Ok(())
7045    }
7046
7047    /// A repair whose rollback could not be confirmed: the run is blocked and
7048    /// resumable, and the unconfirmed state is never reviewed.
7049    async fn stop_recovery(
7050        &mut self,
7051        mut rec: crate::recovery::RecoveryRecord,
7052        head: &str,
7053        why: String,
7054    ) -> Result<()> {
7055        // `Started` stays recorded: a resume retries the restore.
7056        rec.repair = crate::recovery::Repair::Started {
7057            before: head.to_owned(),
7058        };
7059        self.state.recovery = Some(rec);
7060        self.state
7061            .event("recovery", format!("repair could not be undone: {why}"));
7062        self.state.status = RunStatus::Blocked;
7063        self.state.save()?;
7064        bail!("recovery repair could not be rolled back: {why}")
7065    }
7066
7067    async fn review_loop(&mut self) -> Result<()> {
7068        if self.hold_base_sync().await? {
7069            return Ok(());
7070        }
7071        loop {
7072            if self
7073                .state
7074                .base_sync
7075                .as_ref()
7076                .is_some_and(|s| s.conflict.is_some())
7077            {
7078                return Ok(());
7079            }
7080            if self.catchup_repairable() {
7081                let record = self.state.reviews.last().expect("repairable round").clone();
7082                let winner = self
7083                    .state
7084                    .winner()
7085                    .cloned()
7086                    .context("catch-up repair has no winner")?;
7087                if !self.fix_review_round(&winner, record).await? {
7088                    return Ok(());
7089                }
7090            }
7091            self.review_loop_pass().await?;
7092            if !self.catchup_repairable() {
7093                return Ok(());
7094            }
7095        }
7096    }
7097
7098    /// Derive an owed catch-up repair from durable round evidence, including
7099    /// reentry after a crash before the fixer was dispatched.
7100    fn catchup_repairable(&self) -> bool {
7101        self.state.reviews.len() < self.state.config.graph.review_rounds
7102            && self.state.reviews.last().is_some_and(|r| {
7103                !r.clean
7104                    && r.blocking == 0
7105                    && r.fix.is_none()
7106                    && r.e2e_status() == E2eStatus::Failed
7107            })
7108    }
7109
7110    async fn review_loop_pass(&mut self) -> Result<()> {
7111        // A base that would not rebase is a person's decision, not a review
7112        // round: nothing here would change the answer, and reviewers and a
7113        // fixer would be spending real budget on a tree that cannot land
7114        // regardless of what they find.
7115        if self
7116            .state
7117            .base_sync
7118            .as_ref()
7119            .is_some_and(|s| s.conflict.is_some())
7120        {
7121            return Ok(());
7122        }
7123        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7124        // agent files with `magi task add` name the run that paid for it. The
7125        // prompt overlay is cloned alongside it because the waves borrow it
7126        // while `self` is mutably borrowed by the node's own bookkeeping.
7127        let run_id = self.state.id.clone();
7128        let prompts = self.state.config.prompts.clone();
7129        let Some(winner) = self.state.winner().cloned() else {
7130            return Ok(());
7131        };
7132        let granted_review = self
7133            .state
7134            .repair_grants
7135            .last()
7136            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7137            .cloned();
7138        let fresh_review = granted_review
7139            .as_ref()
7140            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7141        let max_rounds = granted_review
7142            .as_ref()
7143            .map_or(self.state.config.graph.review_rounds, |g| {
7144                g.review_start.saturating_add(1)
7145            });
7146        // A clean round, an exhausted round budget, or a stalled tree (see
7147        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7148        // they are recorded — recomputed here, not read off `status`, so a
7149        // reentry into a run that already stopped restates the identical
7150        // verdict instead of silently handing back whatever an earlier node
7151        // in this same walk clobbered `status` to (a solo-candidate
7152        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7153        // below runs an empty range once the budget is spent, and would
7154        // otherwise fall through without touching `status` at all.
7155        if let Some(status) =
7156            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7157        {
7158            // A reentry after a crash between the last round's save and
7159            // `stop_reviewing` reaches the hand-off here, not there.
7160            if status == RunStatus::Gating {
7161                self.record_contested_handoff();
7162            }
7163            self.state.status = status;
7164            self.state.save()?;
7165            return Ok(());
7166        }
7167        self.state.status = RunStatus::Reviewing;
7168        // A last recorded round whose own verification never resolved
7169        // (`ResourceBlocked` — the shared build cache, not the patch) is
7170        // never a concluded round, whatever the round budget says: starting
7171        // a fresh round on top of it would spend a whole new reviewer wave
7172        // re-reading an unchanged patch instead of just retrying the one
7173        // check that actually needs it, and once the budget is spent the
7174        // loop below has nothing left to do at all (its range is empty).
7175        // Retry that check directly instead, exactly the same retry
7176        // `stop_reviewing` already does for its own catch-up case.
7177        if !fresh_review
7178            && self.state.reviews.last().is_some_and(|r| {
7179                r.e2e_status() == E2eStatus::ResourceBlocked
7180                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7181            })
7182        {
7183            let shell = self.state.config.shell();
7184            return self
7185                .stop_unresolved_review(
7186                    "the last round's own verification never resolved",
7187                    &shell,
7188                    &winner.worktree,
7189                )
7190                .await;
7191        }
7192
7193        if granted_review.is_none() {
7194            self.recovery_repair(&winner).await?;
7195        }
7196        if self.resource_halt()? {
7197            return Ok(());
7198        }
7199
7200        let repo = self.state.repo.clone();
7201        let root = self.state.worktree_root();
7202        let language = self.state.config.graph.language.clone();
7203        let sessions = self.state.config.graph.sessions;
7204        let artifacts = self.merge_repair_artifacts();
7205        let base = self.landing_base();
7206        let base_short = short(&base);
7207        let reviewers = self.roles.reviewers.clone();
7208        let shell = self.state.config.shell();
7209
7210        for round in (self.state.reviews.len() + 1)..=max_rounds {
7211            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7212            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7213            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7214            // The prior round's own record, already persisted — never a
7215            // hand-carried variable of just its failing output: that is
7216            // exactly what let a round's e2e result drift out of sync with
7217            // which commit it was actually about (see `SCHEMA`'s doc for
7218            // schema 8). Judged against `head`, the commit reviewers are
7219            // about to look at now, so the summary always reads as "an
7220            // earlier head" here — this round's own patch has not been
7221            // checked yet.
7222            let prev_verification = self
7223                .state
7224                .reviews
7225                .last()
7226                .and_then(|r| r.verification_summary(&head));
7227            // Only the first round of a run that took a branch over: later
7228            // rounds have this run's own history.
7229            let prior_section = match (&self.state.recovery, round) {
7230                (Some(rec), 1) if self.state.reviews.is_empty() => {
7231                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7232                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7233                            Some(crate::recovery::head_of(
7234                                &git::diff(&winner.worktree, reviewed, "HEAD")
7235                                    .await
7236                                    .unwrap_or_default(),
7237                                crate::recovery::SINCE_DIFF_MAX,
7238                            ))
7239                        }
7240                        _ => None,
7241                    };
7242                    Some(prompt::recovery_review(rec, since.as_deref()))
7243                }
7244                _ => None,
7245            };
7246
7247            // Each reviewer gets its own detached checkout of exactly this
7248            // commit: nobody can perturb the winner's tree, and the fixer can
7249            // keep working without racing a reviewer.
7250            let mut jobs = Vec::new();
7251            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7252                self.coordinate(&repo).await?
7253            } else {
7254                None
7255            };
7256            for (r, spec) in reviewers.iter().cloned().enumerate() {
7257                let wt = root.join(format!("review-{}", r + 1));
7258                // `clean -fdx` in a reset wipes setup's products, so every
7259                // round sets the seat up again.
7260                if wt.exists() {
7261                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7262                } else {
7263                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7264                        None
7265                    } else {
7266                        self.coordinate(&repo).await?
7267                    };
7268                    git::worktree_add_detached(&repo, &wt, &head).await?;
7269                }
7270                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7271                    None
7272                } else {
7273                    self.coordinate(&repo).await?
7274                };
7275                self.setup_seat_worktree(&repo, &wt, false).await?;
7276                let seat_key = format!("review-{}", r + 1);
7277                // The seat starts the round on whoever answered it last, not
7278                // on the agent the spec names, so a failure is not re-paid.
7279                let spec = pick_start_spec(
7280                    &self.roles.reviewer_roster,
7281                    spec,
7282                    self.state.seat_history.get(&seat_key),
7283                );
7284                let seat = self.seat(&seat_key, &spec.id);
7285                jobs.push(SeatJob {
7286                    prompt: prompt::review(&prompt::ReviewCtx {
7287                        instruction: &self.state.instruction,
7288                        branch: &winner.branch,
7289                        base_short: &base_short,
7290                        stat: &stat,
7291                        patch: &patch,
7292                        verification: prev_verification.as_ref(),
7293                        prior: prior_section.as_deref(),
7294                        reviewers: reviewers.len(),
7295                        round,
7296                        rounds: max_rounds,
7297                        // A review-only run has no rankings, so nothing
7298                        // competed for this patch and the reviewer is told so.
7299                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7300                        lens: Lens::for_seat(r),
7301                        language: &language,
7302                        github_language: &self.state.config.graph.github_language,
7303                    }),
7304                    spec,
7305                    seat,
7306                    cwd: wt,
7307                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7308                    allow_write: false,
7309                    sessions,
7310                    artifacts: artifacts.clone(),
7311                    stem: format!("review-{round}-{}", r + 1),
7312                    handover: None,
7313                });
7314            }
7315
7316            drop(coordination);
7317            self.state.event(
7318                "review",
7319                format!(
7320                    "round {round}: {} reviewers on {}",
7321                    jobs.len(),
7322                    short(&head)
7323                ),
7324            );
7325            let mut quota_losses = Vec::new();
7326            let review_retries = self.state.config.graph.retries;
7327            let review_cache = self.state.config.cache_dir();
7328            let ctx = WaveCtx {
7329                carry_seats: true,
7330                pauses: Some((&self.pause, &self.interrupt)),
7331                run: &run_id,
7332                node: "review",
7333                prompts: &prompts,
7334                cache: review_cache.as_deref(),
7335                round: Some(round),
7336            };
7337            let results = ask_json_wave::<Review>(
7338                jobs,
7339                Arc::clone(&self.sem),
7340                review_retries,
7341                &self.roles.reviewer_roster,
7342                &ctx,
7343                &mut quota_losses,
7344                &mut self.state,
7345                &|_: &Review| Ok(()),
7346            )
7347            .await;
7348            if self.state.resource_block.is_some() {
7349                return self.stop_for_resource();
7350            }
7351            // Counted before the move below: how many of *this* round's
7352            // reviewer seats were lost to their own rate limit, as opposed to
7353            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7354            let round_quota_missing = quota_losses.len();
7355            self.state.quota.extend(quota_losses);
7356
7357            let mut records = Vec::new();
7358            let mut all_findings = Vec::new();
7359            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7360                let agent_id = seat.agent.clone();
7361                self.state.seats.insert(seat.key.clone(), seat);
7362                let mut record = ReviewRecord {
7363                    reviewer: r + 1,
7364                    agent: agent_id,
7365                    summary: String::new(),
7366                    findings: Vec::new(),
7367                    vote: None,
7368                    failed: None,
7369                    duration_ms: 0,
7370                    // Set for both outcomes: `failed: Some(_)` with
7371                    // `attempts > 0` is a seat every retry still lost, not a
7372                    // recovered one — only `failed: None` with `attempts > 0`
7373                    // reads as "answered after a nudge" (see this field's own
7374                    // doc).
7375                    attempts,
7376                };
7377                match res {
7378                    Ok((review, out)) => {
7379                        // Sanitized here, at the point every other piece of
7380                        // agent prose in this file is (candidate summaries,
7381                        // deliberation turns, vote reasons): a reviewer's own
7382                        // words are the one thing about it that could name
7383                        // it, and reconsideration below broadcasts this same
7384                        // summary and these same findings to every other
7385                        // seat on the panel.
7386                        record.summary =
7387                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7388                        record.vote = Some(review.vote);
7389                        record.duration_ms = out.duration_ms;
7390                        for (n, mut f) in review.findings.into_iter().enumerate() {
7391                            // ids are magi's, never the agent's: the fixer's
7392                            // adoption report is keyed by them.
7393                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7394                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7395                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7396                            // `file` is agent-supplied prose too, never
7397                            // checked against the real tree — the same
7398                            // exposure `title`/`detail` above have, just in
7399                            // a field easy to forget because it looks like a
7400                            // path rather than free text.
7401                            f.file = f
7402                                .file
7403                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7404                            all_findings.push(f.clone());
7405                            record.findings.push(f);
7406                        }
7407                        self.state.event(
7408                            "review",
7409                            format!(
7410                                "round {round}: reviewer {} voted {} with {} finding(s)",
7411                                r + 1,
7412                                review.vote.label(),
7413                                record.findings.len()
7414                            ),
7415                        );
7416                    }
7417                    Err(e) => {
7418                        record.failed = Some(e.to_string());
7419                        self.state.event(
7420                            "review",
7421                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7422                        );
7423                    }
7424                }
7425                records.push(record);
7426            }
7427
7428            // Tally the round's votes and, if they split, spend the one
7429            // round of reconsideration the split -> deliberate -> revote
7430            // shape `judge`/`vote` use for the panel, sized down to what a
7431            // read-only review round can afford: one round, and a revote
7432            // rather than an argument, because the panel already wrote its
7433            // reasoning down as findings the first time around.
7434            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7435            let vote_split =
7436                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7437            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7438            if vote_split {
7439                self.state.event(
7440                    "review",
7441                    format!(
7442                        "round {round}: votes split ({}) — one round of reconsideration",
7443                        initial_votes
7444                            .iter()
7445                            .map(|v| v.label())
7446                            .collect::<Vec<_>>()
7447                            .join(", ")
7448                    ),
7449                );
7450                // Seats read every seat's findings and votes, still numbered
7451                // and never named — the same anonymity `review` itself keeps.
7452                let panel: Vec<ReviewSeatReport<'_>> = records
7453                    .iter()
7454                    .filter_map(|r| {
7455                        r.vote.map(|vote| ReviewSeatReport {
7456                            reviewer: r.reviewer,
7457                            vote,
7458                            summary: &r.summary,
7459                            findings: &r.findings,
7460                        })
7461                    })
7462                    .collect();
7463
7464                let mut jobs = Vec::new();
7465                let mut seats_at = Vec::new();
7466                for (r, spec) in reviewers.iter().cloned().enumerate() {
7467                    // A seat with no initial vote has nothing to reconsider
7468                    // from and stays absent, the same as it stayed absent
7469                    // from `panel` above.
7470                    if records[r].vote.is_none() {
7471                        continue;
7472                    }
7473                    let wt = root.join(format!("review-{}", r + 1));
7474                    let seat_key = format!("review-{}", r + 1);
7475                    let spec = self.occupant(&seat_key, spec);
7476                    let seat = self.seat(&seat_key, &spec.id);
7477                    // A seat with no live session has already forgotten the
7478                    // initial review's prompt — restate the patch it is
7479                    // voting on, the same as `deliberate`/`vote` do for a
7480                    // judge in the same position.
7481                    // The panel already carries this seat's own review and
7482                    // vote, so restating the patch makes the prompt whole for
7483                    // a seat handed to another agent.
7484                    let build = |with_patch: bool| {
7485                        prompt::review_reconsider(&ReviewReconsiderCtx {
7486                            instruction: &self.state.instruction,
7487                            reviewer: r + 1,
7488                            lens: Lens::for_seat(r),
7489                            panel: &panel,
7490                            patch: with_patch.then_some(ReviewPatch {
7491                                branch: &winner.branch,
7492                                base_short: &base_short,
7493                                stat: &stat,
7494                                patch: &patch,
7495                            }),
7496                            round,
7497                            rounds: max_rounds,
7498                            language: &language,
7499                        })
7500                    };
7501                    let full = build(true);
7502                    let prompt = if has_context(&spec, &seat, sessions) {
7503                        build(false)
7504                    } else {
7505                        full.clone()
7506                    };
7507                    jobs.push(SeatJob {
7508                        prompt,
7509                        spec,
7510                        seat,
7511                        cwd: wt,
7512                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7513                        allow_write: false,
7514                        sessions,
7515                        artifacts: artifacts.clone(),
7516                        stem: format!("review-{round}-reconsider-{}", r + 1),
7517                        handover: Some(full),
7518                    });
7519                    seats_at.push(r);
7520                }
7521
7522                let mut recon_quota_losses = Vec::new();
7523                let recon_cache = self.state.config.cache_dir();
7524                let recon_ctx = WaveCtx {
7525                    carry_seats: true,
7526                    pauses: Some((&self.pause, &self.interrupt)),
7527                    run: &run_id,
7528                    node: "review",
7529                    prompts: &prompts,
7530                    cache: recon_cache.as_deref(),
7531                    round: Some(round),
7532                };
7533                let recon_results = ask_json_wave::<ReviewRevote>(
7534                    jobs,
7535                    Arc::clone(&self.sem),
7536                    review_retries,
7537                    &self.roles.reviewer_roster,
7538                    &recon_ctx,
7539                    &mut recon_quota_losses,
7540                    &mut self.state,
7541                    &|_: &ReviewRevote| Ok(()),
7542                )
7543                .await;
7544                if self.state.resource_block.is_some() {
7545                    return self.stop_for_resource();
7546                }
7547                self.state.quota.extend(recon_quota_losses);
7548
7549                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7550                    let agent_id = seat.agent.clone();
7551                    self.state.seats.insert(seat.key.clone(), seat);
7552                    let mut rec = ReviewRevoteRecord {
7553                        reviewer: r + 1,
7554                        agent: agent_id,
7555                        vote: None,
7556                        reason: String::new(),
7557                        failed: None,
7558                    };
7559                    match res {
7560                        Ok((rv, _)) => {
7561                            rec.vote = Some(rv.vote);
7562                            rec.reason =
7563                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7564                            self.state.event(
7565                                "review",
7566                                format!(
7567                                    "round {round}: reviewer {} revoted {}",
7568                                    r + 1,
7569                                    rv.vote.label()
7570                                ),
7571                            );
7572                        }
7573                        Err(e) => {
7574                            rec.failed = Some(e.to_string());
7575                            self.state.event(
7576                                "review",
7577                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7578                            );
7579                        }
7580                    }
7581                    reconsideration.push(rec);
7582                }
7583            } else if initial_votes.len() > 1 {
7584                self.state.event(
7585                    "review",
7586                    format!(
7587                        "round {round}: votes agreed ({}) — no reconsideration",
7588                        initial_votes[0].label()
7589                    ),
7590                );
7591            }
7592
7593            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7594            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7595            // A round that already has a blocking finding and a round left to
7596            // try is going back to the fixer no matter what `verify.e2e`
7597            // says, so running it first only spends the loop's slowest step
7598            // (minutes, for a Rust repo's full test suite) on a head about
7599            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7600            // runs once a round has no blocking findings left (see
7601            // `round_is_clean`, which a deferred — empty — `e2e` can never
7602            // satisfy since `blocking` is nonzero whenever this branch is
7603            // taken), and `stop_reviewing` forces a real run before it will
7604            // ever read a deferred round as green.
7605            let defer_e2e =
7606                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7607            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7608                let reason =
7609                    format!("{blocking} blocking finding(s) already required a fix this round");
7610                self.state.event(
7611                    "verify",
7612                    format!(
7613                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7614                         {}); it will run once a round has none left",
7615                        short(&head)
7616                    ),
7617                );
7618                (Vec::new(), false, true, Some(reason))
7619            } else {
7620                let e2e_commands = self.state.config.verify.e2e.clone();
7621                let cache_dir = self.state.config.cache_dir();
7622                let context = format!("round {round}");
7623                let wait = self.repository_wait();
7624                let (e2e, verify_retried) = with_cache_lease(
7625                    &mut self.state,
7626                    cache_dir.as_deref(),
7627                    "e2e",
7628                    "e2e",
7629                    &winner.worktree,
7630                    &head,
7631                    verify_timeout,
7632                    &context,
7633                    &wait,
7634                    |state, budget| {
7635                        let shell = shell.clone();
7636                        let e2e_commands = e2e_commands.clone();
7637                        let worktree = winner.worktree.clone();
7638                        let context = context.clone();
7639                        async move {
7640                            run_e2e_with_retry(
7641                                state,
7642                                &shell,
7643                                &e2e_commands,
7644                                &worktree,
7645                                budget,
7646                                &context,
7647                            )
7648                            .await
7649                        }
7650                    },
7651                )
7652                .await?;
7653                (e2e, verify_retried, false, None)
7654            };
7655
7656            let expected = records.len();
7657            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7658            let incomplete = answered < expected;
7659            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7660            let policy = self.state.config.graph.incomplete_review;
7661            let clean = round_is_clean(
7662                blocking,
7663                e2e_ok,
7664                answered,
7665                expected,
7666                round_quota_missing,
7667                policy,
7668            );
7669
7670            let mut round_record = ReviewRound {
7671                round,
7672                head: head.clone(),
7673                verified_head: None,
7674                verified_at: None,
7675                reviews: records,
7676                e2e,
7677                verify_retried,
7678                e2e_deferred,
7679                e2e_defer_reason,
7680                fix: None,
7681                blocking,
7682                answered,
7683                expected,
7684                clean,
7685                progressed: false,
7686                vote_split,
7687                reconsideration,
7688                verdict: None,
7689            };
7690            // The final vote per seat is its revote where reconsideration
7691            // ran and answered, its initial vote otherwise — the same
7692            // fallback `tally` uses for a judge whose private vote failed.
7693            round_record.verdict = ReviewVote::worst(
7694                round_record
7695                    .final_votes()
7696                    .into_iter()
7697                    .map(|(_, _, vote)| vote),
7698            );
7699            // Which commit and when magi actually attempted to check —
7700            // known the moment a command was dispatched against `head`,
7701            // whether or not it finished: a resource-blocked attempt still
7702            // targeted a specific commit at a specific time, and leaving
7703            // that unrecorded is exactly what made `verification_summary`
7704            // report a fresh attempt as "commit unknown ... recorded before
7705            // this was tracked", indistinguishable from a genuinely old,
7706            // untracked record. Only a deferred or unconfigured round never
7707            // ran at all and has nothing to record — see
7708            // `ReviewRound::verified_head`'s own doc.
7709            if !matches!(
7710                round_record.e2e_status(),
7711                E2eStatus::Deferred | E2eStatus::NotConfigured
7712            ) {
7713                round_record.verified_head = Some(head.clone());
7714                round_record.verified_at = Some(Timestamp::now());
7715            }
7716
7717            if incomplete {
7718                let missing: Vec<String> = round_record
7719                    .reviews
7720                    .iter()
7721                    .filter(|r| r.failed.is_some())
7722                    .map(|r| format!("review-{}", r.reviewer))
7723                    .collect();
7724                self.state.event(
7725                    "review",
7726                    format!(
7727                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7728                        missing.join(", ")
7729                    ),
7730                );
7731            }
7732
7733            if clean {
7734                self.state.event(
7735                    "review",
7736                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7737                        format!(
7738                            "round {round}: clean (warn policy, incomplete panel) — no \
7739                             blocking findings from the seats that answered, verification green"
7740                        )
7741                    } else if incomplete {
7742                        format!(
7743                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7744                             quorum) — no blocking findings from the seats that answered, \
7745                             verification green",
7746                            expected - answered
7747                        )
7748                    } else {
7749                        format!("round {round}: clean — no blocking findings, verification green")
7750                    },
7751                );
7752                self.state.reviews.push(round_record);
7753                self.state.status = RunStatus::Gating;
7754                self.state.save()?;
7755                return Ok(());
7756            }
7757
7758            // Nothing was raised and verification passed, but not every seat
7759            // answered and `round_is_clean` still refused to call it clean —
7760            // either a seat is missing for a reason other than its own quota
7761            // (a crash, a timeout, unparsable output — worth another try), or
7762            // every seat that could have answered lost its quota and nobody
7763            // is left to decide on: re-review rather than send the fixer
7764            // after a round with nothing to fix.
7765            if incomplete && blocking == 0 && e2e_ok {
7766                self.state.reviews.push(round_record);
7767                self.state.save()?;
7768                if round == max_rounds {
7769                    self.state.status = RunStatus::Blocked;
7770                    self.state.event(
7771                        "review",
7772                        format!(
7773                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7774                             refusing to call it clean",
7775                            expected - answered
7776                        ),
7777                    );
7778                    return Ok(());
7779                }
7780                continue;
7781            }
7782
7783            // Nothing for the fixer to act on (`blocking == 0`) and the only
7784            // reason this round is not clean is that magi itself never got
7785            // a command to run — the shared build cache, not the patch (see
7786            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7787            // the fixer would invite a change to appease contention that has
7788            // nothing to do with the diff, and would leave this attempt
7789            // sitting in the next round's prompt as if it were about an
7790            // earlier, superseded commit rather than what it actually is:
7791            // the same head, still waiting to be checked. Wait for it the
7792            // same way the final round's own contention is already handled,
7793            // whatever round this happens to be.
7794            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7795                self.state.reviews.push(round_record);
7796                return self
7797                    .stop_unresolved_review(
7798                        "the round's own verification could not run",
7799                        &shell,
7800                        &winner.worktree,
7801                    )
7802                    .await;
7803            }
7804
7805            if round == max_rounds {
7806                self.state.reviews.push(round_record);
7807                return self
7808                    .stop_reviewing(
7809                        &format!(
7810                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7811                        ),
7812                        &shell,
7813                        &winner.worktree,
7814                    )
7815                    .await;
7816            }
7817
7818            if !self.fix_review_round(&winner, round_record).await? {
7819                return Ok(());
7820            }
7821        }
7822        Ok(())
7823    }
7824
7825    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7826    /// False means admission or continuation policy stopped the review loop.
7827    async fn fix_review_round(
7828        &mut self,
7829        winner: &Candidate,
7830        mut round_record: ReviewRound,
7831    ) -> Result<bool> {
7832        let round = round_record.round;
7833        let shell = self.state.config.shell();
7834        let step = self.run_fix_round(winner, &mut round_record).await?;
7835        if matches!(step, FixStep::Resource) {
7836            self.stop_for_resource()?;
7837            return Ok(false);
7838        }
7839        if self.state.reviews.len() < round {
7840            self.state.reviews.push(round_record);
7841        } else {
7842            self.state.reviews[round - 1] = round_record;
7843        }
7844        self.state.save()?;
7845
7846        // The fixer's own report never came back this round, even after
7847        // `continue_fix_report`'s own budget was spent on it — not an
7848        // ordinary "no report" (dropped stream, quota, plain failure),
7849        // which already reads that way and is left to the existing round
7850        // budget. Stopping here, rather than opening another round, is
7851        // what keeps a next reviewer/fixer wave from ever being
7852        // dispatched onto `winner.worktree` while whatever the seat's
7853        // last call may still have running there is unaccounted for: no
7854        // process liveness check exists (and none is being added — see
7855        // AGENTS.md/this task's own scope), so the only way to honour
7856        // "nothing starts before a valid report returns" is to not start
7857        // anything further on this worktree from this run at all.
7858        if matches!(step, FixStep::Recorded { report_lost: true }) {
7859            self.stop_reviewing(
7860                "the fixer's adoption report never came back, even after resuming its \
7861                         own seat; refusing to start another round against the same worktree \
7862                         while that is unresolved",
7863                &shell,
7864                &winner.worktree,
7865            )
7866            .await?;
7867            return Ok(false);
7868        }
7869
7870        let streak = self
7871            .state
7872            .reviews
7873            .iter()
7874            .rev()
7875            .take_while(|r| !r.progressed)
7876            .count();
7877        if streak >= STAGNANT_LIMIT {
7878            self.stop_reviewing(
7879                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7880                &shell,
7881                &winner.worktree,
7882            )
7883            .await?;
7884            return Ok(false);
7885        }
7886        Ok(true)
7887    }
7888
7889    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7890    async fn run_fix_round(
7891        &mut self,
7892        winner: &Candidate,
7893        round_record: &mut ReviewRound,
7894    ) -> Result<FixStep> {
7895        let round = round_record.round;
7896        let max_rounds = self.state.config.graph.review_rounds;
7897        let run_id = self.state.id.clone();
7898        let prompts = self.state.config.prompts.clone();
7899        let language = self.state.config.graph.language.clone();
7900        let sessions = self.state.config.graph.sessions;
7901        let artifacts = self.merge_repair_artifacts();
7902        let base = self.landing_base();
7903        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7904        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7905        let this_round_verification = round_record.verification_summary(&head);
7906        let all_findings: Vec<_> = round_record
7907            .reviews
7908            .iter()
7909            .flat_map(|r| r.findings.iter().cloned())
7910            .collect();
7911        // Fix. The winner's own implementer seat continues its conversation:
7912        // the competition is over, so context is pure benefit now.
7913        let blocking_findings: Vec<_> = all_findings
7914            .iter()
7915            .filter(|f| f.severity.blocks())
7916            .cloned()
7917            .collect();
7918        let fix_prompt = prompt::fix_in(
7919            &self.state.instruction,
7920            &blocking_findings,
7921            this_round_verification.as_ref(),
7922            round,
7923            max_rounds,
7924            &language,
7925            &self.state.config.graph.github_language,
7926        );
7927        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7928        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7929        let (job, seat, out) = self
7930            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7931                prompt: fix_prompt.clone(),
7932                spec,
7933                seat,
7934                cwd: winner.worktree.clone(),
7935                timeout,
7936                allow_write: true,
7937                sessions,
7938                artifacts: artifacts.clone(),
7939                stem: format!("fix-{round}"),
7940                handover: None,
7941            })
7942            .await;
7943        // Not asked: the round is not recorded (so a resume repeats it),
7944        // no fixer was handed over to, and the seat is left as it was.
7945        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7946            return Ok(FixStep::Resource);
7947        }
7948        let agent_id = seat.agent.clone();
7949
7950        let mut fix = FixRecord {
7951            agent: agent_id,
7952            addressed: Vec::new(),
7953            rejected: Vec::new(),
7954            notes: String::new(),
7955            committed: false,
7956            failed: None,
7957            duration_ms: 0,
7958            continuation: None,
7959        };
7960        let mut continuation = ContinuationRecord::not_needed();
7961        let mut final_seat = seat.clone();
7962        match out {
7963            AgentOutcome::Ok(o) => {
7964                fix.duration_ms = o.duration_ms;
7965                let parsed = verdict::extract_json::<FixReport>(&o.text);
7966                // A parsed report standing next to a command this same
7967                // reply's own CLI never confirmed the exit status of is
7968                // not a resolved answer — the identical `CommandEvidence`
7969                // `state.jobs` renders, read here instead of only on
7970                // display, per the completion judgment and the shown
7971                // record needing to agree.
7972                let incomplete_reason = match &parsed {
7973                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7974                        "the reply parsed, but it reported a command whose own CLI never \
7975                             confirmed an exit status"
7976                            .to_owned(),
7977                    ),
7978                    Ok(_) => None,
7979                    Err(e) => Some(e.to_string()),
7980                };
7981                match incomplete_reason {
7982                    None => {
7983                        let report = parsed.expect("checked Ok above");
7984                        fix.addressed = report.addressed;
7985                        fix.rejected = report.rejected;
7986                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7987                    }
7988                    Some(reason) => {
7989                        let (resumed_seat, resolved, failure, cont) = self
7990                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7991                            .await;
7992                        fix.duration_ms += cont.cumulative_wait_ms;
7993                        continuation = cont;
7994                        final_seat = resumed_seat;
7995                        match resolved {
7996                            Some(report) => {
7997                                fix.addressed = report.addressed;
7998                                fix.rejected = report.rejected;
7999                                fix.notes =
8000                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
8001                            }
8002                            None => fix.failed = failure,
8003                        }
8004                    }
8005                }
8006            }
8007            // The CLI's raw error JSON is not a fix report to parse.
8008            AgentOutcome::Dropped(o) => {
8009                fix.duration_ms = o.duration_ms;
8010                let why = o
8011                    .dropped
8012                    .as_ref()
8013                    .map(|d| d.why.as_str())
8014                    .unwrap_or("the CLI ended the stream without delivering its answer");
8015                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8016            }
8017            AgentOutcome::Quota(o) => {
8018                self.state.quota.push(QuotaLoss::from_report(
8019                    final_seat.key.clone(),
8020                    "fix",
8021                    o.quota.as_ref(),
8022                ));
8023                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8024            }
8025            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8026        }
8027        if self.state.resource_block.is_some() {
8028            self.state.seats.insert(final_seat.key.clone(), final_seat);
8029            return Ok(FixStep::Resource);
8030        }
8031        fix.continuation = Some(continuation);
8032        self.state.seats.insert(final_seat.key.clone(), final_seat);
8033        let coordination = self.coordinate(&winner.worktree).await?;
8034        if let Ok(r) = git::rescue_commit(
8035            &winner.worktree,
8036            &format!("magi: review round {round} fixes (uncommitted work)"),
8037        )
8038        .await
8039        {
8040            self.state.note_withheld("fix", &r.withheld);
8041        }
8042        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8043        fix.committed = after != before;
8044        // Judged by what `git` says moved against base, never by the
8045        // fixer's own `addressed`/`rejected` count — see
8046        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8047        // `patch` snapshot above: swallowing this error would default
8048        // `diff_after` to empty, which almost always differs from a
8049        // non-empty `patch` and reads as "progressed" — exactly backwards
8050        // for a `git` failure the stagnation check cannot see through.
8051        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8052        drop(coordination);
8053        let progressed = diff_after != patch;
8054        let commit_note = if fix.committed {
8055            "committed"
8056        } else {
8057            "NO new commit"
8058        };
8059        let tree_note = if progressed {
8060            "changed vs base"
8061        } else {
8062            "unchanged vs base"
8063        };
8064        self.state.event(
8065            "fix",
8066            match &fix.failed {
8067                // Distinct on purpose from "0 addressed, 0 rejected": the
8068                // fixer's own diff still landed (blocking counts do keep
8069                // falling round over round), only its adoption report did
8070                // not come back, so this must never read like every
8071                // finding was reviewed and declined.
8072                Some(reason) => {
8073                    format!(
8074                        "round {round}: fixer's adoption report was lost ({reason}); \
8075                             {commit_note}, tree {tree_note}"
8076                    )
8077                }
8078                None => format!(
8079                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8080                         {tree_note}{}",
8081                    fix.addressed.len(),
8082                    fix.rejected.len(),
8083                    if continuation.outcome == ContinuationOutcome::Resumed {
8084                        format!(
8085                            " (adoption report recovered after {} continuation(s))",
8086                            continuation.attempts
8087                        )
8088                    } else {
8089                        String::new()
8090                    },
8091                ),
8092            },
8093        );
8094        round_record.fix = Some(fix);
8095        round_record.progressed = progressed;
8096        Ok(FixStep::Recorded {
8097            report_lost: matches!(
8098                continuation.outcome,
8099                ContinuationOutcome::Exhausted
8100                    | ContinuationOutcome::QuotaLost
8101                    | ContinuationOutcome::NoSession
8102            ),
8103        })
8104    }
8105
8106    /// Decide, from the last recorded round's own verification, whether
8107    /// stopping the review loop is a hand-off or a genuine block.
8108    ///
8109    /// Called once the loop has given up trying — the round budget is spent,
8110    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8111    /// findings still open, never while a round is still clean or the
8112    /// incomplete-panel case handled inline above. Gate and e2e are facts
8113    /// about the tree; a lingering review finding is an opinion, and this
8114    /// workload's own `magi stats` puts reviewer precision low enough
8115    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8116    /// must not by itself stand between a green, verified change and the
8117    /// human who decides what to do with it. A red e2e is not an opinion, so
8118    /// that case still blocks, with the failing command and a tail of its
8119    /// output recorded here rather than left in `run.json` for someone to go
8120    /// find.
8121    ///
8122    /// A round that deferred its own e2e (see [`Config::graph`]'s
8123    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8124    /// only because nothing ran, and treating an empty list as a passing one
8125    /// here is exactly the "deferred painted green" bug this function exists
8126    /// to not have. When the last round's own verification never resolved —
8127    /// deferred on purpose, or a real attempt the shared build cache blocked
8128    /// — this makes (or retries) the real run, on the actual worktree this
8129    /// loop is about to stop touching, before deciding anything. A
8130    /// resource-blocked attempt is likewise never read as either green or
8131    /// red: it is evidence about the machine, not the patch (see
8132    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8133    /// blocked cache leaves this call without deciding rather than guessing
8134    /// — the caller retries on a later reentry.
8135    /// Record, once, that the review loop handed off over a blocking finding
8136    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8137    /// `land` asks the owner even with `land_approval` off. Called from every
8138    /// path that concludes `Gating`; a reentry keeps the first record.
8139    fn record_contested_handoff(&mut self) {
8140        if self.state.contested_handoff.is_some() {
8141            return;
8142        }
8143        let Some(contested) = self
8144            .state
8145            .reviews
8146            .last()
8147            .and_then(ReviewRound::contested_handoff)
8148        else {
8149            return;
8150        };
8151        self.state.event(
8152            "review",
8153            format!(
8154                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8155                 wait for the owner's approval",
8156                contested.findings.len(),
8157                contested.rejecters.len()
8158            ),
8159        );
8160        self.state.contested_handoff = Some(contested);
8161    }
8162
8163    async fn stop_unresolved_review(
8164        &mut self,
8165        why: &str,
8166        shell: &[String],
8167        worktree: &Path,
8168    ) -> Result<()> {
8169        self.stop_reviewing_inner(why, shell, worktree, true).await
8170    }
8171
8172    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8173        self.stop_reviewing_inner(why, shell, worktree, false).await
8174    }
8175
8176    async fn stop_reviewing_inner(
8177        &mut self,
8178        why: &str,
8179        shell: &[String],
8180        worktree: &Path,
8181        repairable: bool,
8182    ) -> Result<()> {
8183        let round_idx = self.state.reviews.len() - 1;
8184        // A deferred round and a resource-blocked one are the same shape
8185        // here: neither has a real result yet, and both get one more
8186        // attempt. Read off `e2e_status` — the single source for this —
8187        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8188        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8189        // still retries instead of being read as a settled result the
8190        // instant it stops being empty.
8191        let needs_catchup_run = matches!(
8192            self.state.reviews[round_idx].e2e_status(),
8193            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8194        );
8195        if needs_catchup_run {
8196            let round = self.state.reviews[round_idx].round;
8197            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8198            let commands = self.state.config.verify.e2e.clone();
8199            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8200            let cache_dir = self.state.config.cache_dir();
8201            let context = format!(
8202                "round {round}: verification unresolved, catching up before the final decision"
8203            );
8204            let wait = self.repository_wait();
8205            let (outcomes, verify_retried) = with_cache_lease(
8206                &mut self.state,
8207                cache_dir.as_deref(),
8208                "e2e",
8209                "e2e",
8210                worktree,
8211                &attempted_head,
8212                timeout,
8213                &context,
8214                &wait,
8215                |state, budget| {
8216                    let shell = shell.to_vec();
8217                    let commands = commands.clone();
8218                    let context = context.clone();
8219                    async move {
8220                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8221                            .await
8222                    }
8223                },
8224            )
8225            .await?;
8226            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8227                self.state.record_base_progress(&attempted_head);
8228            }
8229            let last = &mut self.state.reviews[round_idx];
8230            last.e2e = outcomes;
8231            last.verify_retried = verify_retried;
8232            // Always the commit and time this attempt actually targeted,
8233            // whether or not it happens to equal the reviewed `head` and
8234            // whether or not a command finished — see
8235            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8236            // attempt is recorded too, so a later reader sees "attempted
8237            // again at T2" rather than silence.
8238            last.verified_head = Some(attempted_head);
8239            last.verified_at = Some(Timestamp::now());
8240            if verify_inconclusive(&last.e2e) {
8241                // Still not a real result: `e2e_deferred` is left exactly
8242                // as it was, so `needs_catchup_run` above reads
8243                // `ResourceBlocked` (via `e2e_status`, which checks
8244                // `resource_blocked` before `e2e_deferred`) and retries
8245                // again on the next reentry, rather than recording
8246                // contention as a red e2e and blocking the run on it.
8247                self.state.save()?;
8248                return Ok(());
8249            }
8250            last.e2e_deferred = false;
8251            // A complete panel may have been saved as unclean solely because
8252            // verification could not run. Refresh that verdict with the new
8253            // outcomes before a granted repair consumes the round's result.
8254            // Incomplete panels retain their original policy/quota decision.
8255            if last.answered == last.expected {
8256                last.clean = round_is_clean(
8257                    last.blocking,
8258                    last.e2e.iter().all(CommandOutcome::ok),
8259                    last.answered,
8260                    last.expected,
8261                    0,
8262                    self.state.config.graph.incomplete_review,
8263                );
8264            }
8265        }
8266        let last = &self.state.reviews[round_idx];
8267        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8268
8269        match last.e2e_status() {
8270            E2eStatus::Failed => {
8271                let red: Vec<String> = last
8272                    .e2e
8273                    .iter()
8274                    .filter(|o| !o.ok())
8275                    .map(|o| {
8276                        format!(
8277                            "`{}` -> {:?}\n{}",
8278                            o.command,
8279                            o.code,
8280                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8281                        )
8282                    })
8283                    .collect();
8284                let can_repair = last.blocking == 0 && last.fix.is_none();
8285                self.state.reviews[round_idx].clean = false;
8286                self.state
8287                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8288                self.state.status = RunStatus::Blocked;
8289                if repairable
8290                    && can_repair
8291                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8292                {
8293                    self.state.status = RunStatus::Reviewing;
8294                }
8295            }
8296            // `needs_catchup_run` above already retried once this call; if
8297            // it is still blocked, this is magi's own admission it could
8298            // not get a command to run, never a verdict on the patch — the
8299            // run is left exactly where a later reentry can retry again.
8300            E2eStatus::ResourceBlocked => {
8301                self.state.event(
8302                    "review",
8303                    format!(
8304                        "{why}; e2e could not run (resource unavailable); not \
8305                         deciding yet"
8306                    ),
8307                );
8308            }
8309            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8310                self.state.event(
8311                    "review",
8312                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8313                );
8314                self.record_contested_handoff();
8315                self.state.status = RunStatus::Gating;
8316            }
8317        }
8318        self.state.save()?;
8319        Ok(())
8320    }
8321
8322    // ----------------------------------------------------------------- gate
8323
8324    async fn gate(&mut self) -> Result<()> {
8325        if self.hold_base_sync().await? {
8326            return Ok(());
8327        }
8328        // Judged by the review record itself, not by `status`: a solo
8329        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8330        // reentry (see `judge`), and trusting it here is exactly how a run
8331        // that exhausted its review budget got gated and merged a second
8332        // time around. `review_conclusion` recomputes the review loop's own
8333        // verdict from the round records themselves — `Gating` for a clean
8334        // round or a hand-off (see `stop_reviewing`), anything else means the
8335        // loop is still going or genuinely blocked.
8336        // A base the winner could not be replayed onto is a decision, not a
8337        // round: there is no landing tree to gate. Read as its own record for
8338        // the same reason the review verdict is.
8339        if self.state.status == RunStatus::Failed
8340            || self
8341                .state
8342                .base_sync
8343                .as_ref()
8344                .is_some_and(|s| s.conflict.is_some())
8345            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8346                != Some(RunStatus::Gating)
8347        {
8348            return Ok(());
8349        }
8350        let Some(winner) = self.state.winner().cloned() else {
8351            return Ok(());
8352        };
8353        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8354        self.state.reconcile_verification_head(&head);
8355        self.state.save()?;
8356        let wait = self.repository_wait();
8357        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8358            return Ok(());
8359        }
8360        if self.state.gate_ran {
8361            // `review_loop` derives its conclusion from the clean review
8362            // record on every reentry and therefore puts a completed run back
8363            // in `Gating`. A recorded gate is a stronger, terminal fact:
8364            // retain its original command output (or lack of any, for a repo
8365            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8366            // doc) and restore `Blocked` on a real failure rather than
8367            // pretending the command is still running or running it a second
8368            // time. `gate_ran == false` remains the only shape — unattempted,
8369            // or a resource-blocked retry — that may still need to execute a
8370            // command.
8371            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8372                self.state.status = RunStatus::Blocked;
8373                self.state.save()?;
8374            }
8375            return Ok(());
8376        }
8377        self.state.status = RunStatus::Gating;
8378        let mut outcomes = self.run_gate(&winner).await?;
8379        let mut attempted_head = self.state.gate_head.clone();
8380        loop {
8381            // A resource-blocked outcome means the gate command never actually
8382            // ran - the shared build cache could not be acquired or confirmed
8383            // fresh in time - which is evidence about the machine, not about
8384            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8385            // Recording it as a red gate would mark a run `Blocked` on nothing
8386            // but contention magi has already logged; leaving `self.state.gate`
8387            // empty and `self.state.gate_ran` false instead keeps the shape
8388            // this function already treats as "still needs to run" (see the
8389            // early-return above), so the next call retries the command
8390            // rather than concluding anything.
8391            if verify_inconclusive(&outcomes) {
8392                self.state.save()?;
8393                return Ok(());
8394            }
8395            if outcomes.iter().all(CommandOutcome::ok) {
8396                break;
8397            }
8398            let fixes_before = self.state.gate_fixes.len();
8399            match self.gate_fix_round(&winner, &outcomes).await? {
8400                GateFix::Retry => {
8401                    outcomes = self.run_gate(&winner).await?;
8402                    attempted_head = self.state.gate_head.clone();
8403                }
8404                GateFix::Stop => {
8405                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8406                    if self.state.gate_fixes.len() > fixes_before
8407                        && attempted_head.as_deref() != Some(current.as_str())
8408                        && verify_current_head(&mut self.state, &winner, &wait).await?
8409                    {
8410                        // A new SHA owes a gate even when a spent fixer changed only
8411                        // commit metadata. This does not grant another repair round.
8412                        outcomes = self.run_gate(&winner).await?;
8413                        attempted_head = self.state.gate_head.clone();
8414                    } else {
8415                        break;
8416                    }
8417                }
8418                GateFix::Defer => {
8419                    self.state.save()?;
8420                    return Ok(());
8421                }
8422            }
8423        }
8424        let passed = outcomes.iter().all(CommandOutcome::ok);
8425        self.state.gate = outcomes;
8426        self.state.gate_ran = true;
8427        self.state.gate_head = attempted_head;
8428        if !self.state.gate.is_empty()
8429            && !verify_inconclusive(&self.state.gate)
8430            && let Some(head) = self.state.gate_head.clone()
8431        {
8432            self.state.record_base_progress(&head);
8433        }
8434        self.state.save()?;
8435        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8436            return Ok(());
8437        }
8438        if !passed {
8439            self.state.status = RunStatus::Blocked;
8440            let spent = self.state.gate_fixes.len();
8441            self.state.event(
8442                "gate",
8443                if spent == 0 {
8444                    "gate failed; not merging".to_owned()
8445                } else {
8446                    format!("gate failed after {spent} gate-fix round(s); not merging")
8447                },
8448            );
8449        }
8450        self.state.save()?;
8451        Ok(())
8452    }
8453
8454    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8455    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8456    /// after review is clean and never on a candidate awaiting judging.
8457    ///
8458    /// Never fails the run: a non-zero exit or timeout is a warning and a
8459    /// recorded outcome, and the gate remains the single arbiter. Nothing
8460    /// configured means nothing happens - no event, no commit. `commit_all`
8461    /// commits any leftover change under the neutral identity and returns
8462    /// `false` when the tree is clean, so no empty commit is ever made.
8463    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8464        let commands = state.config.verify.pre_gate.clone();
8465        if commands.is_empty() {
8466            return;
8467        }
8468        let shell = state.config.shell();
8469        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8470        let (outcomes, _) = run_commands(
8471            state,
8472            "pre_gate",
8473            "pre_gate",
8474            0,
8475            &shell,
8476            &commands,
8477            &winner.worktree,
8478            timeout,
8479        )
8480        .await;
8481        for o in &outcomes {
8482            if !o.ok() {
8483                tracing::warn!(
8484                    "pre_gate `{}` failed ({:?}); the gate decides",
8485                    o.command,
8486                    o.code
8487                );
8488            }
8489            state.event(
8490                "pre_gate",
8491                format!(
8492                    "`{}` -> {}",
8493                    o.command,
8494                    if o.ok() {
8495                        "pass".to_owned()
8496                    } else {
8497                        format!(
8498                            "FAIL ({:?})\n{}",
8499                            o.code,
8500                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8501                        )
8502                    }
8503                ),
8504            );
8505        }
8506        state.pre_gate = outcomes;
8507        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8508            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8509                Ok(head) => {
8510                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8511                    state.pre_gate_commit = Some(head);
8512                }
8513                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8514            },
8515            Ok(false) => {}
8516            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8517        }
8518        if let Err(e) = state.save() {
8519            tracing::warn!("could not persist the pre_gate record: {e:#}");
8520        }
8521    }
8522
8523    /// Run `verify.gate` once against the winner's current tree, logging one
8524    /// event per command. Empty when nothing is configured.
8525    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8526        let wait = self.repository_wait();
8527        run_landing_gate(&mut self.state, winner, &wait).await
8528    }
8529
8530    /// One bounded fix round for a failing gate.
8531    ///
8532    /// The fixer is told the failure came from the gate itself, not from a
8533    /// reviewer, and is shown the failed commands, their exit codes and a tail
8534    /// of their output - whatever `[verify].gate` holds, nothing here knows
8535    /// what those commands run. Only a normal non-zero exit that printed
8536    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8537    /// command or a full disk says nothing about the code, and a fixer sent
8538    /// after it can only appease the machine. The round is judged by what git
8539    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8540    /// again before the gate does, so a fix cannot trade a green gate for a
8541    /// red e2e unnoticed.
8542    async fn gate_fix_round(
8543        &mut self,
8544        winner: &Candidate,
8545        outcomes: &[CommandOutcome],
8546    ) -> Result<GateFix> {
8547        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8548            g.phase == crate::repair_budget::Phase::Fixing
8549                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8550        }) {
8551            self.state.gate_fixes.len().saturating_add(1)
8552        } else {
8553            self.state.config.graph.gate_fix_rounds
8554        };
8555        let spent = self.state.gate_fixes.len();
8556        if spent >= cap {
8557            if cap > 0 {
8558                self.state.event(
8559                    "gate",
8560                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8561                );
8562            }
8563            return Ok(GateFix::Stop);
8564        }
8565        if !gate_fixable(outcomes) {
8566            self.state.event(
8567                "gate",
8568                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8569                 command or similar); not spending a fix round on it",
8570            );
8571            return Ok(GateFix::Stop);
8572        }
8573        let min_free = self.state.config.disk.min_free_bytes;
8574        if min_free > 0 {
8575            match crate::disk::free_bytes(&winner.worktree) {
8576                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8577                Ok(free) => {
8578                    self.state.event(
8579                        "gate",
8580                        format!(
8581                            "only {free} bytes free ({min_free} required by `[disk] \
8582                             min_free_bytes`); not spending a fix round on a failure the disk \
8583                             may explain"
8584                        ),
8585                    );
8586                    return Ok(GateFix::Stop);
8587                }
8588                Err(e) => {
8589                    self.state.event(
8590                        "gate",
8591                        format!("free disk space could not be measured ({e:#}); no fix round"),
8592                    );
8593                    return Ok(GateFix::Stop);
8594                }
8595            }
8596        }
8597
8598        let attempt = spent + 1;
8599        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8600        let base = self.landing_base();
8601        let fix_prompt = prompt::gate_fix_in(
8602            &self.state.instruction,
8603            &failed,
8604            attempt,
8605            cap,
8606            &self.state.config.graph.language,
8607            &self.state.config.graph.github_language,
8608        );
8609        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8610        let sessions = self.state.config.graph.sessions;
8611        let artifacts = self.merge_repair_artifacts();
8612        self.state.event(
8613            "gate",
8614            format!("gate failed; gate-fix round {attempt} of {cap}"),
8615        );
8616        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8617        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8618        // Persist before invoking the fixer: it may commit and then be
8619        // interrupted before its result (or the following e2e) is saved.
8620        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8621            head: before.clone(),
8622            passed: false,
8623            outcomes: Vec::new(),
8624        });
8625        self.state.save()?;
8626        let (_, seat, out) = self
8627            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8628                prompt: fix_prompt.clone(),
8629                spec,
8630                seat,
8631                cwd: winner.worktree.clone(),
8632                timeout,
8633                allow_write: true,
8634                sessions,
8635                artifacts: artifacts.clone(),
8636                stem: format!("gate-fix-{attempt}"),
8637                handover: None,
8638            })
8639            .await;
8640        // The fixer was never asked: no round is spent (nothing is recorded)
8641        // and a later reentry asks the same agent again.
8642        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8643            return Ok(GateFix::Defer);
8644        }
8645        let mut record = GateFixRecord {
8646            agent: seat.agent.clone(),
8647            failed,
8648            notes: String::new(),
8649            committed: false,
8650            error: None,
8651        };
8652        match out {
8653            AgentOutcome::Ok(o) => {
8654                // A missing report is not a failed fix: the round is judged
8655                // by the tree below, and the report only carries prose.
8656                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8657                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8658                }
8659            }
8660            AgentOutcome::Dropped(_) => {
8661                record.error = Some("the CLI dropped the stream".to_owned());
8662            }
8663            AgentOutcome::Quota(o) => {
8664                self.state.quota.push(QuotaLoss::from_report(
8665                    seat.key.clone(),
8666                    "gate-fix",
8667                    o.quota.as_ref(),
8668                ));
8669                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8670            }
8671            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8672        }
8673        self.state.seats.insert(seat.key.clone(), seat);
8674        let coordination = self.coordinate(&winner.worktree).await?;
8675        if let Ok(r) = git::rescue_commit(
8676            &winner.worktree,
8677            &format!("magi: gate fix {attempt} (uncommitted work)"),
8678        )
8679        .await
8680        {
8681            self.state.note_withheld("gate-fix", &r.withheld);
8682        }
8683        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8684        record.committed = after != before;
8685        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8686        let note = record.error.clone();
8687        self.state.gate_fixes.push(record);
8688        self.state.save()?;
8689        if !changed {
8690            self.state.event(
8691                "gate",
8692                match note {
8693                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8694                    None => format!("gate-fix round {attempt}: the tree did not change"),
8695                },
8696            );
8697            return Ok(GateFix::Stop);
8698        }
8699        self.state.event(
8700            "gate",
8701            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8702        );
8703
8704        drop(coordination);
8705        let wait = self.repository_wait();
8706        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8707    }
8708
8709    // ---------------------------------------------------------------- merge
8710
8711    /// One read-only rewrite by the summary's author; a text the gate still
8712    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8713    /// applies on their word or on silence. `None` means the run is parked on
8714    /// that question and nothing may be pushed or opened yet.
8715    async fn guarded_pr_message(
8716        &mut self,
8717        winner: &Candidate,
8718        facts: Option<&BranchFacts>,
8719        posting: bool,
8720    ) -> Result<Option<PrMessage>> {
8721        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8722        let head = match final_head.as_deref() {
8723            Some(h) => crate::pr_outcome::FinalHead::At(h),
8724            None => crate::pr_outcome::FinalHead::Unknown,
8725        };
8726        // Only a pull request that is about to be posted pays for a summary.
8727        let summarized = if posting {
8728            Some(self.summarize_change(winner, final_head.as_deref()).await)
8729        } else {
8730            None
8731        };
8732        // A busy shared resource is not a permanent cause: publishing an
8733        // "unavailable" description would stick (a later head move never
8734        // edits the body). Stop before anything is pushed or opened; the
8735        // run is `Blocked` and a resume asks the summary again.
8736        if self.state.resource_block.is_some() {
8737            self.state.event(
8738                "pr-summary",
8739                "a shared resource was busy; nothing was published, resume to summarize again",
8740            );
8741            return Ok(None);
8742        }
8743        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8744            anyhow::bail!(
8745                "the final commit moved during public description generation; retry before publishing"
8746            );
8747        }
8748        let note = match (&summarized, &self.state.change_summary) {
8749            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8750            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8751            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8752            (None, _) => ChangeNote::NotAsked,
8753        };
8754        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8755        if !posting {
8756            let identity = crate::scrub::Identity::current();
8757            return Ok(Some(PrMessage {
8758                title: crate::scrub::scrub(&pr.title, &identity),
8759                body: crate::scrub::scrub(&pr.body, &identity),
8760            }));
8761        }
8762        // This very text was already rejected and asked about: no second
8763        // judge call, rewrite or question, just carry the record on.
8764        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8765        if let Some(known) = self
8766            .state
8767            .github_text
8768            .clone()
8769            .filter(|g| g.fingerprint == raw_fp)
8770        {
8771            return self.settle_withheld(winner, &pr, known).await;
8772        }
8773        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8774        let mut violations =
8775            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8776        let identity = crate::scrub::Identity::current();
8777        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8778            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8779            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8780        {
8781            violations.push(crate::github_text::Violation::SensitiveData);
8782        }
8783        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8784        // only a language violation starts one; sensitive data alone goes
8785        // straight to the owner, with the redacted text as the fallback.
8786        let has_language = violations
8787            .iter()
8788            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8789        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8790            self.state.event(
8791                "github-text",
8792                if has_language {
8793                    format!("description rejected: {violations:?}; requesting one rewrite")
8794                } else {
8795                    format!("description rejected: {violations:?}; asking the owner")
8796                },
8797            );
8798            let mut rewritten = false;
8799            let mut rejected_rewrite = None;
8800            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8801            {
8802                let output = text;
8803                if let Some(title) = summary_title(&output) {
8804                    // Judge the redacted candidate that will actually be posted.
8805                    // Sensitive data alone must not discard a useful rewrite.
8806                    let raw_body = summary_without_title(&output);
8807                    let clean_title = crate::scrub::scrub(&title, &identity);
8808                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8809                    let redacted = clean_title != title || body != raw_body;
8810                    let title = clean_title;
8811                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8812                    if !complete {
8813                        self.state.event(
8814                            "github-text",
8815                            "description rewrite omitted required findings",
8816                        );
8817                    }
8818                    if !body.trim().is_empty() {
8819                        body.push_str("\n\n");
8820                        body.push_str(&public_pr_evidence(&self.state, head));
8821                    }
8822                    let marker = format!("magi:run/{}", self.state.id);
8823                    if !body.trim().is_empty() && !body.contains(&marker) {
8824                        body.push_str(&format!("\n\n{marker}"));
8825                    }
8826                    let rewrite = PrMessage {
8827                        title: title.clone(),
8828                        body: body.clone(),
8829                    };
8830                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8831                    let mut failed = crate::github_text::check_for(
8832                        &self.state.config.graph,
8833                        &title,
8834                        &body,
8835                        redecision,
8836                    );
8837                    if (!crate::github_text::shareable(&title)
8838                        || !crate::github_text::shareable(&body))
8839                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8840                    {
8841                        failed.push(crate::github_text::Violation::SensitiveData);
8842                    }
8843                    if complete
8844                        && !title.trim().is_empty()
8845                        && !body.trim().is_empty()
8846                        && !failed.is_empty()
8847                    {
8848                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8849                            title: title.clone(),
8850                            body: body.clone(),
8851                            failed: failed
8852                                .iter()
8853                                .map(|v| crate::github_text::category(*v).to_owned())
8854                                .collect(),
8855                        });
8856                    }
8857                    if complete
8858                        && !title.trim().is_empty()
8859                        && !raw_body.trim().is_empty()
8860                        && failed.is_empty()
8861                    {
8862                        decision = redecision;
8863                        pr = PrMessage { title, body };
8864                        rewritten = true;
8865                        self.state
8866                            .event("github-text", "description rewrite passed");
8867                        if redacted {
8868                            self.state.event(
8869                                "github-text",
8870                                "sensitive data removed from description rewrite",
8871                            );
8872                        }
8873                    }
8874                }
8875            }
8876            if !rewritten {
8877                self.state.event(
8878                    "github-text",
8879                    "description rewrite unavailable or rejected; asking the owner",
8880                );
8881                if let Some(w) =
8882                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8883                {
8884                    // A different text than the one asked about before: that
8885                    // question no longer describes anything.
8886                    self.retire_github_text_question("the withheld text changed");
8887                    let record = crate::run::GithubTextAsk {
8888                        rewrite: rejected_rewrite,
8889                        fingerprint: raw_fp,
8890                        title: w.title,
8891                        body: w.body,
8892                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8893                        question: None,
8894                        asks: 0,
8895                        resolved: false,
8896                        chosen_title: None,
8897                        choice: None,
8898                        publication: None,
8899                    };
8900                    return self.settle_withheld(winner, &pr, record).await;
8901                }
8902            }
8903        }
8904        let (title, body) =
8905            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8906        Ok(Some(PrMessage { title, body }))
8907    }
8908
8909    /// Describe the whole change from its final diff in either run mode.
8910    ///
8911    /// The summary is tied to the final commit and to the merge base of the
8912    /// resolved remote base; a record for any other pair is dropped and the
8913    /// agent asked again, so a description never outlives the diff it was
8914    /// written from. The seat is read-only and the prompt tells it to read the
8915    /// diff and the code around it. `[roles] synthesizer` selects the agents
8916    /// (unset: the default pick), each asked at most once, advancing on an
8917    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8918    /// carries the reason it is unavailable; nothing is invented then.
8919    async fn summarize_change(
8920        &mut self,
8921        winner: &Candidate,
8922        head: Option<&str>,
8923    ) -> std::result::Result<(), String> {
8924        let Some(head) = head else {
8925            return Err("the final commit could not be read".to_owned());
8926        };
8927        let remote = self.state.config.merge.remote.clone();
8928        let tracking = format!("{remote}/{}", self.state.base_branch);
8929        if !git::rev_exists(&winner.worktree, &tracking).await {
8930            return Err(format!("the remote base `{tracking}` could not be read"));
8931        }
8932        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8933            Ok(b) if !b.is_empty() => b,
8934            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8935        };
8936        let evidence = crate::pr_outcome::summary_evidence(
8937            &self.state,
8938            crate::pr_outcome::FinalHead::At(head),
8939        );
8940        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8941        if self
8942            .state
8943            .change_summary
8944            .as_ref()
8945            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8946        {
8947            return Ok(());
8948        }
8949        if self.state.change_summary.take().is_some() {
8950            self.state.event(
8951                "pr-summary",
8952                "the final diff or evidence changed; the earlier change summary was discarded",
8953            );
8954        }
8955        // Save the source snapshot even if no generation agent is available.
8956        if let Err(e) = write_artifact(
8957            &self.state,
8958            &format!("public-description-{fingerprint}.md"),
8959            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8960        ) {
8961            tracing::warn!("could not save public description evidence: {e:#}");
8962        }
8963        let chain = match agent::pick_chain(
8964            &self.state.config.agents,
8965            self.state.config.roles.synthesizer.as_ref(),
8966            &agent::installed,
8967            "synthesizer",
8968        ) {
8969            Ok(chain) => chain,
8970            Err(e) => {
8971                tracing::warn!("no agent for the change summary: {e:#}");
8972                self.state.event("pr-summary", "no summary agent available");
8973                return Err("no agent was available to summarize it".to_owned());
8974            }
8975        };
8976        let subjects = git::subjects(&winner.worktree, &base, head)
8977            .await
8978            .unwrap_or_default();
8979        let stat = git::diff_stat(&winner.worktree, &base, head)
8980            .await
8981            .unwrap_or_default();
8982        let stat: String = stat.chars().take(6_000).collect();
8983        let mut prompt = crate::prompt::change_summary_in(
8984            &base,
8985            head,
8986            &subjects,
8987            stat.trim(),
8988            &self.state.config.graph.github_language,
8989        );
8990        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8991        let sessions = self.state.config.graph.sessions;
8992        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8993        let prompts = self.state.config.prompts.clone();
8994        let cache = self.state.config.cache_dir();
8995        let run = self.state.id.clone();
8996        for (i, spec) in chain.into_iter().enumerate() {
8997            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8998            let github_spec = match agent::for_github_text(
8999                &spec,
9000                &self.state.config.graph.github_language,
9001                &winner.worktree,
9002            ) {
9003                Ok(spec) => spec,
9004                Err(e) => {
9005                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9006                    continue;
9007                }
9008            };
9009            let job = SeatJob {
9010                spec: github_spec,
9011                seat,
9012                cwd: winner.worktree.clone(),
9013                prompt: prompt.clone(),
9014                timeout,
9015                allow_write: false,
9016                sessions,
9017                artifacts: agent::artifacts_dir(&self.state.dir()),
9018                stem: if i == 0 {
9019                    "change-summary".to_owned()
9020                } else {
9021                    format!("change-summary-{}", spec.id)
9022                },
9023                handover: None,
9024            };
9025            let ctx = WaveCtx {
9026                run: &run,
9027                node: "pr-summary",
9028                prompts: &prompts,
9029                cache: cache.as_deref(),
9030                round: None,
9031                carry_seats: false,
9032                pauses: Some((&self.pause, &self.interrupt)),
9033            };
9034            let (seat, outcome) =
9035                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9036            self.state.seats.insert(seat.key.clone(), seat);
9037            let text = match outcome {
9038                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9039                AgentOutcome::ResourceBlocked(why) => {
9040                    // No agent was asked: the run stops resumably before
9041                    // anything is published (see `guarded_pr_message`).
9042                    if self.state.resource_block.is_none() {
9043                        self.state.resource_block = Some(crate::run::ResourceBlock {
9044                            cause: crate::run::ResourceBlockCause::Other,
9045                            node: "pr-summary".to_owned(),
9046                            kind: "shared resource".to_owned(),
9047                            detail: why,
9048                            at: Timestamp::now(),
9049                        });
9050                    }
9051                    return Err("a shared resource was busy".to_owned());
9052                }
9053                _ => continue,
9054            };
9055            let blind = &self.state.config.blind;
9056            let Some(title) = summary_title(&text) else {
9057                continue;
9058            };
9059            let body: String = summary_without_title(&text)
9060                .lines()
9061                .filter(|l| !l.contains("magi:run/"))
9062                .collect::<Vec<_>>()
9063                .join("\n");
9064            let body = blind::sanitize_prose(body.trim(), blind);
9065            let title = blind::sanitize_prose(&title, blind);
9066            if body.trim().is_empty() || title.trim().is_empty() {
9067                continue;
9068            }
9069            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9070                continue;
9071            }
9072            // The branch must still be the one the agent was shown.
9073            if git::rev_parse(&winner.worktree, "HEAD")
9074                .await
9075                .ok()
9076                .as_deref()
9077                != Some(head)
9078            {
9079                return Err("the branch moved while the summary was written".to_owned());
9080            }
9081            if git::merge_base(&winner.worktree, &tracking, head)
9082                .await
9083                .ok()
9084                .as_deref()
9085                != Some(base.as_str())
9086                || crate::pr_outcome::summary_evidence(
9087                    &self.state,
9088                    crate::pr_outcome::FinalHead::At(head),
9089                ) != evidence
9090            {
9091                return Err("the summary evidence changed while it was written".to_owned());
9092            }
9093            self.state.change_summary = Some(crate::run::ChangeSummary {
9094                head: head.to_owned(),
9095                base,
9096                evidence: fingerprint,
9097                title,
9098                body,
9099            });
9100            self.state.event(
9101                "pr-summary",
9102                format!("change summary written by {} for the final diff", spec.id),
9103            );
9104            if let Err(e) = self.state.save() {
9105                tracing::warn!("could not save the change summary: {e:#}");
9106            }
9107            return Ok(());
9108        }
9109        self.state
9110            .event("pr-summary", "no agent produced a change summary");
9111        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9112    }
9113
9114    /// One read-only rewrite request for a withheld description. The
9115    /// implementer's own conversation is tried first when it exists and can be
9116    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9117    /// writes it in a separate conversation, skipping the implementer's agent.
9118    /// The prompt carries the whole text, so no agent needs any memory. Each
9119    /// agent is asked at most once and the chain advances on an error, quota,
9120    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9121    /// that has run out of quota must not end the rewrite. Nothing is recorded
9122    /// as a handover.
9123    async fn rewrite_pr_text(
9124        &mut self,
9125        winner: &Candidate,
9126        pr: &PrMessage,
9127        violations: &[crate::github_text::Violation],
9128    ) -> Option<String> {
9129        let prompt = format!(
9130            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9131            pr.title, pr.body
9132        );
9133        let sessions = self.state.config.graph.sessions;
9134        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9135        // (spec, the resumed implementer's seat). A fallback seat is created
9136        // only when its turn comes, so an unused fallback leaves no record.
9137        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9138        let key = format!("impl-{}", winner.label);
9139        if let Some(spec) = self
9140            .state
9141            .config
9142            .agents
9143            .iter()
9144            .find(|a| a.id == winner.agent)
9145            .cloned()
9146            && let Some(existing) = self.state.seats.get(&key)
9147            && existing.agent == spec.id
9148            && agent::has_session(spec.kind, existing, sessions)
9149        {
9150            attempts.push((spec, Some(existing.clone())));
9151        }
9152        match agent::pick_chain(
9153            &self.state.config.agents,
9154            self.state.config.roles.language_judge.as_ref(),
9155            &agent::installed,
9156            "language_judge",
9157        ) {
9158            Ok(chain) => {
9159                for spec in chain {
9160                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9161                        continue;
9162                    }
9163                    attempts.push((spec, None));
9164                }
9165            }
9166            Err(e) if attempts.is_empty() => {
9167                tracing::warn!("no agent for the description rewrite: {e:#}");
9168                self.state
9169                    .event("github-text", "no rewrite agent available");
9170                return None;
9171            }
9172            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9173        }
9174        let prompts = self.state.config.prompts.clone();
9175        let cache = self.state.config.cache_dir();
9176        let run = self.state.id.clone();
9177        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9178            let resumed = resumed_seat.is_some();
9179            let seat = resumed_seat.unwrap_or_else(|| {
9180                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9181            });
9182            let github_spec = match agent::for_github_text(
9183                &spec,
9184                &self.state.config.graph.github_language,
9185                &winner.worktree,
9186            ) {
9187                Ok(spec) => spec,
9188                Err(e) => {
9189                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9190                    continue;
9191                }
9192            };
9193            let job = SeatJob {
9194                spec: github_spec,
9195                seat,
9196                cwd: winner.worktree.clone(),
9197                prompt: prompt.clone(),
9198                timeout: retry_budget(timeout_full, resumed),
9199                allow_write: false,
9200                sessions,
9201                artifacts: agent::artifacts_dir(&self.state.dir()),
9202                stem: if i == 0 {
9203                    "github-text-rewrite".to_owned()
9204                } else {
9205                    format!("github-text-rewrite-{}", spec.id)
9206                },
9207                handover: None,
9208            };
9209            let ctx = WaveCtx {
9210                run: &run,
9211                node: "github-text",
9212                prompts: &prompts,
9213                cache: cache.as_deref(),
9214                round: None,
9215                carry_seats: false,
9216                pauses: Some((&self.pause, &self.interrupt)),
9217            };
9218            let (seat, outcome) =
9219                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9220            self.state.seats.insert(seat.key.clone(), seat);
9221            match outcome {
9222                AgentOutcome::Ok(o)
9223                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9224                {
9225                    return Some(o.text);
9226                }
9227                AgentOutcome::ResourceBlocked(_) => return None,
9228                _ => {}
9229            }
9230        }
9231        None
9232    }
9233
9234    /// Abandon the standing posting-gate question, if any.
9235    fn retire_github_text_question(&mut self, why: &str) {
9236        let Some(id) = self
9237            .state
9238            .github_text
9239            .as_ref()
9240            .and_then(|g| g.question.clone())
9241        else {
9242            return;
9243        };
9244        let _ = ask::Questions::open().update(&id, |q| {
9245            q.abandon(why);
9246            Ok(())
9247        });
9248    }
9249
9250    /// Carry a withheld text to its end: file the question, park while it is
9251    /// open, and on an answer, silence or timeout pick the text to post. The
9252    /// decision is saved before it is used, so it applies once.
9253    async fn settle_withheld(
9254        &mut self,
9255        winner: &Candidate,
9256        pr: &PrMessage,
9257        mut g: crate::run::GithubTextAsk,
9258    ) -> Result<Option<PrMessage>> {
9259        use crate::github_text as gt;
9260        let store = ask::Questions::open();
9261        let timeout = self.state.config.graph.answer_timeout;
9262        if !g.resolved {
9263            let mut standing = g
9264                .question
9265                .as_ref()
9266                .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9267            if let Some(q) = standing.as_ref()
9268                && q.status.open()
9269                && !store.held_reply(&q.id)
9270                && gt::expired(q, timeout)
9271            {
9272                let why = format!("no answer within {}s of asking", timeout.max(1));
9273                standing = store
9274                    .update(&q.id, |q| {
9275                        q.abandon(&why);
9276                        Ok(())
9277                    })
9278                    .ok()
9279                    .map(|(q, ())| q);
9280            }
9281            let mut ask_again: Option<Option<Vec<String>>> = None;
9282            match standing {
9283                // Recorded but never written (a stop between the two saves),
9284                // or removed: ask again without spending the retry.
9285                None if g.asks <= 1 => {
9286                    g.asks = 0;
9287                    ask_again = Some(None);
9288                }
9289                None => {
9290                    g.resolved = true;
9291                    g.chosen_title = None;
9292                    g.choice = Some(GithubTextChoice::Fallback);
9293                }
9294                Some(q) if q.status.open() => {
9295                    self.park_on_github_text(&g)?;
9296                    return Ok(None);
9297                }
9298                Some(q) => match gt::read_reply(&q) {
9299                    gt::Reply::Fallback => {
9300                        g.resolved = true;
9301                        g.chosen_title = None;
9302                        g.choice = Some(GithubTextChoice::Fallback);
9303                    }
9304                    gt::Reply::Original => {
9305                        g.resolved = true;
9306                        g.chosen_title = None;
9307                        g.choice = Some(GithubTextChoice::Original);
9308                    }
9309                    gt::Reply::Rewrite => {
9310                        g.resolved = true;
9311                        g.chosen_title = None;
9312                        g.choice = Some(if g.rewrite.is_some() {
9313                            GithubTextChoice::Rewrite
9314                        } else {
9315                            GithubTextChoice::Fallback
9316                        });
9317                    }
9318                    gt::Reply::Title(text) => {
9319                        let first = text
9320                            .lines()
9321                            .map(str::trim)
9322                            .find(|l| !l.is_empty())
9323                            .unwrap_or("");
9324                        let decision = if first.is_empty() {
9325                            None
9326                        } else {
9327                            let candidate = PrMessage {
9328                                title: first.to_owned(),
9329                                body: String::new(),
9330                            };
9331                            self.judge_pr_language(&winner.worktree, &candidate).await
9332                        };
9333                        match gt::vet_title(
9334                            &text,
9335                            decision,
9336                            self.state.config.graph.github_english(),
9337                        ) {
9338                            Ok(title) => {
9339                                g.resolved = true;
9340                                g.chosen_title = Some(title);
9341                                g.choice = Some(GithubTextChoice::MyTitle);
9342                            }
9343                            Err(cats) if g.asks < 2 => {
9344                                ask_again = Some(Some(
9345                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9346                                ))
9347                            }
9348                            Err(_) => {
9349                                g.resolved = true;
9350                                g.chosen_title = None;
9351                                g.choice = Some(GithubTextChoice::Fallback);
9352                            }
9353                        }
9354                    }
9355                },
9356            }
9357            if let Some(rejected) = ask_again {
9358                // A retry (after a rejected replacement title) is still about
9359                // the original text: the same ORIGINAL / FALLBACK comparison.
9360                let w = gt::Withheld {
9361                    title: g.title,
9362                    body: g.body,
9363                    categories: g.categories.clone(),
9364                };
9365                // Named by the original text's fingerprint alone, so a resume
9366                // rewrites the same file and the detail stays identical.
9367                let artifact = match crate::run::write_artifact(
9368                    &self.state,
9369                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9370                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9371                ) {
9372                    Ok(p) => Some(p),
9373                    Err(e) => {
9374                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9375                        None
9376                    }
9377                };
9378                let (detail, comparison) = gt::question_content(
9379                    &w,
9380                    &pr.title,
9381                    &pr.body,
9382                    &self.state.id,
9383                    rejected.as_deref(),
9384                    artifact.as_deref(),
9385                    g.rewrite.as_ref(),
9386                );
9387                let mut q = ask::Question::new(
9388                    self.state.id.clone(),
9389                    gt::ASK_NODE.to_owned(),
9390                    gt::ASK_SEAT.to_owned(),
9391                    gt::question_summary(&self.state.config.graph.language, &w),
9392                    detail,
9393                    gt::question_choices(&w, g.rewrite.as_ref()),
9394                );
9395                q.posting_comparison = Some(comparison);
9396                q.answer_timeout = timeout;
9397                // Recorded before it is written: a stop in between leaves a
9398                // record naming a question the resume then files again.
9399                g.question = Some(q.id.clone());
9400                g.asks += 1;
9401                self.state.github_text = Some(g.clone());
9402                self.state.event(
9403                    "github-text",
9404                    format!("asking the owner about the withheld text ({})", q.short()),
9405                );
9406                self.state.save()?;
9407                store
9408                    .put_executable(&mut q)
9409                    .context("file the posting-gate question")?;
9410                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9411                    tracing::warn!(
9412                        "could not notify about posting-gate question {}: {e:#}",
9413                        q.short()
9414                    );
9415                }
9416                self.park_on_github_text(&g)?;
9417                return Ok(None);
9418            }
9419        }
9420        let resolved = g.chosen_title.clone();
9421        let original = g.choice == Some(GithubTextChoice::Original);
9422        // Field names only: the event log never records the text.
9423        self.state.event(
9424            "github-text",
9425            if original {
9426                format!(
9427                    "posting the original text for: {}",
9428                    match (g.title, g.body) {
9429                        (true, true) => "title, description",
9430                        (true, false) => "title",
9431                        _ => "description",
9432                    }
9433                )
9434            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9435                "posting the saved rewrite title and description".to_owned()
9436            } else if resolved.is_some() {
9437                "posting the owner's replacement title".to_owned()
9438            } else {
9439                "posting the neutral text for the withheld fields".to_owned()
9440            },
9441        );
9442        self.state.github_text = Some(g.clone());
9443        self.state.save()?;
9444        if g.choice == Some(GithubTextChoice::Rewrite)
9445            && let Some(r) = g.rewrite.as_ref()
9446        {
9447            return Ok(Some(PrMessage {
9448                title: r.title.clone(),
9449                body: r.body.clone(),
9450            }));
9451        }
9452        let w = gt::Withheld {
9453            title: g.title,
9454            body: g.body,
9455            categories: g.categories.clone(),
9456        };
9457        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9458        if let Some(t) = resolved {
9459            title = t;
9460        }
9461        // Every withheld field is now neutral or vetted English.
9462        let english = gt::LanguageDecision {
9463            title_english: true,
9464            body_english: true,
9465            body_complete: true,
9466        };
9467        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9468        if original {
9469            // The owner's explicit choice overrides the gate for the fields it
9470            // was made for; the others keep what the gate produced.
9471            if g.title {
9472                title = pr.title.clone();
9473            }
9474            if g.body {
9475                body = pr.body.clone();
9476            }
9477        }
9478        Ok(Some(PrMessage { title, body }))
9479    }
9480
9481    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9482        self.state.github_text = Some(g.clone());
9483        self.state.end_pass_parked(ParkCause::Question);
9484        self.state.parked = true;
9485        self.state.event(
9486            "github-text",
9487            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9488        );
9489        self.state.save()
9490    }
9491
9492    /// Ask `[roles] language_judge` about this text and record which source
9493    /// decided; the text itself is never recorded.
9494    async fn judge_pr_language(
9495        &mut self,
9496        cwd: &Path,
9497        pr: &PrMessage,
9498    ) -> Option<crate::github_text::LanguageDecision> {
9499        self.state.config.roles.language_judge.as_ref()?;
9500        if !self.state.config.graph.language_gate() {
9501            return None;
9502        }
9503        let decision =
9504            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9505        self.state.event(
9506            "github-text",
9507            if decision.is_some() {
9508                "language decided by the language judge"
9509            } else {
9510                "language judge unavailable; using built-in heuristics"
9511            },
9512        );
9513        decision
9514    }
9515
9516    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9517        Box::pin(self.merge_inner())
9518    }
9519
9520    async fn merge_inner(&mut self) -> Result<()> {
9521        if self.resource_halt()? || self.hold_base_sync().await? {
9522            return Ok(());
9523        }
9524        // Same reasoning as `gate`: ask the review and gate records directly
9525        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9526        // skip can rewrite on reentry to something that no longer says
9527        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9528        // so a hand-off (open findings, green verification) reaches merge
9529        // exactly like a genuinely clean round does.
9530        //
9531        // A run resumed mid-`land` never reaches here at all: `execute`
9532        // recognises `RunStatus::Landing` before it even calls `prep`, and
9533        // routes straight to `run_land` instead. That has to happen a level
9534        // up from this function, not with a check in here, because
9535        // `review_loop`'s own status recomputation (see its doc) runs
9536        // *before* `merge` on every reentry and would otherwise overwrite
9537        // the `Landing` marker with `Gating` before this node ever saw it.
9538        if self
9539            .state
9540            .base_sync
9541            .as_ref()
9542            .is_some_and(|s| s.conflict.is_some())
9543            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9544                != Some(RunStatus::Gating)
9545            // `gate_ran == false` is not "passed" - `gate` leaves it false
9546            // both before it has ever run and when its last attempt was
9547            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9548            // permission to merge on nothing but the review record. Only a
9549            // gate that actually ran may proceed; an intentional local
9550            // review can also finish with verification explicitly not configured.
9551            // `RunState::gate_status` computes that distinction.
9552            || !self.state.gate_allows_progress()
9553        {
9554            return Ok(());
9555        }
9556        // This node's own record, not `status`: `status == Ready` is not
9557        // unique to the harmless `MergeMode::None` path this line was
9558        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9559        // run's PR was closed without merging — and on that run `mode` is
9560        // still `Pr`, so a reentry that fell through here would push and
9561        // open a second pull request. `self.state.merge` is set exactly once
9562        // this node (or `land`) has already produced a verdict, under every
9563        // mode, which is what "already done" actually means here.
9564        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9565            return Ok(());
9566        }
9567        let Some(winner) = self.state.winner().cloned() else {
9568            return Ok(());
9569        };
9570        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9571        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9572            self.state.status = RunStatus::Blocked;
9573            self.state.event(
9574                "gate",
9575                "winner changed after gate; fresh verification required",
9576            );
9577            self.state.save()?;
9578            return Ok(());
9579        }
9580        let wait = self.repository_wait();
9581        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9582            return Ok(());
9583        }
9584        let repo = self.state.repo.clone();
9585        let base = self.state.base_branch.clone();
9586        let mode = self.state.config.merge.mode;
9587        let style = self.state.config.merge.style;
9588        let facts = if is_review_run(&self.state) {
9589            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9590            let start = review_base(
9591                &repo,
9592                &self.state.config.merge.remote,
9593                &base,
9594                &self.state.base_commit,
9595                &winner.branch,
9596            )
9597            .await;
9598            branch_facts(&repo, &start, &winner.branch).await
9599        } else {
9600            None
9601        };
9602        // `None` when the base could not be freshly read: then an adopted
9603        // pull request's title is left alone.
9604        let leaked = if is_review_run(&self.state) {
9605            leaked_subjects(&self.state, &winner.branch).await
9606        } else {
9607            Some(Vec::new())
9608        };
9609        // `None`: parked on the owner's word about a withheld title or body.
9610        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9611        // the resume arrives here again.
9612        let Some(pr) = self
9613            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9614            .await?
9615        else {
9616            return Ok(());
9617        };
9618        let message = pr.commit_message();
9619
9620        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9621        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9622            return Ok(());
9623        }
9624        let outcome = match mode {
9625            MergeMode::None => MergeOutcome {
9626                mode,
9627                ok: true,
9628                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9629                empty: false,
9630            },
9631            MergeMode::Pr | MergeMode::Local
9632                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9633            {
9634                MergeOutcome {
9635                    mode,
9636                    ok: false,
9637                    detail: empty_candidate_detail(&self.state, &base),
9638                    empty: true,
9639                }
9640            }
9641            MergeMode::Local => {
9642                let on = git::current_branch(&repo).await?;
9643                if on.as_deref() != Some(base.as_str()) {
9644                    MergeOutcome {
9645                        mode,
9646                        ok: false,
9647                        detail: format!(
9648                            "{} has {} checked out, not the base branch {base}",
9649                            repo.display(),
9650                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9651                        ),
9652                        empty: false,
9653                    }
9654                } else if !git::is_clean(&repo).await? {
9655                    MergeOutcome {
9656                        mode,
9657                        ok: false,
9658                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9659                        empty: false,
9660                    }
9661                } else {
9662                    let out = match style {
9663                        MergeStyle::Merge => {
9664                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9665                        }
9666                        MergeStyle::Squash => {
9667                            git::merge_squash(&repo, &winner.branch, &message).await?
9668                        }
9669                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9670                    };
9671                    MergeOutcome {
9672                        mode,
9673                        ok: out.ok(),
9674                        detail: if out.ok() { out.stdout } else { out.stderr },
9675                        empty: false,
9676                    }
9677                }
9678            }
9679            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9680                self.state.pr_publication_pending = true;
9681                self.state.save()?;
9682                let out =
9683                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9684                self.state.pr_publication_pending = out.is_err();
9685                MergeOutcome {
9686                    mode,
9687                    ok: out.is_ok(),
9688                    detail: match out {
9689                        Ok(url) => url,
9690                        Err(e) => format!("{e:#}"),
9691                    },
9692                    empty: false,
9693                }
9694            }
9695            MergeMode::Pr => {
9696                let remote = self.state.config.merge.remote.clone();
9697                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9698                let pushed = if self.state.foreign_checkout.is_some() {
9699                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9700                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9701                } else {
9702                    git::push(&winner.worktree, &remote, &winner.branch).await?
9703                };
9704                if !pushed.ok() {
9705                    MergeOutcome {
9706                        mode,
9707                        ok: false,
9708                        detail: pushed.stderr,
9709                        empty: false,
9710                    }
9711                } else {
9712                    if self.state.foreign_checkout.is_some() {
9713                        self.state.pr_pushed_head = Some(delivery_head.clone());
9714                    }
9715                    // A retry or resume of a run whose branch already has an
9716                    // open pull request adopts it rather than failing on a
9717                    // duplicate. Only this winner branch into this base:
9718                    // `branch_for` derives the name from the run id, so a
9719                    // different run's pull request never matches.
9720                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9721                    let out = match pr_merge_plan(found) {
9722                        PrPlan::Create => {
9723                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9724                                .await
9725                        }
9726                        PrPlan::Adopt { url, title } => {
9727                            self.state
9728                                .event("merge", format!("Pr: adopted open pull request {url}"));
9729                            if self.state.explicit_github_text_selection() {
9730                                // An explicit publication selection applies the prepared
9731                                // payload; acceptance alone is not publication proof.
9732                                land::set_pr_text_verbatim(
9733                                    &winner.worktree,
9734                                    &url,
9735                                    &pr.title,
9736                                    &pr.body,
9737                                )
9738                                .await
9739                                .map(|()| url)
9740                            } else {
9741                                if title != pr.title
9742                                    && (!is_review_run(&self.state)
9743                                        || leaked
9744                                            .as_deref()
9745                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9746                                    && let Err(e) =
9747                                        if self.state.github_text.as_ref().is_some_and(|g| {
9748                                            g.resolved
9749                                                && g.title
9750                                                && g.choice == Some(GithubTextChoice::Original)
9751                                        }) {
9752                                            // The owner chose to post this title unmodified.
9753                                            land::set_pr_title_verbatim(
9754                                                &winner.worktree,
9755                                                &url,
9756                                                &pr.title,
9757                                            )
9758                                            .await
9759                                        } else {
9760                                            land::set_pr_title(
9761                                                &mut self.state,
9762                                                &winner.worktree,
9763                                                &url,
9764                                                &pr.title,
9765                                            )
9766                                            .await
9767                                        }
9768                                {
9769                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9770                                    self.state
9771                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9772                                }
9773                                Ok(url)
9774                            }
9775                        }
9776                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9777                    };
9778                    match out {
9779                        Ok(url) => MergeOutcome {
9780                            mode,
9781                            ok: true,
9782                            detail: url,
9783                            empty: false,
9784                        },
9785                        Err(e) => MergeOutcome {
9786                            mode,
9787                            ok: false,
9788                            detail: e.to_string(),
9789                            empty: false,
9790                        },
9791                    }
9792                }
9793            }
9794        };
9795
9796        self.state.status = match (mode, outcome.ok) {
9797            (MergeMode::None, _) => RunStatus::Ready,
9798            (_, true) if outcome.empty => RunStatus::Ready,
9799            (MergeMode::Pr, true) => RunStatus::Ready,
9800            (_, true) => RunStatus::Merged,
9801            (_, false) => RunStatus::Blocked,
9802        };
9803        self.state.event(
9804            "merge",
9805            format!(
9806                "{:?}: {}",
9807                mode,
9808                outcome.detail.lines().next().unwrap_or("")
9809            ),
9810        );
9811        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9812            self.state.record_github_text_publication(&outcome.detail);
9813        }
9814        self.state.merge = Some(outcome);
9815        self.state.save()?;
9816
9817        // The PR is open and the run would historically stop here, leaving the
9818        // operator to watch checks, feed review comments back to a fixer, and
9819        // merge. That was done by hand six times in one session before this
9820        // existed. Opt-in, because merging is the one irreversible thing magi
9821        // can do to a repository.
9822        if self.state.config.graph.land
9823            && mode == MergeMode::Pr
9824            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9825        {
9826            self.run_land().await?;
9827        }
9828        // `run_land` may have left `status` at `Landing` - still waiting on
9829        // CI or the owner's approval, not actually settled - so this has to
9830        // read whatever `status` ended up as here, not the `Merged` this
9831        // function set a few lines up.
9832        self.settle_questions();
9833        Ok(())
9834    }
9835
9836    /// Enter `land`.
9837    ///
9838    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9839    /// a resumed run's re-entry. `land::land` itself is what serialises the
9840    /// two git-mutating moments inside the loop — the rebase push and
9841    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9842    /// to hold a lock across the whole call, and doing so would serialise
9843    /// this run's CI wait against a *different* run's land-approval resume
9844    /// in the same repository, which is exactly the "must not wait on
9845    /// another task" property the daemon's slot-freeing exists to give.
9846    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9847        Box::pin(self.run_land_inner())
9848    }
9849
9850    async fn run_land_inner(&mut self) -> Result<()> {
9851        crate::pr_head::ensure_owner(&self.state)?;
9852        let url = self
9853            .state
9854            .pr
9855            .as_ref()
9856            .map(|pr| pr.url.clone())
9857            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9858            .unwrap_or_default();
9859        let url = url.lines().next().unwrap_or("").trim().to_owned();
9860        if !url.starts_with("http") {
9861            return Ok(());
9862        }
9863        // A land failure is not a lost run: the work is on a branch and the
9864        // pull request is open, which is exactly where a human takes over.
9865        loop {
9866            match land::land(&mut self.state, &url).await {
9867                Ok(_)
9868                    if self
9869                        .state
9870                        .merge_repairs
9871                        .last()
9872                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9873                {
9874                    if self.repair_merge_review().await? {
9875                        continue;
9876                    }
9877                }
9878                Ok(_) if self.state.resource_block.is_some() => {
9879                    return self.stop_for_resource();
9880                }
9881                Ok(pr) if self.state.parked => {
9882                    // `land` already saved the parked marker; nothing here
9883                    // overrides `status` back to a terminal value while an
9884                    // approval is still outstanding.
9885                    let _ = pr;
9886                }
9887                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9888                    self.state.status = RunStatus::Superseded;
9889                    self.state.save()?;
9890                }
9891                Ok(pr) => {
9892                    self.state.status = match pr.state {
9893                        land::PrLifecycle::Merged => RunStatus::Merged,
9894                        _ => RunStatus::Blocked,
9895                    };
9896                    // Downstream of a confirmed merge only - see
9897                    // `bump::should_release_bump`'s own doc for why this one
9898                    // check covers all three of `land`'s success paths.
9899                    // Best-effort: the run already landed, so a failure here
9900                    // (the decision call, `gh`, `cargo`) is recorded and never
9901                    // turns a landed run into a failed one.
9902                    if bump::should_release_bump(self.state.status)
9903                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9904                    {
9905                        // The event is the run's own record. Not-eligible cases
9906                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9907                        // `Err` is a bump that was tried and failed:
9908                        // `after_merge` itself raises the operator notice for
9909                        // that, whether or not a release PR exists yet.
9910                        self.state
9911                            .event("bump", format!("release bump skipped: {e:#}"));
9912                    }
9913                    // Independent of the bump, and best-effort in the same way:
9914                    // findings the merge left open become follow-up tasks.
9915                    if self.state.status == RunStatus::Merged {
9916                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9917                    }
9918                    self.state.save()?;
9919                }
9920                Err(e) => {
9921                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9922                        return Err(e);
9923                    }
9924                    self.state.status = RunStatus::Blocked;
9925                    self.state.event("land", format!("gave up: {e}"));
9926                    self.state.save()?;
9927                }
9928            }
9929            break;
9930        }
9931        Ok(())
9932    }
9933
9934    /// One repair cycle stays under the existing driver/task claim. An
9935    /// interrupted fixer is rolled back and held rather than run twice.
9936    async fn repair_merge_review(&mut self) -> Result<bool> {
9937        use crate::run::MergeRepairPhase as Phase;
9938        crate::pr_head::ensure_owner(&self.state)?;
9939        let i = self.state.merge_repairs.len() - 1;
9940        let request = self.state.merge_repairs[i].clone();
9941        if request.phase == Phase::Fixing {
9942            let restored = match self.state.winner().cloned() {
9943                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9944                None => Err(anyhow::anyhow!("repair has no winning branch")),
9945            };
9946            let why = match restored {
9947                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9948                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9949                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9950            };
9951            self.hold_merge_repair(i, why)?;
9952            return Ok(false);
9953        }
9954        // A completed repair gate is a publication checkpoint. On reentry,
9955        // reject external changes before the ordinary gate path can replace
9956        // that evidence by verifying a new head. Requested repairs still
9957        // start their own review generation below.
9958        if request.phase == Phase::Reviewing && self.state.gate_ran {
9959            let winner = self
9960                .state
9961                .winner()
9962                .context("repair has no winning branch")?;
9963            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9964            if self.state.gate_head.as_deref() != Some(head.as_str())
9965                || !git::is_clean(&winner.worktree).await?
9966            {
9967                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9968                return Ok(false);
9969            }
9970        }
9971        let failure = if i >= self.state.config.graph.land_rounds
9972            || self.state.config.graph.review_rounds == 0
9973        {
9974            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9975        } else {
9976            None
9977        };
9978        if let Some(why) = failure {
9979            self.hold_merge_repair(i, why)?;
9980            return Ok(false);
9981        }
9982        if request.phase == Phase::Requested {
9983            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9984                Ok(pr) => pr,
9985                Err(e) => {
9986                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9987                    return Ok(false);
9988                }
9989            };
9990            self.state.merge_repairs[i].phase = Phase::Fixing;
9991            self.state.land_approval = None;
9992            self.state.land_armed_head = None;
9993            self.state.save()?;
9994            // Serialize the exact records, including finding ids/details,
9995            // reconsideration reasons, final votes and verification output.
9996            let evidence = serde_json::to_string_pretty(&request)?;
9997            let reason = format!(
9998                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9999            );
10000            let budget = self.state.config.graph.land_rounds;
10001            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10002            match result {
10003                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10004                Ok(land::Fixed::Deferred) => {
10005                    // An unavailable fixer did not spend this repair request.
10006                    // Restore partial fallback work before retrying it, as for
10007                    // an interrupted repair, without closing the request.
10008                    let worktree = self
10009                        .state
10010                        .winner()
10011                        .context("repair has no winning branch")?
10012                        .worktree
10013                        .clone();
10014                    self.graft_restore(&worktree, &request.head).await?;
10015                    self.state.merge_repairs[i].phase = Phase::Requested;
10016                    self.stop_for_resource()?;
10017                    return Ok(false);
10018                }
10019                Ok(land::Fixed::Failed(why)) => {
10020                    self.hold_merge_repair(
10021                        i,
10022                        format!(
10023                            "repair failed: {why}; inspect the branch and run an explicit review"
10024                        ),
10025                    )?;
10026                    return Ok(false);
10027                }
10028                Err(e) => {
10029                    self.hold_merge_repair(
10030                        i,
10031                        format!(
10032                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10033                        ),
10034                    )?;
10035                    return Ok(false);
10036                }
10037            }
10038            // Even no change gets a real new reviewer wave. The previous
10039            // evidence remains in the request; a former clean/concluded round
10040            // cannot short-circuit the new review or gate.
10041            self.begin_merge_repair_review(i)?;
10042        }
10043        self.review_loop().await?;
10044        self.gate().await?;
10045        if self.resource_halt()? || self.state.parked {
10046            return Ok(false);
10047        }
10048        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10049            != Some(RunStatus::Gating)
10050            || !self.state.gate_allows_progress()
10051        {
10052            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10053            return Ok(false);
10054        }
10055        // Review and gate may have added commits after the initial repair.
10056        // Publish exactly their final head to the existing PR, with a lease.
10057        let winner = self
10058            .state
10059            .winner()
10060            .cloned()
10061            .context("repair has no winning branch")?;
10062        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10063        if self.state.gate_head.as_deref() != Some(after.as_str())
10064            || !git::is_clean(&winner.worktree).await?
10065        {
10066            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10067            return Ok(false);
10068        }
10069        let expected = self
10070            .state
10071            .pr_pushed_head
10072            .as_deref()
10073            .unwrap_or(&request.head)
10074            .to_owned();
10075        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10076        // Push the checked commit itself: a local ref moving after the check
10077        // must not change which commit the forge receives.
10078        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10079            return Ok(false);
10080        }
10081        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10082        let refspec = format!("{after}:refs/heads/{public}");
10083        let push = git::git_raw(
10084            &winner.worktree,
10085            &["push", &lease, &self.state.config.merge.remote, &refspec],
10086        )
10087        .await;
10088        match push {
10089            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10090            Ok(out) => {
10091                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10092                return Ok(false);
10093            }
10094            Err(e) => {
10095                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10096                return Ok(false);
10097            }
10098        }
10099        self.state.merge_repairs[i].phase = Phase::Complete;
10100        self.state.status = RunStatus::Landing;
10101        self.state.event(
10102            "repair",
10103            "new review and gate completed; fresh owner merge decision required",
10104        );
10105        self.state.save()?;
10106        Ok(true)
10107    }
10108
10109    fn merge_repair_artifacts(&self) -> PathBuf {
10110        let artifacts = agent::artifacts_dir(&self.state.dir());
10111        match self
10112            .state
10113            .merge_repairs
10114            .last()
10115            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10116        {
10117            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10118            None => artifacts,
10119        }
10120    }
10121
10122    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10123        if let Some(e2e) = self.state.head_e2e.take() {
10124            self.state
10125                .verification_history
10126                .push(crate::run::VerificationEvidence {
10127                    superseded_by: None,
10128                    archived_at: Some(Timestamp::now()),
10129                    fixes_spent: Some(self.state.gate_fixes.len()),
10130                    gate_head: None,
10131                    gate_ran: false,
10132                    gate: Vec::new(),
10133                    e2e: Some(e2e),
10134                });
10135        }
10136        self.state.reviews.clear();
10137        self.state.gate.clear();
10138        self.state.gate_ran = false;
10139        self.state.gate_head = None;
10140        self.state.gate_fixes.clear();
10141        self.state.gate_repair_e2e = None;
10142        self.state.pre_gate.clear();
10143        self.state.pre_gate_commit = None;
10144        self.state.contested_handoff = None;
10145        self.state.land_reruns = None;
10146        self.state.land_approval = None;
10147        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10148        self.state.status = RunStatus::Reviewing;
10149        self.state.save()
10150    }
10151
10152    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10153        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10154        self.state.merge_repairs[index].failure = Some(why.clone());
10155        self.state.status = RunStatus::Blocked;
10156        self.state.event("repair", why);
10157        self.state.save()
10158    }
10159
10160    // -------------------------------------------------------------- helpers
10161
10162    /// Fetch or create a seat, keeping its conversation across nodes.
10163    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10164        if let Some(existing) = self.state.seats.get(key)
10165            && existing.agent == agent
10166        {
10167            return existing.clone();
10168        }
10169        // A seat that changes agent mints its session id from the agent too,
10170        // like a handover: the old agent's uuid is already taken by the CLI.
10171        let fresh = if self.state.seats.contains_key(key) {
10172            handover_seat(key, agent, self.state.next_seat_seed())
10173        } else {
10174            SeatState::new(key, agent, self.state.seed)
10175        };
10176        self.state.seats.insert(key.to_owned(), fresh.clone());
10177        fresh
10178    }
10179
10180    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10181    /// seat to another roster agent, in which case that agent. Nodes that
10182    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10183    /// reconsideration) must keep talking to whoever answered it, not slip
10184    /// back to the agent that failed it.
10185    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10186        match self.state.seats.get(key) {
10187            Some(s) if s.agent != spec.id => {
10188                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10189            }
10190            _ => spec,
10191        }
10192    }
10193
10194    /// A candidate rendered for judging, with the leak policy applied.
10195    fn view(&self, c: &Candidate) -> CandidateView {
10196        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10197            .unwrap_or_default();
10198        let (patch, _) = blind::sanitize_patch(
10199            &format!("candidate {} patch", c.label),
10200            &raw,
10201            &self.state.config.blind,
10202        );
10203        CandidateView {
10204            label: c.label,
10205            branch: c.branch.clone(),
10206            summary: c.summary.clone(),
10207            stat: c.stat.clone(),
10208            patch,
10209        }
10210    }
10211
10212    /// The full candidate set as prompt text, for seats with no live session.
10213    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10214        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10215        prompt::judge(
10216            "(see above)",
10217            &views,
10218            self.roles.judges.len(),
10219            base_short,
10220            "en",
10221        )
10222    }
10223
10224    /// The final-vote prompt with everything a seat that has no session of its
10225    /// own needs: the candidates, the seat's own ranking and reasons, and the
10226    /// anonymised deliberation it took part in (only when there was one, so a
10227    /// handed-over seat never sees more than the seat it replaces did). The
10228    /// `Final vote` heading stays first.
10229    fn vote_prompt_full(
10230        &self,
10231        j: usize,
10232        labels: &[char],
10233        language: &str,
10234        candidates: &[Candidate],
10235        base_short: &str,
10236    ) -> String {
10237        let mut text = format!(
10238            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10239            prompt::final_vote(labels, language),
10240            self.state.instruction,
10241            self.candidate_block(candidates, base_short)
10242        );
10243        if let Some(own) = self
10244            .state
10245            .judgements
10246            .get(j)
10247            .filter(|r| !r.ranking.is_empty())
10248        {
10249            let reasons = own
10250                .reasons
10251                .iter()
10252                .map(|(k, v)| format!("- {k}: {v}"))
10253                .collect::<Vec<_>>()
10254                .join("\n");
10255            text.push_str(&format!(
10256                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10257                own.ranking.iter().collect::<String>(),
10258                if reasons.is_empty() {
10259                    ""
10260                } else {
10261                    ", because:\n"
10262                }
10263            ));
10264        }
10265        if !self.state.deliberation.is_empty() {
10266            text.push_str("\n# What was argued before this vote\n");
10267            for t in self.transcript(&[], j) {
10268                text.push_str(&format!(
10269                    "\n## {}{}\n\n{}\n",
10270                    t.who,
10271                    if t.is_self { " (you)" } else { "" },
10272                    t.body.trim()
10273                ));
10274            }
10275        }
10276        text
10277    }
10278
10279    /// Anonymised transcript for judge `self_idx`.
10280    ///
10281    /// The initial rankings are always the opening statements. Seeding them
10282    /// only when no turn had been taken yet meant every judge after the first
10283    /// argued against a single voice instead of against the actual split — the
10284    /// disagreement is the information, so it is always on the table.
10285    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10286        let mut turns = Vec::new();
10287        for j in &self.state.judgements {
10288            if j.ranking.is_empty() {
10289                continue;
10290            }
10291            let reasons = j
10292                .reasons
10293                .iter()
10294                .map(|(k, v)| format!("- {k}: {v}"))
10295                .collect::<Vec<_>>()
10296                .join("\n");
10297            turns.push(Turn {
10298                who: format!("Judge {} (opening ranking)", j.judge),
10299                is_self: j.judge == self_idx + 1,
10300                body: format!(
10301                    "Ranked {}{}{reasons}",
10302                    j.ranking.iter().collect::<String>(),
10303                    if reasons.is_empty() {
10304                        ""
10305                    } else {
10306                        ", because:\n"
10307                    }
10308                ),
10309            });
10310        }
10311        for t in self
10312            .state
10313            .deliberation
10314            .iter()
10315            .flat_map(|r| r.turns.iter())
10316            .chain(current)
10317        {
10318            turns.push(Turn {
10319                who: format!("Judge {}", t.judge),
10320                is_self: t.judge == self_idx + 1,
10321                body: t.body.clone(),
10322            });
10323        }
10324        turns
10325    }
10326}
10327
10328/// Does this seat still hold the context a follow-up prompt would rely on?
10329fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10330    agent::has_session(spec.kind, seat, sessions)
10331}
10332
10333/// The next entry in `roster` after `start`, never wrapping back to the
10334/// front, whose id is not in `tried` yet.
10335///
10336/// Starts one past `start` rather than at the front of `roster`: `start` is
10337/// the seat's own original position, and a seat whose candidate slot already
10338/// sits on the roster's second entry must fall through to the third next, not
10339/// restart at the first — which is very likely a different candidate's own
10340/// agent already. Never wraps back past `start`, for the same reason: an
10341/// entry earlier in the roster than the seat's own position is almost
10342/// certainly some *other* candidate slot's own agent, and once the tail of
10343/// the roster is exhausted there are no more untried agents for *this* seat
10344/// to fall through to — the caller's fallback chain ends there, exactly as
10345/// "no further untried agents remain in the list for that seat" asks for.
10346///
10347/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10348/// the same id twice (an operator's `roles.implementers` typo, or a
10349/// `[[agents]]` list reused across roles) must not let
10350/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10351/// over `roster` either finds an untried id or runs out, so this always
10352/// terminates regardless of duplicates.
10353fn next_untried_in_roster<'a>(
10354    roster: &'a [AgentSpec],
10355    start: usize,
10356    tried: &BTreeSet<String>,
10357) -> Option<&'a AgentSpec> {
10358    roster
10359        .get(start + 1..)?
10360        .iter()
10361        .find(|s| !tried.contains(&s.id))
10362}
10363
10364/// The successor for a seat, shared by all four seat kinds (implement, judge,
10365/// review, advise). `others` holds the ids that *currently* occupy the other
10366/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10367/// sees them), so roster entries beyond the seat count act as spares: a failed
10368/// seat goes to an agent no other seat holds whenever the roster permits.
10369///
10370/// `carried` is `Some` only for the review loop's carried failure history.
10371/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10372/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10373/// whole roster: untried and not another seat's occupant; (3) the plain walk
10374/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10375/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10376/// a better outcome than an empty seat, and it keeps the answer to "is there
10377/// a successor at all" exactly what it was before occupants were considered,
10378/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10379/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10380/// carried failure is a cheaper bet than doubling an agent on the panel, and
10381/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10382/// the same round are handled one at a time, so a seat later in the batch
10383/// sees an earlier one's new occupant but not yet its own freed agent.
10384fn pick_successor<'a>(
10385    roster: &'a [AgentSpec],
10386    start: usize,
10387    tried: &BTreeSet<String>,
10388    carried: Option<&BTreeSet<String>>,
10389    others: &BTreeSet<String>,
10390) -> Option<&'a AgentSpec> {
10391    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10392    roster
10393        .get(start + 1..)
10394        .and_then(|tail| {
10395            tail.iter()
10396                .filter(free)
10397                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10398        })
10399        .or_else(|| {
10400            carried
10401                .is_some()
10402                .then(|| roster.iter().find(free))
10403                .flatten()
10404        })
10405        .or_else(|| match carried {
10406            Some(c) => next_for_seat(roster, start, tried, c),
10407            None => next_untried_in_roster(roster, start, tried),
10408        })
10409}
10410
10411/// The next agent for a seat that carries its failure history across rounds
10412/// (the review loop). `round_tried` is this round's own bound and starts
10413/// empty every round; `carried_failed` only decides priority.
10414///
10415/// First: an id walking forward from `start`, never wrapping, that is neither
10416/// tried this round nor failed in an earlier one. Only when that is exhausted
10417/// does it rescue: the first roster id (in roster order, so this one *does*
10418/// look before `start`) not yet tried this round, which is by then a carried
10419/// failure. Each id is rescued at most once per round, so it cannot loop.
10420fn next_for_seat<'a>(
10421    roster: &'a [AgentSpec],
10422    start: usize,
10423    round_tried: &BTreeSet<String>,
10424    carried_failed: &BTreeSet<String>,
10425) -> Option<&'a AgentSpec> {
10426    roster
10427        .get(start + 1..)?
10428        .iter()
10429        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10430        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10431}
10432
10433/// Where a reviewer seat starts a round: the agent that last answered it when
10434/// it is still on the roster and not marked failed, else the spec's own agent
10435/// unless it failed, else the next roster agent that has not failed, else the
10436/// spec's own agent again (the whole roster failed). Ids no longer on the
10437/// roster are ignored. An empty roster has no handover, so the spec stands.
10438fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10439    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10440        return spec;
10441    };
10442    let ok = |id: &str| !h.failed.contains(id);
10443    if let Some(last) = h.last_ok.as_deref()
10444        && ok(last)
10445        && let Some(s) = roster.iter().find(|s| s.id == last)
10446    {
10447        return s.clone();
10448    }
10449    if ok(&spec.id) {
10450        return spec;
10451    }
10452    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10453    roster
10454        .get(from + 1..)
10455        .into_iter()
10456        .flatten()
10457        .chain(roster.iter())
10458        .find(|s| ok(&s.id))
10459        .cloned()
10460        .unwrap_or(spec)
10461}
10462
10463/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10464/// seed so a CLI that mints its session id up front (`--session-id`) never
10465/// reuses the uuid the previous agent already opened under the same seat key.
10466pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10467    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10468}
10469
10470/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10471/// so the classifier below and the code that builds the message cannot drift.
10472const TIMED_OUT: &str = "timed out";
10473
10474impl FailClass {
10475    /// `None` for an answer; otherwise how the turn failed.
10476    fn of(out: &AgentOutcome) -> Option<Self> {
10477        match out {
10478            AgentOutcome::Ok(_) => None,
10479            AgentOutcome::Quota(_) => Some(Self::Quota),
10480            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10481            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10482            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10483            // Not a failure of the agent. Callers check for it first; `None`
10484            // here means "no class to hand over on".
10485            AgentOutcome::ResourceBlocked(_) => None,
10486        }
10487    }
10488
10489    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10490    fn stem_word(&self) -> &'static str {
10491        match self {
10492            Self::Quota => "quota",
10493            _ => "handover",
10494        }
10495    }
10496}
10497
10498/// The message's first line with its variable parts removed — digit runs and
10499/// path-like tokens — so "exit code 2" and "exit code 7" read
10500/// as one kind of failure.
10501fn failure_signature(msg: &str) -> String {
10502    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10503    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10504    // the historical machine key while the process diagnosis uses readable prose.
10505    if let Some(code) = line
10506        .strip_prefix("process exited with no usable output (exit code ")
10507        .and_then(|s| s.strip_suffix(')'))
10508        .and_then(|s| s.parse::<i32>().ok())
10509    {
10510        line = format!("exited with some({code}) and no usable output");
10511    } else if matches!(
10512        line.as_str(),
10513        "process exited with no usable output (no exit code reported)"
10514            | "process exited with no usable output (exit code unavailable)"
10515    ) {
10516        line = "exited with none and no usable output".to_owned();
10517    }
10518    let mut out = Vec::new();
10519    for word in line.split_whitespace() {
10520        if word.contains('/') || word.contains('\\') {
10521            out.push("<path>".to_owned());
10522            continue;
10523        }
10524        let mut w = String::new();
10525        let mut in_digits = false;
10526        for c in word.chars() {
10527            if c.is_ascii_digit() {
10528                if !in_digits {
10529                    w.push('#');
10530                }
10531                in_digits = true;
10532            } else {
10533                in_digits = false;
10534                w.push(c);
10535            }
10536        }
10537        out.push(w);
10538    }
10539    out.join(" ").chars().take(120).collect()
10540}
10541
10542/// Whether a seat that just failed with `cur` may go to the next roster agent.
10543/// A quota or a timeout always may. Any other failure may not when the agent
10544/// before it failed the same way: an error the prompt causes would otherwise
10545/// walk the whole roster. `prev` is the class of the immediately preceding
10546/// agent's failure, so a quota or timeout in between breaks the run of
10547/// identical failures by itself.
10548fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10549    match cur {
10550        FailClass::Quota | FailClass::Timeout => true,
10551        FailClass::Other(_) => prev != Some(cur),
10552    }
10553}
10554
10555/// A short human reason for a failed outcome, for the handover record.
10556fn fail_reason(out: &AgentOutcome) -> String {
10557    match out {
10558        AgentOutcome::Ok(_) => String::new(),
10559        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10560        AgentOutcome::Dropped(o) => format!(
10561            "the CLI dropped the stream ({})",
10562            o.dropped
10563                .as_ref()
10564                .map(|d| d.why.as_str())
10565                .unwrap_or("it ended without delivering its answer")
10566        ),
10567        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10568            e.lines().next().unwrap_or("").chars().take(160).collect()
10569        }
10570    }
10571}
10572
10573/// Note one handover in the run: the structured record and, in the timeline,
10574/// the sentence a person reads. A quota keeps the wording it always had.
10575pub(crate) fn record_handover(
10576    state: &mut RunState,
10577    node: &str,
10578    seat: &str,
10579    from: &str,
10580    to: &str,
10581    class: &FailClass,
10582    reason: &str,
10583) {
10584    let message = if *class == FailClass::Quota {
10585        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10586    } else {
10587        format!("{seat}: handed over {from} -> {to} ({reason})")
10588    };
10589    state.event(node, message);
10590    state.handovers.push(Handover {
10591        at: Timestamp::now(),
10592        node: node.to_owned(),
10593        seat: seat.to_owned(),
10594        from: from.to_owned(),
10595        to: to.to_owned(),
10596        reason: reason.to_owned(),
10597    });
10598}
10599
10600/// Did this reply report running a command whose own CLI never confirmed an
10601/// exit status?
10602///
10603/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10604/// command *finished* (see that type's own doc), so this can only be `true`
10605/// for a command whose completion event carried no readable exit code — not
10606/// for one that simply is not mentioned at all. That is the one signal this
10607/// crate can read, from the same record `state.jobs` renders, about a reply
10608/// standing next to work its own CLI cannot vouch for finishing; it is
10609/// deliberately not a check on the exit code's *value* (a fixer legitimately
10610/// runs a command that fails mid-iteration before it succeeds) and not a
10611/// guess at a command still running in the background (which emits no event
10612/// at all, and so leaves no evidence here to find).
10613fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10614    commands.iter().any(|c| c.exit_code.is_none())
10615}
10616
10617/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10618/// trusted as a verified no-op — the adoption guard's own text-level half.
10619///
10620/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10621/// timed out): a marker only earns the benefit of the doubt from a turn the
10622/// CLI itself vouches for finishing properly, the same house style
10623/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10624/// *fix* report to for `commands`. A candidate that timed out, exited
10625/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10626/// is, whatever prose it wrote — this returns `None` before it ever looks at
10627/// `text`. The remaining guards (the tree really is empty, the evidence is
10628/// non-empty) are the caller's: this only reads what the reply *claimed*.
10629fn verified_noop_claim(
10630    usable: bool,
10631    commands: &[agent::CommandEvidence],
10632    text: &str,
10633) -> Option<String> {
10634    (usable && !has_unconfirmed_command(commands))
10635        .then(|| verdict::verified_noop(text))
10636        .flatten()
10637}
10638
10639fn short(commit: &str) -> String {
10640    commit.chars().take(7).collect()
10641}
10642
10643fn make_executable(path: &Path) -> Result<()> {
10644    #[cfg(unix)]
10645    {
10646        use std::os::unix::fs::PermissionsExt as _;
10647        let mut perms = std::fs::metadata(path)?.permissions();
10648        perms.set_mode(0o755);
10649        std::fs::set_permissions(path, perms)?;
10650    }
10651    #[cfg(not(unix))]
10652    {
10653        let _ = path;
10654    }
10655    Ok(())
10656}
10657
10658/// What every seat in one batch shares: where the answers are attributed, the
10659/// prompt overlay they inherit, and the build cache they are told to use.
10660///
10661/// A struct rather than four more parameters: `wave` also needs the run's
10662/// state (to record who is answering right now) and the attempt number, and
10663/// eight positional arguments is both unreadable and a clippy error.
10664struct WaveCtx<'a> {
10665    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10666    /// paid for it.
10667    run: &'a str,
10668    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10669    node: &'a str,
10670    prompts: &'a Prompts,
10671    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10672    cache: Option<&'a Path>,
10673    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10674    /// `JobRecord::round`. `None` for every other node.
10675    round: Option<usize>,
10676    /// Carry each seat's failed-agent history across waves (the review loop
10677    /// only): start-of-round priority and handover choice read
10678    /// [`RunState::seat_history`], and every answer or failure writes it.
10679    carry_seats: bool,
10680    /// The run's park / interrupt requests, so a wait for a shared resource
10681    /// can be abandoned the moment the operator asks to park.
10682    pauses: Option<(&'a Pause, &'a Pause)>,
10683}
10684
10685/// Shared scheduling policy for operations and spawned seat invocations.
10686fn repository_wait(
10687    state: &RunState,
10688    pauses: Option<(&Pause, &Pause)>,
10689) -> crate::repo_lock::WaitPolicy {
10690    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10691    let inherited = crate::repo_lock::WaitPolicy::current();
10692    crate::repo_lock::WaitPolicy::new(
10693        Duration::from_secs(state.config.graph.timeout_resource_wait),
10694        move || {
10695            inherited.cancelled()
10696                || pauses
10697                    .as_ref()
10698                    .is_some_and(|(a, b)| a.parked() || b.parked())
10699        },
10700    )
10701}
10702
10703fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10704    if error.is::<crate::repo_lock::ResourceBlocked>() {
10705        state.resource_block = Some(crate::run::ResourceBlock {
10706            cause: crate::run::ResourceBlockCause::Other,
10707            node: "repository".into(),
10708            kind: if error.to_string().contains("disk pressure") {
10709                "disk pressure"
10710            } else if error.to_string().contains("disk admission") {
10711                "disk admission"
10712            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10713                "incremental cache admission"
10714            } else {
10715                "repository coordination"
10716            }
10717            .into(),
10718            detail: error.to_string(),
10719            at: Timestamp::now(),
10720        });
10721        state.event("resource", error.to_string());
10722    }
10723}
10724
10725async fn coordinate_operation(
10726    state: &mut RunState,
10727    cwd: &Path,
10728    wait: &crate::repo_lock::WaitPolicy,
10729) -> Result<Option<crate::repo_lock::SeatLease>> {
10730    let result =
10731        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10732            .await;
10733    if let Err(e) = &result {
10734        record_repository_block(state, e);
10735    }
10736    result
10737}
10738
10739/// Run one job, honouring the parallelism budget.
10740async fn run_one(
10741    job: SeatJob,
10742    sem: Arc<Semaphore>,
10743    ctx: &WaveCtx<'_>,
10744    state: &mut RunState,
10745    attempt: usize,
10746) -> (SeatState, AgentOutcome) {
10747    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10748        .await
10749        .pop()
10750        .expect("one job in, one result out");
10751    (seat, out)
10752}
10753
10754/// Run every job concurrently, capped by the semaphore, preserving order.
10755///
10756/// Every seat in the batch is recorded into [`RunState::active`] before the
10757/// wave starts and cleared as each answer lands, so the run's own record says
10758/// who is still being waited on rather than only who finished.
10759async fn wave(
10760    jobs: Vec<SeatJob>,
10761    sem: Arc<Semaphore>,
10762    ctx: &WaveCtx<'_>,
10763    state: &mut RunState,
10764    attempt: usize,
10765) -> Vec<(usize, SeatState, AgentOutcome)> {
10766    let WaveCtx {
10767        run,
10768        node,
10769        prompts,
10770        cache,
10771        round,
10772        carry_seats: _,
10773        pauses,
10774    } = *ctx;
10775    for job in &jobs {
10776        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10777    }
10778    if let Err(e) = state.save() {
10779        // A failed persist of "who is answering right now" must not abort the
10780        // wave: the seats are already being asked, and the alternative is
10781        // losing the answers to save a status line nobody may even be
10782        // watching.
10783        tracing::warn!("could not persist in-progress seats: {e:#}");
10784    }
10785    // Hold the shared build cache's lease for the whole batch, not per job:
10786    // several candidates (an implement wave) or a fixer legitimately share
10787    // one cache concurrently within this run, and that stays untouched — a
10788    // single lease taken once for the whole wave and released once it is
10789    // done is what stops a *different* borrower (another run's own wave, its
10790    // e2e/gate, a human's `magi review`) from interleaving a build into the
10791    // same directory while this one is in flight. Best-effort, not
10792    // all-or-nothing: a wave that cannot get the lease within its own
10793    // longest job's budget still runs — an hour of paid implementer calls is
10794    // not thrown away over cache contention — but every write-allowed seat
10795    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10796    // below), the same fallback a read-only seat always gets, rather than
10797    // building into a directory this run was never granted. The identity
10798    // record is still invalidated below either way, so the next tracked
10799    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10800    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10801    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10802    // never borrowed from a job's agent budget (and never charged to it either,
10803    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10804    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10805    let cache_guard = if let Some(cache_dir) = cache {
10806        if jobs_had_a_writer {
10807            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10808            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10809            match acquire_cache_lease(
10810                state,
10811                cache_dir,
10812                &owner,
10813                resource_limit,
10814                node,
10815                node,
10816                &cancel,
10817            )
10818            .await
10819            {
10820                Ok(Some(g)) => Some(g),
10821                // Parking: leave without spawning anything. The run parks at
10822                // the next boundary and the seats are asked afresh on resume.
10823                Ok(None) => {
10824                    if let Some(block) = &mut state.resource_block {
10825                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10826                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10827                        };
10828                    }
10829                    return block_wave(
10830                        jobs,
10831                        state,
10832                        node,
10833                        attempt,
10834                        "parked while waiting for the shared build cache",
10835                    );
10836                }
10837                // An unreadable lease is not contention: keep the old
10838                // fallback (no shared cache for this wave's seats).
10839                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10840                Err(e) => {
10841                    // Writing seats may not build into a cache this run does
10842                    // not hold, and an agent that cannot be given one is not
10843                    // asked at all: the run stops resumably instead.
10844                    let why = format!("{e:#}");
10845                    state.event(
10846                        "resource",
10847                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10848                    );
10849                    return block_wave(jobs, state, node, attempt, &why);
10850                }
10851            }
10852        } else {
10853            None
10854        }
10855    } else {
10856        None
10857    };
10858    // Every job keeps its whole budget: the wait above was not the agent's.
10859    // Restart the seats' clocks so "started" means "being spawned now".
10860    state.seats_spawning(node, attempt);
10861    if let Err(e) = state.save() {
10862        tracing::warn!("could not persist the start of a wave: {e:#}");
10863    }
10864    // Queueing behind this wave's own seats for a parallelism slot is normal
10865    // (each of them is bounded by its own agent timeout); only a wait beyond
10866    // all of them plus the resource limit means the slots never came back.
10867    let slot_limit = resource_limit
10868        + jobs
10869            .iter()
10870            .map(|j| j.timeout)
10871            .max()
10872            .unwrap_or_default()
10873            .saturating_mul(jobs.len() as u32);
10874    let mut set = tokio::task::JoinSet::new();
10875    // Seats announce themselves here the moment they hold a parallelism
10876    // permit, i.e. just before their CLI is spawned.
10877    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10878    let overlay = prompts.overlay(node);
10879    let repository_wait = repository_wait(state, pauses);
10880    for (i, mut job) in jobs.into_iter().enumerate() {
10881        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10882        if cache.is_some() {
10883            job.prompt.push('\n');
10884            job.prompt
10885                .push_str(&prompt::build_cache_note(node, job.allow_write));
10886        }
10887        let sem = Arc::clone(&sem);
10888        let run = run.to_owned();
10889        let node = node.to_owned();
10890        // Only implementers were told about the task's attachments, so only
10891        // their seats get the directory widened for reading.
10892        let attachments = if node == "implement" {
10893            state.attachments.clone()
10894        } else {
10895            Vec::new()
10896        };
10897        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10898        // `prompt::build_cache_note`'s doc for why setting it anyway is
10899        // exactly how a sandboxed reviewer's write refusal got reported as a
10900        // defect in the patch, not a property of its own seat. And a
10901        // write-allowed one is handed it only when the lease above was
10902        // actually acquired: a wave that could not get it (`cache_guard` is
10903        // `None`, see its own comment) must not send seats to build into a
10904        // directory this run does not hold - that is the exact concurrent,
10905        // unmanaged-write race this module exists to prevent, not something
10906        // "proceeding anyway" is allowed to reintroduce.
10907        let cache = cache
10908            .filter(|_| job.allow_write && cache_guard.is_some())
10909            .map(Path::to_path_buf);
10910        let spawned_tx = spawned_tx.clone();
10911        let repository_wait = repository_wait.clone();
10912        let coordination = state.config.coordination.clone();
10913        let fetched_base = state.fetched_base.clone();
10914        set.spawn(async move {
10915            // A wait for a parallelism slot is scheduling, not the agent: it
10916            // is bounded by its own limit and ends as a resource block, never
10917            // as a timeout of the agent that was not started.
10918            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10919                Ok(p) => p,
10920                Err(_) => {
10921                    return (
10922                        i,
10923                        job.seat,
10924                        AgentOutcome::ResourceBlocked(format!(
10925                            "no parallel slot became free within {}s",
10926                            slot_limit.as_secs()
10927                        )),
10928                        false,
10929                    );
10930                }
10931            };
10932            spawned_tx.send(job.seat.key.clone()).ok();
10933            let mut seat = job.seat;
10934            let out = repository_wait
10935                .scope(agent::invoke(
10936                    &job.spec,
10937                    &mut seat,
10938                    &Invocation {
10939                        coordination: coordination.as_ref(),
10940                        fetched_base: fetched_base.as_ref(),
10941                        cwd: &job.cwd,
10942                        prompt: &job.prompt,
10943                        timeout: job.timeout,
10944                        allow_write: job.allow_write,
10945                        sessions: job.sessions,
10946                        artifacts: &job.artifacts,
10947                        stem: &job.stem,
10948                        run: &run,
10949                        node: &node,
10950                        cache_dir: cache.as_deref(),
10951                        attachments: &attachments,
10952                        writable: &[],
10953                    },
10954                ))
10955                .await;
10956            let interrupted = out
10957                .as_ref()
10958                .err()
10959                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10960            let out = match out {
10961                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10962                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10963                // Billed work the CLI failed to hand over is not an ordinary
10964                // failure, but its text is the CLI's raw error JSON, not an
10965                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10966                // read it as one by forgetting to check. `usable()` is always
10967                // false here (dropped implies an empty response), so this has
10968                // to be checked before the catch-all `Failed` below or the
10969                // one shape this exists for is lost with the rest.
10970                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10971                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10972                Ok(o) => AgentOutcome::Failed(format!(
10973                    "process exited with no usable output ({})",
10974                    crate::notices::exit_status(o.exit_code)
10975                )),
10976                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10977                    AgentOutcome::ResourceBlocked(e.to_string())
10978                }
10979                Err(e) => AgentOutcome::Failed(e.to_string()),
10980            };
10981            (i, seat, out, interrupted)
10982        });
10983    }
10984    drop(spawned_tx);
10985    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10986    loop {
10987        let joined = tokio::select! {
10988            Some(key) = spawned_rx.recv() => {
10989                // The clock of this seat starts now, not when it queued.
10990                state.seat_spawned(&key);
10991                if let Err(e) = state.save() {
10992                    tracing::warn!("could not persist a seat's start: {e:#}");
10993                }
10994                continue;
10995            }
10996            joined = set.join_next() => joined,
10997        };
10998        let Some(joined) = joined else {
10999            break;
11000        };
11001        let (i, seat, out, interrupted) = match joined {
11002            Ok(v) => v,
11003            // No seat to clear: a panicked task never reported which one it
11004            // was. The defensive sweep below this loop is what stops that
11005            // seat's `active` entry from surviving forever.
11006            Err(e) => {
11007                tracing::error!("agent task panicked: {e}");
11008                continue;
11009            }
11010        };
11011        state.seat_finished(&seat.key);
11012        if interrupted
11013            && node == "implement"
11014            && let Some(label) = state
11015                .candidates
11016                .iter()
11017                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11018                .map(|candidate| candidate.label)
11019        {
11020            state.candidate_preservation.insert(
11021                label,
11022                "interrupted implementation awaiting persistence".into(),
11023            );
11024            state.interrupted_implementations.insert(label);
11025        }
11026        if let AgentOutcome::ResourceBlocked(why) = &out
11027            && state.resource_block.is_none()
11028        {
11029            state.resource_block = Some(crate::run::ResourceBlock {
11030                cause: crate::run::ResourceBlockCause::Other,
11031                node: node.to_owned(),
11032                kind: if why.contains("disk pressure") {
11033                    "disk pressure"
11034                } else if why.contains("disk admission") {
11035                    "disk admission"
11036                } else if why.contains(crate::incremental::BLOCKED) {
11037                    "incremental cache admission"
11038                } else if why.contains("repository coordination") {
11039                    "repository coordination"
11040                } else {
11041                    "parallel slot"
11042                }
11043                .to_owned(),
11044                detail: why.clone(),
11045                at: Timestamp::now(),
11046            });
11047            let activity = if interrupted {
11048                "agent execution interrupted"
11049            } else {
11050                "no agent was started"
11051            };
11052            state.event("resource", format!("{node}: {activity} - {why}"));
11053        }
11054        record_jobs(state, node, round, &seat.key, &out);
11055        if let Err(e) = state.save() {
11056            tracing::warn!("could not persist a seat's completion: {e:#}");
11057        }
11058        if collected.len() <= i {
11059            collected.resize_with(i + 1, || None);
11060        }
11061        collected[i] = Some((i, seat, out));
11062    }
11063    // Belt-and-braces for the panic branch above: every seat this exact batch
11064    // started shares this `(node, attempt)` pair, and every seat that finished
11065    // normally already cleared itself, so anything left tagged with it here
11066    // can only be a panicked task's leftover. Cleared unconditionally rather
11067    // than left to read as still answering forever.
11068    if state
11069        .active
11070        .values()
11071        .any(|a| a.node == node && a.attempt == attempt)
11072    {
11073        state
11074            .active
11075            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11076        if let Err(e) = state.save() {
11077            tracing::warn!("could not persist the end of a wave: {e:#}");
11078        }
11079    }
11080    // Whether or not the lease above was actually held, several worktrees
11081    // may just have built into the cache with nothing here able to name one
11082    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11083    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11084    // clean it might not have strictly needed; trusting a stale match would
11085    // cost it a wrong answer.
11086    if let Some(cache_dir) = cache
11087        && jobs_had_a_writer
11088    {
11089        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11090    }
11091    if let Some(guard) = cache_guard {
11092        guard.release();
11093    }
11094    collected.into_iter().flatten().collect()
11095}
11096
11097/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11098/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11099/// this is data collection, not the fix-specific completion contract in
11100/// [`Runner::continue_fix_report`], and applies regardless of which node
11101/// asked.
11102///
11103/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11104/// evidence from; `Failed` does not, and correctly contributes nothing — a
11105/// timeout or crash is not itself evidence about a command the seat may have
11106/// started.
11107fn record_jobs(
11108    state: &mut RunState,
11109    node: &str,
11110    round: Option<usize>,
11111    seat: &str,
11112    out: &AgentOutcome,
11113) {
11114    let commands: &[agent::CommandEvidence] = match out {
11115        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11116        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11117    };
11118    let checked_at = Timestamp::now();
11119    for c in commands {
11120        state.jobs.push(JobRecord {
11121            node: node.to_owned(),
11122            round,
11123            seat: seat.to_owned(),
11124            id: c.id.clone(),
11125            description: c.description.clone(),
11126            checked_at,
11127            status: match c.exit_code {
11128                Some(0) => JobStatus::Completed,
11129                Some(_) => JobStatus::Failed,
11130                None => JobStatus::Unknown,
11131            },
11132            exit_code: c.exit_code,
11133            result_summary: c.result_summary.clone(),
11134            source: c.source.clone(),
11135        });
11136    }
11137}
11138
11139/// Is a review round clean, given how many reviewer seats answered against
11140/// how many the round expected?
11141///
11142/// A seat that never answered (timeout, crash, unparsable output) is not a
11143/// seat that read the patch and found nothing — treating it as such is
11144/// exactly the bug this function exists to close. Under the default `block`
11145/// policy a missing seat can never be clean; `warn` still requires the seats
11146/// that *did* answer to have found nothing blocking and verification to be
11147/// green.
11148///
11149/// `quota_missing` narrows that `block` default for exactly one cause of
11150/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11151/// a session limit lifts by the very next round buys nothing — the seat is
11152/// asked again with the same quota — so once every missing seat is accounted
11153/// for by a quota loss (and at least one seat *did* answer, so a decision has
11154/// something to rest on) the round is decided on the panel that could answer,
11155/// same as `warn` would. A panel that lost every seat to quota is not
11156/// decided here: `answered == 0` falls through to the existing `block`
11157/// fallback so a fully collapsed panel still waits rather than landing on no
11158/// review at all.
11159fn round_is_clean(
11160    blocking: usize,
11161    e2e_ok: bool,
11162    answered: usize,
11163    expected: usize,
11164    quota_missing: usize,
11165    policy: IncompleteReviewPolicy,
11166) -> bool {
11167    if blocking != 0 || !e2e_ok {
11168        return false;
11169    }
11170    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11171        return true;
11172    }
11173    answered > 0 && expected - answered <= quota_missing
11174}
11175
11176/// The review loop's own conclusion, derived entirely from its persisted
11177/// round records and the round budget that produced them — never from
11178/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11179/// recomputes the identical answer regardless of what an earlier node in the
11180/// same walk, or a previous walk, did to `status`.
11181///
11182/// `None` while more rounds remain to try, including when review never ran
11183/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11184/// gone clean, or the budget is spent, or the tree has stopped moving (see
11185/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11186///
11187/// - An incomplete panel that raised nothing is missing input, not a
11188///   verified tree — never a hand-off candidate, whatever verification said
11189///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11190/// - Otherwise, green e2e on the last round hands off (see
11191///   [`Runner::stop_reviewing`]); red e2e blocks.
11192///
11193/// A last round whose own verification is still `ResourceBlocked` — magi
11194/// itself never got a command to run, not evidence the patch is broken —
11195/// is neither: this returns `None` for it too, the same as "more rounds
11196/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11197/// handling of that shape) instead of this cheap recomputation guessing a
11198/// verdict a real attempt never produced.
11199pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11200    if max_rounds == 0
11201        || reviews.last().is_some_and(|r| {
11202            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11203        })
11204    {
11205        return Some(RunStatus::Gating);
11206    }
11207    let last = reviews.last()?;
11208    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11209    if reviews.len() < max_rounds && !stagnant {
11210        return None;
11211    }
11212    if last.incomplete() && last.blocking == 0 {
11213        return Some(RunStatus::Blocked);
11214    }
11215    if last.e2e_status() == E2eStatus::ResourceBlocked
11216        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11217    {
11218        return None;
11219    }
11220    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11221        RunStatus::Gating
11222    } else {
11223        RunStatus::Blocked
11224    })
11225}
11226
11227/// How long a re-ask may take, given the budget the first attempt had.
11228///
11229/// A `nudged` retry is a request to restate an answer the seat has already
11230/// worked out: it carries no new work, so it does not deserve the original
11231/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11232/// 133 seconds while a third sat for over ten minutes on a resumed session
11233/// holding 410 KB of prior output - and because the retry had inherited the
11234/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11235/// judging round whose other seats were long finished.
11236///
11237/// A quarter of the budget, with a floor so that a deliberately short timeout
11238/// does not collapse to nothing. A retry that re-sends the whole prompt
11239/// (because the seat kept no context) is the original job again, and keeps the
11240/// original budget.
11241pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11242    if nudged {
11243        (full / 4).max(Duration::from_secs(120)).min(full)
11244    } else {
11245        full
11246    }
11247}
11248
11249/// Run a wave and parse each reply, re-asking the seats whose reply was
11250/// unusable.
11251///
11252/// The re-ask is a nudge rather than the whole prompt again when the seat still
11253/// holds its conversation, which is the difference between a cheap retry and
11254/// paying for the entire candidate set twice.
11255///
11256/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11257/// successor in `roster` is handed to it instead of being re-asked: the
11258/// handover is the retry. A seat with no successor left (a single-agent
11259/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11260/// the asks to one seat in one node number at most
11261/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11262/// is asked once (a dropped stream is nudged first), and only the last agent
11263/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11264/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11265/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11266/// other failure stops the chain when the previous agent failed the same way
11267/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11268/// [`has_context`] is false and the job's own full prompt and full budget are
11269/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11270/// intermediate ones are not losses) and is returned as a failure like any
11271/// other absent seat — the caller decides whether the panel still has a
11272/// quorum. An empty `roster` disables handover: failures are nudged as they
11273/// always were, and a quota is simply lost. A reply that fails to parse or
11274/// validate is the prompt's doing and is only ever nudged, never handed over.
11275///
11276/// The returned [`SeatState`] names the agent that answered (or tried last).
11277#[allow(clippy::too_many_arguments)]
11278async fn ask_json_wave<T>(
11279    jobs: Vec<SeatJob>,
11280    sem: Arc<Semaphore>,
11281    retries: usize,
11282    roster: &[AgentSpec],
11283    ctx: &WaveCtx<'_>,
11284    losses: &mut Vec<QuotaLoss>,
11285    state: &mut RunState,
11286    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11287) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11288where
11289    T: serde::de::DeserializeOwned + Send + 'static,
11290{
11291    ask_wave_with(
11292        jobs,
11293        sem,
11294        retries,
11295        roster,
11296        ctx,
11297        losses,
11298        state,
11299        &|text: &str| {
11300            let v = verdict::extract_json::<T>(text)?;
11301            validate(&v)?;
11302            Ok(v)
11303        },
11304    )
11305    .await
11306}
11307
11308/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11309/// a node whose answer is prose (deliberation) shares the same handover,
11310/// failure classification, quota bookkeeping and bounds instead of a copy.
11311///
11312/// A seat handed to another roster agent is sent the job's `handover` prompt
11313/// (when it has one) rather than `prompt`: the new agent has no session, so a
11314/// resume-style prompt would be incomplete. That holds for the handover ask
11315/// and for every nudge to that agent whose `has_context` is false.
11316#[allow(clippy::too_many_arguments)]
11317async fn ask_wave_with<T>(
11318    jobs: Vec<SeatJob>,
11319    sem: Arc<Semaphore>,
11320    retries: usize,
11321    roster: &[AgentSpec],
11322    ctx: &WaveCtx<'_>,
11323    losses: &mut Vec<QuotaLoss>,
11324    state: &mut RunState,
11325    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11326) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11327where
11328    T: Send + 'static,
11329{
11330    let n = jobs.len();
11331    let originals: Vec<SeatJob> = jobs;
11332    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11333    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11334    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11335    // answer, N once it has gone through N nudges. Read back once this
11336    // returns, so a caller building a history record (`ReviewRecord`) can
11337    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11338    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11339    // field's own doc.
11340    let mut nudges: Vec<usize> = vec![0; n];
11341    // The agent now occupying each seat, the ids it has already been through,
11342    // where in the roster the walk began, the class of the last failure, and
11343    // the stem word of a handover not yet asked (full prompt, full budget).
11344    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11345    let mut tried: Vec<BTreeSet<String>> = specs
11346        .iter()
11347        .map(|s| BTreeSet::from([s.id.clone()]))
11348        .collect();
11349    let starts: Vec<usize> = specs
11350        .iter()
11351        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11352        .collect();
11353    let carry = ctx.carry_seats && !roster.is_empty();
11354    // Carried across rounds: ids that failed the seat earlier, and how the
11355    // last failure went (so a repeat of it is not handed over again).
11356    let carried: Vec<BTreeSet<String>> = originals
11357        .iter()
11358        .map(|j| {
11359            state
11360                .seat_history
11361                .get(&j.seat.key)
11362                .filter(|_| carry)
11363                .map(|h| h.failed.clone())
11364                .unwrap_or_default()
11365        })
11366        .collect();
11367    let mut prev: Vec<Option<FailClass>> = originals
11368        .iter()
11369        .map(|j| {
11370            state
11371                .seat_history
11372                .get(&j.seat.key)
11373                .filter(|_| carry)
11374                .and_then(|h| h.last_fail.clone())
11375        })
11376        .collect();
11377    let next_agent =
11378        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11379            let others: BTreeSet<String> = specs
11380                .iter()
11381                .enumerate()
11382                .filter(|(j, _)| *j != i)
11383                .map(|(_, s)| s.id.clone())
11384                .collect();
11385            pick_successor(
11386                roster,
11387                starts[i],
11388                tried,
11389                carry.then(|| &carried[i]),
11390                &others,
11391            )
11392            .cloned()
11393        };
11394    let mut fresh: Vec<Option<String>> = vec![None; n];
11395    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11396    let mut pending: Vec<usize> = (0..n).collect();
11397
11398    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11399    // asks: an agent with a successor is asked once and handed over, and only
11400    // a seat with no successor spends `retries` nudges on the same agent. This
11401    // only guarantees the loop's own termination whatever those say.
11402    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11403    for round in 0..max_rounds {
11404        if pending.is_empty() {
11405            break;
11406        }
11407        let mut batch = Vec::with_capacity(pending.len());
11408        let mut renudged: Vec<&str> = Vec::new();
11409        for &i in &pending {
11410            let src = &originals[i];
11411            // A seat now held by another agent than the job named has no
11412            // session of its own: it gets the full-context prompt whenever
11413            // it is asked in full (the handover ask, a nudge it cannot
11414            // resume).
11415            let full: &str = match &src.handover {
11416                Some(h) if specs[i].id != src.spec.id => h,
11417                _ => &src.prompt,
11418            };
11419            // The prompt and the budget are one decision: a nudge restates
11420            // finished work, a re-sent prompt redoes it.
11421            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11422                (
11423                    full.to_owned(),
11424                    src.timeout,
11425                    format!("{}-{word}-{}", src.stem, specs[i].id),
11426                )
11427            } else if nudges[i] == 0 {
11428                (full.to_owned(), src.timeout, src.stem.clone())
11429            } else {
11430                renudged.push(src.seat.key.as_str());
11431                let why = done[i]
11432                    .as_ref()
11433                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11434                    .unwrap_or_else(|| "no parsable answer".to_owned());
11435                let nudge = prompt::nudge(&why);
11436                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11437                let prompt = if nudged {
11438                    nudge
11439                } else {
11440                    format!("{full}\n\n---\n\n{nudge}")
11441                };
11442                (
11443                    prompt,
11444                    retry_budget(src.timeout, nudged),
11445                    format!("{}-retry{}", src.stem, nudges[i]),
11446                )
11447            };
11448            batch.push(SeatJob {
11449                spec: specs[i].clone(),
11450                seat: seats[i].clone(),
11451                cwd: src.cwd.clone(),
11452                prompt,
11453                timeout,
11454                allow_write: src.allow_write,
11455                sessions: src.sessions,
11456                artifacts: src.artifacts.clone(),
11457                stem,
11458                handover: None,
11459            });
11460        }
11461
11462        if !renudged.is_empty() {
11463            state.event(
11464                ctx.node,
11465                format!("retry {round}: re-asking {}", renudged.join(", ")),
11466            );
11467        }
11468        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11469        let mut still = Vec::new();
11470        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11471            seats[i] = seat;
11472            // No agent was asked: nothing to hand over, nudge, remember or
11473            // charge. The caller sees `resource_block` and stops the node.
11474            if let AgentOutcome::ResourceBlocked(why) = &out {
11475                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11476                continue;
11477            }
11478            let class = FailClass::of(&out);
11479            // A dropped stream is nudged first (the conversation is still
11480            // there to pick up); only a seat whose nudges are spent hands over.
11481            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11482                && nudges[i] < retries
11483                && !roster.is_empty();
11484            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11485                let next = should_hand_over(prev[i].as_ref(), &cur)
11486                    .then(|| next_agent(i, &tried[i], &specs))
11487                    .flatten();
11488                if carry {
11489                    let h = state
11490                        .seat_history
11491                        .entry(originals[i].seat.key.clone())
11492                        .or_default();
11493                    h.failed.insert(specs[i].id.clone());
11494                    h.last_fail = Some(cur.clone());
11495                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11496                        h.last_ok = None;
11497                    }
11498                    // Saved before the next agent is asked, so a restart in
11499                    // between does not forget who failed.
11500                    if let Err(e) = state.save() {
11501                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11502                    }
11503                }
11504                if let Some(next) = next {
11505                    record_handover(
11506                        state,
11507                        ctx.node,
11508                        &originals[i].seat.key,
11509                        &specs[i].id,
11510                        &next.id,
11511                        &cur,
11512                        &fail_reason(&out),
11513                    );
11514                    tried[i].insert(next.id.clone());
11515                    prev[i] = Some(cur.clone());
11516                    seats[i] =
11517                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11518                    specs[i] = next;
11519                    fresh[i] = Some(cur.stem_word().to_owned());
11520                    nudges[i] = 0;
11521                    done[i] = Some(Err(anyhow::anyhow!(
11522                        "handed over after: {}",
11523                        fail_reason(&out)
11524                    )));
11525                    still.push(i);
11526                    continue;
11527                }
11528            }
11529            if carry
11530                && !nudge_first
11531                && let Some(cur) = class.clone()
11532            {
11533                // The chain ended here (no successor, or a repeated failure).
11534                let h = state
11535                    .seat_history
11536                    .entry(originals[i].seat.key.clone())
11537                    .or_default();
11538                h.failed.insert(specs[i].id.clone());
11539                h.last_fail = Some(cur);
11540            }
11541            let parsed = match out {
11542                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11543                AgentOutcome::Quota(o) => {
11544                    last_quota[i] = o.quota.clone();
11545                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11546                }
11547                // Not a parseable answer: the nudge loop re-asks it, which is
11548                // exactly what a dropped stream needs. Just don't hand its raw
11549                // error JSON to `extract_json`.
11550                AgentOutcome::Dropped(o) => {
11551                    let why = o
11552                        .dropped
11553                        .as_ref()
11554                        .map(|d| d.why.as_str())
11555                        .unwrap_or("the CLI ended the stream without delivering its answer");
11556                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11557                }
11558                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11559                    Err(anyhow::anyhow!(e))
11560                }
11561            };
11562            let quota = class == Some(FailClass::Quota);
11563            let failed = parsed.is_err();
11564            if !failed && matches!(ctx.node, "judge" | "vote") {
11565                // A validated answer restores this seat's quorum presence,
11566                // including on the normal path after an all-seat block.
11567                state
11568                    .quota
11569                    .retain(|loss| loss.seat != originals[i].seat.key);
11570            }
11571            done[i] = Some(parsed);
11572            if carry && !failed {
11573                let h = state
11574                    .seat_history
11575                    .entry(originals[i].seat.key.clone())
11576                    .or_default();
11577                h.failed.remove(&specs[i].id);
11578                h.last_ok = Some(specs[i].id.clone());
11579                h.last_fail = None;
11580            }
11581            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11582            // fail the same way; and never re-ask a seat that already parsed.
11583            // A failed agent that still has a successor is not re-asked
11584            // either: the handover was its remedy and has just been refused
11585            // (the chain stops on a repeated failure class). A seat with no
11586            // successor left (a single-agent roster, the roster's tail, or an
11587            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11588            let agent_failure = class.is_some()
11589                && !nudge_first
11590                && !roster.is_empty()
11591                && next_agent(i, &tried[i], &specs).is_some();
11592            if failed && !quota && !agent_failure && nudges[i] < retries {
11593                nudges[i] += 1;
11594                still.push(i);
11595            }
11596        }
11597        pending = still;
11598    }
11599
11600    // One loss per seat whose chain ended on a quota: the intermediate ones
11601    // were absorbed by a handover and are not losses.
11602    for (i, q) in last_quota.into_iter().enumerate() {
11603        if let Some(quota) = q {
11604            losses.push(QuotaLoss::from_report(
11605                originals[i].seat.key.clone(),
11606                ctx.node,
11607                Some(&quota),
11608            ));
11609        }
11610    }
11611
11612    // A stalled quorum already has its own recovery checkpoint. Refresh
11613    // its missing seats before deciding whether it can resume.
11614    if state.status != RunStatus::Stalled
11615        && ctx.node != "deliberate"
11616        && n > 0
11617        && losses.len() == n
11618        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11619    {
11620        block_machine_outage(state, ctx.node, losses);
11621        state.quota.append(losses);
11622    }
11623
11624    seats
11625        .into_iter()
11626        .zip(done)
11627        .zip(nudges)
11628        .map(|((seat, res), attempts)| {
11629            (
11630                seat,
11631                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11632                attempts,
11633            )
11634        })
11635        .collect()
11636}
11637
11638/// A node with no available agent has no task verdict. Preserve the node
11639/// checkpoint and record why a subsequent explicit resume is needed.
11640pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11641    let mut auth: Vec<_> = losses
11642        .iter()
11643        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11644        .map(|loss| loss.agent.clone())
11645        .collect();
11646    auth.sort();
11647    auth.dedup();
11648    let (kind, detail) = if auth.is_empty() {
11649        (
11650            "agent usage limit",
11651            "All seats reached their agent usage limits; resume after usage is available"
11652                .to_owned(),
11653        )
11654    } else {
11655        (
11656            "agent authentication",
11657            format!(
11658                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11659                auth.join(", ")
11660            ),
11661        )
11662    };
11663    state.resource_block = Some(crate::run::ResourceBlock {
11664        node: node.to_owned(),
11665        kind: kind.to_owned(),
11666        cause: crate::run::ResourceBlockCause::Other,
11667        detail: detail.clone(),
11668        at: Timestamp::now(),
11669    });
11670    state.event(node, detail);
11671}
11672
11673/// Did the wave end without asking any agent (see [`block_wave`])?
11674fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11675    results
11676        .iter()
11677        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11678}
11679
11680/// Settle a wave that never spawned anything: every job answers
11681/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11682/// else about the seat (session, history, quota) is touched.
11683fn block_wave(
11684    jobs: Vec<SeatJob>,
11685    state: &mut RunState,
11686    node: &str,
11687    attempt: usize,
11688    why: &str,
11689) -> Vec<(usize, SeatState, AgentOutcome)> {
11690    state.resource_wait = None;
11691    state
11692        .active
11693        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11694    if let Err(e) = state.save() {
11695        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11696    }
11697    jobs.into_iter()
11698        .enumerate()
11699        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11700        .collect()
11701}
11702
11703/// Acquire the shared build cache's lease, waiting out contention within
11704/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11705/// unbounded wait is never acceptable).
11706///
11707/// A first, non-blocking check happens before ever waiting; if it finds the
11708/// lease busy, that fact is logged as a `verify` event *and* flushed with
11709/// [`RunState::save`] immediately — not only once the wait finally succeeds
11710/// or gives up — so a `magi show` run by a different process while this one
11711/// is still waiting reads a `run.json` that says so, rather than whatever it
11712/// looked like before the wait started. The same applies to the terminal
11713/// failure: logged and saved before this returns `Err`, so a caller that
11714/// could not get the lease at all still leaves a legible record of why.
11715async fn acquire_cache_lease(
11716    state: &mut RunState,
11717    cache_dir: &Path,
11718    owner: &crate::cache::Owner,
11719    budget: Duration,
11720    node: &str,
11721    context: &str,
11722    cancel: &(dyn Fn() -> bool + Sync),
11723) -> Result<Option<crate::cache::Guard>> {
11724    let home = crate::run::home();
11725    let started = Instant::now();
11726    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11727        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11728        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11729        Err(e) => {
11730            state.event(
11731                "verify",
11732                format!("{context}: could not check the shared build cache: {e:#}"),
11733            );
11734            if let Err(e2) = state.save() {
11735                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11736            }
11737            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11738        }
11739    };
11740    state.event(
11741        "verify",
11742        format!(
11743            "{context}: waiting for the shared build cache at {} ({})",
11744            cache_dir.display(),
11745            busy.describe()
11746        ),
11747    );
11748    // Seats of this node are waiting, not running: say so, so the countdown a
11749    // reader sees only ever measures an agent that has been spawned.
11750    state.resource_wait = Some(crate::run::ResourceWait {
11751        node: context.to_owned(),
11752        kind: "shared build cache".to_owned(),
11753        holder: busy.describe(),
11754        since: Timestamp::now(),
11755        limit_secs: budget.as_secs(),
11756    });
11757    if let Err(e) = state.save() {
11758        tracing::warn!("could not persist a cache wait: {e:#}");
11759    }
11760    let remaining = budget.saturating_sub(started.elapsed());
11761    let waited = crate::cache::wait_for_unless(
11762        &home,
11763        cache_dir,
11764        owner,
11765        remaining,
11766        Duration::from_secs(5),
11767        cancel,
11768    )
11769    .await;
11770    state.resource_wait = None;
11771    let cause = match &waited {
11772        Ok(Some(_)) => None,
11773        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11774        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11775            Some(crate::run::ResourceBlockCause::Timeout {
11776                waited_secs: started.elapsed().as_secs(),
11777                limit_secs: budget.as_secs(),
11778            })
11779        }
11780        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11781    };
11782    if let Some(cause) = cause {
11783        let detail = match &waited {
11784            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11785            Err(error) => format!("{error:#}"),
11786        };
11787        state.resource_block = Some(crate::run::ResourceBlock {
11788            cause,
11789            node: node.to_owned(),
11790            kind: "shared build cache".into(),
11791            detail,
11792            at: Timestamp::now(),
11793        });
11794        state.event(
11795            "resource",
11796            state.resource_block.as_ref().unwrap().description(),
11797        );
11798    }
11799    if let Err(error) = state.save() {
11800        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11801    }
11802    waited
11803}
11804
11805/// The lease state could not be read at all (unreadable lease file, a
11806/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11807/// known to be contended, so a wave degrades to running without a shared cache
11808/// instead of stopping on a hold that [`crate::triage`] could never release.
11809#[derive(Debug)]
11810struct CacheCheckError(String);
11811
11812impl std::fmt::Display for CacheCheckError {
11813    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11814        f.write_str(&self.0)
11815    }
11816}
11817
11818impl std::error::Error for CacheCheckError {}
11819
11820/// Keep a specific acquisition outcome when the caller adds command evidence.
11821fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11822    state
11823        .resource_block
11824        .get_or_insert_with(|| crate::run::ResourceBlock {
11825            cause: crate::run::ResourceBlockCause::Other,
11826            node: node.to_owned(),
11827            kind: "shared build cache".into(),
11828            detail: format!("{error:#}"),
11829            at: Timestamp::now(),
11830        });
11831    if let Err(error) = state.save() {
11832        tracing::warn!("could not persist cache resource block: {error:#}");
11833    }
11834}
11835
11836/// Run `body` — a verify command batch — while holding the shared build
11837/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11838/// never interleave with another borrower's build against the same
11839/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11840/// timeout, or a human's own `magi review`. See the `cache` module doc for
11841/// why this matters more than Cargo's own per-target locking covers — two
11842/// *different* worktrees building the same package name/version into one
11843/// cache directory is a staleness bug, not a lock contention one.
11844///
11845/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11846/// agent waves do. Repository scheduling also has its own bounded,
11847/// cancellable resource-wait policy. Contention must not consume the
11848/// verification budget: `body` receives the full `budget` after both leases
11849/// are acquired and the cache is fresh. Without a shared cache, only
11850/// repository coordination is acquired before handing over the full budget.
11851///
11852/// A lease that cannot be acquired within the resource wait is reported as a single
11853/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11854/// verification — the same shape a spawn failure already takes in
11855/// [`run_commands`], so a caller need not special-case it.
11856#[allow(clippy::too_many_arguments)]
11857async fn with_cache_lease<'s, F, Fut>(
11858    state: &'s mut RunState,
11859    cache_dir: Option<&Path>,
11860    node: &str,
11861    seat: &str,
11862    worktree: &Path,
11863    head: &str,
11864    budget: Duration,
11865    context: &str,
11866    wait: &crate::repo_lock::WaitPolicy,
11867    body: F,
11868) -> Result<(Vec<CommandOutcome>, bool)>
11869where
11870    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11871    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11872{
11873    // Match editing waves: cache first, repository second. In particular,
11874    // pre_gate must never hold the repository while waiting for a wave's cache.
11875    let started = Instant::now();
11876    let home = crate::run::home();
11877    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11878    let guard = if let Some(cache_dir) = cache_dir {
11879        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11880        match acquire_cache_lease(
11881            state,
11882            cache_dir,
11883            &owner,
11884            resource_limit,
11885            node,
11886            context,
11887            &|| wait.cancelled(),
11888        )
11889        .await
11890        .and_then(|g| {
11891            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11892        }) {
11893            Ok(g) => Some(g),
11894            Err(e) => {
11895                record_cache_error(state, node, &e);
11896                return Ok((
11897                    vec![CommandOutcome {
11898                        command: "(waiting for the shared build cache)".into(),
11899                        code: None,
11900                        output_tail: e.to_string(),
11901                        duration_ms: started.elapsed().as_millis() as u64,
11902                        resource_blocked: true,
11903                    }],
11904                    false,
11905                ));
11906            }
11907        }
11908    } else {
11909        None
11910    };
11911    let _coordination = coordinate_operation(state, worktree, wait).await?;
11912    if let Some(cache_dir) = cache_dir {
11913        let identity = crate::cache::Identity::new(worktree, head);
11914        if let Err(e) = crate::cache::ensure_fresh(
11915            &home,
11916            cache_dir,
11917            &identity,
11918            &state.config.verify.cargo_manifests(),
11919        ) {
11920            record_cache_error(state, node, &e);
11921            state.event(
11922                "verify",
11923                format!(
11924                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11925                    worktree.display(),
11926                    short(head)
11927                ),
11928            );
11929            if let Some(guard) = guard {
11930                guard.release();
11931            }
11932            return Ok((
11933                vec![CommandOutcome {
11934                    command: "(confirming the shared build cache is fresh)".into(),
11935                    code: None,
11936                    output_tail: e.to_string(),
11937                    duration_ms: started.elapsed().as_millis() as u64,
11938                    resource_blocked: true,
11939                }],
11940                false,
11941            ));
11942        }
11943    }
11944    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11945    if !timed_out_pids.is_empty() {
11946        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11947    }
11948    // Drop repository coordination before releasing the cache, reversing the
11949    // acquisition order. A cancelled/error future also drops both guards.
11950    drop(_coordination);
11951    if let Some(guard) = guard {
11952        guard.release();
11953    }
11954    Ok((outcomes, retried))
11955}
11956
11957/// Poll `pids` — commands [`run_commands`] reports as still running when its
11958/// own timeout elapsed — until every one is confirmed gone, or
11959/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11960///
11961/// Real confirmation where confirmation is possible, not a substitute for
11962/// full process-tree observation: a grandchild the timed-out process spawned
11963/// and that survives independently of it is invisible to a pid check the
11964/// same way it always was, and continuing to observe and collect *that*
11965/// stays a different piece of work with its own owner. This only narrows a
11966/// fixed blind wait into an actual check of the pids this process does know
11967/// about.
11968async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11969    wait_for_pids_with(
11970        pids,
11971        crate::proc::pid_alive,
11972        LEASE_RELEASE_POLL,
11973        LEASE_RELEASE_MAX_WAIT,
11974    )
11975    .await;
11976}
11977
11978/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11979/// interval and ceiling supplied by the caller, so the polling *logic* -
11980/// returns as soon as every pid reports dead, gives up at the ceiling
11981/// otherwise - is testable on millisecond durations without asking the real
11982/// OS about a pid at all.
11983async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11984    pids: &[u32],
11985    alive: F,
11986    poll: Duration,
11987    max_wait: Duration,
11988) {
11989    let deadline = Instant::now() + max_wait;
11990    loop {
11991        if pids.iter().all(|&pid| !alive(pid)) {
11992            return;
11993        }
11994        if Instant::now() >= deadline {
11995            return;
11996        }
11997        tokio::time::sleep(poll).await;
11998    }
11999}
12000
12001/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
12002/// admission that it could not even get a verify command to run, as opposed
12003/// to evidence the command actually produced? A caller that would otherwise
12004/// read a resource-blocked outcome as a red command must check this first:
12005/// see [`Runner::gate`], which retries rather than records `Blocked` when
12006/// this is true.
12007fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12008    outcomes.iter().any(|o| o.resource_blocked)
12009}
12010
12011/// What [`Runner::gate_fix_round`] decided.
12012pub(crate) enum GateFix {
12013    /// The tree changed and `verify.e2e` is still green: run the gate again.
12014    Retry,
12015    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12016    /// last failure stands and the run ends blocked.
12017    Stop,
12018    /// `verify.e2e` could not run after the fix (magi's own contention):
12019    /// decide nothing now, a later reentry retries.
12020    Defer,
12021}
12022
12023/// Is every red command in `outcomes` an ordinary failure the code could
12024/// explain: it ran, exited non-zero, and said something?
12025///
12026/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12027/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12028/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12029/// what the command is stays the gate's business.
12030pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12031    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12032    red.peek().is_some()
12033        && red.all(|o| {
12034            !o.resource_blocked
12035                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12036                && !o.output_tail.trim().is_empty()
12037        })
12038}
12039
12040/// Describe one verify command's outcome for the event log, distinguishing a
12041/// build/link failure — the toolchain never produced a binary to run — from
12042/// an actual test failure, since only the latter is a verdict on the patch.
12043fn e2e_outcome_label(o: &CommandOutcome) -> String {
12044    if o.ok() {
12045        return "pass".to_owned();
12046    }
12047    let reason = if o.build_failed() {
12048        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12049    } else {
12050        format!("FAIL ({:?})", o.code)
12051    };
12052    format!(
12053        "{reason}\n{}",
12054        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12055    )
12056}
12057
12058/// Run `verify.e2e`, retrying once if the first attempt could not build or
12059/// link — a build/link failure is frequently a race against a shared
12060/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12061/// `verify` event per command, tagged with `context` (normally `"round N"`)
12062/// so the two call sites that need this — the ordinary per-round leg in
12063/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12064/// it will ever call a round green — read identically in the event log.
12065async fn run_e2e_with_retry(
12066    state: &mut RunState,
12067    shell: &[String],
12068    commands: &[String],
12069    worktree: &Path,
12070    timeout: Duration,
12071    context: &str,
12072) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12073    let (mut e2e, mut timed_out_pids) = run_commands(
12074        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12075    )
12076    .await;
12077    for o in &e2e {
12078        state.event(
12079            "verify",
12080            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12081        );
12082    }
12083    // A build/link failure is not a verdict on the patch — it is frequently a
12084    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12085    // one retry before letting a red like that decide the round.
12086    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12087    if verify_retried {
12088        state.event(
12089            "verify",
12090            format!(
12091                "{context}: verify could not build/link, not a test result — retrying once \
12092                 before concluding"
12093            ),
12094        );
12095        let retried = run_commands(
12096            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12097        )
12098        .await;
12099        e2e = retried.0;
12100        // Both attempts' timeouts matter, not just the last one: the first
12101        // attempt's descendants may still be alive alongside the retry's.
12102        timed_out_pids.extend(retried.1);
12103        for o in &e2e {
12104            state.event(
12105                "verify",
12106                format!(
12107                    "{context}: retry `{}` -> {}",
12108                    o.command,
12109                    e2e_outcome_label(o)
12110                ),
12111            );
12112        }
12113    }
12114    (e2e, verify_retried, timed_out_pids)
12115}
12116
12117/// Run configured shell commands in `cwd`, in order. The second element is
12118/// the pid of every command that hit `timeout` and was still running when
12119/// this stopped waiting on it (best-effort: `None` when the platform did not
12120/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12121/// that releases a shared resource afterward needs to know.
12122///
12123/// Records `task` into [`RunState::active`] at every command boundary
12124/// (`RunState::task_command`) and clears it once the whole list has run
12125/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12126/// for minutes with no seat and no output of its own to show for it (see
12127/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12128/// apart from "not yet run" without this), and this is the only place that
12129/// knows which command is running right now and how many are left. Three
12130/// saves per command — start, not per second — matching the same "only at a
12131/// boundary" rule [`wave`] already follows for seats.
12132#[allow(clippy::too_many_arguments)]
12133async fn run_commands(
12134    state: &mut RunState,
12135    node: &str,
12136    task: &str,
12137    attempt: usize,
12138    shell: &[String],
12139    commands: &[String],
12140    cwd: &Path,
12141    timeout: Duration,
12142) -> (Vec<CommandOutcome>, Vec<u32>) {
12143    if commands.is_empty() {
12144        // Nothing to mark as running and nothing to clear — an empty list
12145        // means "not configured", and touching `active` (or the disk) over
12146        // that would be a write for every round of a repo with no
12147        // `verify.e2e` / `verify.gate` commands at all.
12148        return (Vec::new(), Vec::new());
12149    }
12150    let mut out = Vec::new();
12151    let mut timed_out_pids = Vec::new();
12152    let total = commands.len();
12153    for (idx, command) in commands.iter().enumerate() {
12154        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12155        if let Err(e) = state.save() {
12156            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12157        }
12158        let started = Instant::now();
12159        let mut cmd = tokio::process::Command::new(&shell[0]);
12160        cmd.quiet();
12161        // Verify the checkout as a standalone caller. Worker ownership and
12162        // pinned executable selection belong to this driver, not to tests or
12163        // tools it runs. Preserve home/seat context and Running's containment.
12164        for key in crate::worker::INVOCATION_ENV {
12165            cmd.env_remove(key);
12166        }
12167        cmd.args(&shell[1..])
12168            .arg(command)
12169            .current_dir(cwd)
12170            .stdin(std::process::Stdio::null())
12171            .stdout(std::process::Stdio::piped())
12172            .stderr(std::process::Stdio::piped())
12173            .kill_on_drop(true);
12174        let mut outputs = Vec::new();
12175        let protected = (|| -> Result<()> {
12176            crate::incremental::configure(&mut cmd, cwd)?;
12177            let mut paths = vec![cwd.to_owned()];
12178            if let Some(cache) = state.config.cache_dir() {
12179                paths.push(cache);
12180            }
12181            outputs.extend(crate::retention::OutputLease::admit(
12182                &crate::run::home(),
12183                &paths,
12184                state.config.disk.min_free_bytes,
12185            )?);
12186            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12187                outputs.push(guard);
12188            }
12189            if let Some(cache) = state.config.cache_dir() {
12190                outputs.push(crate::retention::OutputLease::cache(
12191                    &crate::run::home(),
12192                    &cache,
12193                    false,
12194                )?);
12195            }
12196            Ok(())
12197        })();
12198        if let Err(e) = protected {
12199            let detail = format!("execution admission unavailable: {e:#}");
12200            state.resource_block = Some(crate::run::ResourceBlock {
12201                cause: crate::run::ResourceBlockCause::Other,
12202                node: node.to_owned(),
12203                kind: if detail.contains("disk admission") {
12204                    "disk admission"
12205                } else {
12206                    "output ownership"
12207                }
12208                .to_owned(),
12209                detail: detail.clone(),
12210                at: Timestamp::now(),
12211            });
12212            out.push(CommandOutcome {
12213                command: command.clone(),
12214                code: None,
12215                output_tail: detail,
12216                duration_ms: 0,
12217                resource_blocked: true,
12218            });
12219            break;
12220        }
12221        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12222        let (code, body) = match spawned {
12223            Ok(mut running) => {
12224                let pid = running.child().id();
12225                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12226                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12227                let mut paths = vec![cwd.to_owned()];
12228                if let Some(cache) = state.config.cache_dir() {
12229                    paths.push(cache);
12230                }
12231                let mut pressure = None;
12232                let status = tokio::select! {
12233                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12234                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12235                        pressure = Some(reason); None
12236                    }
12237                };
12238                let (code, failure) = match status {
12239                    Some(Ok(Ok(status))) => (
12240                        status.code(),
12241                        status
12242                            .code()
12243                            .is_none()
12244                            .then(|| format!("command terminated: {status}")),
12245                    ),
12246                    Some(Ok(Err(error))) => {
12247                        (None, Some(format!("failed waiting for command: {error}")))
12248                    }
12249                    Some(Err(_)) => {
12250                        // Leased cleanup confirms process-tree death itself.
12251                        // Unleased commands still need the caller's PID check
12252                        // after kill-on-drop, before releasing cache ownership.
12253                        if !running.coordinated()
12254                            && let Some(pid) = pid
12255                        {
12256                            timed_out_pids.push(pid);
12257                        }
12258                        (
12259                            None,
12260                            Some(format!("timed out after {}s", timeout.as_secs())),
12261                        )
12262                    }
12263                    None => (None, None),
12264                };
12265                running.finish().await;
12266                if let Some(reason) = pressure {
12267                    state.resource_block = Some(crate::run::ResourceBlock {
12268                        cause: crate::run::ResourceBlockCause::Other,
12269                        node: node.to_owned(),
12270                        kind: "disk pressure".to_owned(),
12271                        detail: reason.clone(),
12272                        at: Timestamp::now(),
12273                    });
12274                    out.push(CommandOutcome {
12275                        command: command.clone(),
12276                        code: None,
12277                        output_tail: reason,
12278                        duration_ms: started.elapsed().as_millis() as u64,
12279                        resource_blocked: true,
12280                    });
12281                    break;
12282                }
12283                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12284                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12285                if let Some(failure) = failure {
12286                    body.push('\n');
12287                    body.push_str(&failure);
12288                }
12289                (code, body)
12290            }
12291            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12292        };
12293        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12294        if !body.is_empty() {
12295            // The full text is kept whatever the summary dropped. A fresh
12296            // name per command so a resume or retry never overwrites an
12297            // earlier run's output. A failed write changes no verdict.
12298            match crate::run::write_verification_artifact(state, task, &body) {
12299                Ok(name) if summary != body => {
12300                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12301                }
12302                Ok(_) => {}
12303                Err(e) => {
12304                    let warning = format!("could not save the full {task} output: {e:#}");
12305                    tracing::warn!("{warning}");
12306                    state.event(node, warning);
12307                    if summary != body {
12308                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12309                    }
12310                }
12311            }
12312        }
12313        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12314        out.push(CommandOutcome {
12315            command: command.clone(),
12316            code,
12317            output_tail: summary,
12318            duration_ms: started.elapsed().as_millis() as u64,
12319            resource_blocked,
12320        });
12321        if resource_blocked {
12322            state.resource_block = Some(crate::run::ResourceBlock {
12323                cause: crate::run::ResourceBlockCause::Other,
12324                node: node.to_owned(),
12325                kind: "incremental cache admission".into(),
12326                detail: body,
12327                at: Timestamp::now(),
12328            });
12329            break;
12330        }
12331    }
12332    state.task_finished(task);
12333    if let Err(e) = state.save() {
12334        tracing::warn!("could not persist the end of {task}: {e:#}");
12335    }
12336    (out, timed_out_pids)
12337}
12338
12339/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12340/// section (`report::run`) and in the `merge` event this node records — for
12341/// the operator to run by hand.
12342///
12343/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12344/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12345/// merge commits", or "require linear history") rejects the push a `--no-ff`
12346/// merge would produce, which is exactly the guidance this function replaces.
12347/// `message`'s first line becomes the squash commit's subject, matching the
12348/// note `report::run` prints alongside this command — see that function for
12349/// why an explicit subject is not optional there.
12350fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12351    let repo = repo.display();
12352    match style {
12353        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12354        MergeStyle::Squash => {
12355            // The subject sits inside double quotes, and a title an agent
12356            // wrote may carry the characters that break out of them.
12357            let subject = message
12358                .lines()
12359                .next()
12360                .unwrap_or(branch)
12361                .replace(['\\', '"', '$', '`'], "");
12362            format!(
12363                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12364            )
12365        }
12366        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12367    }
12368}
12369
12370/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12371/// under the hood, rejects a `title` over 256 characters and the whole
12372/// command fails — no PR at all, for a run whose body was otherwise fine
12373/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12374/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12375/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12376/// this bound. It is a margin, not a guarantee — a title packed
12377/// with multi-unit characters could still in principle land close to the
12378/// edge, but a real task title's occasional emoji or accented letter fits
12379/// comfortably inside it.
12380const PR_TITLE_MAX: usize = 240;
12381
12382/// A pull request title from the opening line of `text`, or `None` when that
12383/// line is not English (GitHub text is) or has no letters.
12384///
12385/// A line within `max` is kept as is. A longer one is cut at the end of its
12386/// first sentence when that falls inside `max`, else at a word boundary with a
12387/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12388/// make every merely-truncated title look non-English). The language check
12389/// runs on the kept text before any mark is added, so only what GitHub will
12390/// show is judged: an English opening followed by non-ASCII far past the cut
12391/// still passes.
12392fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12393    let line = queue::first_line(text)?;
12394    let chars: Vec<char> = line.chars().collect();
12395    let (kept, mark) = if chars.len() <= max {
12396        (line.to_owned(), "")
12397    } else if let Some(end) = sentence_end(&chars, max) {
12398        (chars[..end].iter().collect::<String>(), "")
12399    } else {
12400        let room = max.saturating_sub(3);
12401        // Cut at the last space inside the room; when the char just past the
12402        // room is a space the room already ends on a word.
12403        let cut = if chars[room].is_whitespace() {
12404            room
12405        } else {
12406            chars[..room]
12407                .iter()
12408                .rposition(|c| c.is_whitespace())
12409                .unwrap_or(room)
12410        };
12411        let head: String = chars[..cut].iter().collect();
12412        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12413        (head.to_owned(), "...")
12414    };
12415    if kept.is_empty() {
12416        return None;
12417    }
12418    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12419        return None;
12420    }
12421    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12422        return None;
12423    }
12424    Some(format!("{kept}{mark}"))
12425}
12426
12427/// The char length of the first sentence of `chars` when it ends within `max`
12428/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12429/// abbreviations so `e.g. foo` does not end a title early.
12430fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12431    const MIN: usize = 20;
12432    for i in MIN..max.min(chars.len()) {
12433        if !matches!(chars[i], '.' | '!' | '?') {
12434            continue;
12435        }
12436        let Some(&next) = chars.get(i + 1) else {
12437            continue;
12438        };
12439        if !next.is_whitespace() {
12440            continue;
12441        }
12442        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12443        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12444            continue;
12445        }
12446        let word: String = chars[..i]
12447            .iter()
12448            .rev()
12449            .take_while(|c| !c.is_whitespace())
12450            .collect::<Vec<_>>()
12451            .into_iter()
12452            .rev()
12453            .collect();
12454        let word = word.to_ascii_lowercase();
12455        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12456            continue;
12457        }
12458        let end = chars[..i]
12459            .iter()
12460            .rposition(|c| !c.is_whitespace())
12461            .map_or(i, |p| p + 1);
12462        return Some(end);
12463    }
12464    None
12465}
12466
12467/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12468/// change: a title and a body describing what was *implemented*, not the task
12469/// that asked for it. A task reads as a request; a reader of the merged
12470/// history wants the change.
12471struct PrMessage {
12472    title: String,
12473    body: String,
12474}
12475
12476impl PrMessage {
12477    /// Title, blank line, body. The first line is the squash/merge commit
12478    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12479    /// has to stay one sensible line.
12480    fn commit_message(&self) -> String {
12481        format!("{}\n\n{}", self.title, self.body)
12482    }
12483}
12484
12485/// The text after a leading `TITLE:` (any case) on `line`.
12486fn title_marker(line: &str) -> Option<&str> {
12487    let line = line.trim();
12488    let head = line.get(..6)?;
12489    head.eq_ignore_ascii_case("title:")
12490        .then(|| line[6..].trim())
12491}
12492
12493/// The implementer's own one-line title: the `TITLE:` line the implement
12494/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12495/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12496/// source, and a title that says as much is refused here too.
12497fn summary_title(summary: &str) -> Option<String> {
12498    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12499    let raw = title_marker(first)?;
12500    if raw.is_empty() {
12501        return None;
12502    }
12503    let title = queue::title_from(raw, PR_TITLE_MAX);
12504    let lower = title.to_ascii_lowercase();
12505    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12506        return None;
12507    }
12508    Some(title)
12509}
12510
12511/// How `open_review`'s instruction begins; see [`landing_title`].
12512const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12513
12514/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12515pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12516
12517/// Does this run review work that already existed, rather than implement a
12518/// task? Runs recorded before `reviewed_commits` existed carry only the
12519/// candidate marker.
12520pub(crate) fn is_review_run(state: &RunState) -> bool {
12521    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12522}
12523
12524/// The title of a review-only run: the subject of the oldest commit under
12525/// review. Later commits are usually fixups, and `instruction` is the review
12526/// prompt, which says nothing about the change. GitHub text is English, so a
12527/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12528fn review_title(state: &RunState) -> Option<String> {
12529    english_subject(
12530        state.reviewed_commits.as_ref()?.first()?,
12531        state.config.graph.github_english(),
12532    )
12533}
12534
12535/// `raw` as a pull request title, or `None` when it is blank, not English
12536/// (GitHub text is), or one of magi's own candidate commit subjects.
12537fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12538    let raw = raw.trim();
12539    if raw.is_empty() {
12540        return None;
12541    }
12542    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12543        return None;
12544    }
12545    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12546        return None;
12547    }
12548    let title = queue::title_from(raw, PR_TITLE_MAX);
12549    let lower = title.to_ascii_lowercase();
12550    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12551        return None;
12552    }
12553    Some(title)
12554}
12555
12556/// What a review-only run's branch says about itself, read at the moment the
12557/// pull request is opened.
12558#[derive(Debug, Clone, PartialEq, Eq)]
12559struct BranchFacts {
12560    /// `(subject, body)` of each commit, oldest first.
12561    commits: Vec<(String, String)>,
12562    /// Trimmed `git diff --stat`.
12563    stat: String,
12564}
12565
12566/// Longest diff stat shown: this many file lines plus the summary line.
12567const STAT_FILE_LINES: usize = 25;
12568/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12569const COMMITS_MAX_CHARS: usize = 20_000;
12570
12571/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12572/// say or finds nothing, so the caller falls back to what the run recorded.
12573async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12574    let commits = git::commit_log(repo, base, branch).await.ok()?;
12575    if commits.is_empty() {
12576        return None;
12577    }
12578    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12579    let lines: Vec<&str> = stat.lines().collect();
12580    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12581        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12582        let more = format!(" ... {omitted} more file(s)");
12583        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12584        kept.push(&more);
12585        kept.push(lines[lines.len() - 1]);
12586        kept.join("\n")
12587    } else {
12588        lines.join("\n")
12589    };
12590    Some(BranchFacts { commits, stat })
12591}
12592
12593/// Defang what would break the surrounding markdown: a closing `</details>`
12594/// and a code fence.
12595fn markdown_safe(text: &str) -> String {
12596    text.replace("</details>", "&lt;/details&gt;")
12597        .replace("\x60\x60\x60", "~~~")
12598}
12599
12600fn neutral_title(state: &RunState, winner: char) -> String {
12601    format!(
12602        "chore: land candidate {} of run {}",
12603        winner.to_ascii_uppercase(),
12604        state.id
12605    )
12606}
12607
12608/// Reuse the graph's bounded fixer without starting another graph or driver.
12609/// Copy the working state back even on errors so spent rounds remain visible.
12610pub(crate) async fn repair_landing_gate(
12611    state: &mut RunState,
12612    winner: &Candidate,
12613    outcomes: &[CommandOutcome],
12614) -> Result<GateFix> {
12615    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12616        return Ok(GateFix::Stop);
12617    }
12618    let mut runner = Runner {
12619        roles: state.config.resolve_roles()?,
12620        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12621        state: state.clone(),
12622        pause: Pause::new(),
12623        interrupt: Pause::new(),
12624    };
12625    let result = runner.gate_fix_round(winner, outcomes).await;
12626    *state = runner.state;
12627    result
12628}
12629
12630/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12631/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12632pub(crate) async fn verify_current_head(
12633    state: &mut RunState,
12634    winner: &Candidate,
12635    wait: &crate::repo_lock::WaitPolicy,
12636) -> Result<bool> {
12637    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12638    // Preserve the previous record before replacing it, including on crash recovery.
12639    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12640        state
12641            .verification_history
12642            .push(crate::run::VerificationEvidence {
12643                superseded_by: Some(head.clone()),
12644                archived_at: Some(Timestamp::now()),
12645                fixes_spent: Some(state.gate_fixes.len()),
12646                gate_head: None,
12647                gate_ran: false,
12648                gate: Vec::new(),
12649                e2e: state.head_e2e.take(),
12650            });
12651    }
12652    if let Some(e) = &state.head_e2e
12653        && e.head == head
12654    {
12655        if e.passed {
12656            return Ok(true);
12657        }
12658        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12659            state.status = RunStatus::Blocked;
12660            state.save()?;
12661            return Ok(false);
12662        }
12663    } else if state.reviews.last().is_some_and(|r| {
12664        r.verified_head.as_deref() == Some(head.as_str())
12665            && (r.e2e_status() == E2eStatus::Passed
12666                || (r.e2e_status() == E2eStatus::NotConfigured
12667                    && state.config.verify.e2e.is_empty()))
12668    }) || state
12669        .gate_repair_e2e
12670        .as_ref()
12671        .is_some_and(|e| e.head == head && e.passed)
12672    {
12673        return Ok(true);
12674    }
12675    if let Some(e) = &state.gate_repair_e2e
12676        && e.head == head
12677        && !e.passed
12678        && !e.outcomes.is_empty()
12679        && !verify_inconclusive(&e.outcomes)
12680    {
12681        state.status = RunStatus::Blocked;
12682        state.save()?;
12683        return Ok(false);
12684    }
12685    state.head_e2e = Some(crate::run::GateRepairE2e {
12686        head: head.clone(),
12687        passed: false,
12688        outcomes: Vec::new(),
12689    });
12690    state.save()?;
12691    let commands = state.config.verify.e2e.clone();
12692    let shell = state.config.shell();
12693    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12694    let cache_dir = state.config.cache_dir();
12695    let outcomes = if commands.is_empty() {
12696        Vec::new()
12697    } else {
12698        let (outcomes, _) = with_cache_lease(
12699            state,
12700            cache_dir.as_deref(),
12701            "e2e",
12702            "e2e",
12703            &winner.worktree,
12704            &head,
12705            timeout,
12706            "current head e2e",
12707            wait,
12708            |state, budget| async move {
12709                run_e2e_with_retry(
12710                    state,
12711                    &shell,
12712                    &commands,
12713                    &winner.worktree,
12714                    budget,
12715                    "current head e2e",
12716                )
12717                .await
12718            },
12719        )
12720        .await?;
12721        outcomes
12722    };
12723    let unresolved = verify_inconclusive(&outcomes);
12724    let passed = !unresolved
12725        && outcomes.iter().all(CommandOutcome::ok)
12726        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12727    if !outcomes.is_empty() && !unresolved {
12728        state.record_base_progress(&head);
12729    }
12730    state.head_e2e = Some(crate::run::GateRepairE2e {
12731        head: head.clone(),
12732        passed,
12733        outcomes,
12734    });
12735    if !passed && !unresolved {
12736        state.status = RunStatus::Blocked;
12737        state.event(
12738            "e2e",
12739            format!("verify.e2e failed on head {head}; nothing published"),
12740        );
12741    }
12742    state.save()?;
12743    Ok(passed)
12744}
12745
12746/// Recheck a repair's e2e requirement across failures, resource deferrals and
12747/// process interruptions. A successful result only applies to its exact SHA.
12748pub(crate) async fn check_gate_repair_e2e(
12749    state: &mut RunState,
12750    winner: &Candidate,
12751    wait: &crate::repo_lock::WaitPolicy,
12752) -> Result<GateFix> {
12753    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12754        return Ok(GateFix::Retry);
12755    }
12756    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12757    if state
12758        .gate_repair_e2e
12759        .as_ref()
12760        .is_some_and(|r| r.head == head && r.passed)
12761    {
12762        return Ok(GateFix::Retry);
12763    }
12764    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12765        head: head.clone(),
12766        passed: false,
12767        outcomes: Vec::new(),
12768    });
12769    state.save()?;
12770    let commands = state.config.verify.e2e.clone();
12771    let outcomes = if commands.is_empty() {
12772        Vec::new()
12773    } else {
12774        let shell = state.config.shell();
12775        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12776        let cache_dir = state.config.cache_dir();
12777        let (outcomes, _) = with_cache_lease(
12778            state,
12779            cache_dir.as_deref(),
12780            "e2e",
12781            "e2e",
12782            &winner.worktree,
12783            &head,
12784            timeout,
12785            "gate repair e2e",
12786            wait,
12787            |state, budget| async move {
12788                run_e2e_with_retry(
12789                    state,
12790                    &shell,
12791                    &commands,
12792                    &winner.worktree,
12793                    budget,
12794                    "gate repair e2e",
12795                )
12796                .await
12797            },
12798        )
12799        .await?;
12800        outcomes
12801    };
12802    let deferred = verify_inconclusive(&outcomes);
12803    let passed = outcomes.iter().all(CommandOutcome::ok)
12804        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12805    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12806        head,
12807        passed,
12808        outcomes,
12809    });
12810    state.save()?;
12811    if deferred {
12812        Ok(GateFix::Defer)
12813    } else if !passed {
12814        state.event(
12815            "gate",
12816            "verify.e2e failed after gate repair; nothing published",
12817        );
12818        Ok(GateFix::Stop)
12819    } else {
12820        Ok(GateFix::Retry)
12821    }
12822}
12823
12824/// Run the same configured gate for a landing head and a fresh graph.
12825pub(crate) async fn run_landing_gate(
12826    state: &mut RunState,
12827    winner: &Candidate,
12828    wait: &crate::repo_lock::WaitPolicy,
12829) -> Result<Vec<CommandOutcome>> {
12830    let shell = state.config.shell();
12831    let gate_commands = state.config.verify.gate.clone();
12832    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12833    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12834        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12835        Vec::new()
12836    } else {
12837        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12838        let cache_dir = state.config.cache_dir();
12839        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12840        let (outcomes, _) = with_cache_lease(
12841            state,
12842            cache_dir.as_deref(),
12843            "gate",
12844            "gate",
12845            &winner.worktree,
12846            &head,
12847            timeout,
12848            "final gate",
12849            wait,
12850            |state, budget| {
12851                let shell = shell.clone();
12852                let gate_commands = gate_commands.clone();
12853                let winner = winner.clone();
12854                async move {
12855                    Runner::run_pre_gate(state, &winner).await;
12856                    // Bind command evidence after mechanical fixes, before commands start.
12857                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12858                    let (outcomes, timed_out_pids) = run_commands(
12859                        state,
12860                        "gate",
12861                        "gate",
12862                        0,
12863                        &shell,
12864                        &gate_commands,
12865                        &winner.worktree,
12866                        budget,
12867                    )
12868                    .await;
12869                    (outcomes, false, timed_out_pids)
12870                }
12871            },
12872        )
12873        .await?;
12874        outcomes
12875    };
12876    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12877    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12878        let mut outcomes = outcomes;
12879        outcomes.push(CommandOutcome {
12880            command: "gate head stability".into(), code: None,
12881            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12882            duration_ms: 0, resource_blocked: false,
12883        });
12884        return Ok(outcomes);
12885    }
12886    if outcomes.is_empty() {
12887        // Nothing configured to check — distinct from every other
12888        // silence in this run's event log, since an empty `gate` alone
12889        // no longer says whether the gate ran at all (see
12890        // `RunState::gate_ran`'s own doc).
12891        state.event("gate", "no gate commands configured; not tested");
12892    }
12893    for o in &outcomes {
12894        state.event(
12895            "gate",
12896            format!(
12897                "`{}` -> {}",
12898                o.command,
12899                if o.ok() {
12900                    "pass".to_owned()
12901                } else {
12902                    format!(
12903                        "FAIL ({:?})\n{}",
12904                        o.code,
12905                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12906                    )
12907                }
12908            ),
12909        );
12910    }
12911    Ok(outcomes)
12912}
12913
12914/// The pull request title to hand to `land::merge_subject`. A review-only run
12915/// opened by an earlier build titled its pull request with the review prompt;
12916/// that title is dropped (empty, so the fallback applies) rather than landed.
12917/// Any other title, including an operator's rename, passes through untouched,
12918/// and so does every title of a run that implements a task.
12919pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12920    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12921        ""
12922    } else {
12923        pr_title
12924    }
12925}
12926
12927/// What the squash subject falls back to when the pull request title is empty
12928/// or candidate-shaped: for a review-only run the derived title, never the
12929/// review prompt held in `instruction`.
12930pub fn landing_subject_source(state: &RunState) -> String {
12931    if is_review_run(state) {
12932        let winner = state.candidates.first().map_or('A', |c| c.label);
12933        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12934    }
12935    state.instruction.clone()
12936}
12937
12938/// `summary` without its `TITLE:` line, which the pull request title already
12939/// carries.
12940fn summary_without_title(summary: &str) -> String {
12941    let mut lines = summary.trim().lines().peekable();
12942    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12943        lines.next();
12944    }
12945    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12946}
12947
12948/// The pull request title and body for the winning candidate.
12949///
12950/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12951/// the task's own opening line via [`queue::title_from`] when there is none.
12952/// `state.instruction` can open with blank lines (`task_text` only rejects a
12953/// body that is blank *entirely*), which `title_from` skips.
12954///
12955/// Publishing uses an independent description generated from the final diff
12956/// and structured evidence. Internal summaries and reports remain local.
12957/// Non-publishing previews may display the original task and candidate summary.
12958/// The footer retains the run and candidate provenance.
12959#[cfg(test)]
12960fn pr_message(state: &RunState, winner: char) -> PrMessage {
12961    pr_message_with(state, winner, None)
12962}
12963
12964/// [`pr_message`] with what the branch of a review-only run says about itself.
12965/// `facts` is ignored for a run that implements a task.
12966#[cfg(test)]
12967fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12968    let raw = pr_message_raw(
12969        state,
12970        winner,
12971        facts,
12972        crate::pr_outcome::FinalHead::Unreported,
12973        ChangeNote::NotAsked,
12974    );
12975    let id = crate::scrub::Identity::current();
12976    PrMessage {
12977        title: crate::scrub::scrub(&raw.title, &id),
12978        body: crate::scrub::scrub(&raw.body, &id),
12979    }
12980}
12981
12982/// What the public description may say about the change itself.
12983#[derive(Debug, Clone, Copy)]
12984enum ChangeNote<'a> {
12985    /// A local preview or non-publishing merge mode: no public generation needed.
12986    NotAsked,
12987    /// A summary written from the final diff.
12988    Summary(&'a crate::run::ChangeSummary),
12989    /// A summary was asked for and could not be produced; the text says why.
12990    Unavailable(&'a str),
12991}
12992
12993fn pr_message_raw(
12994    state: &RunState,
12995    winner: char,
12996    facts: Option<&BranchFacts>,
12997    head: crate::pr_outcome::FinalHead<'_>,
12998    change: ChangeNote<'_>,
12999) -> PrMessage {
13000    if !matches!(change, ChangeNote::NotAsked) {
13001        let (title, mut body) = match change {
13002            ChangeNote::Summary(c) => (
13003                c.title.clone(),
13004                format!("## Summary\n\n{}\n\n", c.body.trim()),
13005            ),
13006            ChangeNote::Unavailable(why) => (
13007                neutral_title(state, winner),
13008                format!(
13009                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13010                ),
13011            ),
13012            ChangeNote::NotAsked => unreachable!(),
13013        };
13014        body.push_str(&public_pr_evidence(state, head));
13015        body.push_str(&format!(
13016            "\n---\nmagi:run/{} magi:candidate-{}\n",
13017            state.id,
13018            winner.to_ascii_lowercase()
13019        ));
13020        return PrMessage { title, body };
13021    }
13022    pr_preview(state, winner, facts, head)
13023}
13024
13025/// Append public facts from the run records after every prose generation,
13026/// including posting-gate rewrites. The agent cannot drop this provenance.
13027fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13028    let mut body = crate::pr_outcome::review_rounds(state);
13029    body.push_str(&crate::pr_outcome::final_verification(state, head));
13030    let open = state.last_round_findings();
13031    if !open.is_empty() {
13032        body.push_str("## Open review findings\n\n");
13033        for f in open {
13034            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13035        }
13036        body.push('\n');
13037    }
13038    if let Some(c) = state.change_summary.as_ref()
13039        && head == crate::pr_outcome::FinalHead::At(&c.head)
13040    {
13041        body.push_str(&format!(
13042            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13043            c.base, c.head
13044        ));
13045    }
13046    body
13047}
13048
13049/// Local-only preview of the operator's task and candidate summary.
13050fn pr_preview(
13051    state: &RunState,
13052    winner: char,
13053    facts: Option<&BranchFacts>,
13054    head: crate::pr_outcome::FinalHead<'_>,
13055) -> PrMessage {
13056    let summary = state
13057        .candidates
13058        .iter()
13059        .find(|c| c.label == winner)
13060        .map(|c| c.summary.as_str())
13061        .unwrap_or_default();
13062    // The fallback is the operator's own words and may not be English; GitHub
13063    // text always is, so a non-English task gets a neutral title instead.
13064    let review = is_review_run(state);
13065    let title = if review {
13066        facts
13067            .and_then(|f| {
13068                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13069            })
13070            .or_else(|| review_title(state))
13071            .or_else(|| {
13072                state
13073                    .candidates
13074                    .iter()
13075                    .find(|c| c.label == winner)
13076                    .filter(|c| !c.branch.starts_with("magi/"))
13077                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13078            })
13079            .unwrap_or_else(|| neutral_title(state, winner))
13080    } else {
13081        summary_title(summary).unwrap_or_else(|| {
13082            english_title(
13083                &state.instruction,
13084                PR_TITLE_MAX,
13085                state.config.graph.github_english(),
13086            )
13087            .unwrap_or_else(|| neutral_title(state, winner))
13088        })
13089    };
13090
13091    let mut body = String::new();
13092    let what = summary_without_title(summary);
13093    if !what.is_empty() {
13094        body.push_str("## Summary\n\n");
13095        body.push_str(&what);
13096        body.push_str("\n\n");
13097    }
13098
13099    // Review history uses structured facts across all rounds, never reports.
13100    body.push_str(&crate::pr_outcome::review_rounds(state));
13101    body.push_str(&crate::pr_outcome::final_verification(state, head));
13102    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13103
13104    let open = state.open_findings();
13105    if !open.is_empty() {
13106        body.push_str("## Open review findings\n\n");
13107        for f in &open {
13108            body.push_str(&format!(
13109                "- `{}` [{:?}] {}\n",
13110                f.id, f.severity, "Context retained in local review records."
13111            ));
13112        }
13113        body.push('\n');
13114    }
13115
13116    if let Some(fix) = fix
13117        && !fix.rejected.is_empty()
13118    {
13119        body.push_str("## Declined by the fixer\n\n");
13120        for r in &fix.rejected {
13121            body.push_str(&format!(
13122                "- `{}`: {}\n",
13123                r.id, "Reason retained in local review records."
13124            ));
13125        }
13126        body.push('\n');
13127    }
13128
13129    if review {
13130        // The review prompt is not the task; list what the branch carries.
13131        body.push_str("## Commits under review\n\n");
13132        if let Some(facts) = facts {
13133            let mut left = COMMITS_MAX_CHARS;
13134            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13135                let mut entry = format!("- {}\n", markdown_safe(subject));
13136                for l in markdown_safe(text).lines() {
13137                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13138                }
13139                if left == 0 {
13140                    body.push_str(&format!(
13141                        "- ... {} more commit(s)\n",
13142                        facts.commits.len() - i
13143                    ));
13144                    break;
13145                }
13146                if entry.len() > left {
13147                    // Even the first commit is cut: one huge body must not
13148                    // push the whole description past GitHub's limit.
13149                    let mut end = left;
13150                    while !entry.is_char_boundary(end) {
13151                        end -= 1;
13152                    }
13153                    entry.truncate(end);
13154                    entry.push_str("\n  ... (truncated)\n");
13155                    left = 0;
13156                } else {
13157                    left -= entry.len();
13158                }
13159                body.push_str(&entry);
13160            }
13161            if !facts.stat.trim().is_empty() {
13162                body.push_str(&format!(
13163                    "\n## Diff stat\n\n```\n{}\n```\n",
13164                    markdown_safe(facts.stat.trim())
13165                ));
13166            }
13167        } else {
13168            match &state.reviewed_commits {
13169                Some(subjects) => {
13170                    for s in subjects {
13171                        body.push_str(&format!("- {}\n", s.trim()));
13172                    }
13173                }
13174                None => {
13175                    // An older run kept only the prompt, with the commit list
13176                    // after its first paragraph.
13177                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13178                    body.push_str(rest.trim());
13179                    body.push('\n');
13180                }
13181            }
13182        }
13183    } else {
13184        let task = state.instruction.trim();
13185        let task = if task.is_empty() {
13186            "(empty task)"
13187        } else {
13188            task
13189        };
13190        body.push_str(&format!(
13191            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13192            task.replace("</details>", "&lt;/details&gt;")
13193        ));
13194    }
13195
13196    body.push_str(&format!(
13197        "\n---\nmagi:run/{} magi:candidate-{}\n",
13198        state.id,
13199        winner.to_ascii_lowercase()
13200    ));
13201
13202    PrMessage { title, body }
13203}
13204
13205/// Source imports have already been prepared from structured inputs. Legacy
13206/// inferred references must never instruct a seat to reconstruct those imports.
13207fn seeded_instruction(state: &RunState) -> String {
13208    let mut inputs = Vec::new();
13209    if let Some(start) = &state.sources.start_from {
13210        inputs.push(format!(
13211            "Start from `{}` at `{}` (including ancestry).",
13212            start.token, start.sha
13213        ));
13214    }
13215    for pick in &state.sources.pick_commits {
13216        inputs.push(format!(
13217            "Pick `{}` at `{}` (single commit).",
13218            pick.token, pick.sha
13219        ));
13220    }
13221    if inputs.is_empty() {
13222        state.instruction.clone()
13223    } else {
13224        format!(
13225            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13226            state.instruction,
13227            inputs.join("\n")
13228        )
13229    }
13230}
13231
13232/// Does the winner have no commits ahead of the base it would land on?
13233/// Any failure to find out reads as "not empty": the merge then behaves as it
13234/// always did rather than refusing on a guess.
13235async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13236    let base = &state.base_branch;
13237    let mut against = base.clone();
13238    if mode == MergeMode::Pr {
13239        // A pull request lands on the remote's base, never the local branch
13240        // of the same name: if that cannot be read, "not empty" is the safe
13241        // answer.
13242        let remote = &state.config.merge.remote;
13243        let tracking = format!("{remote}/{base}");
13244        let fetched = git::fetch(repo, remote, base).await;
13245        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13246            against = tracking;
13247        } else {
13248            return false;
13249        }
13250    }
13251    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13252}
13253
13254/// Why nothing was opened for an empty winner, with what the task's own
13255/// references resolved to.
13256fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13257    let mut detail = format!(
13258        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13259         nothing to open a pull request for"
13260    );
13261    detail.push_str(
13262        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13263    );
13264    detail
13265}
13266
13267/// What the `Pr` merge does once it knows whether the branch already has an
13268/// open pull request.
13269#[derive(Debug, PartialEq, Eq)]
13270enum PrPlan {
13271    Create,
13272    Adopt { url: String, title: String },
13273    Stop(String),
13274}
13275
13276/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13277/// a failed lookup -> stop with the real reason. Never guesses.
13278fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13279    match found {
13280        Ok(land::OpenPr::None) => PrPlan::Create,
13281        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13282        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13283            "several open pull requests exist for this branch, not picking one: {}",
13284            urls.join(" ")
13285        )),
13286        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13287    }
13288}
13289
13290/// `gh pr create`, returning the PR url.
13291pub(crate) async fn gh_pr_create(
13292    cwd: &Path,
13293    base: &str,
13294    head: &str,
13295    title: &str,
13296    body: &str,
13297) -> Result<String> {
13298    let out = tokio::process::Command::new("gh")
13299        .args([
13300            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13301        ])
13302        .current_dir(cwd)
13303        .quiet()
13304        .stdin(std::process::Stdio::null())
13305        .output()
13306        .await
13307        .context("spawn gh")?;
13308    if out.status.success() {
13309        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13310    } else {
13311        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13312    }
13313}
13314
13315/// Never force deletion of uncommitted work, including unknown legacy trees.
13316async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13317    match std::fs::symlink_metadata(path) {
13318        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13319            // A previous cleanup may have removed the tree before a branch
13320            // deletion or state save failed. Unregister it if still present.
13321            git::worktree_remove_clean(repo, path).await?;
13322            return Ok(());
13323        }
13324        Err(e) => return Err(e.into()),
13325        Ok(_) => {}
13326    }
13327    if !git::is_clean(path).await? {
13328        bail!(
13329            "worktree has uncommitted work; refusing removal: {}",
13330            path.display()
13331        );
13332    }
13333    if !git::worktree_remove_clean(repo, path).await? {
13334        bail!("worktree removal refused: {}", path.display());
13335    }
13336    Ok(())
13337}
13338
13339/// Read seats return replies, not working-tree changes. Only registered,
13340/// detached seats directly inside this run's root may discard those changes.
13341async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13342    if !crate::retention::read_seat(path) || !path.exists() {
13343        return remove_preserved_worktree(&state.repo, path).await;
13344    }
13345    let owned = std::fs::symlink_metadata(path)?.is_dir()
13346        && path.parent().map(Path::canonicalize).transpose()?
13347            == Some(state.worktree_root().canonicalize()?)
13348        && crate::retention::worktrees(state)
13349            .await?
13350            .contains(&path.canonicalize()?);
13351    let common = crate::repo_lock::common_dir(&state.repo).await?;
13352    if !owned
13353        || common.is_none()
13354        || crate::repo_lock::common_dir(path).await? != common
13355        || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13356            .await?
13357            .ok()
13358        || state.candidates.iter().any(|c| c.worktree == path)
13359    {
13360        return remove_preserved_worktree(&state.repo, path).await;
13361    }
13362    let changes = git::status_porcelain(path).await?;
13363    if !git::worktree_remove(&state.repo, path).await? {
13364        bail!(
13365            "read-only seat worktree removal refused: {}",
13366            path.display()
13367        );
13368    }
13369    if !changes.is_empty() {
13370        let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13371        let paths: String = paths.chars().take(1024).collect();
13372        state.event(
13373            "fold",
13374            format!(
13375                "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13376                path.file_name().unwrap_or_default().to_string_lossy(),
13377                changes.lines().count()
13378            ),
13379        );
13380    }
13381    Ok(())
13382}
13383
13384/// Tear a run's worktrees and branches down.
13385///
13386/// `home` is where the updated `run.json` is saved (via
13387/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13388/// a housekeeping pass already has its own honest `home` handed to it, and
13389/// falling through to the global here would write back through whichever
13390/// directory some other process or test pinned into that `OnceLock` first,
13391/// not the one the caller actually resolved its `runs` and `state` from.
13392pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13393    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13394        bail!("refusing fold: {reason}");
13395    }
13396    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13397        bail!("candidate preservation incomplete; refusing cleanup");
13398    }
13399    // Serialize ordinary folding with the same starts/resumes as target-only
13400    // eviction. Otherwise a second janitor could remove the whole worktree
13401    // around a live output borrower even though target eviction was fenced.
13402    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13403    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13404    let _driver = crate::driver::maintenance(home, &state.id)?;
13405    let mut output_guards = Vec::new();
13406    for entry in std::fs::read_dir(state.worktree_root())
13407        .into_iter()
13408        .flatten()
13409        .flatten()
13410    {
13411        if entry.path().join(".git").is_file() {
13412            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13413        }
13414    }
13415    let repo = state.repo.clone();
13416    // CLI, Web and automatic cleanup all enter here, independently of
13417    // Runner::fold_losers. Protect deletion and shared config release too.
13418    let wait = repository_wait(state, None);
13419    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13420        Ok(lease) => lease,
13421        Err(e) => {
13422            state.save_under(home)?;
13423            return Err(e);
13424        }
13425    };
13426    // Confirmed merges dispose of their owned sources after durable evidence
13427    // capture. Preflight every source before removing anything.
13428    let _source_outputs = if state.status == RunStatus::Merged
13429        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13430    {
13431        verify_merged_fold_sources(state, drop_winner, home).await?
13432    } else {
13433        Vec::new()
13434    };
13435    let root = state.worktree_root();
13436    let winner = state.tally.as_ref().map(|t| t.winner);
13437    let mut removed = Vec::new();
13438
13439    for i in 0..state.candidates.len() {
13440        let c = state.candidates[i].clone();
13441        let is_winner = Some(c.label) == winner;
13442        if is_winner && !drop_winner {
13443            continue;
13444        }
13445        let existed = c.worktree.exists();
13446        // A vanished directory can still have a Git registration that keeps
13447        // its branch checked out. Remove that registration before the branch.
13448        remove_preserved_worktree(&repo, &c.worktree).await?;
13449        if existed {
13450            removed.push(c.worktree.to_string_lossy().into_owned());
13451        }
13452        // A branch handed to a later run (and its pull request) is not this
13453        // run's to delete.
13454        let handed_over = state.released_branches.contains(&c.branch);
13455        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13456            if !git::branch_delete(&repo, &c.branch).await? {
13457                bail!("candidate branch removal refused");
13458            }
13459            removed.push(c.branch.clone());
13460        }
13461        state.candidates[i].folded = true;
13462    }
13463
13464    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13465        let path = name.path();
13466        let keep = !drop_winner
13467            && winner.is_some_and(|w| {
13468                path.file_name()
13469                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13470            });
13471        if keep {
13472            continue;
13473        }
13474        remove_read_seat_worktree(state, &path).await?;
13475        removed.push(path.to_string_lossy().into_owned());
13476    }
13477
13478    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13479    // judge worktrees, so once the loop above has cleared all of them out,
13480    // the parent is a bare directory nobody else was ever going to remove -
13481    // git only ever managed what was inside it. Left alone, one of these
13482    // accumulates per fully-folded run; the operator's own machine had 74.
13483    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13484    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13485    // so does anything a run never claimed that happens to share the bay.
13486    remove_if_empty(&root);
13487
13488    if state.enabled_worktree_config && drop_winner {
13489        // A release, not a raw disable: some sibling run in this repository
13490        // may still hold its own reference (see `git::acquire_worktree_config`),
13491        // and only the last release actually turns the setting back off.
13492        git::release_worktree_config(&repo).await.ok();
13493        state.enabled_worktree_config = false;
13494    }
13495    state.save_under(home)?;
13496    Ok(removed)
13497}
13498
13499async fn verify_merged_fold_sources(
13500    state: &RunState,
13501    drop_winner: bool,
13502    home: &Path,
13503) -> Result<Vec<std::fs::File>> {
13504    let winner = state.tally.as_ref().map(|t| t.winner);
13505    let mut heads = BTreeSet::new();
13506    for candidate in &state.candidates {
13507        if Some(candidate.label) == winner && !drop_winner {
13508            continue;
13509        }
13510        anyhow::ensure!(
13511            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13512            "candidate recovery source is outside the inventoried run root: {}",
13513            candidate.worktree.display()
13514        );
13515        match std::fs::symlink_metadata(&candidate.worktree) {
13516            Ok(_) => {}
13517            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13518            Err(error) => return Err(error.into()),
13519        }
13520        if !state.released_branches.contains(&candidate.branch)
13521            && git::branch_exists(&state.repo, &candidate.branch).await?
13522        {
13523            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13524        }
13525    }
13526    let mut cargo_guards = Vec::new();
13527    let entries = match std::fs::read_dir(state.worktree_root()) {
13528        Ok(entries) => Some(entries),
13529        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13530        Err(error) => return Err(error.into()),
13531    };
13532    for entry in entries.into_iter().flatten() {
13533        let path = entry?.path();
13534        let metadata = std::fs::symlink_metadata(&path)?;
13535        anyhow::ensure!(
13536            metadata.is_dir() && !metadata.file_type().is_symlink(),
13537            "non-directory or symlink recovery source retained: {}",
13538            path.display()
13539        );
13540        if !drop_winner
13541            && winner.is_some_and(|w| {
13542                path.file_name()
13543                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13544            })
13545        {
13546            continue;
13547        }
13548        anyhow::ensure!(
13549            path.join(".git").is_file(),
13550            "unregistered recovery source retained: {}",
13551            path.display()
13552        );
13553        anyhow::ensure!(
13554            git::git(
13555                &path,
13556                &[
13557                    "status",
13558                    "--porcelain",
13559                    "--untracked-files=all",
13560                    "--ignored",
13561                    "--",
13562                    ".",
13563                    ":(exclude)target"
13564                ]
13565            )
13566            .await?
13567            .is_empty(),
13568            "source changes or ignored recovery files retained: {}",
13569            path.display()
13570        );
13571        match std::fs::symlink_metadata(path.join("target")) {
13572            Ok(_) => {
13573                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13574            }
13575            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13576            Err(error) => return Err(error.into()),
13577        }
13578        for checkpoint in [
13579            "rebase-merge",
13580            "rebase-apply",
13581            "sequencer",
13582            "CHERRY_PICK_HEAD",
13583            "MERGE_HEAD",
13584            "REBASE_HEAD",
13585            "REVERT_HEAD",
13586            "BISECT_LOG",
13587            "AUTO_MERGE",
13588            "MERGE_MSG",
13589        ] {
13590            let location = git::git(
13591                &path,
13592                &[
13593                    "rev-parse",
13594                    "--path-format=absolute",
13595                    "--git-path",
13596                    checkpoint,
13597                ],
13598            )
13599            .await?;
13600            anyhow::ensure!(
13601                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13602                if e.kind() == std::io::ErrorKind::NotFound),
13603                "Git recovery checkpoint retained: {checkpoint}"
13604            );
13605        }
13606        heads.insert(git::rev_parse(&path, "HEAD").await?);
13607    }
13608    crate::fold_evidence::preserve(state, home, &heads).await?;
13609
13610    Ok(cargo_guards)
13611}
13612
13613/// Remove `dir` if it exists and has nothing in it.
13614///
13615/// Best-effort and silent by design: a directory that is not empty (a run
13616/// whose winner is still parked there, a stray file some other process left)
13617/// is exactly the case this must refuse, and a directory that is already gone
13618/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13619/// already refuses a non-empty directory, so the emptiness check below is
13620/// belt, not suspenders - it is what keeps this from ever attempting the
13621/// removal in the case that matters, rather than trusting `remove_dir`'s
13622/// error path to have no side effects if it ever changed.
13623fn remove_if_empty(dir: &Path) {
13624    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13625        std::fs::remove_dir(dir).ok();
13626    }
13627}
13628
13629/// Severity of the worst open finding in the last review round, for reporting.
13630pub fn worst_open(state: &RunState) -> Option<Severity> {
13631    state
13632        .reviews
13633        .last()?
13634        .reviews
13635        .iter()
13636        .flat_map(|r| r.findings.iter())
13637        .map(|f| f.severity)
13638        .max()
13639}
13640
13641/// `[worktree] setup` hides its products per worktree, which needs
13642/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13643/// worktrees, like the hook's; every place that runs setup calls this first.
13644async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13645    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13646        git::acquire_worktree_config(repo).await?;
13647        state.enabled_worktree_config = true;
13648    }
13649    Ok(())
13650}
13651
13652#[cfg(test)]
13653mod tests {
13654    #[tokio::test]
13655    async fn launcher_base_fetch_requires_remote_success() {
13656        let tmp = tempfile::tempdir().unwrap();
13657        let repo = tmp.path().join("repo");
13658        let remote = tmp.path().join("remote");
13659        std::fs::create_dir(&repo).unwrap();
13660        std::fs::create_dir(&remote).unwrap();
13661        git::git(&remote, &["init", "--bare", "-b", "main"])
13662            .await
13663            .unwrap();
13664        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13665        git::git(
13666            &repo,
13667            &[
13668                "-c",
13669                "user.name=Fixture",
13670                "-c",
13671                "user.email=fixture@example.invalid",
13672                "commit",
13673                "--allow-empty",
13674                "-m",
13675                "fixture",
13676            ],
13677        )
13678        .await
13679        .unwrap();
13680        git::git(
13681            &repo,
13682            &["remote", "add", "origin", remote.to_str().unwrap()],
13683        )
13684        .await
13685        .unwrap();
13686        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13687        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13688        assert_eq!(
13689            resolve_base(&repo, "main", "origin", None)
13690                .await
13691                .unwrap()
13692                .commit,
13693            head
13694        );
13695        #[cfg(unix)]
13696        {
13697            let policy: crate::config::Coordination = toml::from_str(
13698                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13699            ).unwrap();
13700            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13701            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13702            let held = crate::repo_lock::SeatLease::operation_with(
13703                &repo,
13704                Some(&policy),
13705                &crate::repo_lock::WaitPolicy::default(),
13706            )
13707            .await
13708            .unwrap();
13709            assert!(held.is_some());
13710            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13711            tokio::pin!(fetch);
13712            assert!(
13713                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13714                    .await
13715                    .is_err()
13716            );
13717            assert!(
13718                !repo.join(".git/FETCH_HEAD").exists(),
13719                "no fetch before admission"
13720            );
13721            drop(held);
13722            let fetched = fetch.await.unwrap();
13723            assert_eq!(fetched.commit, head);
13724            assert_eq!(fetched.remote, "origin");
13725            assert_eq!(fetched.branch, "main");
13726        }
13727        // A surviving tracking ref is not successful fetch evidence.
13728        git::git(
13729            &repo,
13730            &[
13731                "remote",
13732                "set-url",
13733                "origin",
13734                tmp.path().join("missing").to_str().unwrap(),
13735            ],
13736        )
13737        .await
13738        .unwrap();
13739        let error = resolve_base(&repo, "main", "origin", None)
13740            .await
13741            .unwrap_err()
13742            .to_string();
13743        assert!(error.contains("cannot read origin/main"));
13744        assert!(error.contains("refusing to branch off"));
13745    }
13746
13747    #[test]
13748    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13749        let leaked = vec!["chore(deps): update a crate".to_owned()];
13750        let own = "fix(daemon): apply a chosen action";
13751        assert!(should_retitle("", own, &leaked));
13752        assert!(should_retitle(
13753            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13754            own,
13755            &leaked
13756        ));
13757        assert!(should_retitle(
13758            "chore: land candidate A of run 1",
13759            own,
13760            &leaked
13761        ));
13762        assert!(should_retitle(
13763            "magi: candidate A (uncommitted work)",
13764            own,
13765            &leaked
13766        ));
13767        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13768        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13769        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13770    }
13771
13772    #[test]
13773    fn pr_merge_plan_creates_adopts_or_stops() {
13774        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13775        assert_eq!(
13776            pr_merge_plan(Ok(land::OpenPr::One {
13777                url: "u".into(),
13778                title: "t".into()
13779            })),
13780            PrPlan::Adopt {
13781                url: "u".into(),
13782                title: "t".into()
13783            }
13784        );
13785        let PrPlan::Stop(many) =
13786            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13787        else {
13788            panic!("many must stop");
13789        };
13790        assert!(many.contains('a') && many.contains('b'));
13791        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13792            panic!("a failed lookup must stop");
13793        };
13794        assert!(err.contains("bad token"));
13795    }
13796
13797    use super::*;
13798    use crate::run::GateStatus;
13799    use std::collections::BTreeMap;
13800    use std::time::Duration;
13801
13802    fn conductor() -> AgentSpec {
13803        AgentSpec {
13804            id: "conductor".to_owned(),
13805            kind: crate::config::AgentKind::Command,
13806            model: None,
13807            command: vec!["true".to_owned()],
13808            extra_args: Vec::new(),
13809            env: BTreeMap::new(),
13810            prompt_delivery: None,
13811            max_turn_secs: None,
13812        }
13813    }
13814
13815    fn spec(id: &str) -> AgentSpec {
13816        AgentSpec {
13817            id: id.to_owned(),
13818            kind: crate::config::AgentKind::Command,
13819            model: None,
13820            command: vec!["true".to_owned()],
13821            extra_args: Vec::new(),
13822            env: BTreeMap::new(),
13823            prompt_delivery: None,
13824            max_turn_secs: None,
13825        }
13826    }
13827
13828    fn ids(xs: &[&str]) -> BTreeSet<String> {
13829        xs.iter().map(|s| (*s).to_owned()).collect()
13830    }
13831
13832    #[test]
13833    fn pick_successor_skips_an_agent_another_seat_holds() {
13834        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13835        let roster = [spec("a"), spec("b"), spec("c")];
13836        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13837        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13838    }
13839
13840    #[test]
13841    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13842        // The other seat started on c but was handed to d; c is free again.
13843        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13844        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13845        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13846        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13847        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13848    }
13849
13850    #[test]
13851    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13852        let roster = [spec("a"), spec("b")];
13853        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13854        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13855        let carried = ids(&["b"]);
13856        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13857        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13858    }
13859
13860    #[test]
13861    fn pick_successor_returns_none_without_any_untried_successor() {
13862        let roster = [spec("a"), spec("b")];
13863        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13864        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13865        let carried = ids(&["a"]);
13866        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13867    }
13868
13869    #[test]
13870    fn pick_successor_rescue_avoids_another_seats_occupant() {
13871        // b is a carried failure and free; a is held by the other seat.
13872        let roster = [spec("a"), spec("b"), spec("c")];
13873        let carried = ids(&["b", "c"]);
13874        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13875        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13876    }
13877
13878    #[test]
13879    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13880        let roster = [spec("a"), spec("b"), spec("c")];
13881        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13882        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13883    }
13884
13885    #[test]
13886    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13887        let roster = [spec("a"), spec("b"), spec("c")];
13888        let failed = ids(&["a", "b", "c"]);
13889        // Rescue looks at the whole roster, once per id, then runs out.
13890        let mut tried = ids(&["b"]);
13891        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13892        assert_eq!(first.id, "a");
13893        tried.insert(first.id.clone());
13894        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13895        assert_eq!(second.id, "c");
13896        tried.insert(second.id.clone());
13897        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13898    }
13899
13900    #[test]
13901    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13902        let roster = [spec("a"), spec("b")];
13903        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13904        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13905    }
13906
13907    #[test]
13908    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13909        let roster = [spec("a"), spec("b"), spec("c")];
13910        let h = SeatHistory {
13911            failed: ids(&["a"]),
13912            last_ok: Some("b".to_owned()),
13913            last_fail: None,
13914        };
13915        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13916        // A last answerer that left the roster, or later failed, is ignored.
13917        let gone = SeatHistory {
13918            last_ok: Some("zzz".to_owned()),
13919            ..h.clone()
13920        };
13921        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13922        let failed = SeatHistory {
13923            failed: ids(&["a", "b"]),
13924            last_ok: Some("b".to_owned()),
13925            last_fail: None,
13926        };
13927        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13928    }
13929
13930    #[test]
13931    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13932        let first = SeatState::new("review-1", "alpha", 7);
13933        let next = handover_seat("review-1", "gamma", 7);
13934        assert_ne!(first.claude_session, next.claude_session);
13935    }
13936
13937    #[test]
13938    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13939        let mut state = state_with_summary("x", "y");
13940        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13941        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13942        assert_ne!(a.claude_session, b.claude_session);
13943    }
13944
13945    #[test]
13946    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13947        let roster = [spec("a"), spec("b")];
13948        let h = SeatHistory {
13949            failed: ids(&["a", "b"]),
13950            ..SeatHistory::default()
13951        };
13952        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13953        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13954        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13955    }
13956
13957    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13958    // fallback loop depends on to terminate: it must walk forward from the
13959    // seat's own position, never restart at the front of the roster, and it
13960    // must never hand back an id already tried, however many times that id
13961    // happens to appear.
13962
13963    #[test]
13964    fn failure_signature_ignores_numbers_and_paths() {
13965        assert_eq!(
13966            failure_signature("exited with Some(2) and no usable output"),
13967            failure_signature("exited with Some(137) and no usable output")
13968        );
13969        assert_eq!(
13970            failure_signature("process exited with no usable output (exit code 2)"),
13971            failure_signature("process exited with no usable output (exit code 137)")
13972        );
13973        for (old, new) in [
13974            (
13975                "exited with Some(2) and no usable output",
13976                "process exited with no usable output (exit code 2)",
13977            ),
13978            (
13979                "exited with None and no usable output",
13980                "process exited with no usable output (exit code unavailable)",
13981            ),
13982        ] {
13983            let previous = FailClass::Other(failure_signature(old));
13984            let current = FailClass::Other(failure_signature(new));
13985            assert_eq!(previous, current, "resumption retains the failure class");
13986            assert!(!should_hand_over(Some(&previous), &current));
13987        }
13988        assert_eq!(
13989            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13990            failure_signature("cannot open /var/x.txt: denied")
13991        );
13992        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13993    }
13994
13995    #[test]
13996    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13997        let other = FailClass::Other("x".into());
13998        assert!(should_hand_over(None, &FailClass::Quota));
13999        assert!(should_hand_over(Some(&other), &FailClass::Quota));
14000        assert!(should_hand_over(
14001            Some(&FailClass::Timeout),
14002            &FailClass::Timeout
14003        ));
14004        assert!(should_hand_over(None, &other));
14005        assert!(!should_hand_over(Some(&other), &other));
14006        assert!(should_hand_over(
14007            Some(&other),
14008            &FailClass::Other("y".into())
14009        ));
14010        // A quota or timeout in between ends the run of identical failures.
14011        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14012        assert!(should_hand_over(Some(&FailClass::Quota), &other));
14013    }
14014
14015    #[test]
14016    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14017        let a = SeatState::new("judge-1", "alpha", 7);
14018        let b = handover_seat("judge-1", "beta", 7);
14019        assert_ne!(a.claude_session, b.claude_session);
14020        assert_eq!(b.turns, 0);
14021    }
14022
14023    #[test]
14024    fn a_timeout_is_classified_apart_from_other_failures() {
14025        assert_eq!(
14026            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14027            Some(FailClass::Timeout)
14028        );
14029        assert!(matches!(
14030            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14031            Some(FailClass::Other(_))
14032        ));
14033    }
14034
14035    #[test]
14036    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14037        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14038        let tried = BTreeSet::from(["beta".to_owned()]);
14039        // beta sits at index 1; the next candidate is gamma, never alpha —
14040        // which is very likely a different candidate slot's own agent.
14041        let next = next_untried_in_roster(&roster, 1, &tried);
14042        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14043    }
14044
14045    #[test]
14046    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14047        let roster = vec![spec("alpha"), spec("beta")];
14048        let tried = BTreeSet::from(["beta".to_owned()]);
14049        // beta is the roster's last entry: nothing follows it, and alpha —
14050        // earlier in the roster, almost certainly a different candidate
14051        // slot's own agent — must not be reached by wrapping back to it.
14052        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14053    }
14054
14055    #[test]
14056    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14057        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14058        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14059        // beta (index 1) and gamma (index 2, the only entry after it) have
14060        // both been tried; alpha (index 0) never has, but it comes before
14061        // beta's own position, so there is nothing further for this seat.
14062        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14063    }
14064
14065    #[test]
14066    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14067        let roster = vec![spec("a"), spec("a"), spec("b")];
14068        let tried = BTreeSet::from(["a".to_owned()]);
14069        let next = next_untried_in_roster(&roster, 0, &tried);
14070        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14071    }
14072
14073    #[test]
14074    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14075        let roster = vec![spec("a"), spec("b")];
14076        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14077        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14078    }
14079
14080    #[test]
14081    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14082        let dir = tempfile::tempdir().unwrap();
14083        let bay = dir.path().join("ffff");
14084
14085        // Not there yet: nothing to do, nothing to panic on.
14086        remove_if_empty(&bay);
14087        assert!(!bay.exists());
14088
14089        // Something still inside - the winner's worktree, or a stray file -
14090        // keeps the directory standing.
14091        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14092        remove_if_empty(&bay);
14093        assert!(bay.exists(), "non-empty directory must survive");
14094
14095        // Once the last entry is gone, so is the directory itself.
14096        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14097        remove_if_empty(&bay);
14098        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14099    }
14100
14101    // `round_is_clean` is the exact decision this task fixed: a round with a
14102    // seat that never answered must not read the same as a round every seat
14103    // actually reviewed. These are deterministic and process-free by design —
14104    // the equivalent end-to-end check (a real reviewer timing out under a
14105    // live graph run) is a genuine race against wall-clock contention, and a
14106    // spawn slow enough to blow even a generous budget under a loaded test
14107    // run must not turn this specific regression check flaky.
14108
14109    #[test]
14110    fn a_full_panel_that_found_nothing_is_clean() {
14111        assert!(round_is_clean(
14112            0,
14113            true,
14114            2,
14115            2,
14116            0,
14117            IncompleteReviewPolicy::Block
14118        ));
14119    }
14120
14121    #[test]
14122    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14123        assert!(!round_is_clean(
14124            0,
14125            true,
14126            1,
14127            2,
14128            0,
14129            IncompleteReviewPolicy::Block
14130        ));
14131    }
14132
14133    #[test]
14134    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14135        assert!(!round_is_clean(
14136            1,
14137            true,
14138            1,
14139            2,
14140            0,
14141            IncompleteReviewPolicy::Warn
14142        ));
14143    }
14144
14145    #[test]
14146    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14147        assert!(round_is_clean(
14148            0,
14149            true,
14150            1,
14151            2,
14152            0,
14153            IncompleteReviewPolicy::Warn
14154        ));
14155    }
14156
14157    #[test]
14158    fn a_full_panel_with_an_open_finding_is_not_clean() {
14159        assert!(!round_is_clean(
14160            1,
14161            true,
14162            2,
14163            2,
14164            0,
14165            IncompleteReviewPolicy::Block
14166        ));
14167    }
14168
14169    #[test]
14170    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14171        assert!(!round_is_clean(
14172            0,
14173            false,
14174            2,
14175            2,
14176            0,
14177            IncompleteReviewPolicy::Block
14178        ));
14179    }
14180
14181    // The stall this task closes: under the default `block` policy, a seat
14182    // missing only because it was rate limited must not force a wait for a
14183    // session limit that will not lift by the next round. `round_is_clean`
14184    // is where that quorum carve-out lives; the review loop around it never
14185    // changes what a reviewer's vote or a finding's severity means.
14186
14187    #[test]
14188    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14189        // 1 of 2 answered, and the one missing was quota'd — the exact
14190        // "review-2 rate limited (quota)" shape from the field report.
14191        assert!(round_is_clean(
14192            0,
14193            true,
14194            1,
14195            2,
14196            1,
14197            IncompleteReviewPolicy::Block
14198        ));
14199    }
14200
14201    #[test]
14202    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14203        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14204        // not a quota loss (`quota_missing` stays 0) — worth another try.
14205        assert!(!round_is_clean(
14206            0,
14207            true,
14208            1,
14209            2,
14210            0,
14211            IncompleteReviewPolicy::Block
14212        ));
14213    }
14214
14215    #[test]
14216    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14217        assert!(!round_is_clean(
14218            1,
14219            true,
14220            1,
14221            2,
14222            1,
14223            IncompleteReviewPolicy::Block
14224        ));
14225        assert!(!round_is_clean(
14226            0,
14227            false,
14228            1,
14229            2,
14230            1,
14231            IncompleteReviewPolicy::Block
14232        ));
14233    }
14234
14235    #[test]
14236    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14237        // Every seat quota'd, nobody answered: there is no panel to decide
14238        // on, so this must fall through to the existing block-and-retry
14239        // fallback rather than call an unreviewed patch clean.
14240        assert!(!round_is_clean(
14241            0,
14242            true,
14243            0,
14244            2,
14245            2,
14246            IncompleteReviewPolicy::Block
14247        ));
14248    }
14249
14250    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14251        CommandOutcome {
14252            command: "test".to_owned(),
14253            code,
14254            output_tail: String::new(),
14255            duration_ms: 0,
14256            resource_blocked,
14257        }
14258    }
14259
14260    #[cfg(unix)]
14261    #[tokio::test]
14262    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14263        ask_test_home();
14264        let temp = tempfile::tempdir().unwrap();
14265        let mut runner = runner_at(RunStatus::Gating);
14266        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14267        let commands = vec![
14268            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14269            "touch must-not-run".to_owned(),
14270        ];
14271        let (outcomes, retried, pids) = run_e2e_with_retry(
14272            &mut runner.state,
14273            &shell,
14274            &commands,
14275            temp.path(),
14276            Duration::from_secs(5),
14277            "fixture",
14278        )
14279        .await;
14280        assert_eq!(outcomes.len(), 1);
14281        assert!(outcomes[0].resource_blocked);
14282        assert!(!outcomes[0].build_failed());
14283        assert!(!retried);
14284        assert!(pids.is_empty());
14285        assert!(!temp.path().join("must-not-run").exists());
14286        assert_eq!(
14287            runner.state.resource_block.as_ref().unwrap().kind,
14288            "incremental cache admission"
14289        );
14290    }
14291
14292    #[test]
14293    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14294        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14295        assert!(
14296            !verify_inconclusive(&[outcome(Some(1), false)]),
14297            "an ordinary failure is still evidence about the patch"
14298        );
14299        assert!(verify_inconclusive(&[outcome(None, true)]));
14300        assert!(
14301            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14302            "one inconclusive outcome taints the whole batch"
14303        );
14304        assert!(!verify_inconclusive(&[]));
14305    }
14306
14307    #[tokio::test]
14308    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14309        // Alive for the first two checks, then dead - confirms the loop
14310        // actually re-polls rather than deciding once and sleeping out the
14311        // ceiling regardless.
14312        let calls = std::sync::atomic::AtomicUsize::new(0);
14313        let started = Instant::now();
14314        wait_for_pids_with(
14315            &[123],
14316            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14317            Duration::from_millis(5),
14318            Duration::from_secs(5),
14319        )
14320        .await;
14321        assert!(
14322            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14323            "must keep checking rather than deciding on the first answer"
14324        );
14325        assert!(
14326            started.elapsed() < Duration::from_secs(1),
14327            "must return the moment it is confirmed dead, not wait out the ceiling"
14328        );
14329    }
14330
14331    #[tokio::test]
14332    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14333        let started = Instant::now();
14334        wait_for_pids_with(
14335            &[123],
14336            |_| true, // never reports dead
14337            Duration::from_millis(5),
14338            Duration::from_millis(30),
14339        )
14340        .await;
14341        let elapsed = started.elapsed();
14342        assert!(
14343            elapsed >= Duration::from_millis(30),
14344            "must not give up before its own ceiling: {elapsed:?}"
14345        );
14346        assert!(
14347            elapsed < Duration::from_secs(1),
14348            "must not wait past its own ceiling either: {elapsed:?}"
14349        );
14350    }
14351
14352    #[tokio::test]
14353    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14354        let started = Instant::now();
14355        wait_for_pids_with(
14356            &[],
14357            |_| true,
14358            Duration::from_secs(5),
14359            Duration::from_secs(5),
14360        )
14361        .await;
14362        assert!(
14363            started.elapsed() < Duration::from_millis(200),
14364            "an empty pid list has nothing to confirm"
14365        );
14366    }
14367
14368    // `review_conclusion` is the exact decision the review hand-off task
14369    // fixed: a round budget spent (or a tree that stopped moving) must not
14370    // collapse into `Blocked` regardless of what verification actually
14371    // said. Deterministic and process-free for the same reason the
14372    // `round_is_clean` family above is.
14373    fn review_round(
14374        clean: bool,
14375        blocking: usize,
14376        answered: usize,
14377        expected: usize,
14378        progressed: bool,
14379        e2e_ok: bool,
14380    ) -> ReviewRound {
14381        ReviewRound {
14382            round: 1,
14383            head: "h".to_owned(),
14384            verified_head: None,
14385            verified_at: None,
14386            reviews: Vec::new(),
14387            e2e: vec![CommandOutcome {
14388                command: "test".to_owned(),
14389                code: Some(if e2e_ok { 0 } else { 1 }),
14390                output_tail: String::new(),
14391                duration_ms: 0,
14392                resource_blocked: false,
14393            }],
14394            verify_retried: false,
14395            e2e_deferred: false,
14396            e2e_defer_reason: None,
14397            fix: None,
14398            blocking,
14399            answered,
14400            expected,
14401            clean,
14402            progressed,
14403            vote_split: false,
14404            reconsideration: Vec::new(),
14405            verdict: None,
14406        }
14407    }
14408
14409    #[test]
14410    fn review_conclusion_is_none_when_nothing_has_run() {
14411        assert_eq!(review_conclusion(&[], 3), None);
14412    }
14413
14414    #[test]
14415    fn review_conclusion_is_none_while_rounds_remain() {
14416        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14417        assert_eq!(review_conclusion(&rounds, 3), None);
14418    }
14419
14420    #[test]
14421    fn review_conclusion_is_gating_once_a_round_is_clean() {
14422        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14423        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14424    }
14425
14426    #[test]
14427    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14428        let rounds = vec![
14429            review_round(false, 1, 2, 2, true, true),
14430            review_round(false, 1, 2, 2, true, true),
14431        ];
14432        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14433    }
14434
14435    #[test]
14436    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14437        let rounds = vec![
14438            review_round(false, 1, 2, 2, true, true),
14439            review_round(false, 1, 2, 2, true, false),
14440        ];
14441        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14442    }
14443
14444    #[test]
14445    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14446        // Magi never got a command to run against this round's own head — a
14447        // resource-blocked attempt, not a red one — so this must never
14448        // settle on `Blocked` the way a genuine e2e failure would. `None`
14449        // here is what tells `Runner::review_loop` to retry the check
14450        // itself rather than trust this cheap recomputation with a verdict
14451        // it cannot actually produce.
14452        let mut blocked = review_round(false, 1, 2, 2, true, false);
14453        blocked.e2e[0].resource_blocked = true;
14454        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14455        assert_eq!(review_conclusion(&rounds, 2), None);
14456    }
14457
14458    #[test]
14459    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14460        // Missing input, not a verified tree — never a hand-off candidate.
14461        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14462        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14463    }
14464
14465    #[test]
14466    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14467        let rounds = vec![
14468            review_round(false, 1, 2, 2, false, true),
14469            review_round(false, 1, 2, 2, false, true),
14470        ];
14471        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14472    }
14473
14474    fn secs(n: u64) -> Duration {
14475        Duration::from_secs(n)
14476    }
14477
14478    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14479    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14480    #[tokio::test]
14481    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14482        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14483        if std::env::var_os(CHILD).is_none() {
14484            // Other tests reconcile every run in the process-global home,
14485            // briefly taking owner locks. This ancestry test expects an idle
14486            // predecessor, so give it its own process and pinned test home.
14487            let output = std::process::Command::new(std::env::current_exe().unwrap())
14488                .args([
14489                    "--exact",
14490                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14491                    "--nocapture",
14492                ])
14493                .env(CHILD, "1")
14494                .env_remove("MAGI_WORKER_HOME")
14495                .env_remove("MAGI_WORKER_ID")
14496                .output()
14497                .unwrap();
14498            assert!(
14499                output.status.success()
14500                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14501                "{}\n{}",
14502                String::from_utf8_lossy(&output.stdout),
14503                String::from_utf8_lossy(&output.stderr)
14504            );
14505            return;
14506        }
14507        ask_test_home();
14508        let tmp = tempfile::tempdir().unwrap();
14509        let repo = tmp.path().join("repo");
14510        std::fs::create_dir_all(&repo).unwrap();
14511        init_repo(&repo);
14512        let g = |args: &[&str]| {
14513            let out = std::process::Command::new("git")
14514                .args(args)
14515                .current_dir(&repo)
14516                .quiet()
14517                .output()
14518                .unwrap();
14519            assert!(
14520                out.status.success(),
14521                "{}",
14522                String::from_utf8_lossy(&out.stderr)
14523            );
14524        };
14525        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14526        g(&["checkout", "-b", "historical"]);
14527        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14528        g(&["add", "."]);
14529        g(&["commit", "-m", "old alternative"]);
14530        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14531        g(&["checkout", "-b", "magi/task-selected", &base]);
14532        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14533        g(&["add", "."]);
14534        g(&["commit", "-m", "measurement infrastructure"]);
14535        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14536        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14537        g(&["add", "."]);
14538        g(&["commit", "-m", "dependent implementation"]);
14539        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14540        g(&["checkout", "main"]);
14541        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14542        g(&["add", "."]);
14543        g(&["commit", "-m", "unrelated base change"]);
14544        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14545        let origin = tmp.path().join("origin.git");
14546        g(&[
14547            "clone",
14548            "--bare",
14549            repo.to_str().unwrap(),
14550            origin.to_str().unwrap(),
14551        ]);
14552        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14553        g(&["fetch", "origin"]);
14554        let selection = crate::pr_recovery::Selection {
14555            url: "https://example.test/project/pull/1".into(),
14556            branch: "magi/task-selected".into(),
14557            head: head.clone(),
14558            source_head: None,
14559            source_run: None,
14560            source_proof: None,
14561            disposition: Some(crate::pr_recovery::Disposition::Review),
14562            instruction: "preserve measured behavior".into(),
14563            approved_at: jiff::Timestamp::now(),
14564        };
14565        let mut template = runner_at(RunStatus::Reviewing);
14566        template.state.repo = repo.clone();
14567        template.state.base_commit = current.clone();
14568        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14569        template.state.origin = Some(crate::run::Origin::queue("selected"));
14570        template.state.pr_head = Some(selection.branch.clone());
14571        template.state.pr_recovery = Some(selection.clone());
14572        let mut runner =
14573            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14574                .await
14575                .unwrap();
14576        let wt = runner.state.winner().unwrap().worktree.clone();
14577        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14578        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14579        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14580        assert!(runner.state.seeds.is_empty());
14581        assert_eq!(runner.state.instruction, "preserve measured behavior");
14582        runner.sync_to_base().await.unwrap();
14583        assert_eq!(
14584            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14585            "prerequisite"
14586        );
14587        assert_eq!(
14588            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14589            "measured implementation"
14590        );
14591        assert_eq!(
14592            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14593            "current main change"
14594        );
14595        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14596        assert_eq!(
14597            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14598            head
14599        );
14600
14601        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14602        let mut partial = runner_at(RunStatus::Reviewing);
14603        partial.state.repo = repo.clone();
14604        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14605        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14606            run: Some("exit 1".into()),
14607            ..Default::default()
14608        }];
14609        let selection = runner.state.pr_recovery.clone().unwrap();
14610        partial.state.pr_recovery = Some(selection.clone());
14611        partial.state.pr_head = Some(selection.branch.clone());
14612        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14613        let id = partial.state.id.clone();
14614        let error = match Runner::open_selected(
14615            &repo,
14616            partial.state,
14617            partial.roles,
14618            1,
14619            runner.state.base_commit.clone(),
14620            selection,
14621        )
14622        .await
14623        {
14624            Ok(_) => panic!("setup must fail"),
14625            Err(e) => e,
14626        };
14627        assert_eq!(
14628            error
14629                .downcast_ref::<crate::run::StartupFailure>()
14630                .unwrap()
14631                .run,
14632            id
14633        );
14634        let failed = RunState::load(&id).unwrap();
14635        assert_eq!(failed.status, RunStatus::Failed);
14636        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14637        assert!(Runner::resume(&id).is_err());
14638        runner.state.status = RunStatus::Ready;
14639        runner.state.driver_exited = true;
14640        runner.state.save().unwrap();
14641        let mut retry = runner_at(RunStatus::Reviewing);
14642        retry.state.repo = repo.clone();
14643        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14644        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14645        reselected.approved_at = jiff::Timestamp::now();
14646        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14647        retry.state.pr_head = Some(reselected.branch.clone());
14648        retry.state.pr_recovery = Some(reselected.clone());
14649        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14650            .await
14651            .unwrap();
14652        let restarted = Runner::open_selected(
14653            &repo,
14654            retry.state,
14655            retry.roles,
14656            1,
14657            runner.state.base_commit.clone(),
14658            reselected,
14659        )
14660        .await
14661        .unwrap();
14662        assert_eq!(
14663            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14664                .await
14665                .unwrap(),
14666            head
14667        );
14668        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14669        assert_eq!(
14670            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14671            head
14672        );
14673    }
14674
14675    #[tokio::test]
14676    #[cfg(unix)]
14677    async fn fallback_and_interrupted_restoration_require_coordination() {
14678        use std::os::fd::AsRawFd as _;
14679        ask_test_home();
14680        let tmp = tempfile::tempdir().unwrap();
14681        let mut runner = registered_runner(tmp.path()).await;
14682        runner.state.config.graph.timeout_resource_wait = 1;
14683        let winner = runner.state.candidates[0].clone();
14684        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14685        let leftover = winner.worktree.join("leftover");
14686        std::fs::write(&leftover, "untrusted edits").unwrap();
14687        let next = tmp.path().join("next-fixer");
14688        let mut failed = spec("first");
14689        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14690        let mut successor = spec("second");
14691        successor.command = vec![
14692            "sh".into(),
14693            "-c".into(),
14694            format!("touch '{}'; echo done", next.display()),
14695        ];
14696        let attempts = vec![
14697            (failed.clone(), "first".into()),
14698            (successor.clone(), "second".into()),
14699        ];
14700        let lock_path = runner.state.repo.join(".git/updater.lock");
14701        let external = std::fs::File::create(&lock_path).unwrap();
14702        assert_eq!(
14703            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14704            0
14705        );
14706        let artifacts = tmp.path().join("artifacts");
14707        let build = |spec, seat| SeatJob {
14708            spec,
14709            seat,
14710            cwd: winner.worktree.clone(),
14711            prompt: "fixture".into(),
14712            timeout: Duration::from_secs(10),
14713            allow_write: true,
14714            sessions: false,
14715            artifacts: artifacts.clone(),
14716            stem: "fix".into(),
14717            handover: None,
14718        };
14719        let (_, _, out) = runner
14720            .ask_fixer_among(
14721                attempts.clone(),
14722                "graft",
14723                None,
14724                Some((&winner.worktree, &head)),
14725                build,
14726            )
14727            .await;
14728        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14729        assert!(leftover.exists(), "contention must prevent clean");
14730        assert!(!next.exists());
14731        assert!(runner.state.handovers.is_empty());
14732        // Early interrupted-repair rollback uses the same protected route.
14733        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14734            from_run: "earlier".into(),
14735            generation: 1,
14736            mode: crate::recovery::Mode::VerifyRepair,
14737            reasons: Vec::new(),
14738            reviewed_head: None,
14739            rounds: Vec::new(),
14740            open_findings: Vec::new(),
14741            evidence: None,
14742            criteria_inherited: true,
14743            repair: crate::recovery::Repair::Started {
14744                before: head.clone(),
14745            },
14746        });
14747        assert!(
14748            runner
14749                .recovery_repair(&winner)
14750                .await
14751                .unwrap_err()
14752                .is::<crate::repo_lock::ResourceBlocked>()
14753        );
14754        assert!(matches!(
14755            runner.state.recovery.as_ref().unwrap().repair,
14756            crate::recovery::Repair::Started { .. }
14757        ));
14758        assert!(leftover.exists());
14759        drop(external);
14760        runner.recovery_repair(&winner).await.unwrap();
14761        assert!(!leftover.exists());
14762        assert!(matches!(
14763            runner.state.recovery.as_ref().unwrap().repair,
14764            crate::recovery::Repair::Done { .. }
14765        ));
14766        // A hard lock-open failure also leaves HEAD and untracked work intact.
14767        std::fs::remove_file(&lock_path).unwrap();
14768        std::fs::create_dir(&lock_path).unwrap();
14769        std::fs::write(&leftover, "untrusted edits").unwrap();
14770        let (_, _, out) = runner
14771            .ask_fixer_among(
14772                attempts,
14773                "graft",
14774                None,
14775                Some((&winner.worktree, &head)),
14776                build,
14777            )
14778            .await;
14779        assert!(matches!(out, AgentOutcome::Failed(_)));
14780        assert!(leftover.exists());
14781        assert!(!next.exists());
14782        assert_eq!(
14783            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14784            head
14785        );
14786    }
14787
14788    #[cfg(unix)]
14789    async fn registered_runner(root: &Path) -> Runner {
14790        repository_runner(root, crate::config::test_coordination()).await
14791    }
14792
14793    async fn repository_runner(
14794        root: &Path,
14795        coordination: Option<crate::config::Coordination>,
14796    ) -> Runner {
14797        ask_test_home();
14798        let repo = root.join("repo");
14799        std::fs::create_dir_all(&repo).unwrap();
14800        init_repo(&repo);
14801        if coordination.is_some() {
14802            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14803        }
14804        let mut runner = runner_at(RunStatus::Prep);
14805        runner.state.repo = repo.clone();
14806        runner.state.config.coordination = coordination;
14807        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14808        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14809        runner.state.config.graph.advise = false;
14810        runner.roles.implementers = vec![spec("implement")];
14811        runner.roles.judges = vec![spec("judge")];
14812        runner.prep().await.unwrap();
14813        runner
14814    }
14815
14816    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14817
14818    #[tokio::test]
14819    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14820        let tmp = tempfile::tempdir().unwrap();
14821        let mut runner = repository_runner(tmp.path(), None).await;
14822        let prompts = runner.state.config.prompts.clone();
14823        let run = runner.state.id.clone();
14824        let ctx = WaveCtx {
14825            carry_seats: false,
14826            pauses: None,
14827            run: &run,
14828            node: "advise",
14829            prompts: &prompts,
14830            cache: None,
14831            round: None,
14832        };
14833        let jobs: Vec<_> = ["alpha", "beta"]
14834            .into_iter()
14835            .map(|id| SeatJob {
14836                spec: scripted(id, AUTH_OUTAGE),
14837                seat: SeatState::new(id, id, 7),
14838                prompt: "propose".into(),
14839                cwd: runner.state.candidates[0].worktree.clone(),
14840                timeout: Duration::from_secs(10),
14841                allow_write: false,
14842                sessions: false,
14843                artifacts: agent::artifacts_dir(&runner.state.dir()),
14844                stem: id.into(),
14845                handover: None,
14846            })
14847            .collect();
14848        let mut losses = Vec::new();
14849        let results = ask_wave_with(
14850            jobs.clone(),
14851            runner.sem.clone(),
14852            2,
14853            &[],
14854            &ctx,
14855            &mut losses,
14856            &mut runner.state,
14857            &|_: &str| Ok(()),
14858        )
14859        .await;
14860        assert!(
14861            results
14862                .iter()
14863                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14864        );
14865        assert_eq!(runner.state.quota.len(), 2);
14866        assert!(
14867            runner
14868                .state
14869                .quota
14870                .iter()
14871                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14872        );
14873        assert!(!runner.state.advise_attempted);
14874        runner.stop_for_resource().unwrap();
14875        assert_eq!(runner.state.status, RunStatus::Blocked);
14876        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14877        assert_eq!(
14878            runner.state.resource_block.as_ref().unwrap().kind,
14879            "agent authentication"
14880        );
14881        // A machine loss followed by an ordinary failure is not evidence
14882        // that the whole node was unavailable, and must not earn a refund.
14883        runner.state.resource_block = None;
14884        runner.state.quota.clear();
14885        let mut mixed = jobs;
14886        mixed[1].spec.command = vec!["false".into()];
14887        let results = ask_wave_with(
14888            mixed,
14889            runner.sem.clone(),
14890            0,
14891            &[],
14892            &ctx,
14893            &mut losses,
14894            &mut runner.state,
14895            &|_: &str| Ok(()),
14896        )
14897        .await;
14898        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14899        assert!(runner.state.resource_block.is_none());
14900        assert_eq!(losses.len(), 1);
14901    }
14902
14903    #[tokio::test]
14904    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14905        let tmp = tempfile::tempdir().unwrap();
14906        let mut runner = repository_runner(tmp.path(), None).await;
14907        let implementer = scripted("implement", AUTH_OUTAGE);
14908        runner.roles.implementers = vec![implementer.clone()];
14909        runner.state.config.agents = vec![implementer];
14910        runner.implement().await.unwrap();
14911        assert_eq!(runner.state.status, RunStatus::Blocked);
14912        assert_eq!(runner.state.quota.len(), 1);
14913        assert_eq!(
14914            runner.state.quota[0].cause,
14915            agent::QuotaCause::Authentication
14916        );
14917        let candidate = &runner.state.candidates[0];
14918        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14919        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14920    }
14921
14922    #[tokio::test]
14923    #[cfg(unix)]
14924    async fn parked_implementer_persists_completed_work() {
14925        for registered in [false, true] {
14926            let tmp = tempfile::tempdir().unwrap();
14927            let mut runner = registered_runner(tmp.path()).await;
14928            if !registered {
14929                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14930            }
14931            let marker = tmp.path().join("started");
14932            let release = tmp.path().join("release");
14933            let mut implementer = spec("implement");
14934            implementer.command = vec![
14935                "sh".into(),
14936                "-c".into(),
14937                format!(
14938                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14939                    marker.display(),
14940                    release.display()
14941                ),
14942            ];
14943            runner.roles.implementers = vec![implementer.clone()];
14944            runner.state.config.agents = vec![implementer];
14945            let pause = runner.pause.clone();
14946            let (result, ()) = tokio::join!(runner.implement(), async {
14947                tokio::time::timeout(Duration::from_secs(10), async {
14948                    while !marker.exists() {
14949                        tokio::time::sleep(Duration::from_millis(10)).await;
14950                    }
14951                })
14952                .await
14953                .unwrap();
14954                pause.park();
14955                std::fs::write(&release, "go").unwrap();
14956            });
14957            result.unwrap();
14958            let candidate = &runner.state.candidates[0];
14959            assert!(candidate.commits > 0, "{candidate:?}");
14960            assert!(candidate.summary.contains("saved-result"));
14961            assert_eq!(candidate.files, 1);
14962            let before = candidate.commits;
14963            runner.implement().await.unwrap();
14964            assert_eq!(runner.state.candidates[0].commits, before);
14965        }
14966    }
14967
14968    #[tokio::test]
14969    #[cfg(unix)]
14970    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14971        let tmp = tempfile::tempdir().unwrap();
14972        let mut runner = registered_runner(tmp.path()).await;
14973        let winner = runner.state.candidates[0].clone();
14974        let cache = tmp.path().join("cache");
14975        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14976        runner.state.config.graph.timeout_resource_wait = 0;
14977        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14978        let crate::cache::AcquireOutcome::Acquired(held) =
14979            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14980        else {
14981            panic!("cache must be free")
14982        };
14983        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14984            from_run: "earlier".into(),
14985            generation: 1,
14986            mode: crate::recovery::Mode::VerifyRepair,
14987            reasons: Vec::new(),
14988            reviewed_head: None,
14989            rounds: Vec::new(),
14990            open_findings: Vec::new(),
14991            evidence: None,
14992            criteria_inherited: true,
14993            repair: crate::recovery::Repair::NotStarted,
14994        });
14995        runner.recovery_repair(&winner).await.unwrap();
14996        assert!(matches!(
14997            runner.state.recovery.as_ref().unwrap().repair,
14998            crate::recovery::Repair::WaitingForRestore { .. }
14999        ));
15000        drop(held);
15001        runner.state.resource_block = None;
15002        runner.state.config.graph.timeout_resource_wait = 1;
15003        runner.recovery_repair(&winner).await.unwrap();
15004        assert!(matches!(
15005            runner.state.recovery.as_ref().unwrap().repair,
15006            crate::recovery::Repair::Done { .. }
15007        ));
15008        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15009    }
15010
15011    #[tokio::test]
15012    #[cfg(unix)]
15013    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15014        use std::os::fd::AsRawFd as _;
15015        for graft in [false, true] {
15016            let tmp = tempfile::tempdir().unwrap();
15017            let mut runner = registered_runner(tmp.path()).await;
15018            runner.state.config.graph.timeout_resource_wait = 1;
15019            let winner = runner.state.candidates[0].clone();
15020            let marker = tmp.path().join("fixer-started");
15021            let mut fixer = spec("fixer");
15022            fixer.command = vec![
15023                "sh".into(),
15024                "-c".into(),
15025                format!("echo started >> '{}'; echo done", marker.display()),
15026            ];
15027            runner.roles.fixer = Some(vec![fixer]);
15028            if graft {
15029                let mut loser = winner.clone();
15030                loser.label = 'B';
15031                loser.index = 1;
15032                runner.state.candidates.push(loser);
15033                runner.state.config.graph.graft_runner_up = true;
15034                runner.state.tally = Some(Tally {
15035                    first_choice: BTreeMap::new(),
15036                    borda: BTreeMap::new(),
15037                    winner: winner.label,
15038                    rankings: 1,
15039                    unanimous_initial: true,
15040                    deliberated: false,
15041                    changed_votes: 0,
15042                    unanimous_final: true,
15043                    tie_break: None,
15044                    judges: 1,
15045                    present: 1,
15046                    quorum: 1,
15047                    met_quorum: true,
15048                    uncontested: None,
15049                });
15050                runner.state.judgements.push(crate::run::Judgement {
15051                    judge: 1,
15052                    seat: "judge-1".into(),
15053                    agent: "judge".into(),
15054                    ranking: vec![winner.label, 'B'],
15055                    reasons: BTreeMap::new(),
15056                    confidence: None,
15057                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15058                    order: vec![0, 1],
15059                    failed: None,
15060                    duration_ms: 0,
15061                });
15062            } else {
15063                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15064                    from_run: "earlier".into(),
15065                    generation: 1,
15066                    mode: crate::recovery::Mode::VerifyRepair,
15067                    reasons: Vec::new(),
15068                    reviewed_head: None,
15069                    rounds: Vec::new(),
15070                    open_findings: Vec::new(),
15071                    evidence: None,
15072                    criteria_inherited: true,
15073                    repair: crate::recovery::Repair::NotStarted,
15074                });
15075            }
15076            let reader = crate::repo_lock::SeatLease::for_checkout(
15077                &winner.worktree,
15078                false,
15079                crate::config::test_coordination().as_ref(),
15080            )
15081            .await
15082            .unwrap();
15083            let pause = runner.pause.clone();
15084            let (result, ()) = tokio::join!(
15085                async {
15086                    if graft {
15087                        runner.graft().await
15088                    } else {
15089                        runner.recovery_repair(&winner).await
15090                    }
15091                },
15092                async {
15093                    tokio::time::sleep(Duration::from_millis(100)).await;
15094                    pause.park();
15095                }
15096            );
15097            result.unwrap();
15098            assert!(!marker.exists(), "a blocked seat must not spawn");
15099            // Exercise the persisted state, as a binary replacement/resume does.
15100            runner.state = RunState::load(&runner.state.id).unwrap();
15101            if graft {
15102                assert_eq!(
15103                    runner.state.graft.as_ref().unwrap().outcome,
15104                    GraftOutcome::WaitingForRestore
15105                );
15106            } else {
15107                assert!(matches!(
15108                    runner.state.recovery.as_ref().unwrap().repair,
15109                    crate::recovery::Repair::WaitingForRestore { .. }
15110                ));
15111            }
15112            drop(reader);
15113            runner.pause = Pause::new();
15114            runner.state.resource_block = None;
15115            // Deferred restoration must still respect an external lock owner.
15116            let external =
15117                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15118            assert_eq!(
15119                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15120                0
15121            );
15122            let leftover = winner.worktree.join("leftover");
15123            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15124            let result = if graft {
15125                runner.graft().await
15126            } else {
15127                runner.recovery_repair(&winner).await
15128            };
15129            assert!(
15130                result
15131                    .unwrap_err()
15132                    .is::<crate::repo_lock::ResourceBlocked>()
15133            );
15134            assert!(leftover.exists());
15135            assert!(!marker.exists());
15136            drop(external);
15137            runner.state.resource_block = None;
15138            let result = if graft {
15139                runner.graft().await
15140            } else {
15141                runner.recovery_repair(&winner).await
15142            };
15143            result.unwrap();
15144            assert!(!leftover.exists(), "restore must precede retry");
15145            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15146            if graft {
15147                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15148            } else {
15149                assert!(matches!(
15150                    runner.state.recovery.as_ref().unwrap().repair,
15151                    crate::recovery::Repair::Done { .. }
15152                ));
15153            }
15154        }
15155    }
15156
15157    #[tokio::test]
15158    #[cfg(unix)]
15159    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15160        use std::os::fd::AsRawFd as _;
15161        let tmp = tempfile::tempdir().unwrap();
15162        let mut runner = registered_runner(tmp.path()).await;
15163        runner.state.config.graph.timeout_resource_wait = 1;
15164        let winner = runner.state.candidates[0].clone();
15165        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15166        runner.state.tally = Some(Tally {
15167            first_choice: BTreeMap::new(),
15168            borda: BTreeMap::new(),
15169            winner: winner.label,
15170            rankings: 0,
15171            unanimous_initial: false,
15172            deliberated: false,
15173            changed_votes: 0,
15174            unanimous_final: false,
15175            tie_break: None,
15176            judges: 0,
15177            present: 0,
15178            quorum: 0,
15179            met_quorum: true,
15180            uncontested: Some("fixture".into()),
15181        });
15182        runner.state.graft = Some(GraftRecord {
15183            pre: pre.clone(),
15184            post: None,
15185            lines: None,
15186            limit: None,
15187            agent: None,
15188            outcome: GraftOutcome::Running,
15189            requested: Vec::new(),
15190            adoption: None,
15191            adoption_error: None,
15192        });
15193        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15194        git::commit_all(&winner.worktree, "unfinished graft")
15195            .await
15196            .unwrap();
15197        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15198        let leftover = winner.worktree.join("leftover");
15199        std::fs::write(&leftover, "untrusted edits").unwrap();
15200        let path = runner.state.repo.join(".git/updater.lock");
15201        let external = std::fs::File::create(&path).unwrap();
15202        assert_eq!(
15203            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15204            0
15205        );
15206        assert!(
15207            runner
15208                .graft()
15209                .await
15210                .unwrap_err()
15211                .is::<crate::repo_lock::ResourceBlocked>()
15212        );
15213        assert!(matches!(
15214            runner.state.graft.as_ref().unwrap().outcome,
15215            GraftOutcome::Running
15216        ));
15217        assert_eq!(
15218            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15219            unfinished
15220        );
15221        assert!(leftover.exists());
15222        drop(external);
15223        std::fs::remove_file(&path).unwrap();
15224        std::fs::create_dir(&path).unwrap();
15225        assert!(runner.graft().await.is_err());
15226        assert!(matches!(
15227            runner.state.graft.as_ref().unwrap().outcome,
15228            GraftOutcome::Running
15229        ));
15230        assert_eq!(
15231            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15232            unfinished
15233        );
15234        assert!(leftover.exists());
15235        std::fs::remove_dir(&path).unwrap();
15236        runner.graft().await.unwrap();
15237        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15238        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15239        assert!(!leftover.exists());
15240    }
15241
15242    #[tokio::test]
15243    #[cfg(unix)]
15244    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15245        use std::os::fd::AsRawFd as _;
15246        let tmp = tempfile::tempdir().unwrap();
15247        let mut runner = registered_runner(tmp.path()).await;
15248        runner.state.prep_complete = Some(false);
15249        runner.state.config.graph.timeout_resource_wait = 1;
15250        runner.state.save().unwrap();
15251        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15252        assert_eq!(
15253            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15254            0
15255        );
15256        runner.execute().await.unwrap();
15257        let disk = RunState::load(&runner.state.id).unwrap();
15258        assert!(disk.driver_exited);
15259        assert_eq!(disk.driver_pid, None);
15260        assert_eq!(runner.state.status, RunStatus::Blocked);
15261        assert_eq!(
15262            runner.state.resource_block.as_ref().unwrap().kind,
15263            "repository coordination"
15264        );
15265        assert!(runner.state.prep_failure.is_none());
15266        assert!(runner.state.seats.is_empty());
15267        runner.pause.park();
15268        // Include headroom for Git subprocesses under parallel test load;
15269        // the repository contention deadline itself remains one second.
15270        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15271            .await
15272            .unwrap()
15273            .unwrap();
15274        assert!(runner.state.parked);
15275        assert!(runner.state.prep_failure.is_none());
15276        let disk = RunState::load(&runner.state.id).unwrap();
15277        assert!(disk.driver_exited);
15278        assert_eq!(disk.driver_pid, None);
15279    }
15280
15281    #[tokio::test]
15282    #[cfg(unix)]
15283    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15284        use std::os::fd::AsRawFd as _;
15285        let tmp = tempfile::tempdir().unwrap();
15286        let mut runner = registered_runner(tmp.path()).await;
15287        runner.state.prep_complete = Some(false);
15288        runner.state.config.graph.timeout_resource_wait = 30;
15289        runner.state.save().unwrap();
15290        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15291        assert_eq!(
15292            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15293            0
15294        );
15295        assert!(
15296            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15297                .await
15298                .is_err()
15299        );
15300        let disk = RunState::load(&runner.state.id).unwrap();
15301        assert!(disk.driver_exited);
15302        assert_eq!(disk.driver_pid, None);
15303        assert!(disk.status.resumable());
15304        assert!(runner.state.driver_exited);
15305        assert_eq!(runner.state.driver_pid, None);
15306        runner.state.save().unwrap();
15307        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15308    }
15309
15310    #[tokio::test]
15311    #[cfg(unix)]
15312    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15313        let tmp = tempfile::tempdir().unwrap();
15314        let mut runner = registered_runner(tmp.path()).await;
15315        let winner = runner.state.candidates[0].clone();
15316        let cache = tmp.path().join("cache");
15317        let started = tmp.path().join("pre-gate-started");
15318        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15319        runner.state.config.verify.gate =
15320            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15321        let home = crate::run::home();
15322        let owner =
15323            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15324        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15325            crate::cache::AcquireOutcome::Acquired(g) => g,
15326            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15327        };
15328        let mut gate = Box::pin(runner.run_gate(&winner));
15329        assert!(
15330            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15331                .await
15332                .is_err()
15333        );
15334        assert!(!started.exists(), "pre-gate must wait for the cache too");
15335        // Simulate the wave that already owns the cache entering invoke.
15336        // It must obtain the editor reservation and finish without waiting
15337        // for the gate that is queued behind its cache.
15338        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15339        let editor = crate::repo_lock::SeatLease::checkout_with(
15340            &winner.worktree,
15341            true,
15342            crate::config::test_coordination().as_ref(),
15343            &wait,
15344        )
15345        .await
15346        .unwrap();
15347        drop(editor);
15348        guard.release();
15349        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15350            .await
15351            .unwrap()
15352            .unwrap();
15353        drop(gate);
15354        assert!(outcomes.iter().all(CommandOutcome::ok));
15355        assert!(started.exists());
15356    }
15357
15358    #[tokio::test]
15359    #[cfg(unix)]
15360    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15361        use std::os::fd::AsRawFd as _;
15362
15363        ask_test_home();
15364        let tmp = tempfile::tempdir().unwrap();
15365        let repo = tmp.path().join("repo");
15366        std::fs::create_dir_all(&repo).unwrap();
15367        init_repo(&repo);
15368        let common = repo.join(".git");
15369        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15370        let mut runner = runner_at(RunStatus::Prep);
15371        runner.state.repo = repo.clone();
15372        runner.state.config.coordination = crate::config::test_coordination();
15373        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15374        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15375        runner.state.config.graph.advise = false;
15376        runner.roles.implementers = vec![spec("implement")];
15377        runner.roles.judges = vec![spec("judge")];
15378        runner.prep().await.unwrap();
15379        let winner = runner.state.candidates[0].clone();
15380        // Emulate edits already left by a completed seat. The fixture CLI
15381        // only reports completion; it performs no repository operations.
15382        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15383        let completed = tmp.path().join("seat-completed");
15384        runner.roles.implementers[0].command = vec![
15385            "sh".into(),
15386            "-c".into(),
15387            format!("touch '{}'; echo completed", completed.display()),
15388        ];
15389        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15390        assert_eq!(
15391            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15392            0
15393        );
15394        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15395        let mut implement = Box::pin(runner.implement());
15396        tokio::select! {
15397            result = &mut implement => panic!("rescue must wait: {result:?}"),
15398            result = tokio::time::timeout(Duration::from_secs(10), async {
15399                while !completed.exists() {
15400                    tokio::time::sleep(Duration::from_millis(20)).await;
15401                }
15402            }) => result.unwrap(),
15403        }
15404        assert!(
15405            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15406                .await
15407                .is_err()
15408        );
15409        assert_eq!(
15410            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15411            before
15412        );
15413        assert_eq!(
15414            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15415            0
15416        );
15417        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15418            .await
15419            .unwrap()
15420            .unwrap();
15421        drop(implement);
15422        assert_ne!(
15423            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15424            before
15425        );
15426        assert!(git::is_clean(&winner.worktree).await.unwrap());
15427
15428        let started = tmp.path().join("pre-gate-started");
15429        let proceed = tmp.path().join("pre-gate-proceed");
15430        let checked = tmp.path().join("gate-ran");
15431        runner.state.config.verify.pre_gate = vec![format!(
15432            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15433            started.display(),
15434            proceed.display()
15435        )];
15436        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15437        assert_eq!(
15438            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15439            0
15440        );
15441        let mut gate = Box::pin(runner.run_gate(&winner));
15442        assert!(
15443            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15444                .await
15445                .is_err()
15446        );
15447        assert!(!started.exists());
15448        assert!(!checked.exists());
15449        assert!(git::is_clean(&winner.worktree).await.unwrap());
15450        assert_eq!(
15451            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15452            0
15453        );
15454        tokio::select! {
15455            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15456            result = tokio::time::timeout(Duration::from_secs(10), async {
15457                while !started.exists() {
15458                    tokio::time::sleep(Duration::from_millis(20)).await;
15459                }
15460            }) => result.unwrap(),
15461        }
15462        assert_eq!(
15463            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15464            -1,
15465            "mechanical edits must hold the real lock"
15466        );
15467        std::fs::write(proceed, "ready").unwrap();
15468        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15469            .await
15470            .unwrap()
15471            .unwrap();
15472        drop(gate);
15473        assert!(outcomes.iter().all(CommandOutcome::ok));
15474        assert!(checked.exists());
15475        assert!(runner.state.pre_gate_commit.is_some());
15476        assert!(git::is_clean(&winner.worktree).await.unwrap());
15477        assert_eq!(
15478            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15479            0,
15480            "gate completion must release coordination"
15481        );
15482        drop(outsider);
15483        std::fs::remove_file(common.join("updater.lock")).unwrap();
15484        std::fs::create_dir(common.join("updater.lock")).unwrap();
15485        std::fs::remove_file(&started).unwrap();
15486        std::fs::remove_file(&checked).unwrap();
15487        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15488        let error = runner.run_gate(&winner).await.unwrap_err();
15489        assert!(
15490            error
15491                .to_string()
15492                .contains("acquire mandatory repository lock")
15493        );
15494        assert!(
15495            !started.exists(),
15496            "failed acquisition must prevent pre-gate"
15497        );
15498        assert!(!checked.exists(), "failed acquisition must prevent gate");
15499        assert!(git::is_clean(&winner.worktree).await.unwrap());
15500        assert_eq!(
15501            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15502            head
15503        );
15504    }
15505
15506    #[tokio::test]
15507    #[cfg(unix)]
15508    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15509        use std::os::fd::AsRawFd as _;
15510        use std::time::Duration;
15511
15512        ask_test_home();
15513        let tmp = tempfile::tempdir().unwrap();
15514        let repo = tmp.path().join("repo");
15515        std::fs::create_dir_all(&repo).unwrap();
15516        init_repo(&repo);
15517        let common = repo.join(".git");
15518        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15519        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15520        assert_eq!(
15521            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15522            0
15523        );
15524        let mut runner = runner_at(RunStatus::Prep);
15525        runner.state.repo = repo.clone();
15526        runner.state.config.coordination = crate::config::test_coordination();
15527        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15528        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15529        runner.state.config.graph.advise = true;
15530        runner.state.config.graph.advisors = 1;
15531        runner.roles.implementers = vec![spec("implement")];
15532        runner.roles.judges = vec![spec("judge")];
15533        let root = runner.state.worktree_root();
15534        let started = tmp.path().join("setup-started");
15535        let proceed = tmp.path().join("setup-proceed");
15536        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15537            run: Some(format!(
15538                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15539                started.display(),
15540                proceed.display()
15541            )),
15542            ..Default::default()
15543        }];
15544        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15545        let config = git::git(&repo, &["config", "--local", "--list"])
15546            .await
15547            .unwrap();
15548        let mut prep = Box::pin(runner.prep());
15549        assert!(
15550            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15551                .await
15552                .is_err()
15553        );
15554        assert!(
15555            !root.exists(),
15556            "contention must prevent even checkout creation"
15557        );
15558        assert!(!started.exists(), "contention must prevent setup");
15559        assert_eq!(
15560            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15561            branches
15562        );
15563        assert_eq!(
15564            git::git(&repo, &["config", "--local", "--list"])
15565                .await
15566                .unwrap(),
15567            config
15568        );
15569        drop(lock);
15570        tokio::select! {
15571            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15572            result = tokio::time::timeout(Duration::from_secs(10), async {
15573                while !started.exists() {
15574                    tokio::time::sleep(Duration::from_millis(20)).await;
15575                }
15576            }) => result.unwrap(),
15577        }
15578        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15579        assert_eq!(
15580            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15581            -1
15582        );
15583        assert!(
15584            tokio::time::timeout(
15585                Duration::from_millis(100),
15586                crate::repo_lock::SeatLease::for_checkout(
15587                    &repo,
15588                    true,
15589                    crate::config::test_coordination().as_ref()
15590                )
15591            )
15592            .await
15593            .is_err(),
15594            "preparation must also exclude local editors"
15595        );
15596        std::fs::write(proceed, "ready").unwrap();
15597        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15598            .await
15599            .unwrap()
15600            .unwrap();
15601        drop(prep);
15602        assert_eq!(runner.state.prep_complete, Some(true));
15603        assert!(root.join("judge-1").exists());
15604        assert!(root.join("advisor-1").exists());
15605        assert_eq!(
15606            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15607            0
15608        );
15609    }
15610
15611    #[tokio::test]
15612    #[cfg(unix)]
15613    async fn preparation_lock_failure_prevents_repository_changes() {
15614        ask_test_home();
15615        let tmp = tempfile::tempdir().unwrap();
15616        let repo = tmp.path().join("repo");
15617        std::fs::create_dir_all(&repo).unwrap();
15618        init_repo(&repo);
15619        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15620        // A directory at the required lock path makes opening the real lock
15621        // fail, rather than merely exercising an invalid registration.
15622        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15623        let mut runner = runner_at(RunStatus::Prep);
15624        runner.state.repo = repo.clone();
15625        runner.state.config.coordination = crate::config::test_coordination();
15626        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15627        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15628        runner.roles.implementers = vec![spec("implement")];
15629        let error = runner.prep().await.unwrap_err();
15630        assert!(
15631            error
15632                .to_string()
15633                .contains("acquire mandatory repository lock")
15634        );
15635        assert!(!runner.state.worktree_root().exists());
15636        assert!(runner.state.candidates.is_empty());
15637        assert!(!repo.join(".git/config.worktree").exists());
15638        assert_eq!(
15639            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15640            "* main"
15641        );
15642    }
15643
15644    #[tokio::test]
15645    async fn preparation_records_creation_intent_before_git_can_fail() {
15646        ask_test_home();
15647        let tmp = tempfile::tempdir().unwrap();
15648        let repo = tmp.path().join("repo");
15649        std::fs::create_dir_all(&repo).unwrap();
15650        init_repo(&repo);
15651        let mut runner = runner_at(RunStatus::Prep);
15652        runner.state.repo = repo.clone();
15653        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15654        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15655        runner.state.config.blind.commit_msg_hook = false;
15656        runner.roles.implementers = vec![spec("one")];
15657        let label = blind::assign_labels(1, runner.state.seed)[0];
15658        let branch = runner.state.branch_for(label);
15659        // A conflicting branch makes Git refuse creation before a candidate
15660        // can be registered. The durable intent must still identify it.
15661        git::git(&repo, &["branch", &branch]).await.unwrap();
15662        assert!(runner.prep().await.is_err());
15663        let saved = RunState::load(&runner.state.id).unwrap();
15664        assert!(saved.candidates.is_empty());
15665        let event = saved
15666            .events
15667            .iter()
15668            .find(|e| e.node == "prep-intent")
15669            .unwrap();
15670        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15671        assert_eq!(intent["branch"], branch);
15672        assert_eq!(intent["start"], runner.state.base_commit);
15673        assert_eq!(
15674            intent["worktree"],
15675            runner
15676                .state
15677                .worktree_root()
15678                .join(format!("cand-{label}"))
15679                .to_string_lossy()
15680                .as_ref()
15681        );
15682        assert_eq!(intent["picks"], serde_json::json!([]));
15683    }
15684
15685    #[tokio::test]
15686    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15687        ask_test_home();
15688        let tmp = tempfile::tempdir().unwrap();
15689        let repo = tmp.path().join("repo");
15690        std::fs::create_dir_all(&repo).unwrap();
15691        init_repo(&repo);
15692        let mut runner = runner_at(RunStatus::Prep);
15693        runner.state.repo = repo.clone();
15694        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15695        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15696        runner.state.sources.pick_commits.push(refs::Input {
15697            token: "missing".into(),
15698            sha: "0".repeat(40),
15699            provenance: refs::Provenance::ExplicitCli,
15700        });
15701        assert!(runner.prep().await.is_err());
15702        assert!(runner.state.candidates.is_empty());
15703        assert!(!runner.state.worktree_root().exists());
15704    }
15705
15706    #[tokio::test]
15707    async fn prose_and_legacy_inference_never_import_into_candidates() {
15708        ask_test_home();
15709        let tmp = tempfile::tempdir().unwrap();
15710        let repo = tmp.path().join("repo");
15711        std::fs::create_dir_all(&repo).unwrap();
15712        init_repo(&repo);
15713        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15714        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15715            .await
15716            .unwrap();
15717        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15718        git::git(&repo, &["add", "."]).await.unwrap();
15719        git::git(&repo, &["commit", "-m", "historical implementation"])
15720            .await
15721            .unwrap();
15722        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15723        git::git(&repo, &["checkout", "main"]).await.unwrap();
15724        for instruction in [
15725            format!("Example: magi/history/A at {source}"),
15726            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15727            format!("Never import magi/history/A or {source}"),
15728            format!("Conversation answer: I mentioned {source} only as background"),
15729        ] {
15730            let mut runner = runner_at(RunStatus::Prep);
15731            runner.state.repo = repo.clone();
15732            runner.state.base_commit = base.clone();
15733            runner.state.instruction = instruction.clone();
15734            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15735            runner.state.config.blind.commit_msg_hook = false;
15736            runner.state.seeds = vec![refs::Seed {
15737                token: "magi/history/A".into(),
15738                sha: source.clone(),
15739                kind: refs::SeedKind::Unmerged,
15740                branch: true,
15741                detail: String::new(),
15742            }];
15743            runner.roles.implementers = vec![spec("one")];
15744            assert_eq!(seeded_instruction(&runner.state), instruction);
15745            // Model an old record with inferred seeds but no structured input field.
15746            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15747            legacy.as_object_mut().unwrap().remove("sources");
15748            runner.state = serde_json::from_value(legacy).unwrap();
15749            assert_eq!(runner.state.sources, refs::Sources::default());
15750            runner.state.save().unwrap();
15751            runner.state = RunState::load(&runner.state.id).unwrap();
15752            runner.prep().await.unwrap();
15753            for candidate in &runner.state.candidates {
15754                assert_eq!(
15755                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15756                    base
15757                );
15758                assert!(!candidate.worktree.join("unwanted").exists());
15759            }
15760        }
15761    }
15762
15763    #[tokio::test]
15764    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15765        ask_test_home();
15766        let tmp = tempfile::tempdir().unwrap();
15767        let repo = tmp.path().join("repo");
15768        std::fs::create_dir_all(&repo).unwrap();
15769        init_repo(&repo);
15770        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15771        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15772        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15773        git::git(&repo, &["add", "."]).await.unwrap();
15774        git::git(&repo, &["commit", "-m", "required seed"])
15775            .await
15776            .unwrap();
15777        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15778        git::git(&repo, &["checkout", "main"]).await.unwrap();
15779        let mut runner = runner_at(RunStatus::Prep);
15780        runner.state.repo = repo.clone();
15781        runner.state.base_commit = base.clone();
15782        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15783        runner.state.config.graph.advise = false;
15784        runner.state.config.blind.commit_msg_hook = false;
15785        runner.state.sources =
15786            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15787                .await
15788                .unwrap();
15789        runner.roles.implementers = vec![spec("one"), spec("two")];
15790        runner.roles.judges = vec![spec("judge")];
15791        let label = blind::assign_labels(2, runner.state.seed)[0];
15792        let branch = runner.state.branch_for(label);
15793        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15794        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15795            .await
15796            .unwrap();
15797        runner.state.candidates.push(Candidate {
15798            index: 0,
15799            label,
15800            agent: "one".into(),
15801            branch,
15802            worktree,
15803            summary: String::new(),
15804            stat: String::new(),
15805            files: 0,
15806            commits: 0,
15807            empty: false,
15808            failed: None,
15809            verified_noop: None,
15810            duration_ms: 0,
15811            folded: false,
15812        });
15813        runner.state.save().unwrap();
15814        // Reload the checkpoint immediately before its first cherry-pick.
15815        runner.state = RunState::load(&runner.state.id).unwrap();
15816        assert_eq!(runner.state.prep_complete, Some(false));
15817        runner.prep().await.unwrap();
15818        assert_eq!(runner.state.prep_complete, Some(true));
15819        assert_eq!(runner.state.candidates.len(), 2);
15820        for candidate in &runner.state.candidates {
15821            assert_eq!(
15822                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15823                git::tree_of(&repo, &seed).await.unwrap()
15824            );
15825            assert_eq!(
15826                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15827                "required commit"
15828            );
15829        }
15830        assert!(runner.state.worktree_root().join("judge-1").exists());
15831        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15832        runner.prep().await.unwrap();
15833        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15834
15835        runner.state.prep_complete = Some(false);
15836        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15837            run: Some(format!(
15838                "test -f '{}'",
15839                tmp.path().join("dependency-ready").display()
15840            )),
15841            ..Default::default()
15842        }];
15843        assert!(runner.prep().await.is_err());
15844        assert!(runner.state.prep_failure.is_some());
15845        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15846        runner.state = RunState::load(&runner.state.id).unwrap();
15847        assert!(
15848            runner.prep().await.is_err(),
15849            "correcting the cause alone must not auto-retry"
15850        );
15851        let failed_at = runner.state.events.last().unwrap().at;
15852        assert!(
15853            runner.retry_preparation(failed_at).is_err(),
15854            "old approvals cannot unlock the hold"
15855        );
15856        let mut task = crate::queue::Task::new(
15857            "retry".into(),
15858            "retry setup".into(),
15859            repo.clone(),
15860            crate::queue::Source::Human,
15861        );
15862        task.start(runner.state.id.clone());
15863        task.hold_machine(Some("setup failed".into()));
15864        task.release_operator("cli", None);
15865        runner
15866            .retry_preparation(task.resumption.as_ref().unwrap().at)
15867            .unwrap();
15868        runner.prep().await.unwrap();
15869        assert!(runner.state.prep_failure.is_none());
15870        assert_eq!(runner.state.prep_complete, Some(true));
15871        assert_eq!(runner.state.candidates.len(), 2);
15872        assert!(runner.state.worktree_root().join("judge-1").exists());
15873        for candidate in &runner.state.candidates {
15874            assert_eq!(
15875                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15876                git::tree_of(&repo, &seed).await.unwrap()
15877            );
15878        }
15879    }
15880
15881    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15882        let mut heads = Vec::new();
15883        for candidate in candidates {
15884            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15885        }
15886        heads
15887    }
15888
15889    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15890        ask_test_home();
15891        let tmp = tempfile::tempdir().unwrap();
15892        let repo = tmp.path().join("repo");
15893        std::fs::create_dir(&repo).unwrap();
15894        init_repo(&repo);
15895        let mut runner = runner_at(RunStatus::Reviewing);
15896        runner.state.repo = repo.clone();
15897        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15898        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15899        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15900            let branch = format!("candidate-{label}");
15901            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15902            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15903                .await
15904                .unwrap();
15905            if label == 'B' {
15906                std::fs::write(worktree.join("change"), "winner").unwrap();
15907                git::rescue_commit(&worktree, "candidate change")
15908                    .await
15909                    .unwrap();
15910            }
15911            runner.state.candidates.push(Candidate {
15912                index,
15913                label,
15914                agent: "alpha".into(),
15915                branch,
15916                worktree,
15917                summary: "completed reply".into(),
15918                stat: String::new(),
15919                files: usize::from(label == 'B'),
15920                commits: usize::from(label == 'B'),
15921                empty: label == 'A',
15922                failed: None,
15923                verified_noop: None,
15924                duration_ms: 1,
15925                folded: false,
15926            });
15927        }
15928        runner.state.judge_skipped = true;
15929        runner.state.tally = Some(Tally {
15930            first_choice: BTreeMap::new(),
15931            borda: BTreeMap::new(),
15932            winner: 'B',
15933            rankings: 0,
15934            unanimous_initial: false,
15935            deliberated: false,
15936            changed_votes: 0,
15937            unanimous_final: false,
15938            tie_break: None,
15939            judges: 0,
15940            present: 0,
15941            quorum: 0,
15942            met_quorum: true,
15943            uncontested: Some("only B produced a change".into()),
15944        });
15945        runner.state.save().unwrap();
15946        (tmp, runner)
15947    }
15948
15949    #[tokio::test]
15950    async fn fold_losers_persists_discard_before_later_teardown_failure() {
15951        let (_tmp, mut runner) = preservation_resume_fixture().await;
15952        runner.state.candidates[0].folded = true;
15953        runner.state.config.graph.advise = true;
15954        runner.state.config.graph.advisors = 2;
15955        let repo = runner.state.repo.clone();
15956        let root = runner.state.worktree_root();
15957        let discarded = root.join("advisor-1");
15958        git::worktree_add_detached(&repo, &discarded, "main")
15959            .await
15960            .unwrap();
15961        std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
15962        let protected = root.join("advisor-2");
15963        git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
15964            .await
15965            .unwrap();
15966        std::fs::write(protected.join("keep"), "uncommitted").unwrap();
15967        runner.state.save().unwrap();
15968
15969        assert!(runner.fold_losers().await.is_err());
15970        assert!(!discarded.exists());
15971        assert!(protected.join("keep").exists());
15972        let saved = RunState::load(&runner.state.id).unwrap();
15973        assert!(saved.events.iter().any(|event| {
15974            event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
15975        }));
15976    }
15977
15978    #[tokio::test]
15979    async fn read_seat_teardown_discards_only_owned_detached_seats() {
15980        let (tmp, mut runner) = preservation_resume_fixture().await;
15981        let repo = runner.state.repo.clone();
15982        let root = runner.state.worktree_root();
15983        for name in ["advisor-3", "judge-1", "review-2"] {
15984            let path = root.join(name);
15985            git::worktree_add_detached(&repo, &path, "main")
15986                .await
15987                .unwrap();
15988            std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
15989            std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
15990            remove_read_seat_worktree(&mut runner.state, &path)
15991                .await
15992                .unwrap();
15993            assert!(!path.exists());
15994        }
15995        let outside = tmp.path().join("advisor-1");
15996        let unknown = root.join("unknown");
15997        let attached = root.join("advisor-4");
15998        git::worktree_add_detached(&repo, &outside, "main")
15999            .await
16000            .unwrap();
16001        git::worktree_add_detached(&repo, &unknown, "main")
16002            .await
16003            .unwrap();
16004        git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16005            .await
16006            .unwrap();
16007        let foreign = tmp.path().join("foreign");
16008        std::fs::create_dir(&foreign).unwrap();
16009        init_repo(&foreign);
16010        let wrong_repo = root.join("judge-2");
16011        git::worktree_add_detached(&foreign, &wrong_repo, "main")
16012            .await
16013            .unwrap();
16014        let candidate = runner.state.candidates[0].worktree.clone();
16015        for path in [outside, unknown, attached, wrong_repo, candidate] {
16016            std::fs::write(path.join("uncommitted"), "keep").unwrap();
16017            assert!(
16018                remove_read_seat_worktree(&mut runner.state, &path)
16019                    .await
16020                    .is_err()
16021            );
16022            assert_eq!(
16023                std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16024                "keep"
16025            );
16026        }
16027    }
16028
16029    #[tokio::test]
16030    #[cfg(unix)]
16031    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16032        for (solo, failed_checkpoint) in
16033            [(true, false), (false, false), (true, true), (false, true)]
16034        {
16035            let (_tmp, mut runner) = preservation_resume_fixture().await;
16036            if solo {
16037                runner.state.candidates.retain(|c| c.label == 'A');
16038            }
16039            runner.state.candidates[0].empty = false;
16040            runner.state.candidates[0].summary.clear();
16041            runner.state.tally = None;
16042            runner.state.judge_skipped = false;
16043            runner.state.status = RunStatus::Implementing;
16044            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16045            let mut implementer = spec("alpha");
16046            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16047            runner.roles.implementers = vec![implementer];
16048            let patch = runner.state.dir().join("artifacts/cand-A.patch");
16049            if failed_checkpoint {
16050                std::fs::create_dir_all(&patch).unwrap();
16051            }
16052            runner.state.save().unwrap();
16053            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16054            drop(pressure);
16055            runner.state = RunState::load(&runner.state.id).unwrap();
16056            assert_eq!(runner.state.status, RunStatus::Blocked);
16057            assert!(runner.state.interrupted_implementations.contains(&'A'));
16058            assert_eq!(runner.state.candidates[0].commits, 0);
16059            assert!(!runner.state.candidates[0].empty);
16060            assert!(runner.state.candidates[0].failed.is_none());
16061            if failed_checkpoint {
16062                assert!(runner.state.candidate_preservation.contains_key(&'A'));
16063                std::fs::remove_dir(patch).unwrap();
16064            }
16065            runner.state.resource_block = None;
16066            runner.state.config.disk.min_free_bytes = 0;
16067            runner.roles.implementers[0].command = vec![
16068                "sh".into(),
16069                "-c".into(),
16070                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16071            ];
16072            runner.implement().await.unwrap();
16073            assert!(runner.state.interrupted_implementations.is_empty());
16074            assert!(runner.state.candidate_preservation.is_empty());
16075            assert_eq!(runner.state.candidates[0].commits, 1);
16076            assert!(!runner.state.candidates[0].empty);
16077            assert!(
16078                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16079                    .unwrap()
16080                    .contains("retried empty interrupted seat")
16081            );
16082            if !solo {
16083                assert_eq!(runner.state.candidates[1].commits, 1);
16084            }
16085        }
16086    }
16087
16088    #[tokio::test]
16089    #[cfg(unix)]
16090    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16091        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16092            let (_tmp, mut runner) = preservation_resume_fixture().await;
16093            runner.state.candidates.retain(|c| c.label == 'A');
16094            runner.state.candidates[0].empty = false;
16095            runner.state.candidates[0].summary.clear();
16096            runner.state.tally = None;
16097            runner.state.judge_skipped = false;
16098            runner.state.status = RunStatus::Implementing;
16099            // Force only the real post-spawn watchdog, without failing the
16100            // separate admission check. Existing edits predate its measurement.
16101            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16102            let wt = runner.state.candidates[0].worktree.clone();
16103            std::fs::write(
16104                wt.join("interrupted-source"),
16105                "retain implementation work\n",
16106            )
16107            .unwrap();
16108            if committed {
16109                git::rescue_commit(&wt, "implementation checkpoint")
16110                    .await
16111                    .unwrap();
16112            }
16113            let lock = git::git(
16114                &wt,
16115                &[
16116                    "rev-parse",
16117                    "--path-format=absolute",
16118                    "--git-path",
16119                    "index.lock",
16120                ],
16121            )
16122            .await
16123            .unwrap();
16124            if block_rescue {
16125                std::fs::write(&lock, "fixture-owned lock").unwrap();
16126            }
16127            let mut implementer = spec("alpha");
16128            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16129            runner.roles.implementers = vec![implementer];
16130            runner.state.save().unwrap();
16131            let result = runner.implement().await;
16132            drop(pressure);
16133            assert_eq!(result.is_err(), block_rescue);
16134            assert_eq!(
16135                runner.state.resource_block.as_ref().unwrap().kind,
16136                "disk pressure"
16137            );
16138            assert!(
16139                runner
16140                    .state
16141                    .events
16142                    .iter()
16143                    .any(|e| e.message.contains("agent execution interrupted"))
16144            );
16145            assert!(
16146                runner
16147                    .state
16148                    .dir()
16149                    .join("artifacts/candidate-A-recovery/staged.patch")
16150                    .exists()
16151            );
16152            if block_rescue {
16153                let saved = RunState::load(&runner.state.id).unwrap();
16154                assert!(saved.candidate_preservation.contains_key(&'A'));
16155                assert!(!saved.candidates[0].empty);
16156                assert_eq!(
16157                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16158                    "retain implementation work\n"
16159                );
16160                std::fs::remove_file(lock).unwrap();
16161                runner.state = saved;
16162            } else {
16163                assert_eq!(runner.state.status, RunStatus::Blocked);
16164                assert_eq!(runner.state.candidates[0].commits, 1);
16165            }
16166            // Recovery must save the candidate before deciding whether another
16167            // implementation is needed. No implementer is available on retry.
16168            runner.roles.implementers.clear();
16169            runner.state.config.disk.min_free_bytes = 0;
16170            runner.state.resource_block = None;
16171            runner.implement().await.unwrap();
16172            assert!(runner.state.candidate_preservation.is_empty());
16173            assert_eq!(runner.state.candidates[0].commits, 1);
16174            assert!(!runner.state.candidates[0].empty);
16175            assert!(git::is_clean(&wt).await.unwrap());
16176            assert!(
16177                crate::run::read_artifact(&runner.state, "cand-A.patch")
16178                    .unwrap()
16179                    .contains("retain implementation work")
16180            );
16181        }
16182    }
16183
16184    #[tokio::test]
16185    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16186        let (_tmp, mut runner) = preservation_resume_fixture().await;
16187        // Schema 15 already confirmed the unfolded no-change result.
16188        runner.implement().await.unwrap();
16189        assert!(runner.state.candidate_preservation.is_empty());
16190        assert!(
16191            !runner
16192                .state
16193                .dir()
16194                .join("artifacts/candidate-A-recovery")
16195                .exists()
16196        );
16197        runner.fold_losers().await.unwrap();
16198        assert!(!runner.state.candidates[0].worktree.exists());
16199        for schema in [14, crate::run::SCHEMA] {
16200            runner.state.schema = schema;
16201            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16202            runner.state.save().unwrap();
16203            runner.state = RunState::load(&runner.state.id).unwrap();
16204            runner.implement().await.unwrap();
16205            assert!(runner.state.candidate_preservation.is_empty());
16206            assert!(runner.state.candidates[0].folded);
16207            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16208        }
16209    }
16210
16211    #[tokio::test]
16212    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16213        let (_tmp, mut runner) = preservation_resume_fixture().await;
16214        runner.state.candidates.retain(|c| c.label == 'A');
16215        runner.state.tally = None;
16216        runner.state.judge_skipped = false;
16217        let candidate = &mut runner.state.candidates[0];
16218        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16219        candidate.failed = Some("deployment validation unavailable".into());
16220        std::fs::write(
16221            candidate.worktree.join("repair"),
16222            "future launcher policy\n",
16223        )
16224        .unwrap();
16225        runner
16226            .state
16227            .candidate_preservation
16228            .insert('A', "awaiting persistence".into());
16229        runner.persist_candidates().await.unwrap();
16230        let candidate = &runner.state.candidates[0];
16231        assert!(!candidate.empty);
16232        assert_eq!(candidate.commits, 1);
16233        assert_eq!(candidate.files, 1);
16234        assert!(candidate.failed.is_none());
16235        assert!(candidate.summary.contains("EPERM"));
16236        assert!(
16237            crate::run::read_artifact(&runner.state, "cand-A.patch")
16238                .unwrap()
16239                .contains("future launcher policy")
16240        );
16241        assert!(runner.state.candidate_preservation.is_empty());
16242    }
16243
16244    #[tokio::test]
16245    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16246        let (_tmp, mut runner) = preservation_resume_fixture().await;
16247        let wt = runner.state.candidates[0].worktree.clone();
16248        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16249        runner.state.schema = 14;
16250        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16251        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16252        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16253        for _ in 0..2 {
16254            assert!(runner.implement().await.is_err());
16255            assert_eq!(runner.state.status, RunStatus::Blocked);
16256            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16257            assert!(
16258                runner.state.candidates[0].empty,
16259                "old verdict must remain unusable"
16260            );
16261            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16262            assert!(runner.state.judge_skipped);
16263            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16264            assert!(
16265                crate::run::read_artifact(&runner.state, "cand-A.patch")
16266                    .unwrap()
16267                    .contains("owned edits")
16268            );
16269            assert!(wt.exists());
16270            assert_eq!(
16271                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16272                    .await
16273                    .unwrap(),
16274                1
16275            );
16276            runner.state = RunState::load(&runner.state.id).unwrap();
16277        }
16278    }
16279
16280    #[tokio::test]
16281    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16282        let (_tmp, mut runner) = preservation_resume_fixture().await;
16283        let wt = runner.state.candidates[0].worktree.clone();
16284        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16285        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16286        let index_lock = PathBuf::from(
16287            git::git(
16288                &wt,
16289                &[
16290                    "rev-parse",
16291                    "--path-format=absolute",
16292                    "--git-path",
16293                    "index.lock",
16294                ],
16295            )
16296            .await
16297            .unwrap(),
16298        );
16299        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16300        runner.state.schema = 14;
16301        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16302        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16303        assert!(runner.implement().await.is_err());
16304        assert!(runner.after_implement().is_err());
16305        assert!(runner.fold_losers().await.is_err());
16306        assert_eq!(
16307            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16308            "failed seat edits"
16309        );
16310        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16311        runner.state = RunState::load(&runner.state.id).unwrap();
16312        std::fs::remove_file(index_lock).unwrap();
16313        // Recovery finds changes that the old uncontested result never judged.
16314        assert!(runner.implement().await.is_err());
16315        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16316        assert!(runner.state.candidates[0].empty);
16317        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16318        assert!(
16319            crate::run::read_artifact(&runner.state, "cand-A.patch")
16320                .unwrap()
16321                .contains("failed seat edits")
16322        );
16323        assert!(wt.exists());
16324    }
16325
16326    #[tokio::test]
16327    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16328        let (_tmp, mut runner) = preservation_resume_fixture().await;
16329        let repo = runner.state.repo.clone();
16330        let common = git::git(
16331            &repo,
16332            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16333        )
16334        .await
16335        .unwrap();
16336        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16337        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16338        assert!(runner.fold_losers().await.is_err());
16339        assert!(!runner.state.candidates[0].worktree.exists());
16340        assert!(!runner.state.candidates[0].folded);
16341        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16342        runner.state = RunState::load(&runner.state.id).unwrap();
16343        std::fs::remove_file(branch_lock).unwrap();
16344        let judge = runner.state.worktree_root().join("judge-1");
16345        git::worktree_add_detached(&repo, &judge, "main")
16346            .await
16347            .unwrap();
16348        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16349        git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16350            .await
16351            .unwrap();
16352        runner.roles.judges = vec![spec("judge")];
16353        assert!(runner.fold_losers().await.is_err());
16354        runner.state = RunState::load(&runner.state.id).unwrap();
16355        assert!(
16356            runner.state.candidates[0].folded,
16357            "candidate progress saved before judge failure"
16358        );
16359        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16360        assert!(judge.join("uncommitted").exists());
16361        git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16362            .await
16363            .unwrap();
16364        runner.fold_losers().await.unwrap();
16365        assert!(!judge.exists());
16366        runner.fold_losers().await.unwrap();
16367    }
16368
16369    #[tokio::test]
16370    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16371        let (_tmp, mut runner) = preservation_resume_fixture().await;
16372        runner.state.candidates.retain(|c| c.label == 'B');
16373        runner.state.tally = None;
16374        runner.state.judge_skipped = false;
16375        runner.state.prep_complete = Some(true);
16376        runner.state.config.graph.advise = false;
16377        runner.state.config.graph.selected_implementer = Some("alpha".into());
16378        let wt = runner.state.candidates[0].worktree.clone();
16379        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16380        runner
16381            .state
16382            .candidate_preservation
16383            .insert('B', "awaiting persistence".into());
16384        let index_lock = git::git(
16385            &wt,
16386            &[
16387                "rev-parse",
16388                "--path-format=absolute",
16389                "--git-path",
16390                "index.lock",
16391            ],
16392        )
16393        .await
16394        .unwrap();
16395        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16396        runner.state.save().unwrap();
16397        assert!(runner.execute_graph().await.is_err());
16398        assert_eq!(runner.state.status, RunStatus::Blocked);
16399        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16400        std::fs::remove_file(index_lock).unwrap();
16401        // Park at the next graph boundary: this tests the real selected-seat
16402        // hold guard without dispatching reviewers or publishing anything.
16403        for _ in 0..2 {
16404            runner.state = RunState::load(&runner.state.id).unwrap();
16405            runner.pause.park();
16406            runner.execute_graph().await.unwrap();
16407            assert_eq!(runner.state.status, RunStatus::Implementing);
16408            assert!(runner.state.candidate_preservation.is_empty());
16409            assert_eq!(runner.state.candidates[0].commits, 2);
16410            assert!(runner.state.tally.is_none());
16411        }
16412        runner.implement().await.unwrap();
16413        assert_eq!(runner.state.status, RunStatus::Judging);
16414        // A genuine selected-implementer hold must still require release.
16415        runner.state.status = RunStatus::Blocked;
16416        runner.state.candidates[0].failed = Some("implementation failed".into());
16417        runner.state.save().unwrap();
16418        let error = runner.execute_graph().await.unwrap_err();
16419        assert!(
16420            error
16421                .to_string()
16422                .contains("selected implementation is held")
16423        );
16424    }
16425
16426    #[tokio::test]
16427    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16428        let (_tmp, mut runner) = preservation_resume_fixture().await;
16429        let repo = runner.state.repo.clone();
16430        let candidate = runner.state.candidates[0].clone();
16431        // Remove only the disposable fixture directory, leaving Git's linked
16432        // worktree registration and branch intact, as in a vanished checkout.
16433        let registered = git::worktree_holding(&repo, &candidate.branch)
16434            .await
16435            .unwrap()
16436            .unwrap();
16437        assert_eq!(
16438            registered.canonicalize().unwrap(),
16439            candidate.worktree.canonicalize().unwrap()
16440        );
16441        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16442        assert_eq!(
16443            git::worktree_holding(&repo, &candidate.branch)
16444                .await
16445                .unwrap(),
16446            Some(registered)
16447        );
16448        assert!(
16449            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16450            "the stale registration initially keeps the branch checked out"
16451        );
16452        runner.fold_losers().await.unwrap();
16453        assert!(runner.state.candidates[0].folded);
16454        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16455        assert_eq!(
16456            git::worktree_holding(&repo, &candidate.branch)
16457                .await
16458                .unwrap(),
16459            None
16460        );
16461        runner.state = RunState::load(&runner.state.id).unwrap();
16462        runner.fold_losers().await.unwrap();
16463    }
16464
16465    #[tokio::test]
16466    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16467        let (_tmp, mut runner) = preservation_resume_fixture().await;
16468        let repo = runner.state.repo.clone();
16469        let candidate = runner.state.candidates[0].clone();
16470        let winner = runner.state.candidates[1].clone();
16471        let registered = git::worktree_holding(&repo, &candidate.branch)
16472            .await
16473            .unwrap()
16474            .unwrap();
16475        assert_eq!(
16476            registered.canonicalize().unwrap(),
16477            candidate.worktree.canonicalize().unwrap()
16478        );
16479        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16480        assert_eq!(
16481            git::worktree_holding(&repo, &candidate.branch)
16482                .await
16483                .unwrap(),
16484            Some(registered)
16485        );
16486        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16487
16488        for _ in 0..2 {
16489            fold_run(&mut runner.state, false, &crate::run::home())
16490                .await
16491                .unwrap();
16492            runner.state = RunState::load(&runner.state.id).unwrap();
16493            assert!(runner.state.candidates[0].folded);
16494            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16495            assert_eq!(
16496                git::worktree_holding(&repo, &candidate.branch)
16497                    .await
16498                    .unwrap(),
16499                None
16500            );
16501            assert!(!runner.state.candidates[1].folded);
16502            assert!(winner.worktree.exists());
16503            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16504        }
16505    }
16506
16507    #[tokio::test]
16508    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16509        ask_test_home();
16510        let tmp = tempfile::tempdir().unwrap();
16511        let repo = tmp.path().join("repo");
16512        std::fs::create_dir(&repo).unwrap();
16513        init_repo(&repo);
16514        let mut runner = runner_at(RunStatus::Implementing);
16515        runner.state.repo = repo.clone();
16516        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16517        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16518        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16519            let branch = format!("candidate-{label}");
16520            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16521            git::worktree_add_branch(&repo, &wt, &branch, "main")
16522                .await
16523                .unwrap();
16524            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16525            git::git(&wt, &["add", "README.md"]).await.unwrap();
16526            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16527            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16528            runner.state.candidates.push(Candidate {
16529                index,
16530                label,
16531                agent: "alpha".to_owned(),
16532                branch,
16533                worktree: wt,
16534                summary: "completed implementation".to_owned(),
16535                stat: String::new(),
16536                files: 0,
16537                commits: 0,
16538                empty: false,
16539                failed: None,
16540                verified_noop: None,
16541                duration_ms: 42,
16542                folded: false,
16543            });
16544            runner
16545                .state
16546                .candidate_preservation
16547                .insert(label, "awaiting persistence".to_owned());
16548        }
16549        let wt = runner.state.candidates[0].worktree.clone();
16550        let index_lock = git::git(
16551            &wt,
16552            &[
16553                "rev-parse",
16554                "--path-format=absolute",
16555                "--git-path",
16556                "index.lock",
16557            ],
16558        )
16559        .await
16560        .unwrap();
16561        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16562        runner.state.save().unwrap();
16563        assert!(runner.persist_candidates().await.is_err());
16564        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16565        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16566        assert_eq!(
16567            runner.state.candidates[1].commits, 1,
16568            "other results are still collected"
16569        );
16570        assert!(
16571            !runner.state.candidates[0].empty,
16572            "failed rescue is not no change"
16573        );
16574        assert_eq!(
16575            runner.state.candidates[0].summary,
16576            "completed implementation"
16577        );
16578        assert!(runner.after_implement().is_err());
16579        let home = tmp.path().join("fold-home");
16580        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16581        assert!(wt.join("new-file").exists());
16582        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16583        assert!(recovery.join("staged.patch").exists());
16584        assert!(recovery.join("unstaged.patch").exists());
16585        assert_eq!(
16586            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16587            "untracked edits\n"
16588        );
16589        runner.state.begin_pass();
16590        runner.state.end_pass_parked(ParkCause::Upgrade {
16591            from: "1.0".into(),
16592            to: Some("1.1".into()),
16593        });
16594        runner.state.save().unwrap();
16595        runner.state = RunState::load(&runner.state.id).unwrap();
16596        assert!(matches!(
16597            runner.state.passes.last().unwrap().end,
16598            Some(crate::run::PassEnd::Parked {
16599                cause: ParkCause::Upgrade { .. }
16600            })
16601        ));
16602        std::fs::remove_file(index_lock).unwrap();
16603        // No implementer is configured: a persistence retry cannot dispatch a seat.
16604        runner.implement().await.unwrap();
16605        assert!(runner.state.candidate_preservation.is_empty());
16606        assert!(
16607            runner
16608                .state
16609                .candidates
16610                .iter()
16611                .all(|c| c.commits == 1 && !c.empty)
16612        );
16613        assert!(git::is_clean(&wt).await.unwrap());
16614        assert!(
16615            crate::run::read_artifact(&runner.state, "cand-A.patch")
16616                .unwrap()
16617                .contains("later tracked edits")
16618        );
16619    }
16620
16621    #[tokio::test]
16622    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16623        ask_test_home();
16624        let tmp = tempfile::tempdir().unwrap();
16625        let repo = tmp.path().join("repo");
16626        std::fs::create_dir(&repo).unwrap();
16627        init_repo(&repo);
16628        let wt = tmp.path().join("candidate");
16629        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16630            .await
16631            .unwrap();
16632        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16633        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16634        assert!(wt.join("uncommitted").exists());
16635        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16636        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16637        let mut runner = runner_at(RunStatus::Implementing);
16638        runner.state.repo = repo.clone();
16639        runner.state.base_commit = "missing-base".to_owned();
16640        runner.state.candidates.push(Candidate {
16641            index: 0,
16642            label: 'A',
16643            agent: "alpha".into(),
16644            branch: "candidate".into(),
16645            worktree: wt.clone(),
16646            summary: "completed".into(),
16647            stat: String::new(),
16648            files: 0,
16649            commits: 0,
16650            empty: false,
16651            failed: None,
16652            verified_noop: None,
16653            duration_ms: 1,
16654            folded: false,
16655        });
16656        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16657        assert!(
16658            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16659                .await
16660                .is_err()
16661        );
16662        assert!(!runner.state.candidates[0].folded);
16663        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16664        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16665        runner
16666            .state
16667            .candidate_preservation
16668            .insert('A', "pending".into());
16669        assert!(runner.persist_candidates().await.is_err());
16670        assert!(!runner.state.candidates[0].empty);
16671        assert!(!runner.state.candidates[0].folded);
16672        assert!(wt.exists());
16673        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16674        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16675        std::fs::create_dir_all(&patch_path).unwrap();
16676        assert!(
16677            runner.persist_candidates().await.is_err(),
16678            "artifact failure stays pending"
16679        );
16680        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16681        assert!(!runner.state.candidates[0].empty);
16682        std::fs::remove_dir(patch_path).unwrap();
16683        runner.persist_candidates().await.unwrap();
16684        assert!(
16685            runner.state.candidates[0].empty,
16686            "only successful inspection confirms no change"
16687        );
16688    }
16689
16690    fn init_repo(dir: &Path) {
16691        let run = |args: &[&str]| {
16692            let out = std::process::Command::new("git")
16693                .args(args)
16694                .current_dir(dir)
16695                .quiet()
16696                .output()
16697                .expect("spawn git");
16698            assert!(
16699                out.status.success(),
16700                "git {args:?} failed: {}",
16701                String::from_utf8_lossy(&out.stderr)
16702            );
16703        };
16704        run(&["init", "-b", "main"]);
16705        run(&["config", "user.name", "magi test"]);
16706        run(&["config", "user.email", "magi@example.com"]);
16707        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16708        run(&["add", "-A"]);
16709        run(&["commit", "-m", "init"]);
16710    }
16711
16712    // `settle_questions` is what closes the ghost the phone showed: a run's
16713    // seat asked something, the run then ended, and nothing was left to
16714    // abandon the question it left `open`. `HOME` is a process-wide
16715    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16716    // first time it runs in the binary — every test below still reaches the
16717    // same directory whichever call won, and each gets its own run id from
16718    // `RunState::new`, so they never collide there.
16719    fn ask_test_home() {
16720        crate::run::pin_test_home();
16721    }
16722
16723    #[test]
16724    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16725        ask_test_home();
16726        let mut runner = runner_at(RunStatus::Prep);
16727        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16728        // A blocked temporary output path makes the terminal save fail while
16729        // preserving the already published reservation.
16730        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16731        let error = guard
16732            .finish(
16733                &mut runner.state,
16734                Err(
16735                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16736                        .into(),
16737                ),
16738            )
16739            .unwrap_err();
16740        assert_eq!(
16741            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16742            std::io::ErrorKind::PermissionDenied
16743        );
16744        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16745        let message = format!("{error:#}");
16746        assert!(message.contains("original refusal"));
16747        assert!(!message.contains("could not save terminal"));
16748        assert_eq!(
16749            error.chain().count(),
16750            2,
16751            "only startup attribution wraps the original error"
16752        );
16753        let saved = RunState::load(&runner.state.id).unwrap();
16754        assert_eq!(saved.status, RunStatus::Blocked);
16755        assert!(!saved.startup.unwrap().complete);
16756    }
16757
16758    #[test]
16759    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16760        ask_test_home();
16761        let mut runner = runner_at(RunStatus::Prep);
16762        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16763        let id = runner.state.id.clone();
16764        drop(guard);
16765        let saved = RunState::load(&id).unwrap();
16766        assert_eq!(saved.status, RunStatus::Blocked);
16767        assert!(!saved.startup.as_ref().unwrap().complete);
16768        assert!(
16769            saved
16770                .startup
16771                .unwrap()
16772                .failure
16773                .unwrap()
16774                .contains("interrupted")
16775        );
16776        assert!(Runner::resume(&id).is_err());
16777    }
16778
16779    #[test]
16780    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16781        ask_test_home();
16782        let mut runner = runner_at(RunStatus::Prep);
16783        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16784        let error = guard
16785            .finish(
16786                &mut runner.state,
16787                Err(
16788                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16789                        .into(),
16790                ),
16791            )
16792            .unwrap_err();
16793        assert_eq!(
16794            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16795            std::io::ErrorKind::PermissionDenied
16796        );
16797        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16798        let mut task = crate::queue::Task::new(
16799            "review".into(),
16800            "criteria".into(),
16801            runner.state.repo.clone(),
16802            crate::queue::Source::Human,
16803        );
16804        task.record_startup_failure(failure.clone());
16805        task.record_startup_failure(failure.clone());
16806        assert_eq!(task.startup_failures.len(), 1);
16807        assert!(task.runs.is_empty());
16808        assert_eq!(task.attempts, 0);
16809    }
16810
16811    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16812        ask_test_home();
16813        let tmp = tempfile::tempdir().unwrap();
16814        let repo = tmp.path().join("repo");
16815        std::fs::create_dir(&repo).unwrap();
16816        init_repo(&repo);
16817        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16818        let origin = tmp.path().join("origin.git");
16819        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16820            .await
16821            .unwrap();
16822        git::git(
16823            &repo,
16824            &["remote", "add", "origin", origin.to_str().unwrap()],
16825        )
16826        .await
16827        .unwrap();
16828        git::git(&repo, &["checkout", "-b", "fix/retained"])
16829            .await
16830            .unwrap();
16831        std::fs::write(repo.join("retained"), "retained change").unwrap();
16832        git::git(&repo, &["add", "."]).await.unwrap();
16833        git::git(&repo, &["commit", "-m", "retained implementation"])
16834            .await
16835            .unwrap();
16836        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16837        git::git(&repo, &["checkout", "main"]).await.unwrap();
16838        std::fs::write(repo.join("base-change"), "new base").unwrap();
16839        git::git(&repo, &["add", "."]).await.unwrap();
16840        git::git(&repo, &["commit", "-m", "base moved"])
16841            .await
16842            .unwrap();
16843        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16844        git::git(&repo, &["checkout", "fix/retained"])
16845            .await
16846            .unwrap();
16847        let mut runner = runner_at(RunStatus::Voting);
16848        runner.state.repo = repo.clone();
16849        runner.state.base_commit = base;
16850        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16851        runner.state.config.disk.min_free_bytes = 0;
16852        runner.state.config.graph.review_rounds = 1;
16853        runner.state.config.verify.e2e = vec!["true".into()];
16854        runner.state.config.verify.gate = vec!["true".into()];
16855        runner.state.candidates = vec![Candidate {
16856            index: 0,
16857            label: 'A',
16858            agent: "fixture".into(),
16859            branch: "fix/retained".into(),
16860            worktree: repo.clone(),
16861            summary: String::new(),
16862            stat: String::new(),
16863            files: 1,
16864            commits: 1,
16865            empty: false,
16866            failed: None,
16867            verified_noop: None,
16868            duration_ms: 0,
16869            folded: false,
16870        }];
16871        runner.state.tally = Some(Tally {
16872            first_choice: BTreeMap::new(),
16873            borda: BTreeMap::new(),
16874            winner: 'A',
16875            rankings: 0,
16876            unanimous_initial: true,
16877            deliberated: false,
16878            changed_votes: 0,
16879            unanimous_final: true,
16880            tie_break: None,
16881            judges: 0,
16882            present: 0,
16883            quorum: 0,
16884            met_quorum: true,
16885            uncontested: Some("review-only".into()),
16886        });
16887        let mut task = crate::queue::Task::new(
16888            "fixture".into(),
16889            "fixture".into(),
16890            repo,
16891            crate::queue::Source::Human,
16892        );
16893        task.link_run(&runner.state.id);
16894        crate::queue::Queue::open().put(&mut task).unwrap();
16895        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16896        runner.state.base_sync = Some(BaseSync {
16897            tip: String::new(),
16898            behind: 1,
16899            attempts: BASE_SYNC_ROUNDS,
16900            conflict: None,
16901            already_in: None,
16902        });
16903        runner.sync_to_base().await.unwrap();
16904        assert_eq!(runner.state.status, RunStatus::Blocked);
16905        (tmp, runner, head, task.id)
16906    }
16907
16908    #[tokio::test]
16909    async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
16910        let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
16911        runner.state.status = RunStatus::Voting;
16912        runner.state.base_sync = None;
16913        runner.state.base_sync_recovery = Default::default();
16914        let scratch = runner.state.dir().join("base-sync");
16915        std::fs::create_dir_all(&scratch).unwrap();
16916        std::fs::write(scratch.join("obstruction"), "keep").unwrap();
16917        runner.sync_to_base().await.unwrap();
16918        assert!(runner.state.resource_block.is_some());
16919        assert!(!runner.state.base_sync_blocked());
16920        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
16921        assert_eq!(runner.state.base_episode_used(), 0);
16922        assert!(runner.state.base_sync_recovery.operations.is_empty());
16923        assert!(runner.state.rebase_fixes.is_empty());
16924        assert_eq!(
16925            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16926            head
16927        );
16928        std::fs::remove_dir_all(&scratch).unwrap();
16929        runner.state.resource_block = None;
16930        runner.sync_to_base().await.unwrap();
16931        assert!(!runner.state.base_sync_blocked());
16932        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
16933        assert_eq!(runner.state.base_episode_used(), 1);
16934        assert!(runner.state.base_sync_recovery.operations[0].finished);
16935        assert!(runner.state.rebase_fixes.is_empty());
16936        assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
16937        assert!(git::is_clean(&runner.state.repo).await.unwrap());
16938    }
16939
16940    #[cfg(unix)]
16941    #[tokio::test]
16942    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16943        for routed in [false, true] {
16944            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16945            let home = crate::run::home();
16946            let queue = crate::queue::Queue::open();
16947            let task = queue.get(&owner).unwrap();
16948            runner.state.task_instruction = Some(task.instruction.clone());
16949            runner.state.driver_exited = true;
16950            runner.state.save().unwrap();
16951            let questions = crate::ask::Questions::at(home.join("questions"));
16952            let mut q = crate::ask::Question::new(
16953                runner.state.id.clone(),
16954                "base-recovery".into(),
16955                "owner".into(),
16956                "Recover bounded base movement?".into(),
16957                String::new(),
16958                vec!["recover".into()],
16959            );
16960            q.actions.insert(
16961                "recover".into(),
16962                crate::ask::ChoiceAction::RecoverBase {
16963                    run: runner.state.id.clone(),
16964                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16965                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16966                },
16967            );
16968            q.routed_task = Some(owner.clone());
16969            q.answer(crate::ask::Answer::Choice("recover".into()))
16970                .unwrap();
16971            questions.put(&mut q).unwrap();
16972            crate::base_recovery_control::recover(
16973                &queue,
16974                &home,
16975                crate::base_recovery_control::Request {
16976                    run: runner.state.id.clone(),
16977                    head: head.clone(),
16978                    actor: "decisions".into(),
16979                    request: q.id.clone(),
16980                    decision: Some((owner.clone(), q.answered_at.unwrap())),
16981                    expected_stop: Some((
16982                        runner.state.base_sync_recovery.stop.clone().unwrap(),
16983                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
16984                    )),
16985                },
16986            )
16987            .await
16988            .unwrap();
16989            if routed {
16990                questions
16991                    .update(&q.id, |q| {
16992                        q.routed_task = Some("another-owner".into());
16993                        Ok(())
16994                    })
16995                    .unwrap();
16996            } else {
16997                let mut task = queue.get(&owner).unwrap();
16998                task.hold_manual(Some("new owner decision".into()));
16999                // Keep the generation to exercise the separate owner-decision binding.
17000                task.status = crate::queue::TaskStatus::Queued;
17001                queue.put(&mut task).unwrap();
17002            }
17003            runner.state = RunState::load(&runner.state.id).unwrap();
17004            runner.sync_to_base().await.unwrap();
17005            let approval = runner
17006                .state
17007                .base_sync_recovery
17008                .authorizations
17009                .last()
17010                .unwrap();
17011            assert!(approval.consumed);
17012            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17013                "routing"
17014            } else {
17015                "owner decision"
17016            }));
17017            assert_eq!(
17018                runner.state.base_sync.as_ref().unwrap().attempts,
17019                BASE_SYNC_ROUNDS
17020            );
17021            assert_eq!(
17022                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17023                head
17024            );
17025        }
17026    }
17027
17028    #[cfg(unix)]
17029    #[tokio::test]
17030    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17031        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17032        if std::env::var_os(CHILD).is_none() {
17033            let home = tempfile::tempdir().unwrap();
17034            let output = std::process::Command::new(std::env::current_exe().unwrap())
17035                .args([
17036                    "--exact",
17037                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17038                    "--nocapture",
17039                ])
17040                .env(CHILD, "1")
17041                .env("MAGI_HOME", home.path())
17042                .env("MAGI_WORKER_HOME", home.path())
17043                .env("MAGI_WORKER_ID", "abcdef")
17044                .output()
17045                .unwrap();
17046            assert!(
17047                output.status.success()
17048                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17049                "{}\n{}",
17050                String::from_utf8_lossy(&output.stdout),
17051                String::from_utf8_lossy(&output.stderr)
17052            );
17053            return;
17054        }
17055        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17056        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17057        let home = crate::run::home();
17058        let queue = crate::queue::Queue::open();
17059        let mut owner = queue.get(&task).unwrap();
17060        owner.release_operator("test-release", None);
17061        owner.start(runner.state.id.clone());
17062        queue.put(&mut owner).unwrap();
17063        runner.state.driver_exited = true;
17064        runner.state.task_instruction = Some(owner.instruction.clone());
17065        runner.state.save().unwrap();
17066        let _claim = queue.claim(&task).unwrap();
17067        let reserve = |id: &str| {
17068            let directory = home.join("workers/jobs").join(id);
17069            std::fs::create_dir_all(&directory).unwrap();
17070            let job = crate::worker::Job {
17071                id: id.into(),
17072                sequence: 1,
17073                kind: "task".into(),
17074                key: format!("task:{task}"),
17075                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17076                generation: "fixture-generation".into(),
17077                version: None,
17078                status: crate::worker::Status::Running,
17079                result: None,
17080                pid: Some(std::process::id()),
17081                created_at: 0,
17082                finished_at: None,
17083            };
17084            std::fs::write(
17085                directory.join("job.json"),
17086                serde_json::to_vec(&job).unwrap(),
17087            )
17088            .unwrap();
17089            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17090        };
17091        let _own = reserve("abcdef");
17092        let other = reserve("fedcba");
17093        assert!(
17094            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17095                .await
17096                .is_err()
17097        );
17098        assert!(
17099            RunState::load(&runner.state.id)
17100                .unwrap()
17101                .base_sync_recovery
17102                .authorizations
17103                .is_empty()
17104        );
17105        drop(other);
17106        assert!(crate::daemon::is_working_on(
17107            &home,
17108            &runner.state.id,
17109            Timestamp::now()
17110        ));
17111        assert!(
17112            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17113        );
17114        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17115            .await
17116            .unwrap();
17117        assert!(
17118            RunState::load(&runner.state.id)
17119                .unwrap()
17120                .base_recovery_pending()
17121        );
17122    }
17123
17124    #[cfg(unix)]
17125    #[tokio::test]
17126    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17127        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17128        let home = crate::run::home();
17129        runner.state.driver_pid = Some(std::process::id());
17130        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17131        runner.state.driver_exited = false;
17132        runner.state.begin_pass();
17133        runner.state.save().unwrap();
17134        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17135        driver.record(&runner.state).unwrap();
17136        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17137        // Preserve the durable claim as if the process died before Drop saved.
17138        drop(driver);
17139        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17140        assert_eq!(
17141            runner.state.liveness_under(&home, false),
17142            crate::run::Liveness::Dead
17143        );
17144        let queue = crate::queue::Queue::open();
17145        let _claim = queue.claim(&task).unwrap();
17146        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17147            .await
17148            .unwrap();
17149        let disk = RunState::load(&runner.state.id).unwrap();
17150        assert!(!disk.driver_exited);
17151        assert!(disk.base_recovery_pending());
17152    }
17153
17154    #[cfg(unix)]
17155    #[tokio::test]
17156    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17157        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17158        if std::env::var_os(CHILD).is_none() {
17159            use std::os::unix::fs::PermissionsExt;
17160            let tmp = tempfile::tempdir().unwrap();
17161            let gh = tmp.path().join("gh");
17162            std::fs::write(&gh, r#"#!/bin/sh
17163case "$1 $2" in
17164  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17165  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17166  *) exit 2 ;;
17167esac
17168"#).unwrap();
17169            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17170            let mut paths = vec![tmp.path().to_path_buf()];
17171            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17172            let output = std::process::Command::new(std::env::current_exe().unwrap())
17173                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17174                .env(CHILD, "1")
17175                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17176                .env("PATH", std::env::join_paths(paths).unwrap())
17177                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17178                .output().unwrap();
17179            assert!(
17180                output.status.success()
17181                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17182                "{}\n{}",
17183                String::from_utf8_lossy(&output.stdout),
17184                String::from_utf8_lossy(&output.stderr)
17185            );
17186            return;
17187        }
17188        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17189        let repo = runner.state.repo.clone();
17190        git::git(&repo, &["checkout", "main"]).await.unwrap();
17191        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17192        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17193        git::git(&repo, &["checkout", "fix/retained"])
17194            .await
17195            .unwrap();
17196        runner.state.pr = Some(crate::run::PrRecord {
17197            url: "https://example.test/project/pull/1".into(),
17198            number: 1,
17199            state: "open".into(),
17200            checks: "unknown".into(),
17201            round: 0,
17202            rounds: 1,
17203            red_at_merge: Vec::new(),
17204        });
17205        runner.sync_to_base().await.unwrap();
17206        assert_eq!(runner.state.status, RunStatus::Blocked);
17207        assert_eq!(
17208            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17209            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17210        );
17211        assert!(runner.state.base_sync_closure_pending());
17212        assert!(crate::handover::can_resume(&runner.state));
17213        let queue = crate::queue::Queue::open();
17214        assert!(!crate::base_sync_recovery::held_task(
17215            &queue,
17216            &queue.get(&task).unwrap()
17217        ));
17218        assert!(runner.state.authorize_base_sync(&task).is_err());
17219        // Legacy closure stops have no typed stop but must retain the checkpoint.
17220        runner.state.base_sync_recovery.stop = None;
17221        runner.state.status = RunStatus::Voting;
17222        runner.state.gate_ran = true;
17223        runner.state.config.graph.review_rounds = 0;
17224        runner.state.save().unwrap();
17225        assert!(crate::handover::can_resume(&runner.state));
17226        runner.state = RunState::load(&runner.state.id).unwrap();
17227        runner.execute_graph().await.unwrap();
17228        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17229        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17230        assert_eq!(
17231            runner.state.base_sync.as_ref().unwrap().attempts,
17232            BASE_SYNC_ROUNDS
17233        );
17234        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17235        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17236        assert!(runner.state.reviews.is_empty());
17237        assert!(runner.state.gate.is_empty());
17238    }
17239
17240    #[cfg(unix)]
17241    #[tokio::test]
17242    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17243        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17244        runner.state.driver_exited = true;
17245        runner.state.save().unwrap();
17246        let home = crate::run::home();
17247        let queue = crate::queue::Queue::open();
17248        let _claim = queue.claim(&task).unwrap();
17249        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17250        assert!(
17251            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17252                .await
17253                .is_err()
17254        );
17255        drop(lease);
17256        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17257        assert!(
17258            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17259                .await
17260                .is_err()
17261        );
17262        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17263        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17264            .await
17265            .unwrap();
17266        let disk = RunState::load(&runner.state.id).unwrap();
17267        assert!(disk.base_recovery_pending());
17268        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17269        assert!(
17270            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17271                .await
17272                .is_err()
17273        );
17274        assert_eq!(
17275            RunState::load(&runner.state.id)
17276                .unwrap()
17277                .base_sync_recovery
17278                .authorizations
17279                .len(),
17280            1
17281        );
17282        let mut owner = queue.get(&task).unwrap();
17283        owner.runs.push("20260101-000000-ffff".into());
17284        queue.put(&mut owner).unwrap();
17285        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17286            .await
17287            .unwrap_err();
17288        assert!(error.to_string().contains("run task ownership changed"));
17289    }
17290
17291    #[cfg(unix)]
17292    #[tokio::test]
17293    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17294        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17295        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17296        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17297        legacy["status"] = serde_json::json!("voting");
17298        runner.state = serde_json::from_value(legacy).unwrap();
17299        runner.state.save().unwrap();
17300        for _ in 0..3 {
17301            runner.execute_graph().await.unwrap();
17302            assert_eq!(runner.state.status, RunStatus::Blocked);
17303            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17304            assert!(!crate::handover::can_resume(&runner.state));
17305            assert_eq!(
17306                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17307                head
17308            );
17309            assert!(runner.state.reviews.is_empty());
17310            assert!(!runner.state.gate_ran);
17311        }
17312        let notice = crate::notices::run_ended(&runner.state).unwrap();
17313        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17314        runner.state.begin_pass();
17315        runner.state.status = RunStatus::Voting;
17316        runner.deliberate().await.unwrap();
17317        runner.vote().await.unwrap();
17318        runner.finish_after_tally().await.unwrap();
17319        assert_eq!(runner.state.status, RunStatus::Blocked);
17320        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17321        assert_eq!(
17322            crate::notices::run_ended(&runner.state).unwrap().since,
17323            notice.since
17324        );
17325    }
17326
17327    #[cfg(unix)]
17328    #[tokio::test]
17329    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17330        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17331        for detail in [
17332            "fix/retained conflicts with origin/main and did not rebase: conflict",
17333            "rebased fix/retained locally but origin refused the push",
17334            "unknown failure",
17335        ] {
17336            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17337            runner.state.record_base_sync_stop(&head);
17338            assert!(runner.state.authorize_base_sync(&task).is_err());
17339            runner.state.status = RunStatus::Voting;
17340            runner.execute_graph().await.unwrap();
17341            assert_eq!(runner.state.status, RunStatus::Blocked);
17342            assert!(!crate::handover::can_resume(&runner.state));
17343            assert_eq!(
17344                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17345                head
17346            );
17347        }
17348    }
17349
17350    #[cfg(unix)]
17351    #[tokio::test]
17352    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17353        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17354        runner.state.authorize_base_sync(&task).unwrap();
17355        assert!(runner.state.consume_base_recovery(&head));
17356        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17357        runner.state.reserve_base_operation(&head, &tip);
17358        runner.state.save().unwrap();
17359        runner.state = RunState::load(&runner.state.id).unwrap();
17360        assert!(!runner.state.consume_base_recovery(&head));
17361        runner.sync_to_base().await.unwrap();
17362        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17363        assert_eq!(runner.state.base_episode_used(), 1);
17364        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17365        assert!(
17366            runner
17367                .state
17368                .base_sync_recovery
17369                .operations
17370                .last()
17371                .unwrap()
17372                .finished
17373        );
17374        assert!(!runner.state.base_sync_blocked());
17375        runner.sync_to_base().await.unwrap();
17376        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17377    }
17378
17379    #[cfg(unix)]
17380    #[tokio::test]
17381    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17382        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17383        // Both historical diagnostic spellings retain an exhaustion identity.
17384        let original = runner
17385            .state
17386            .base_sync
17387            .as_ref()
17388            .unwrap()
17389            .conflict
17390            .clone()
17391            .unwrap();
17392        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17393            original
17394                .replace("synchronization(s)", "rebase(s)")
17395                .replace("synchronizing", "rebasing"),
17396        );
17397        runner.state.base_sync_recovery = Default::default();
17398        runner.state.record_base_sync_stop(&head);
17399        assert_eq!(
17400            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17401            crate::base_sync_recovery::StopKind::Exhausted
17402        );
17403        runner.state.authorize_base_sync(&task).unwrap();
17404        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17405        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17406            .as_object_mut()
17407            .unwrap();
17408        auth.remove("interval");
17409        auth.remove("scope");
17410        runner.state = serde_json::from_value(legacy).unwrap();
17411        assert!(runner.state.consume_base_recovery(&head));
17412        assert_eq!(runner.state.base_episode_used(), 3);
17413        runner.sync_to_base().await.unwrap();
17414        assert_eq!(runner.state.base_episode_used(), 4);
17415        runner.state.resource_block = Some(crate::run::ResourceBlock {
17416            node: "gate".into(),
17417            kind: "cache".into(),
17418            detail: "legacy wait".into(),
17419            at: Timestamp::now(),
17420            cause: crate::run::ResourceBlockCause::Unrecorded,
17421        });
17422        runner.state.save().unwrap();
17423        assert!(
17424            runner
17425                .state
17426                .base_sync_recovery
17427                .episodes
17428                .last()
17429                .unwrap()
17430                .boundary
17431                .is_none()
17432        );
17433        runner.state.resource_block = None;
17434        advance_base_fixture(&runner).await;
17435        runner.sync_to_base().await.unwrap();
17436        assert!(runner.state.base_sync_blocked());
17437        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17438    }
17439
17440    #[cfg(unix)]
17441    async fn advance_base_fixture(runner: &Runner) {
17442        let repo = &runner.state.repo;
17443        for args in [
17444            vec!["checkout", "main"],
17445            vec!["commit", "--allow-empty", "-m", "base advanced"],
17446            vec!["push", "origin", "main"],
17447            vec!["checkout", "fix/retained"],
17448        ] {
17449            git::git(repo, &args).await.unwrap();
17450        }
17451    }
17452
17453    #[cfg(unix)]
17454    fn fresh_base_interval_fixture(runner: &mut Runner) {
17455        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17456        runner.state.base_sync.as_mut().unwrap().conflict = None;
17457        runner.state.base_sync_recovery = Default::default();
17458        runner.state.status = RunStatus::Voting;
17459    }
17460
17461    #[cfg(unix)]
17462    #[tokio::test]
17463    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17464        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17465        fresh_base_interval_fixture(&mut runner);
17466        runner.state.begin_pass();
17467        runner.sync_to_base().await.unwrap();
17468        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17469        runner.state.resource_block = Some(crate::run::ResourceBlock {
17470            node: "gate".into(),
17471            kind: "build cache".into(),
17472            detail: "busy".into(),
17473            at: Timestamp::now(),
17474            cause: crate::run::ResourceBlockCause::Timeout {
17475                waited_secs: 1,
17476                limit_secs: 1,
17477            },
17478        });
17479        runner.state.save().unwrap();
17480        let queue = crate::queue::Queue::open();
17481        let mut owner = queue.get(&task).unwrap();
17482        runner.state.task_instruction = Some(owner.instruction.clone());
17483        owner.release_operator("fixture-owner", None);
17484        queue.put(&mut owner).unwrap();
17485        let approval = owner.resumption.clone().unwrap();
17486        runner.state.resource_block = None;
17487        runner.state.save().unwrap();
17488        runner.state = RunState::load(&runner.state.id).unwrap();
17489        advance_base_fixture(&runner).await;
17490        runner.sync_to_base().await.unwrap();
17491        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17492        assert_eq!(runner.state.base_episode_used(), 1);
17493        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17494        assert!(
17495            runner.state.base_sync_recovery.episodes[0]
17496                .boundary
17497                .as_ref()
17498                .unwrap()
17499                .consumed
17500        );
17501        assert!(
17502            !runner
17503                .state
17504                .renew_base_episode(&task, &approval, &head, "base")
17505        );
17506        for _ in 0..3 {
17507            owner.release_operator("fixture-owner", None);
17508            queue.put(&mut owner).unwrap();
17509            runner.state = RunState::load(&runner.state.id).unwrap();
17510            runner.state.begin_pass();
17511            runner.sync_to_base().await.unwrap();
17512            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17513            assert_eq!(runner.state.base_episode_used(), 1);
17514        }
17515        runner.state.resource_block = Some(crate::run::ResourceBlock {
17516            node: "gate".into(),
17517            kind: "disk pressure".into(),
17518            detail: "free space required".into(),
17519            at: Timestamp::now(),
17520            cause: crate::run::ResourceBlockCause::Other,
17521        });
17522        runner.state.end_pass_parked(ParkCause::Interrupt {
17523            reason: "disk pressure".into(),
17524        });
17525        runner.state.save().unwrap();
17526        owner.release_operator("fixture-owner-after-disk", None);
17527        queue.put(&mut owner).unwrap();
17528        runner.state.resource_block = None;
17529        advance_base_fixture(&runner).await;
17530        runner.sync_to_base().await.unwrap();
17531        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17532        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17533        assert_eq!(runner.state.base_episode_used(), 1);
17534        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17535    }
17536
17537    #[cfg(unix)]
17538    #[tokio::test]
17539    async fn base_recovery_continuous_advancement_stops_at_four() {
17540        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17541        fresh_base_interval_fixture(&mut runner);
17542        for spent in 1..=BASE_SYNC_ROUNDS {
17543            runner.sync_to_base().await.unwrap();
17544            assert!(!runner.state.base_sync_blocked());
17545            assert_eq!(runner.state.base_episode_used(), spent);
17546            advance_base_fixture(&runner).await;
17547        }
17548        runner.sync_to_base().await.unwrap();
17549        assert_eq!(
17550            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17551            crate::base_sync_recovery::StopKind::Exhausted
17552        );
17553        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17554        assert!(stop.unverified_sync);
17555        let action = runner.state.base_sync_next_action();
17556        assert!(action.contains("never reached its gate"));
17557        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17558        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17559        runner.state = RunState::load(&runner.state.id).unwrap();
17560        runner.sync_to_base().await.unwrap();
17561        assert_eq!(runner.state.base_episode_used(), 4);
17562        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17563    }
17564
17565    #[cfg(unix)]
17566    #[tokio::test]
17567    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17568        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17569        runner.state.ensure_base_episode();
17570        // Verification progress alone is not evidence of a successful sync.
17571        runner.state.record_base_progress(&head);
17572        runner.state.base_sync_recovery.stop = None;
17573        runner.state.record_base_sync_stop(&head);
17574        assert!(
17575            !runner
17576                .state
17577                .base_sync_recovery
17578                .stop
17579                .as_ref()
17580                .unwrap()
17581                .unverified_sync
17582        );
17583
17584        runner.state.finish_base_operation(Some(&head));
17585        // A gate for another head cannot verify this synchronization.
17586        runner.state.gate_ran = true;
17587        runner.state.gate_head = Some("older-head".into());
17588        runner.state.base_sync_recovery.stop = None;
17589        runner.state.record_base_sync_stop(&head);
17590        runner.state.save().unwrap();
17591        runner.state = RunState::load(&runner.state.id).unwrap();
17592        assert!(
17593            runner
17594                .state
17595                .base_sync_recovery
17596                .stop
17597                .as_ref()
17598                .unwrap()
17599                .unverified_sync
17600        );
17601
17602        runner.state.gate_head = Some(head.clone());
17603        runner.state.base_sync_recovery.stop = None;
17604        runner.state.record_base_sync_stop(&head);
17605        assert!(
17606            !runner
17607                .state
17608                .base_sync_recovery
17609                .stop
17610                .as_ref()
17611                .unwrap()
17612                .unverified_sync
17613        );
17614        assert!(
17615            !runner
17616                .state
17617                .base_sync_next_action()
17618                .contains("never reached its gate")
17619        );
17620
17621        // Historical records cannot establish the missing synchronization evidence.
17622        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17623        legacy["base_sync_recovery"]["stop"]
17624            .as_object_mut()
17625            .unwrap()
17626            .remove("unverified_sync");
17627        legacy["base_sync_recovery"]["episodes"][0]
17628            .as_object_mut()
17629            .unwrap()
17630            .remove("last_sync_head");
17631        runner.state = serde_json::from_value(legacy).unwrap();
17632        assert!(
17633            !runner
17634                .state
17635                .base_sync_recovery
17636                .stop
17637                .as_ref()
17638                .unwrap()
17639                .unverified_sync
17640        );
17641        assert!(
17642            runner.state.base_sync_recovery.episodes[0]
17643                .last_sync_head
17644                .is_none()
17645        );
17646    }
17647
17648    #[cfg(unix)]
17649    #[tokio::test]
17650    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17651        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17652        runner.state.authorize_base_sync(&task).unwrap();
17653        assert!(
17654            runner
17655                .state
17656                .base_sync_next_action()
17657                .contains("Approval recorded")
17658        );
17659        assert!(
17660            runner
17661                .state
17662                .base_sync_next_action()
17663                .contains("4/4 in episode 0, 4 lifetime")
17664        );
17665        assert!(runner.state.consume_base_recovery(&head));
17666        assert!(
17667            runner
17668                .state
17669                .base_sync_next_action()
17670                .contains("0/4 in episode 1, 4 lifetime")
17671        );
17672    }
17673
17674    #[cfg(unix)]
17675    #[tokio::test]
17676    async fn base_recovery_changed_head_invalidates_without_spending() {
17677        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17678        runner.state.authorize_base_sync(&task).unwrap();
17679        git::git(
17680            &runner.state.repo,
17681            &["commit", "--allow-empty", "-m", "external head change"],
17682        )
17683        .await
17684        .unwrap();
17685        runner.sync_to_base().await.unwrap();
17686        let a = runner
17687            .state
17688            .base_sync_recovery
17689            .authorizations
17690            .last()
17691            .unwrap();
17692        assert!(a.consumed);
17693        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17694        runner.state = RunState::load(&runner.state.id).unwrap();
17695        assert!(
17696            runner
17697                .state
17698                .base_sync_next_action()
17699                .contains("Approval invalidated")
17700        );
17701        assert!(!runner.state.base_recovery_pending());
17702        assert!(runner.state.base_sync_blocked());
17703        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17704    }
17705
17706    #[cfg(unix)]
17707    #[tokio::test]
17708    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17709        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17710        runner.state.driver_exited = true;
17711        runner.state.save().unwrap();
17712        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17713            .await
17714            .unwrap();
17715        runner.state = RunState::load(&runner.state.id).unwrap();
17716        advance_base_fixture(&runner).await;
17717        runner.sync_to_base().await.unwrap();
17718        let a = runner
17719            .state
17720            .base_sync_recovery
17721            .authorizations
17722            .last()
17723            .unwrap();
17724        assert!(
17725            a.consumed
17726                && a.invalidated
17727                    .as_deref()
17728                    .unwrap()
17729                    .contains("fetched base changed")
17730        );
17731        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17732        assert_eq!(
17733            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17734            head
17735        );
17736        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17737            .await
17738            .unwrap();
17739        runner.state = RunState::load(&runner.state.id).unwrap();
17740        runner.sync_to_base().await.unwrap();
17741        assert!(!runner.state.base_sync_blocked());
17742        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17743    }
17744
17745    #[cfg(unix)]
17746    #[tokio::test]
17747    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17748        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17749        runner.state.authorize_base_sync(&task).unwrap();
17750        assert!(runner.state.consume_base_recovery(&head));
17751        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17752        runner.state.reserve_base_operation(&head, &tip);
17753        runner.state.save().unwrap();
17754        let scratch = runner.state.dir().join("base-sync");
17755        assert_eq!(
17756            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17757                .await
17758                .unwrap(),
17759            crate::rebase::Rebased::Applied
17760        );
17761        runner.state = RunState::load(&runner.state.id).unwrap();
17762        runner.sync_to_base().await.unwrap();
17763        assert!(
17764            runner
17765                .state
17766                .base_sync_recovery
17767                .operations
17768                .last()
17769                .unwrap()
17770                .finished
17771        );
17772        assert_eq!(runner.state.base_episode_used(), 1);
17773        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17774        assert!(!runner.state.base_sync_blocked());
17775    }
17776
17777    #[cfg(unix)]
17778    #[tokio::test]
17779    async fn base_recovery_crash_preserves_untracked_files() {
17780        for ignored in [false, true] {
17781            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17782            runner.state.authorize_base_sync(&task).unwrap();
17783            assert!(runner.state.consume_base_recovery(&head));
17784            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17785            runner.state.reserve_base_operation(&head, &tip);
17786            runner.state.save().unwrap();
17787            let scratch = runner.state.dir().join("base-sync");
17788            assert_eq!(
17789                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17790                    .await
17791                    .unwrap(),
17792                crate::rebase::Rebased::Applied
17793            );
17794            let repo = &runner.state.repo;
17795            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17796            assert_ne!(result, head);
17797            if ignored {
17798                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17799                    .await
17800                    .unwrap();
17801                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17802            }
17803            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17804            assert!(
17805                git::git_raw(repo, &["diff", "--quiet", &head])
17806                    .await
17807                    .unwrap()
17808                    .ok()
17809            );
17810            // Resume after the branch moved, before its checkout was reset.
17811            runner.state = RunState::load(&runner.state.id).unwrap();
17812            runner.sync_to_base().await.unwrap();
17813            assert!(runner.state.base_sync_blocked());
17814            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17815            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17816            assert_eq!(
17817                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17818                result
17819            );
17820            assert_eq!(
17821                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17822                "retained work\n"
17823            );
17824            assert!(
17825                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17826                    .await
17827                    .unwrap()
17828                    .ok()
17829            );
17830        }
17831    }
17832
17833    #[cfg(unix)]
17834    #[tokio::test]
17835    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17836        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17837        runner.state.authorize_base_sync(&task).unwrap();
17838        assert!(runner.state.consume_base_recovery(&head));
17839        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17840        runner.state.reserve_base_operation(&head, &tip);
17841        runner.state.save().unwrap();
17842        let scratch = runner.state.dir().join("base-sync");
17843        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17844            .await
17845            .unwrap();
17846        git::sync_to_head(&runner.state.repo).await.unwrap();
17847        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17848        git::git(&runner.state.repo, &["add", "foreign"])
17849            .await
17850            .unwrap();
17851        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17852            .await
17853            .unwrap();
17854        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17855        runner.state = RunState::load(&runner.state.id).unwrap();
17856        runner.sync_to_base().await.unwrap();
17857        assert!(runner.state.base_sync_blocked());
17858        assert!(
17859            !runner
17860                .state
17861                .base_sync_recovery
17862                .operations
17863                .last()
17864                .unwrap()
17865                .finished
17866        );
17867        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17868        assert_eq!(
17869            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17870            foreign_head
17871        );
17872    }
17873
17874    #[cfg(unix)]
17875    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17876    {
17877        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17878        fresh_base_interval_fixture(&mut runner);
17879        let repo = &runner.state.repo;
17880        git::git(repo, &["checkout", "main"]).await.unwrap();
17881        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17882        git::git(repo, &["add", "retained"]).await.unwrap();
17883        git::git(repo, &["commit", "-m", "upstream retained change"])
17884            .await
17885            .unwrap();
17886        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17887        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17888        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17889        let scratch = runner.state.dir().join("base-sync");
17890        assert!(matches!(
17891            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17892                .await
17893                .unwrap(),
17894            git::RebaseStart::Conflicted(_)
17895        ));
17896        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17897            agent: "fixture".into(),
17898            paths: vec!["retained".into()],
17899            from: Some(head.clone()),
17900            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17901            index_outside: None,
17902            finished: false,
17903            error: None,
17904            launcher: None,
17905            infra: None,
17906            outage: None,
17907        });
17908        (tmp, runner, head, tip, scratch)
17909    }
17910
17911    #[cfg(unix)]
17912    #[tokio::test]
17913    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17914        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17915        // A resource stop from before reservations existed, with resolved
17916        // contents left for the launcher's ordinary staging continuation.
17917        std::fs::write(
17918            scratch.join("retained"),
17919            "upstream and retained resolution\n",
17920        )
17921        .unwrap();
17922        runner.state.resource_block = Some(crate::run::ResourceBlock {
17923            node: "rebase".into(),
17924            kind: "repository coordination".into(),
17925            detail: "busy".into(),
17926            at: Timestamp::now(),
17927            cause: crate::run::ResourceBlockCause::Other,
17928        });
17929        runner.state.save().unwrap();
17930        // Advance the remote without moving either checkout participating
17931        // in the standing rebase.
17932        let newer = git::git(
17933            &runner.state.repo,
17934            &[
17935                "commit-tree",
17936                "main^{tree}",
17937                "-p",
17938                "main",
17939                "-m",
17940                "base advanced during wait",
17941            ],
17942        )
17943        .await
17944        .unwrap();
17945        git::git(
17946            &runner.state.repo,
17947            &[
17948                "push",
17949                "origin",
17950                &format!("{}:refs/heads/main", newer.trim()),
17951            ],
17952        )
17953        .await
17954        .unwrap();
17955        runner.state = RunState::load(&runner.state.id).unwrap();
17956        runner.state.resource_block = None;
17957        runner.sync_to_base().await.unwrap();
17958        assert!(!runner.state.base_sync_blocked());
17959        let op = &runner.state.base_sync_recovery.operations[0];
17960        assert_eq!(op.head, head);
17961        assert_eq!(op.tip, tip);
17962        assert!(op.finished);
17963        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17964        assert_eq!(runner.state.rebase_fixes.len(), 1);
17965        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17966            .await
17967            .unwrap();
17968        assert_ne!(latest, tip);
17969        runner.state = RunState::load(&runner.state.id).unwrap();
17970        runner.sync_to_base().await.unwrap();
17971        assert!(!runner.state.base_sync_blocked());
17972        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17973        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17974        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17975    }
17976
17977    #[cfg(unix)]
17978    #[tokio::test]
17979    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17980        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17981        runner.state.reserve_base_operation(&head, &tip);
17982        runner.state.save().unwrap();
17983        std::fs::write(
17984            scratch.join("retained"),
17985            "upstream and retained resolution\n",
17986        )
17987        .unwrap();
17988        git::git(&scratch, &["add", "retained"]).await.unwrap();
17989        git::git(
17990            &scratch,
17991            &["-c", "core.editor=true", "rebase", "--continue"],
17992        )
17993        .await
17994        .unwrap();
17995        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17996        assert!(
17997            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17998                .await
17999                .unwrap()
18000                .is_empty()
18001        );
18002        assert_eq!(
18003            crate::rebase::finish(
18004                &mut runner.state,
18005                &scratch,
18006                "fix/retained",
18007                &head,
18008                &tip,
18009                &["retained".into()],
18010                ""
18011            )
18012            .await
18013            .unwrap(),
18014            crate::rebase::Rebased::Applied
18015        );
18016        assert!(!scratch.exists());
18017        // Simulate losing the caller before finish_base_operation/save.
18018        runner.state = RunState::load(&runner.state.id).unwrap();
18019        let op = &runner.state.base_sync_recovery.operations[0];
18020        assert!(!op.finished);
18021        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18022        runner.sync_to_base().await.unwrap();
18023        assert!(!runner.state.base_sync_blocked());
18024        assert!(runner.state.base_sync_recovery.operations[0].finished);
18025        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18026        assert_eq!(runner.state.rebase_fixes.len(), 1);
18027        let winner = runner.state.winner().unwrap().clone();
18028        let wait = runner.repository_wait();
18029        assert!(
18030            verify_current_head(&mut runner.state, &winner, &wait)
18031                .await
18032                .unwrap()
18033        );
18034        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18035    }
18036
18037    #[cfg(unix)]
18038    #[tokio::test]
18039    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18040        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18041            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18042            runner.state.reserve_base_operation(&head, &tip);
18043            runner.state.save().unwrap();
18044            std::fs::write(
18045                scratch.join("retained"),
18046                "upstream and retained resolution\n",
18047            )
18048            .unwrap();
18049            git::git(&scratch, &["add", "retained"]).await.unwrap();
18050            git::git(
18051                &scratch,
18052                &["-c", "core.editor=true", "rebase", "--continue"],
18053            )
18054            .await
18055            .unwrap();
18056            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18057            assert!(!git::rebase_in_progress(&scratch).await);
18058            assert!(
18059                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18060                    .await
18061                    .unwrap()
18062                    .is_empty()
18063            );
18064            match damage {
18065                Some("record") => {
18066                    let path =
18067                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18068                            .await
18069                            .unwrap();
18070                    std::fs::write(scratch.join(path), "").unwrap();
18071                }
18072                Some("foreign") => {
18073                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18074                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
18075                    git::git(&scratch, &["commit", "-m", "foreign work"])
18076                        .await
18077                        .unwrap();
18078                }
18079                Some("dirty") => {
18080                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18081                }
18082                _ => {}
18083            }
18084            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18085            // Crash immediately after Git continuation, before finish runs.
18086            runner.state = RunState::load(&runner.state.id).unwrap();
18087            assert!(
18088                runner.state.base_sync_recovery.operations[0]
18089                    .validated_head
18090                    .is_none()
18091            );
18092            runner.sync_to_base().await.unwrap();
18093            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18094            assert_eq!(runner.state.base_episode_used(), 1);
18095            assert_eq!(runner.state.rebase_fixes.len(), 1);
18096            assert_eq!(
18097                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18098                retained_head
18099            );
18100            if damage.is_some() {
18101                assert!(runner.state.base_sync_blocked());
18102                assert!(scratch.exists());
18103                assert!(!runner.state.base_sync_recovery.operations[0].finished);
18104                assert!(
18105                    runner.state.base_sync_recovery.operations[0]
18106                        .validated_head
18107                        .is_none()
18108                );
18109            } else {
18110                assert!(!runner.state.base_sync_blocked());
18111                assert!(!scratch.exists());
18112                assert!(runner.state.base_sync_recovery.operations[0].finished);
18113                assert_eq!(
18114                    runner.state.base_sync_recovery.operations[0]
18115                        .validated_head
18116                        .as_deref(),
18117                    Some(result.as_str())
18118                );
18119                let winner = runner.state.winner().unwrap().clone();
18120                let wait = runner.repository_wait();
18121                assert!(
18122                    verify_current_head(&mut runner.state, &winner, &wait)
18123                        .await
18124                        .unwrap()
18125                );
18126                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18127                runner.state = RunState::load(&runner.state.id).unwrap();
18128                runner.sync_to_base().await.unwrap();
18129                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18130            }
18131        }
18132    }
18133
18134    #[cfg(unix)]
18135    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18136        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18137        // This is an ordinary gate stop, not an authorized base-budget recovery.
18138        runner.state.prep_complete = Some(true);
18139        runner.state.base_sync = None;
18140        runner.state.base_sync_recovery = Default::default();
18141        runner.state.config.graph.gate_fix_rounds = 1;
18142        let mut review = review_round(true, 0, 1, 1, false, true);
18143        review.head = head.clone();
18144        review.verified_head = Some(head.clone());
18145        runner.state.reviews = vec![review];
18146        runner.state.gate = vec![CommandOutcome {
18147            command: "old gate".into(),
18148            code: Some(1),
18149            output_tail: "old failure".into(),
18150            duration_ms: 0,
18151            resource_blocked: false,
18152        }];
18153        runner.state.gate_head = Some(head.clone());
18154        runner.state.gate_ran = true;
18155        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18156            agent: "fixture".into(),
18157            failed: runner.state.gate.clone(),
18158            notes: "spent".into(),
18159            committed: false,
18160            error: None,
18161        }];
18162        runner.state.land_approval = Some(crate::run::LandApproval {
18163            question: "old approval".into(),
18164            head: head.clone(),
18165        });
18166        runner.state.status = RunStatus::Blocked;
18167        (tmp, runner, head)
18168    }
18169
18170    #[cfg(unix)]
18171    #[tokio::test]
18172    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18173        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18174            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18175            let phases = tmp.path().join("phases");
18176            runner.state.config.verify.e2e = vec![format!(
18177                "echo e2e >> '{}'; {}",
18178                phases.display(),
18179                if e2e_ok {
18180                    "true"
18181                } else {
18182                    "echo new-e2e-failure; false"
18183                }
18184            )];
18185            runner.state.config.verify.gate = vec![format!(
18186                "echo gate >> '{}'; {}",
18187                phases.display(),
18188                if gate_ok {
18189                    "true"
18190                } else {
18191                    "echo new-gate-failure; false"
18192                }
18193            )];
18194            runner.state.save().unwrap();
18195            runner.state = RunState::load(&runner.state.id).unwrap();
18196            runner.sync_to_base().await.unwrap();
18197            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18198            assert_ne!(head, old);
18199            // Reentry after Git updated but before verification was invalidated.
18200            runner.state = RunState::load(&runner.state.id).unwrap();
18201            runner.execute_graph().await.unwrap();
18202            assert_eq!(runner.state.gate_fixes.len(), 1);
18203            assert_eq!(runner.state.reviews.len(), 1);
18204            assert_eq!(runner.state.reviews[0].head, old);
18205            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18206            assert_eq!(runner.state.verification_history.len(), 1);
18207            let history = &runner.state.verification_history[0];
18208            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18209            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18210            assert!(history.archived_at.is_some());
18211            assert_eq!(history.fixes_spent, Some(1));
18212            assert_eq!(history.gate[0].output_tail, "old failure");
18213            let archive_events: Vec<_> = runner
18214                .state
18215                .events
18216                .iter()
18217                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18218                .collect();
18219            assert_eq!(archive_events.len(), 1);
18220            assert!(archive_events[0].message.contains(&short(&old)));
18221            assert!(archive_events[0].message.contains(&short(&head)));
18222            assert!(
18223                archive_events[0]
18224                    .message
18225                    .contains("1 gate-fix round(s) already spent stay spent")
18226            );
18227            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18228            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18229            if e2e_ok {
18230                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18231                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18232            } else {
18233                assert!(!runner.state.gate_ran);
18234            }
18235            assert_eq!(
18236                runner.state.status,
18237                if e2e_ok && gate_ok {
18238                    RunStatus::Ready
18239                } else {
18240                    RunStatus::Blocked
18241                }
18242            );
18243            if !e2e_ok || !gate_ok {
18244                let cause = runner.state.verification_blocking_cause().unwrap();
18245                assert!(cause.contains(&head));
18246                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18247                // A later successful progress event must not become the cause.
18248                runner.state.event("land", "rebased successfully");
18249                let notice = crate::notices::run_ended(&runner.state).unwrap();
18250                assert!(!format!("{notice:?}").contains("rebased successfully"));
18251                assert!(runner.state.merge.is_none());
18252            }
18253            let before = std::fs::read_to_string(&phases).unwrap();
18254            runner.state.save().unwrap();
18255            runner.state = RunState::load(&runner.state.id).unwrap();
18256            runner.review_loop().await.unwrap();
18257            runner.gate().await.unwrap();
18258            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18259            assert_eq!(runner.state.verification_history.len(), 1);
18260            let history = &runner.state.verification_history[0];
18261            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18262            assert!(history.archived_at.is_some());
18263            assert_eq!(history.fixes_spent, Some(1));
18264        }
18265    }
18266
18267    #[cfg(unix)]
18268    #[tokio::test]
18269    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18270        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18271        let marker = tmp.path().join("gate-ran");
18272        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18273        runner.review_loop().await.unwrap();
18274        runner.gate().await.unwrap();
18275        assert_eq!(runner.state.status, RunStatus::Blocked);
18276        assert!(!marker.exists());
18277        assert!(runner.state.verification_history.is_empty());
18278        runner.state.gate_head = None;
18279        runner.gate().await.unwrap();
18280        assert!(marker.exists());
18281        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18282        assert_eq!(runner.state.verification_history.len(), 1);
18283        assert_eq!(runner.state.gate_fixes.len(), 1);
18284    }
18285
18286    #[cfg(unix)]
18287    #[tokio::test]
18288    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18289        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18290        runner.sync_to_base().await.unwrap();
18291        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18292        runner.state.reconcile_verification_head(&head);
18293        // Both crash-before-command and resource wait are unresolved evidence.
18294        for outcomes in [
18295            Vec::new(),
18296            vec![CommandOutcome {
18297                command: "true".into(),
18298                code: None,
18299                output_tail: "cache unavailable".into(),
18300                duration_ms: 0,
18301                resource_blocked: true,
18302            }],
18303        ] {
18304            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18305                head: head.clone(),
18306                passed: false,
18307                outcomes,
18308            });
18309            runner.state.gate_ran = false;
18310            runner.state.gate.clear();
18311            runner.state.save().unwrap();
18312            runner.state = RunState::load(&runner.state.id).unwrap();
18313            runner.merge().await.unwrap();
18314            assert!(runner.state.merge.is_none());
18315            runner.review_loop().await.unwrap();
18316            runner.gate().await.unwrap();
18317            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18318            assert!(runner.state.gate_allows_progress());
18319            assert_eq!(runner.state.gate_fixes.len(), 1);
18320            assert!(runner.state.merge.is_none());
18321        }
18322        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18323        git::git(
18324            &runner.state.repo,
18325            &["commit", "--allow-empty", "-m", "external update"],
18326        )
18327        .await
18328        .unwrap();
18329        runner.merge().await.unwrap();
18330        assert!(runner.state.merge.is_none());
18331        assert_eq!(runner.state.status, RunStatus::Blocked);
18332    }
18333
18334    #[cfg(unix)]
18335    #[tokio::test]
18336    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18337        for waiting_on_gate in [false, true] {
18338            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18339            runner.sync_to_base().await.unwrap();
18340            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18341            runner.state.reconcile_verification_head(&head);
18342            if waiting_on_gate {
18343                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18344                    head: head.clone(),
18345                    passed: true,
18346                    outcomes: Vec::new(),
18347                });
18348            }
18349            let cache = tmp.path().join("cache");
18350            runner.state.config.verify.e2e =
18351                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18352            runner.state.config.verify.gate = vec!["true".into()];
18353            runner.state.config.graph.timeout_resource_wait = 1;
18354            let owner =
18355                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18356            let guard =
18357                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18358                    crate::cache::AcquireOutcome::Acquired(g) => g,
18359                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18360                };
18361            runner.review_loop().await.unwrap();
18362            runner.gate().await.unwrap();
18363            assert!(!runner.state.gate_ran);
18364            assert_eq!(runner.state.gate_fixes.len(), 1);
18365            assert!(runner.state.resource_block.is_some());
18366            runner.merge().await.unwrap();
18367            assert!(runner.state.merge.is_none());
18368            drop(guard);
18369            runner.state = RunState::load(&runner.state.id).unwrap();
18370            // The driver clears the previous pass's resource stop on resume.
18371            runner.state.resource_block = None;
18372            runner.review_loop().await.unwrap();
18373            runner.gate().await.unwrap();
18374            assert!(runner.state.gate_allows_progress());
18375            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18376            assert_eq!(runner.state.gate_fixes.len(), 1);
18377        }
18378    }
18379
18380    #[cfg(unix)]
18381    #[tokio::test]
18382    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18383        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18384        runner.sync_to_base().await.unwrap();
18385        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18386        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18387        // Verification passes before pre_gate and fails on the mechanically changed head.
18388        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18389        runner.review_loop().await.unwrap();
18390        runner.gate().await.unwrap();
18391        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18392        assert_ne!(before, after);
18393        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18394        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18395        assert!(!runner.state.gate_allows_progress());
18396        runner.merge().await.unwrap();
18397        assert!(runner.state.merge.is_none());
18398        assert!(
18399            runner
18400                .state
18401                .verification_history
18402                .iter()
18403                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18404        );
18405
18406        // An incomplete review is still an independent hold on a changed head.
18407        runner.state.reviews[0].clean = false;
18408        runner.state.reviews[0].answered = 0;
18409        runner.state.config.verify.e2e = vec!["true".into()];
18410        runner.review_loop().await.unwrap();
18411        runner.gate().await.unwrap();
18412        runner.merge().await.unwrap();
18413        assert_eq!(runner.state.status, RunStatus::Blocked);
18414        assert!(runner.state.merge.is_none());
18415        assert!(
18416            runner
18417                .state
18418                .verification_blocking_cause()
18419                .unwrap()
18420                .contains("review blocked")
18421        );
18422    }
18423
18424    #[cfg(unix)]
18425    #[tokio::test]
18426    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18427        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18428        runner.state.gate_head = None;
18429        runner.state.config.verify.gate =
18430            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18431        runner.review_loop().await.unwrap();
18432        runner.gate().await.unwrap();
18433        assert_ne!(
18434            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18435            old
18436        );
18437        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18438        assert!(!runner.state.gate_allows_progress());
18439        assert_eq!(runner.state.status, RunStatus::Blocked);
18440        runner.merge().await.unwrap();
18441        assert!(runner.state.merge.is_none());
18442    }
18443
18444    #[cfg(unix)]
18445    #[tokio::test]
18446    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18447        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18448        let mut old_review = review_round(true, 0, 1, 1, false, true);
18449        old_review.head = head.clone();
18450        old_review.verified_head = Some(head.clone());
18451        runner.state.reviews = vec![old_review];
18452        runner.state.gate = vec![CommandOutcome {
18453            command: "old gate".into(),
18454            code: None,
18455            output_tail: "timed out after 2400s".into(),
18456            duration_ms: 2_400_000,
18457            resource_blocked: false,
18458        }];
18459        runner.state.gate_ran = true;
18460        runner.state.gate_head = Some(head.clone());
18461        runner.state.land_approval = Some(crate::run::LandApproval {
18462            question: "old-head-approval".into(),
18463            head: head.clone(),
18464        });
18465        runner.state.authorize_base_sync(&task).unwrap();
18466        runner.sync_to_base().await.unwrap();
18467        assert!(!runner.state.base_sync_blocked());
18468        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18469        assert_ne!(head, fresh);
18470        assert_eq!(runner.state.reviews.len(), 1);
18471        assert!(!runner.state.gate_ran);
18472        let saved = &runner.state.base_sync_recovery.authorizations[0];
18473        assert_eq!(saved.reviews[0].head, head);
18474        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18475        assert!(saved.gate_ran);
18476        assert!(!saved.gate[0].ok());
18477        runner.review_loop().await.unwrap();
18478        runner.gate().await.unwrap();
18479        assert_eq!(runner.state.reviews[0].head, head);
18480        assert_eq!(
18481            runner.state.reviews[0].verified_head.as_deref(),
18482            Some(head.as_str())
18483        );
18484        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18485        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18486        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18487        assert!(runner.state.gate_ran);
18488        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18489        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18490        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18491        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18492        assert_eq!(
18493            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18494            "retained change"
18495        );
18496    }
18497
18498    #[cfg(unix)]
18499    #[tokio::test]
18500    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18501        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18502        let queue = crate::queue::Queue::open();
18503        let mut owner = queue.get(&task).unwrap();
18504        owner.release_operator("test-resume", None);
18505        queue.put(&mut owner).unwrap();
18506        let generation = owner.resumption.as_ref().unwrap().generation;
18507        runner.state.task_instruction = Some(owner.instruction.clone());
18508        runner.state.driver_exited = true;
18509        runner.state.save().unwrap();
18510        let home = crate::run::home();
18511        let _claim = queue.claim(&task).unwrap();
18512        owner.resumption.as_mut().unwrap().at =
18513            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18514        queue.put(&mut owner).unwrap();
18515        assert!(
18516            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18517                .await
18518                .is_err(),
18519            "an approval from before the stop grants nothing"
18520        );
18521        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18522        queue.put(&mut owner).unwrap();
18523        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18524            .await
18525            .unwrap();
18526        assert!(
18527            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18528                .await
18529                .is_err()
18530        );
18531        runner.state = RunState::load(&runner.state.id).unwrap();
18532        assert_eq!(
18533            runner.state.base_sync_recovery.authorizations[0].generation,
18534            Some(generation)
18535        );
18536        runner.sync_to_base().await.unwrap();
18537        let repo = runner.state.repo.clone();
18538        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18539        git::git(&repo, &["checkout", "main"]).await.unwrap();
18540        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18541        git::git(&repo, &["add", "."]).await.unwrap();
18542        git::git(&repo, &["commit", "-m", "move base again"])
18543            .await
18544            .unwrap();
18545        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18546        git::git(&repo, &["checkout", "fix/retained"])
18547            .await
18548            .unwrap();
18549        runner.sync_to_base().await.unwrap();
18550        assert!(!runner.state.base_sync_blocked());
18551        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18552        assert_eq!(runner.state.base_episode_used(), 2);
18553        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18554        assert_ne!(fresh, head);
18555        assert!(!runner.state.base_recovery_pending());
18556        // Even replaying the generation with a later timestamp grants nothing.
18557        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18558        queue.put(&mut owner).unwrap();
18559        assert!(
18560            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18561                .await
18562                .is_err()
18563        );
18564        runner.state = RunState::load(&runner.state.id).unwrap();
18565        runner.sync_to_base().await.unwrap();
18566        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18567        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18568        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18569    }
18570
18571    #[cfg(unix)]
18572    #[tokio::test]
18573    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18574        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18575        runner.state.base_sync = None;
18576        runner.state.status = RunStatus::Blocked;
18577        let mut second = runner.state.candidates[0].clone();
18578        second.label = 'B';
18579        runner.state.candidates.push(second);
18580        assert!(runner.state.votes.is_empty());
18581        assert!(runner.state.deliberation.is_empty());
18582        runner.deliberate().await.unwrap();
18583        runner.vote().await.unwrap();
18584        assert_eq!(runner.state.status, RunStatus::Blocked);
18585        assert!(runner.state.votes.is_empty());
18586        assert!(runner.state.deliberation.is_empty());
18587    }
18588
18589    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18590    /// reads nothing else off it.
18591    fn runner_at(status: RunStatus) -> Runner {
18592        let mut state = RunState::new(
18593            PathBuf::from("/nonexistent/repo"),
18594            "main".to_owned(),
18595            "deadbeef".to_owned(),
18596            "task".to_owned(),
18597            Config::default(),
18598        );
18599        // These fixtures test execution flow, independent of host disk capacity.
18600        state.config.disk.min_free_bytes = 0;
18601        // These fixtures share one home and save concurrently. Short clock
18602        // IDs can collide, making an ownership test read another test's run.
18603        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18604        state.id = format!(
18605            "20000101-000000-{:04x}",
18606            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18607        );
18608        state.status = status;
18609        Runner {
18610            state,
18611            roles: ResolvedRoles {
18612                implementers: Vec::new(),
18613                judges: Vec::new(),
18614                reviewers: Vec::new(),
18615                fixer: None,
18616                conductor: conductor(),
18617                implementer_roster: Vec::new(),
18618                judge_roster: Vec::new(),
18619                reviewer_roster: Vec::new(),
18620            },
18621            sem: Arc::new(Semaphore::new(1)),
18622            pause: Pause::new(),
18623            interrupt: Pause::new(),
18624        }
18625    }
18626
18627    #[tokio::test]
18628    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18629        crate::run::pin_test_home();
18630        let temp = tempfile::tempdir().unwrap();
18631        let root = temp.path().canonicalize().unwrap();
18632        let repo = root.join("repo");
18633        std::fs::create_dir(&repo).unwrap();
18634        init_repo(&repo);
18635        git::git(&repo, &["config", "core.autocrlf", "false"])
18636            .await
18637            .unwrap();
18638        let mut runner = runner_at(RunStatus::Blocked);
18639        runner.state.repo = repo.clone();
18640        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18641        runner.state.config.agents = vec![conductor()];
18642        runner.state.config.graph.selected_implementer = Some("conductor".into());
18643        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18644        let (shell, command, setup_output) = if cfg!(windows) {
18645            (
18646                vec!["cmd".into(), "/C".into()],
18647                "echo ready>seat-ready",
18648                "ready\r\n",
18649            )
18650        } else {
18651            (
18652                vec!["sh".into(), "-c".into()],
18653                "printf ready > seat-ready",
18654                "ready",
18655            )
18656        };
18657        runner.state.config.verify.shell = Some(shell);
18658        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18659            run: Some(command.into()),
18660            ..Default::default()
18661        }];
18662        runner.state.save().unwrap();
18663        let path = runner.state.worktree_root().join("judge-1");
18664        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18665            .await
18666            .unwrap();
18667        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18668        git::git(&path, &["add", "."]).await.unwrap();
18669        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18670        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18671            .await
18672            .unwrap();
18673        git::git(&path, &["checkout", "--detach", &head])
18674            .await
18675            .unwrap();
18676        let evidence = crate::fold_evidence::snapshot(
18677            &runner.state,
18678            &crate::run::home().canonicalize().unwrap(),
18679            &head,
18680            64 * 1024 * 1024,
18681        )
18682        .await
18683        .unwrap();
18684        let mut contents = BTreeMap::new();
18685        contents.insert(
18686            repo.join("logs/HEAD"),
18687            format!("{head} {head} fixture\n").into_bytes(),
18688        );
18689        let record = serde_json::json!({
18690            "version":1, "run":runner.state.id, "source":path, "head":head,
18691            "evidence":evidence, "admin":{}, "admin_contents":contents,
18692            "history":{head.clone():evidence}, "phase":"evicted"
18693        });
18694        std::fs::write(
18695            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18696            serde_json::to_vec(&record).unwrap(),
18697        )
18698        .unwrap();
18699        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18700        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18701            .await
18702            .unwrap();
18703        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18704        assert!(
18705            !git::git_raw(&repo, &["cat-file", "-e", &head])
18706                .await
18707                .unwrap()
18708                .ok()
18709        );
18710        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18711        let error = resumed.execute().await.unwrap_err();
18712        assert!(
18713            error
18714                .to_string()
18715                .contains("selected implementation is held"),
18716            "{error:#}"
18717        );
18718        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18719        assert_eq!(
18720            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18721            setup_output
18722        );
18723        assert_eq!(
18724            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18725            "unrelated seat"
18726        );
18727        let error = resumed.execute().await.unwrap_err();
18728        assert!(
18729            error
18730                .to_string()
18731                .contains("selected implementation is held"),
18732            "{error:#}"
18733        );
18734        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18735        git::git(&path, &["add", "."]).await.unwrap();
18736        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18737        let next = git::git(
18738            &path,
18739            &[
18740                "commit-tree",
18741                &next_tree,
18742                "-p",
18743                &head,
18744                "-m",
18745                "next review checkpoint",
18746            ],
18747        )
18748        .await
18749        .unwrap();
18750        git::git(&path, &["checkout", "--detach", &next])
18751            .await
18752            .unwrap();
18753        resumed.state.reviews.push(
18754            serde_json::from_value(serde_json::json!({
18755                "round":1, "head":next, "reviews":[]
18756            }))
18757            .unwrap(),
18758        );
18759        resumed.state.save().unwrap();
18760        let error = resumed.execute().await.unwrap_err();
18761        assert!(
18762            error
18763                .to_string()
18764                .contains("selected implementation is held"),
18765            "{error:#}"
18766        );
18767        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18768    }
18769
18770    #[tokio::test]
18771    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18772        crate::run::pin_test_home();
18773        let mut runner = runner_at(RunStatus::Gating);
18774        runner.state.save().unwrap();
18775        let id = runner.state.id.clone();
18776        let passes = runner.state.passes.len();
18777        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18778        let mut claiming = Box::pin(runner.claim_driver());
18779        std::future::poll_fn(|cx| {
18780            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18781            std::task::Poll::Ready(())
18782        })
18783        .await;
18784        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18785        drop(owner);
18786        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18787            .await
18788            .unwrap()
18789            .unwrap();
18790        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18791        assert!(execution.runner.state.resource_block.is_none());
18792        assert_eq!(
18793            execution.runner.state.liveness(false),
18794            crate::run::Liveness::Live
18795        );
18796        drop(execution);
18797        assert!(RunState::load(&id).unwrap().driver_exited);
18798    }
18799
18800    #[tokio::test]
18801    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18802        crate::run::pin_test_home();
18803        let mut runner = runner_at(RunStatus::Gating);
18804        runner.state.save().unwrap();
18805        // run.json can commit, but the subsequent lease metadata write fails.
18806        // Cleanup must run after releasing the short ownership lock.
18807        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18808        assert!(runner.claim_driver().await.is_err());
18809        assert!(runner.state.driver_exited);
18810        assert_eq!(runner.state.driver_pid, None);
18811        let disk = RunState::load(&runner.state.id).unwrap();
18812        assert!(disk.driver_exited);
18813        assert_eq!(disk.driver_pid, None);
18814        assert_eq!(disk.status, RunStatus::Gating);
18815    }
18816
18817    #[tokio::test]
18818    async fn an_early_graph_error_releases_its_driver() {
18819        crate::run::pin_test_home();
18820        let mut runner = runner_at(RunStatus::Blocked);
18821        runner.state.config.graph.selected_implementer = Some("selected".into());
18822        runner.state.save().unwrap();
18823        assert!(runner.execute().await.is_err());
18824        let disk = RunState::load(&runner.state.id).unwrap();
18825        assert!(disk.driver_exited);
18826        assert_eq!(disk.driver_pid, None);
18827        assert_eq!(disk.status, RunStatus::Blocked);
18828    }
18829
18830    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18831        crate::run::MergeRepair {
18832            question: "repair-question".into(),
18833            head: "old-head".into(),
18834            pr: "https://github.com/o/r/pull/42".into(),
18835            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18836            gate: vec![outcome(Some(0), false)],
18837            gate_head: Some("old-head".into()),
18838            gate_fixes: Vec::new(),
18839            pre_gate: Vec::new(),
18840            phase,
18841            failure: None,
18842        }
18843    }
18844
18845    #[tokio::test]
18846    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18847        crate::run::pin_test_home();
18848        let mut runner = runner_at(RunStatus::Landing);
18849        runner.state.merge_repairs.push(merge_repair_request(
18850            crate::run::MergeRepairPhase::Requested,
18851        ));
18852        runner.state.save().unwrap();
18853        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18854        assert!(
18855            runner
18856                .execute()
18857                .await
18858                .unwrap_err()
18859                .to_string()
18860                .contains("already running")
18861        );
18862        assert_eq!(
18863            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18864            crate::run::MergeRepairPhase::Requested
18865        );
18866        drop(claim);
18867        let mut disk = runner.state.clone();
18868        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18869        disk.save().unwrap();
18870        assert!(
18871            runner
18872                .execute()
18873                .await
18874                .unwrap_err()
18875                .to_string()
18876                .contains("decision changed")
18877        );
18878        assert_eq!(
18879            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18880            crate::run::MergeRepairPhase::Complete
18881        );
18882    }
18883
18884    #[test]
18885    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18886        crate::run::pin_test_home();
18887        let mut runner = runner_at(RunStatus::Landing);
18888        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18889        runner.state.reviews = request.reviews.clone();
18890        runner.state.gate = request.gate.clone();
18891        runner.state.gate_ran = true;
18892        runner.state.land_approval = Some(crate::run::LandApproval {
18893            question: "old".into(),
18894            head: "old-head".into(),
18895        });
18896        runner.state.merge_repairs.push(request);
18897        runner.begin_merge_repair_review(0).unwrap();
18898        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18899        assert!(!runner.state.gate_status().ok());
18900        assert!(runner.state.land_approval.is_none());
18901        let disk = RunState::load(&runner.state.id).unwrap();
18902        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18903        assert!(disk.merge_repairs[0].gate[0].ok());
18904        assert_eq!(
18905            disk.merge_repairs[0].phase,
18906            crate::run::MergeRepairPhase::Reviewing
18907        );
18908        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18909        assert!(disk.reviews.is_empty());
18910    }
18911
18912    #[tokio::test]
18913    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18914        crate::run::pin_test_home();
18915        for phase in [
18916            crate::run::MergeRepairPhase::Fixing,
18917            crate::run::MergeRepairPhase::Requested,
18918        ] {
18919            let mut runner = runner_at(RunStatus::Landing);
18920            runner.state.config.graph.land_rounds = 0;
18921            runner.state.merge_repairs.push(merge_repair_request(phase));
18922            runner.state.save().unwrap();
18923            assert!(!runner.repair_merge_review().await.unwrap());
18924            assert_eq!(runner.state.status, RunStatus::Blocked);
18925            assert_eq!(
18926                runner.state.merge_repairs[0].phase,
18927                crate::run::MergeRepairPhase::Failed
18928            );
18929            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18930            assert!(
18931                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18932                    "interrupted"
18933                } else {
18934                    "budget exhausted"
18935                })
18936            );
18937            assert!(runner.state.merge.is_none());
18938        }
18939    }
18940
18941    /// `park_here` folding in the reason `Pause::park_because` recorded -
18942    /// this is what lets an operator reading a run's events tell an
18943    /// interrupt-driven park from an ordinary shutdown park.
18944    #[test]
18945    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18946        crate::run::pin_test_home();
18947        let mut runner = runner_at(RunStatus::Implementing);
18948        let interrupt = Pause::new();
18949        runner.watch_interrupt(interrupt.clone());
18950
18951        interrupt.park_because("task a1b2 asked to run first");
18952
18953        assert!(runner.park_here().expect("park_here"));
18954        assert!(runner.state.parked);
18955        let last = runner.state.events.last().expect("a park event");
18956        assert_eq!(last.node, "park");
18957        assert!(
18958            last.message.contains("task a1b2 asked to run first"),
18959            "expected the interrupt reason in {:?}",
18960            last.message
18961        );
18962    }
18963
18964    /// The cause is stored on the pass when it parks, and the first cause a
18965    /// shared pause was given is the one that stays.
18966    #[test]
18967    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18968        crate::run::pin_test_home();
18969        let mut runner = runner_at(RunStatus::Implementing);
18970        runner.state.begin_pass();
18971        let shutdown = Pause::new();
18972        runner.on_pause(shutdown.clone());
18973        let upgrade = ParkCause::Upgrade {
18974            from: "0.121.0".to_owned(),
18975            to: Some("0.121.1".to_owned()),
18976        };
18977        shutdown.park_for(upgrade.clone());
18978        shutdown.park_for(ParkCause::Operator);
18979
18980        assert!(runner.park_here().expect("park_here"));
18981        assert_eq!(
18982            runner.state.passes[0].end,
18983            Some(crate::run::PassEnd::Parked { cause: upgrade })
18984        );
18985        let last = runner.state.events.last().expect("a park event");
18986        assert!(last.message.contains("0.121.0"), "{}", last.message);
18987    }
18988
18989    #[tokio::test]
18990    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18991        crate::run::pin_test_home();
18992        let mut runner = runner_at(RunStatus::Reviewing);
18993        runner.state.save().unwrap();
18994        let mut execution = runner.claim_driver().await.expect("first execution");
18995        assert_eq!(execution.runner.state.passes.len(), 1);
18996        execution
18997            .driver
18998            .finish(&mut execution.runner.state)
18999            .await
19000            .unwrap();
19001    }
19002
19003    #[tokio::test]
19004    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19005        crate::run::pin_test_home();
19006        let mut runner = runner_at(RunStatus::Reviewing);
19007        runner.state.config.agents = vec![conductor()];
19008        runner.state.config.graph.timeout_resource_wait = 0;
19009        runner.state.save().unwrap();
19010        let id = runner.state.id.clone();
19011        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19012        assert!(runner.claim_driver().await.is_err());
19013        drop(owner);
19014        drop(runner);
19015        let mut resumed = Runner::resume(&id).unwrap();
19016        assert_eq!(resumed.state.driver_claimed, Some(false));
19017        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19018        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19019        let mut execution = resumed
19020            .claim_driver()
19021            .await
19022            .expect("retry initial claim after reload");
19023        assert_eq!(execution.runner.state.driver_claimed, Some(true));
19024        assert_eq!(execution.runner.state.passes.len(), 1);
19025        execution
19026            .driver
19027            .finish(&mut execution.runner.state)
19028            .await
19029            .unwrap();
19030    }
19031
19032    #[tokio::test]
19033    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19034        crate::run::pin_test_home();
19035        let mut runner = runner_at(RunStatus::Reviewing);
19036        runner.state.config.agents = vec![conductor()];
19037        runner.state.save().unwrap();
19038        let id = runner.state.id.clone();
19039        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19040        let mut resumed = Runner::resume(&id).unwrap();
19041        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19042        assert!(resumed.claim_driver().await.is_err());
19043        assert!(RunState::load(&id).unwrap().passes.is_empty());
19044        drop(lease);
19045        let mut execution = resumed.claim_driver().await.unwrap();
19046        execution
19047            .driver
19048            .finish(&mut execution.runner.state)
19049            .await
19050            .unwrap();
19051    }
19052
19053    #[tokio::test]
19054    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19055        crate::run::pin_test_home();
19056        let mut runner = runner_at(RunStatus::Reviewing);
19057        let state = &mut runner.state;
19058        state.config.agents = vec![conductor()];
19059        state.driver_claimed = None;
19060        state.driver_pid = Some(std::process::id());
19061        state.driver_started_at = None;
19062        state.begin_pass();
19063        state.save().unwrap();
19064        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19065        std::fs::write(
19066            state.dir().join("driver.json"),
19067            serde_json::to_vec(&serde_json::json!({
19068                "pid": state.driver_pid, "started_at": null,
19069                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19070            }))
19071            .unwrap(),
19072        )
19073        .unwrap();
19074        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19075        drop(lease);
19076        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19077        let mut resumed = Runner::resume(&state.id).unwrap();
19078        assert!(!resumed.state.driver_exited);
19079        let mut execution = resumed
19080            .claim_driver()
19081            .await
19082            .expect("lease proves previous execution ended");
19083        assert_eq!(execution.runner.state.passes.len(), 2);
19084        execution
19085            .driver
19086            .finish(&mut execution.runner.state)
19087            .await
19088            .unwrap();
19089    }
19090
19091    #[tokio::test]
19092    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19093        crate::run::pin_test_home();
19094        let mut runner = runner_at(RunStatus::Reviewing);
19095        let mut json = serde_json::to_value(&runner.state).unwrap();
19096        json.as_object_mut().unwrap().remove("driver_claimed");
19097        runner.state = serde_json::from_value(json).unwrap();
19098        assert_eq!(runner.state.driver_claimed, None);
19099        runner.state.save().unwrap();
19100        let mut execution = runner
19101            .claim_driver()
19102            .await
19103            .expect("the execution lease, not a missing marker, excludes a second driver");
19104        assert_eq!(execution.runner.state.passes.len(), 1);
19105        execution
19106            .driver
19107            .finish(&mut execution.runner.state)
19108            .await
19109            .unwrap();
19110    }
19111
19112    #[tokio::test]
19113    async fn driver_admission_refuses_a_proven_live_driver() {
19114        crate::run::pin_test_home();
19115        let mut runner = runner_at(RunStatus::Reviewing);
19116        let pid = std::process::id();
19117        runner.state.driver_pid = Some(pid);
19118        runner.state.driver_started_at = crate::proc::process_started_at(pid);
19119        assert!(runner.state.driver_started_at.is_some());
19120        runner.state.save().unwrap();
19121        let error = runner
19122            .claim_driver()
19123            .await
19124            .err()
19125            .expect("a live driver must refuse");
19126        assert!(error.to_string().contains("live driver"), "{error:#}");
19127        let saved = RunState::load(&runner.state.id).unwrap();
19128        assert_eq!(saved.driver_pid, Some(pid));
19129        assert_eq!(saved.status, RunStatus::Reviewing);
19130        assert!(saved.passes.is_empty());
19131    }
19132
19133    #[test]
19134    fn parking_resource_halt_persists_the_pass_once() {
19135        crate::run::pin_test_home();
19136        let mut runner = runner_at(RunStatus::Reviewing);
19137        runner.state.begin_pass();
19138        runner.state.resource_block = Some(crate::run::ResourceBlock {
19139            node: "fix".into(),
19140            kind: "shared build cache".into(),
19141            detail: "cancelled".into(),
19142            at: Timestamp::now(),
19143            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19144        });
19145        let cause = ParkCause::Upgrade {
19146            from: "old".into(),
19147            to: Some("new".into()),
19148        };
19149        runner.pause.park_for(cause.clone());
19150        // Both direct fixer exits and graph resource boundaries must complete
19151        // parking, rather than returning a stale Reviewing record.
19152        runner.stop_for_resource().unwrap();
19153        assert!(runner.resource_halt().unwrap());
19154        assert!(runner.park_here().unwrap());
19155        let saved = RunState::load(&runner.state.id).unwrap();
19156        assert!(saved.parked);
19157        assert_eq!(saved.status, RunStatus::Reviewing);
19158        assert!(saved.passes[0].ended_at.is_some());
19159        assert_eq!(
19160            saved.passes[0].end,
19161            Some(crate::run::PassEnd::Parked {
19162                cause: cause.clone()
19163            })
19164        );
19165        assert_eq!(
19166            saved.resource_block.unwrap().cause,
19167            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19168        );
19169        assert_eq!(
19170            saved
19171                .events
19172                .iter()
19173                .filter(|event| event.node == "park")
19174                .count(),
19175            1
19176        );
19177    }
19178
19179    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19180    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19181    /// cleared) must not make a *different* run - one only watching its own,
19182    /// unshared interrupt `Pause` - see itself as parked. If a future change
19183    /// ever collapsed these back into one handle, the interrupt scheduler
19184    /// would park every run for the rest of the daemon's life, not just the
19185    /// one it meant to interrupt.
19186    #[test]
19187    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19188        crate::run::pin_test_home();
19189        let mut runner = runner_at(RunStatus::Implementing);
19190        let shutdown = Pause::new();
19191        runner.on_pause(shutdown.clone());
19192        let interrupt = Pause::new();
19193        runner.watch_interrupt(interrupt.clone());
19194
19195        // Nobody has asked for anything yet.
19196        assert!(!runner.park_here().expect("park_here"));
19197        assert!(!runner.state.parked);
19198
19199        // Only the interrupt handle fires; the shutdown handle stays clear.
19200        interrupt.park_because("test");
19201        assert!(!shutdown.parked());
19202        assert!(runner.park_here().expect("park_here"));
19203    }
19204
19205    /// The property every prior attempt at this feature failed to pin down:
19206    /// asking a run to park while one of its nodes has a real, in-flight
19207    /// async operation running (an agent call, in production) must not cut
19208    /// that operation short. `park_here` is only ever consulted *between*
19209    /// `execute`'s node calls - see its own doc - so nothing inside a node
19210    /// can observe a park request until the node itself returns. This proves
19211    /// that structurally, with real `tokio` concurrency and a channel
19212    /// handshake (never a sleep, which would only prove "usually", not
19213    /// "cannot"): the "node" below reports that it has genuinely started,
19214    /// and only then is the park requested; the node still has to be told to
19215    /// finish before `park_here` is ever called, exactly mirroring every
19216    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19217    /// in `execute`.
19218    #[tokio::test]
19219    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19220        crate::run::pin_test_home();
19221        let mut runner = runner_at(RunStatus::Implementing);
19222        let interrupt = Pause::new();
19223        runner.watch_interrupt(interrupt.clone());
19224
19225        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19226        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19227
19228        // Stands in for one node's in-flight agent call: it proves it has
19229        // genuinely started, then blocks - exactly as a spawned CLI process
19230        // does - until told to finish.
19231        let node = async move {
19232            started_tx.send(()).expect("send started");
19233            finish_rx.await.expect("recv finish");
19234            "node finished"
19235        };
19236
19237        let interrupter = async move {
19238            started_rx.await.expect("recv started");
19239            // The call is now genuinely in flight. Ask it to park.
19240            interrupt.park_because("higher-priority task waiting");
19241            // Nothing the node does can observe this yet - there is no
19242            // check inside it, by construction - so let the executor run
19243            // anything pending and then let the node finish on its own.
19244            tokio::task::yield_now().await;
19245            finish_tx.send(()).expect("send finish");
19246        };
19247
19248        let (node_result, ()) = tokio::join!(node, interrupter);
19249        assert_eq!(
19250            node_result, "node finished",
19251            "the in-flight call ran to completion"
19252        );
19253
19254        // Only now, at the boundary the real `execute` would check right
19255        // after this node, does the park take effect.
19256        assert!(runner.park_here().expect("park_here"));
19257        assert!(runner.state.parked);
19258    }
19259
19260    /// A run parked mid-competition carries every field it had accumulated
19261    /// through the exact same disk round-trip an ordinary resume uses -
19262    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19263    /// Nothing about parking for an interrupt is a special case of that path;
19264    /// this is what proves it rather than assuming it.
19265    #[test]
19266    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19267        crate::run::pin_test_home();
19268        let mut runner = runner_at(RunStatus::Judging);
19269        // `Runner::resume` re-resolves roles from the saved config, which
19270        // refuses an empty roster - give it the same minimal one `conductor`
19271        // itself uses.
19272        runner.state.config.agents = vec![conductor()];
19273        runner.state.candidates = vec![Candidate {
19274            index: 0,
19275            label: 'A',
19276            agent: "alpha".to_owned(),
19277            branch: "magi/x/A".to_owned(),
19278            worktree: PathBuf::from("/nonexistent/worktree"),
19279            summary: "did the thing".to_owned(),
19280            stat: "1 file changed".to_owned(),
19281            files: 1,
19282            commits: 1,
19283            empty: false,
19284            failed: None,
19285            verified_noop: None,
19286            duration_ms: 1234,
19287            folded: false,
19288        }];
19289        let run_id = runner.state.id.clone();
19290
19291        let interrupt = Pause::new();
19292        runner.watch_interrupt(interrupt.clone());
19293        interrupt.park_because("task c3d4 asked to run first");
19294        assert!(runner.park_here().expect("park_here"));
19295
19296        let resumed = Runner::resume(&run_id).expect("resume");
19297        assert_eq!(resumed.state.candidates.len(), 1);
19298        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19299        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19300        assert_eq!(resumed.state.status, runner.state.status);
19301        assert!(
19302            resumed.state.parked,
19303            "still parked until `execute` actually walks the graph again"
19304        );
19305        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19306    }
19307
19308    /// A fresh open question on `run`, stored and handed back for assertions.
19309    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19310        let mut q = ask::Question::new(
19311            run.to_owned(),
19312            "implement".to_owned(),
19313            "impl-A".to_owned(),
19314            "Which storage backend should the cache use?".to_owned(),
19315            String::new(),
19316            vec!["SQLite".to_owned(), "Redis".to_owned()],
19317        );
19318        store.put(&mut q).unwrap();
19319        q
19320    }
19321
19322    #[test]
19323    fn a_failed_runs_open_question_is_abandoned() {
19324        ask_test_home();
19325        let store = ask::Questions::open();
19326        let mut runner = runner_at(RunStatus::Failed);
19327        let run = runner.state.id.clone();
19328        let q = ask_open_question(&store, &run);
19329
19330        runner.settle_questions();
19331
19332        let back = store.get(&q.id).unwrap();
19333        assert!(
19334            !back.status.open(),
19335            "the seat that asked died with the run; nobody is left to read an answer"
19336        );
19337        assert!(
19338            back.detail.contains(&run) && back.detail.contains("failed"),
19339            "the reason names what the run became, not just that it is gone: {}",
19340            back.detail
19341        );
19342    }
19343
19344    #[test]
19345    fn a_merged_runs_open_question_is_abandoned_too() {
19346        ask_test_home();
19347        let store = ask::Questions::open();
19348        // A run that finishes cleanly still leaves nobody to read an answer -
19349        // this is not only a failure-path cleanup.
19350        for status in [RunStatus::Merged, RunStatus::Ready] {
19351            let mut runner = runner_at(status);
19352            let run = runner.state.id.clone();
19353            let q = ask_open_question(&store, &run);
19354
19355            runner.settle_questions();
19356
19357            let back = store.get(&q.id).unwrap();
19358            assert!(
19359                !back.status.open(),
19360                "{status:?} run's question must not outlive the run"
19361            );
19362        }
19363    }
19364
19365    #[test]
19366    fn a_still_resumable_runs_open_question_is_left_alone() {
19367        ask_test_home();
19368        let store = ask::Questions::open();
19369        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19370        // review round and the seat sessions are all still on disk — so a
19371        // question asked mid-round may yet get a real answer from a real
19372        // resume. Sweeping it here would be exactly the failure mode this
19373        // whole feature exists to avoid on the other side.
19374        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19375            let mut runner = runner_at(status);
19376            let run = runner.state.id.clone();
19377            let q = ask_open_question(&store, &run);
19378
19379            runner.settle_questions();
19380
19381            let back = store.get(&q.id).unwrap();
19382            assert!(
19383                back.status.open(),
19384                "{status:?} is still alive; the question must still be waiting"
19385            );
19386        }
19387    }
19388
19389    #[test]
19390    fn settle_questions_never_touches_an_already_answered_question() {
19391        ask_test_home();
19392        let store = ask::Questions::open();
19393        let mut runner = runner_at(RunStatus::Failed);
19394        let run = runner.state.id.clone();
19395        let mut q = ask_open_question(&store, &run);
19396        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19397            .unwrap();
19398        store.put(&mut q).unwrap();
19399
19400        // Called twice, the way a crash-recovered daemon reclaim and the
19401        // graph's own cleanup both can for the same run — `abandon_for_run`
19402        // only ever touches what is still open, so this must be inert both
19403        // times, not merely the second.
19404        runner.settle_questions();
19405        runner.settle_questions();
19406
19407        let back = store.get(&q.id).unwrap();
19408        assert_eq!(
19409            back.status,
19410            ask::QuestionStatus::Answered,
19411            "a real answer is a decision on record, never overwritten by a sweep"
19412        );
19413    }
19414
19415    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19416    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19417    /// without merging, whose winner is still the operator's answer to read.
19418    /// Nothing previously called `fold_run` itself with a real `tally`, so
19419    /// this is the first test to pin down the one distinction the whole
19420    /// automatic-fold feature depends on: the winner's worktree and branch
19421    /// must survive, everything else sharing the run's worktree bay - a
19422    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19423    /// second sweep treats every non-winner directory under the bay alike -
19424    /// must not.
19425    #[tokio::test]
19426    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19427        crate::run::pin_test_home();
19428        let tmp = tempfile::tempdir().expect("tempdir");
19429        let repo = tmp.path().join("repo");
19430        std::fs::create_dir_all(&repo).unwrap();
19431        init_repo(&repo);
19432
19433        let mut config = Config::default();
19434        config.graph.worktree_root = Some(tmp.path().join("wt"));
19435
19436        let mut state = RunState::new(
19437            repo.clone(),
19438            "main".to_owned(),
19439            "deadbeef".to_owned(),
19440            "task".to_owned(),
19441            config,
19442        );
19443        let root = state.worktree_root();
19444        let wt_a = root.join("cand-A");
19445        let wt_b = root.join("cand-B");
19446        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19447            .await
19448            .expect("worktree A");
19449        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19450            .await
19451            .expect("worktree B");
19452
19453        state.candidates = vec![
19454            Candidate {
19455                index: 0,
19456                label: 'A',
19457                agent: "alpha".to_owned(),
19458                branch: "magi/x/A".to_owned(),
19459                worktree: wt_a.clone(),
19460                summary: String::new(),
19461                stat: String::new(),
19462                files: 0,
19463                commits: 0,
19464                empty: false,
19465                failed: None,
19466                verified_noop: None,
19467                duration_ms: 0,
19468                folded: false,
19469            },
19470            Candidate {
19471                index: 1,
19472                label: 'B',
19473                agent: "beta".to_owned(),
19474                branch: "magi/x/B".to_owned(),
19475                worktree: wt_b.clone(),
19476                summary: String::new(),
19477                stat: String::new(),
19478                files: 0,
19479                commits: 0,
19480                empty: false,
19481                failed: None,
19482                verified_noop: None,
19483                duration_ms: 0,
19484                folded: false,
19485            },
19486        ];
19487        state.tally = Some(Tally {
19488            first_choice: BTreeMap::from([('A', 1)]),
19489            borda: BTreeMap::new(),
19490            winner: 'A',
19491            rankings: 1,
19492            unanimous_initial: true,
19493            deliberated: false,
19494            changed_votes: 0,
19495            unanimous_final: true,
19496            tie_break: None,
19497            judges: 1,
19498            present: 1,
19499            quorum: 1,
19500            met_quorum: true,
19501            uncontested: None,
19502        });
19503        state.status = RunStatus::Ready;
19504
19505        fold_run(&mut state, false, &crate::run::home())
19506            .await
19507            .expect("fold_run");
19508
19509        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19510        assert!(
19511            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19512            "the unmerged winner's branch survives"
19513        );
19514        assert!(
19515            !state.candidates[0].folded,
19516            "the winner is not marked folded"
19517        );
19518
19519        assert!(!wt_b.exists(), "the loser's worktree is removed");
19520        assert!(
19521            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19522            "the loser's branch is removed"
19523        );
19524        assert!(state.candidates[1].folded, "the loser is marked folded");
19525    }
19526
19527    #[tokio::test]
19528    #[cfg(unix)]
19529    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19530        use std::os::fd::AsRawFd as _;
19531        use std::time::Duration;
19532
19533        let tmp = tempfile::tempdir().unwrap();
19534        let repo = tmp.path().join("repo");
19535        std::fs::create_dir_all(&repo).unwrap();
19536        init_repo(&repo);
19537        let common = repo.join(".git");
19538        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19539        let home = tmp.path().join("home");
19540        let mut config = Config::default();
19541        config.coordination = crate::config::test_coordination();
19542        config.graph.worktree_root = Some(tmp.path().join("wt"));
19543        config.graph.timeout_resource_wait = 1;
19544        let mut state = RunState::new(
19545            repo.clone(),
19546            "main".into(),
19547            "deadbeef".into(),
19548            "task".into(),
19549            config,
19550        );
19551        let root = state.worktree_root();
19552        let candidate = root.join("cand-A");
19553        let reviewer = root.join("review-1");
19554        let branch = state.branch_for('A');
19555        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19556            .await
19557            .unwrap();
19558        git::worktree_add_detached(&repo, &reviewer, "main")
19559            .await
19560            .unwrap();
19561        git::acquire_worktree_config(&repo).await.unwrap();
19562        state.enabled_worktree_config = true;
19563        state.status = RunStatus::Ready;
19564        state.candidates.push(Candidate {
19565            index: 0,
19566            label: 'A',
19567            agent: "implement".into(),
19568            branch: branch.clone(),
19569            worktree: candidate.clone(),
19570            summary: String::new(),
19571            stat: String::new(),
19572            files: 0,
19573            commits: 0,
19574            empty: false,
19575            failed: None,
19576            verified_noop: None,
19577            duration_ms: 0,
19578            folded: false,
19579        });
19580        state.save_under(&home).unwrap();
19581        let config_before = std::fs::read(common.join("config")).unwrap();
19582        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19583            .await
19584            .unwrap();
19585        // A separate OS handle bypasses the launcher's local gate, exactly as
19586        // an external updater does, and must still prevent all folding work.
19587        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19588        assert_eq!(
19589            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19590            0
19591        );
19592        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19593        assert!(
19594            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19595                .await
19596                .is_err()
19597        );
19598        assert!(candidate.exists());
19599        assert!(reviewer.exists());
19600        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19601        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19602        assert_eq!(
19603            git::git(&repo, &["worktree", "list", "--porcelain"])
19604                .await
19605                .unwrap(),
19606            worktrees_before
19607        );
19608        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19609            .await
19610            .unwrap()
19611            .unwrap_err();
19612        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19613        drop(fold);
19614        assert!(candidate.exists() && reviewer.exists());
19615        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19616        assert_eq!(
19617            state.resource_block.as_ref().unwrap().kind,
19618            "repository coordination"
19619        );
19620        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19621        drop(external);
19622        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19623        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19624            .await
19625            .unwrap()
19626            .unwrap();
19627        drop(fold);
19628        assert!(removed.contains(&branch));
19629        assert!(!root.exists());
19630        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19631        assert!(state.candidates[0].folded);
19632        assert!(!state.enabled_worktree_config);
19633        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19634        let enabled = git::git_raw(
19635            &repo,
19636            &["config", "--local", "--get", "extensions.worktreeConfig"],
19637        )
19638        .await
19639        .unwrap();
19640        assert!(!enabled.ok(), "the last config reference was released");
19641    }
19642
19643    /// A branch handed to a later run is that run's (and its pull request's):
19644    /// folding the run that released it must not delete it.
19645    #[tokio::test]
19646    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19647        let tmp = tempfile::tempdir().expect("tempdir");
19648        let repo = tmp.path().join("repo");
19649        std::fs::create_dir_all(&repo).unwrap();
19650        init_repo(&repo);
19651        let home = tmp.path().join("home");
19652
19653        let mut config = Config::default();
19654        config.graph.worktree_root = Some(tmp.path().join("wt"));
19655        let mut state = RunState::new(
19656            repo.clone(),
19657            "main".to_owned(),
19658            "deadbeef".to_owned(),
19659            "task".to_owned(),
19660            config,
19661        );
19662        // The worktree is already gone (released); the branch survives.
19663        git::git(&repo, &["branch", "magi/x/A", "main"])
19664            .await
19665            .expect("branch");
19666        state.candidates = vec![Candidate {
19667            index: 0,
19668            label: 'A',
19669            agent: "alpha".to_owned(),
19670            branch: "magi/x/A".to_owned(),
19671            worktree: state.worktree_root().join("cand-A"),
19672            summary: String::new(),
19673            stat: String::new(),
19674            files: 0,
19675            commits: 0,
19676            empty: false,
19677            failed: None,
19678            verified_noop: None,
19679            duration_ms: 0,
19680            folded: true,
19681        }];
19682        state.released_to = Some("20260901-000000-new1".to_owned());
19683        state.released_branches = vec!["magi/x/A".to_owned()];
19684
19685        fold_run(&mut state, true, &home).await.expect("fold_run");
19686        state.status = RunStatus::Merged;
19687        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19688        state.merge = Some(crate::run::MergeOutcome {
19689            mode: MergeMode::Pr,
19690            ok: true,
19691            detail: "confirmed merged".into(),
19692            empty: false,
19693        });
19694        fold_run(&mut state, true, &home)
19695            .await
19696            .expect("merged fold");
19697
19698        assert!(
19699            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19700            "the handed-over branch survives a fold"
19701        );
19702    }
19703
19704    /// A winner with nothing ahead of the base is caught before `gh` is ever
19705    /// asked for a pull request, and the message carries what the task's
19706    /// references resolved to.
19707    #[tokio::test]
19708    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19709        let tmp = tempfile::tempdir().expect("tempdir");
19710        let repo = tmp.path().join("repo");
19711        std::fs::create_dir_all(&repo).unwrap();
19712        init_repo(&repo);
19713        let run = |args: &[&str]| {
19714            let out = std::process::Command::new("git")
19715                .quiet()
19716                .args(args)
19717                .current_dir(&repo)
19718                .output()
19719                .expect("spawn git");
19720            assert!(out.status.success(), "git {args:?}");
19721        };
19722        run(&["branch", "magi/x/A"]);
19723        run(&["checkout", "-q", "-b", "magi/x/B"]);
19724        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19725        run(&["add", "-A"]);
19726        run(&["commit", "-q", "-m", "work"]);
19727        run(&["checkout", "-q", "main"]);
19728
19729        // A pull request is compared against the remote's base, so the
19730        // fixture needs one. Before it exists nothing can be read, and the
19731        // answer must be "not empty".
19732        let probe = RunState::new(
19733            repo.clone(),
19734            "main".to_owned(),
19735            "deadbeef".to_owned(),
19736            "task".to_owned(),
19737            Config::default(),
19738        );
19739        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19740        let bare = tmp.path().join("origin.git");
19741        let out = std::process::Command::new("git")
19742            .quiet()
19743            .args(["init", "-q", "--bare"])
19744            .arg(&bare)
19745            .output()
19746            .expect("spawn git");
19747        assert!(out.status.success(), "git init --bare");
19748        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19749        run(&["push", "-q", "origin", "main"]);
19750
19751        let mut state = RunState::new(
19752            repo.clone(),
19753            "main".to_owned(),
19754            "deadbeef".to_owned(),
19755            "task".to_owned(),
19756            Config::default(),
19757        );
19758        state.seeds = vec![refs::Seed {
19759            token: "magi/27b2/A".to_owned(),
19760            kind: refs::SeedKind::Unresolved,
19761            sha: String::new(),
19762            branch: true,
19763            detail: "no branch or commit named magi/27b2/A".to_owned(),
19764        }];
19765
19766        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19767        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19768        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19769        let detail = empty_candidate_detail(&state, "main");
19770        assert!(detail.starts_with("empty candidate"), "{detail}");
19771        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19772        assert!(detail.contains("--start-from"), "{detail}");
19773    }
19774
19775    /// `status == Ready` used to be read as "this is the harmless
19776    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19777    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19778    /// run's PR was closed without merging — and reentering `merge` with
19779    /// `mode` still `Pr` does not know the difference, so it pushed and
19780    /// opened a second pull request. `mode == Local` reproduces the same
19781    /// blind spot without a network call: reentry must not attempt another
19782    /// git merge once this node has already recorded an outcome.
19783    #[tokio::test]
19784    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19785        let tmp = tempfile::tempdir().expect("tempdir");
19786        let repo = tmp.path().join("repo");
19787        std::fs::create_dir_all(&repo).unwrap();
19788        init_repo(&repo);
19789
19790        let mut config = Config::default();
19791        config.merge.mode = MergeMode::Local;
19792
19793        let mut state = RunState::new(
19794            repo.clone(),
19795            "main".to_owned(),
19796            "deadbeef".to_owned(),
19797            "task".to_owned(),
19798            config,
19799        );
19800        state.candidates = vec![Candidate {
19801            index: 0,
19802            label: 'A',
19803            agent: "alpha".to_owned(),
19804            branch: "does-not-exist".to_owned(),
19805            worktree: repo.clone(),
19806            summary: String::new(),
19807            stat: String::new(),
19808            files: 0,
19809            commits: 0,
19810            empty: false,
19811            failed: None,
19812            verified_noop: None,
19813            duration_ms: 0,
19814            folded: false,
19815        }];
19816        state.tally = Some(Tally {
19817            first_choice: BTreeMap::from([('A', 1)]),
19818            borda: BTreeMap::new(),
19819            winner: 'A',
19820            rankings: 1,
19821            unanimous_initial: true,
19822            deliberated: false,
19823            changed_votes: 0,
19824            unanimous_final: true,
19825            tie_break: None,
19826            judges: 0,
19827            present: 0,
19828            quorum: 0,
19829            met_quorum: true,
19830            uncontested: Some("only candidate A produced a change".to_owned()),
19831        });
19832        state.reviews = vec![ReviewRound {
19833            round: 1,
19834            head: "deadbeef".to_owned(),
19835            verified_head: None,
19836            verified_at: None,
19837            reviews: Vec::new(),
19838            e2e: Vec::new(),
19839            fix: None,
19840            blocking: 0,
19841            answered: 0,
19842            expected: 0,
19843            clean: true,
19844            verify_retried: false,
19845            e2e_deferred: false,
19846            e2e_defer_reason: None,
19847            progressed: false,
19848            vote_split: false,
19849            reconsideration: Vec::new(),
19850            verdict: None,
19851        }];
19852        state.gate = vec![CommandOutcome {
19853            command: "test".to_owned(),
19854            code: Some(0),
19855            output_tail: String::new(),
19856            duration_ms: 0,
19857            resource_blocked: false,
19858        }];
19859        state.gate_ran = true;
19860        // Reached its conclusion already — e.g. `land` closing the PR without
19861        // merging it, which (like the honest `MergeMode::None` path) leaves
19862        // `status` at `Ready`. The recorded outcome is what actually marks
19863        // this node done.
19864        state.status = RunStatus::Ready;
19865        state.merge = Some(MergeOutcome {
19866            mode: MergeMode::Local,
19867            ok: false,
19868            detail: "already concluded".to_owned(),
19869            empty: false,
19870        });
19871
19872        let mut runner = Runner {
19873            state,
19874            roles: ResolvedRoles {
19875                implementers: Vec::new(),
19876                judges: Vec::new(),
19877                reviewers: Vec::new(),
19878                fixer: None,
19879                conductor: conductor(),
19880                implementer_roster: Vec::new(),
19881                judge_roster: Vec::new(),
19882                reviewer_roster: Vec::new(),
19883            },
19884            sem: Arc::new(Semaphore::new(1)),
19885            pause: Pause::new(),
19886            interrupt: Pause::new(),
19887        };
19888
19889        runner.merge().await.expect("merge");
19890
19891        assert_eq!(
19892            runner.state.status,
19893            RunStatus::Ready,
19894            "a concluded run's status must not change on reentry"
19895        );
19896        assert_eq!(
19897            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19898            Some("already concluded"),
19899            "merge must not run again once the node already recorded an outcome"
19900        );
19901    }
19902
19903    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19904    /// when its last attempt was resource-blocked (the shared build cache
19905    /// could not be acquired or confirmed fresh in time - see
19906    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19907    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19908    /// a run merge a tree the gate never actually checked - exactly the case
19909    /// a contended cache produces on every retry until it clears. `merge`
19910    /// must refuse until `gate` has actually recorded an attempt.
19911    #[tokio::test]
19912    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19913        let tmp = tempfile::tempdir().expect("tempdir");
19914        let repo = tmp.path().join("repo");
19915        std::fs::create_dir_all(&repo).unwrap();
19916        init_repo(&repo);
19917
19918        let mut config = Config::default();
19919        config.merge.mode = MergeMode::Local;
19920
19921        let mut state = RunState::new(
19922            repo.clone(),
19923            "main".to_owned(),
19924            "deadbeef".to_owned(),
19925            "task".to_owned(),
19926            config,
19927        );
19928        state.candidates = vec![Candidate {
19929            index: 0,
19930            label: 'A',
19931            agent: "alpha".to_owned(),
19932            branch: "does-not-exist".to_owned(),
19933            worktree: repo.clone(),
19934            summary: String::new(),
19935            stat: String::new(),
19936            files: 0,
19937            commits: 0,
19938            empty: false,
19939            failed: None,
19940            verified_noop: None,
19941            duration_ms: 0,
19942            folded: false,
19943        }];
19944        state.tally = Some(Tally {
19945            first_choice: BTreeMap::from([('A', 1)]),
19946            borda: BTreeMap::new(),
19947            winner: 'A',
19948            rankings: 1,
19949            unanimous_initial: true,
19950            deliberated: false,
19951            changed_votes: 0,
19952            unanimous_final: true,
19953            tie_break: None,
19954            judges: 0,
19955            present: 0,
19956            quorum: 0,
19957            met_quorum: true,
19958            uncontested: Some("only candidate A produced a change".to_owned()),
19959        });
19960        state.reviews = vec![ReviewRound {
19961            round: 1,
19962            head: "deadbeef".to_owned(),
19963            verified_head: None,
19964            verified_at: None,
19965            reviews: Vec::new(),
19966            e2e: Vec::new(),
19967            fix: None,
19968            blocking: 0,
19969            answered: 0,
19970            expected: 0,
19971            clean: true,
19972            verify_retried: false,
19973            e2e_deferred: false,
19974            e2e_defer_reason: None,
19975            progressed: false,
19976            vote_split: false,
19977            reconsideration: Vec::new(),
19978            verdict: None,
19979        }];
19980        // The point: `gate` has not recorded anything yet.
19981        state.gate = Vec::new();
19982        state.gate_ran = false;
19983        state.status = RunStatus::Gating;
19984
19985        let mut runner = Runner {
19986            state,
19987            roles: ResolvedRoles {
19988                implementers: Vec::new(),
19989                judges: Vec::new(),
19990                reviewers: Vec::new(),
19991                fixer: None,
19992                conductor: conductor(),
19993                implementer_roster: Vec::new(),
19994                judge_roster: Vec::new(),
19995                reviewer_roster: Vec::new(),
19996            },
19997            sem: Arc::new(Semaphore::new(1)),
19998            pause: Pause::new(),
19999            interrupt: Pause::new(),
20000        };
20001
20002        runner.merge().await.expect("merge");
20003
20004        assert!(
20005            runner.state.merge.is_none(),
20006            "an empty gate must never be read as a passing one: {:?}",
20007            runner.state.merge
20008        );
20009    }
20010
20011    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
20012    /// commands configured and `merge.mode` is `none` (a review-only run).
20013    /// `gate` must still record a real attempt — zero commands, vacuously
20014    /// passed — rather than leaving `state.gate` empty in a way `merge`
20015    /// cannot tell apart from "never ran"; otherwise the run reaches
20016    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
20017    #[tokio::test]
20018    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20019        let tmp = tempfile::tempdir().expect("tempdir");
20020        let repo = tmp.path().join("repo");
20021        std::fs::create_dir_all(&repo).unwrap();
20022        init_repo(&repo);
20023
20024        // Default config: `verify.gate` empty, `merge.mode` is `none`.
20025        let config = Config::default();
20026
20027        let mut state = RunState::new(
20028            repo.clone(),
20029            "main".to_owned(),
20030            "deadbeef".to_owned(),
20031            "task".to_owned(),
20032            config,
20033        );
20034        state.candidates = vec![Candidate {
20035            index: 0,
20036            label: 'A',
20037            agent: "alpha".to_owned(),
20038            branch: "does-not-exist".to_owned(),
20039            worktree: repo.clone(),
20040            summary: String::new(),
20041            stat: String::new(),
20042            files: 0,
20043            commits: 0,
20044            empty: false,
20045            failed: None,
20046            verified_noop: None,
20047            duration_ms: 0,
20048            folded: false,
20049        }];
20050        state.tally = Some(Tally {
20051            first_choice: BTreeMap::from([('A', 1)]),
20052            borda: BTreeMap::new(),
20053            winner: 'A',
20054            rankings: 1,
20055            unanimous_initial: true,
20056            deliberated: false,
20057            changed_votes: 0,
20058            unanimous_final: true,
20059            tie_break: None,
20060            judges: 0,
20061            present: 0,
20062            quorum: 0,
20063            met_quorum: true,
20064            uncontested: Some("only candidate A produced a change".to_owned()),
20065        });
20066        state.reviews = vec![ReviewRound {
20067            round: 1,
20068            head: "deadbeef".to_owned(),
20069            verified_head: None,
20070            verified_at: None,
20071            reviews: Vec::new(),
20072            e2e: Vec::new(),
20073            fix: None,
20074            blocking: 0,
20075            answered: 0,
20076            expected: 0,
20077            clean: true,
20078            verify_retried: false,
20079            e2e_deferred: false,
20080            e2e_defer_reason: None,
20081            progressed: false,
20082            vote_split: false,
20083            reconsideration: Vec::new(),
20084            verdict: None,
20085        }];
20086
20087        let mut runner = Runner {
20088            state,
20089            roles: ResolvedRoles {
20090                implementers: Vec::new(),
20091                judges: Vec::new(),
20092                reviewers: Vec::new(),
20093                fixer: None,
20094                conductor: conductor(),
20095                implementer_roster: Vec::new(),
20096                judge_roster: Vec::new(),
20097                reviewer_roster: Vec::new(),
20098            },
20099            sem: Arc::new(Semaphore::new(1)),
20100            pause: Pause::new(),
20101            interrupt: Pause::new(),
20102        };
20103
20104        runner.gate().await.expect("gate");
20105        assert!(
20106            runner.state.gate_ran,
20107            "zero configured commands is still a real attempt, not an unrun gate"
20108        );
20109        assert!(runner.state.gate.is_empty());
20110        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20111        assert_ne!(
20112            runner.state.status,
20113            RunStatus::Blocked,
20114            "a gate with nothing to check must not read as failed"
20115        );
20116
20117        runner.merge().await.expect("merge");
20118        assert_eq!(
20119            runner.state.status,
20120            RunStatus::Ready,
20121            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20122        );
20123    }
20124
20125    /// `Config::cache_dir` is derived from `verify.e2e` as well as
20126    /// `verify.gate` (so the e2e leg and the final gate never build against
20127    /// different directories). With zero `verify.gate` commands but a
20128    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
20129    /// that lease before discovering it had nothing to run - so a repo with
20130    /// no gate commands could come back `resource_blocked` (and therefore
20131    /// still `gate_ran == false`) on nothing but an unrelated run holding the
20132    /// cache, exactly the contention this run's own zero commands could
20133    /// never have touched. `gate` must recognise there is nothing to check
20134    /// before it ever asks for the lease.
20135    #[tokio::test]
20136    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20137        crate::run::pin_test_home();
20138        let home = crate::run::home();
20139
20140        let tmp = tempfile::tempdir().expect("tempdir");
20141        let repo = tmp.path().join("repo");
20142        std::fs::create_dir_all(&repo).unwrap();
20143        init_repo(&repo);
20144        // Unique to this test, so holding its lease cannot collide with
20145        // another test sharing the same process-wide `home`.
20146        let cache_dir = tmp.path().join("target");
20147
20148        let mut config = Config::default();
20149        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20150        // `verify.gate` stays empty (the default). Any accidental cache
20151        // acquisition must report contention immediately, regardless of how
20152        // long unrelated Git subprocesses take under parallel test load.
20153        config.graph.timeout_verify = Some(2);
20154        config.graph.timeout_resource_wait = 0;
20155
20156        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20157        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20158            .expect("no io error acquiring directly")
20159        {
20160            crate::cache::AcquireOutcome::Acquired(g) => g,
20161            crate::cache::AcquireOutcome::Busy(b) => {
20162                panic!("expected the direct acquire to win the lease first: {b:?}")
20163            }
20164        };
20165
20166        let mut state = RunState::new(
20167            repo.clone(),
20168            "main".to_owned(),
20169            "deadbeef".to_owned(),
20170            "task".to_owned(),
20171            config,
20172        );
20173        state.candidates = vec![Candidate {
20174            index: 0,
20175            label: 'A',
20176            agent: "alpha".to_owned(),
20177            branch: "does-not-exist".to_owned(),
20178            worktree: repo.clone(),
20179            summary: String::new(),
20180            stat: String::new(),
20181            files: 0,
20182            commits: 0,
20183            empty: false,
20184            failed: None,
20185            verified_noop: None,
20186            duration_ms: 0,
20187            folded: false,
20188        }];
20189        state.tally = Some(Tally {
20190            first_choice: BTreeMap::from([('A', 1)]),
20191            borda: BTreeMap::new(),
20192            winner: 'A',
20193            rankings: 1,
20194            unanimous_initial: true,
20195            deliberated: false,
20196            changed_votes: 0,
20197            unanimous_final: true,
20198            tie_break: None,
20199            judges: 0,
20200            present: 0,
20201            quorum: 0,
20202            met_quorum: true,
20203            uncontested: Some("only candidate A produced a change".to_owned()),
20204        });
20205        state.reviews = vec![ReviewRound {
20206            round: 1,
20207            head: "deadbeef".to_owned(),
20208            verified_head: None,
20209            verified_at: None,
20210            reviews: Vec::new(),
20211            e2e: Vec::new(),
20212            fix: None,
20213            blocking: 0,
20214            answered: 0,
20215            expected: 0,
20216            clean: true,
20217            verify_retried: false,
20218            e2e_deferred: false,
20219            e2e_defer_reason: None,
20220            progressed: false,
20221            vote_split: false,
20222            reconsideration: Vec::new(),
20223            verdict: None,
20224        }];
20225
20226        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20227        state.reviews[0].e2e = vec![CommandOutcome {
20228            command: state.config.verify.e2e[0].clone(),
20229            code: Some(0),
20230            output_tail: String::new(),
20231            duration_ms: 0,
20232            resource_blocked: false,
20233        }];
20234        let mut runner = Runner {
20235            state,
20236            roles: ResolvedRoles {
20237                implementers: Vec::new(),
20238                judges: Vec::new(),
20239                reviewers: Vec::new(),
20240                fixer: None,
20241                conductor: conductor(),
20242                implementer_roster: Vec::new(),
20243                judge_roster: Vec::new(),
20244                reviewer_roster: Vec::new(),
20245            },
20246            sem: Arc::new(Semaphore::new(1)),
20247            pause: Pause::new(),
20248            interrupt: Pause::new(),
20249        };
20250
20251        runner.gate().await.expect("gate");
20252        assert!(runner.state.resource_block.is_none());
20253        assert!(
20254            !runner
20255                .state
20256                .events
20257                .iter()
20258                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20259            "a gate with nothing to run must never request the held cache lease"
20260        );
20261        assert!(
20262            runner.state.gate_ran,
20263            "zero commands is still a real, immediate attempt"
20264        );
20265        assert!(runner.state.gate.is_empty());
20266        assert_ne!(
20267            runner.state.status,
20268            RunStatus::Blocked,
20269            "must not read as resource-blocked on a lease it never asked for"
20270        );
20271    }
20272
20273    /// A separate process carries a live worker's environment throughout all
20274    /// verification entry points; the parallel harness never changes its env.
20275    #[cfg(unix)]
20276    #[tokio::test]
20277    async fn verification_is_standalone_inside_a_live_worker() {
20278        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20279        if std::env::var_os(CHILD).is_none() {
20280            let home = tempfile::tempdir().unwrap();
20281            let generation = crate::worker::pin_current(home.path()).unwrap();
20282            let output = std::process::Command::new(std::env::current_exe().unwrap())
20283                .args([
20284                    "--exact",
20285                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20286                    "--nocapture",
20287                ])
20288                .env(CHILD, "1")
20289                .env("MAGI_HOME", home.path())
20290                .env("MAGI_WORKER_HOME", home.path())
20291                .env("MAGI_WORKER_ID", "abcdef")
20292                .env("MAGI_WORKER_GENERATION", &generation.id)
20293                .env("MAGI_WORKER_EXE", &generation.executable)
20294                .env("MAGI_RUN", "verification-parent")
20295                .env("MAGI_NODE", "gate")
20296                .env("MAGI_SEAT", "worker-fixture")
20297                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20298                .env("MAGI_NO_AUTOUPDATE", "0")
20299                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20300                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20301                .env(
20302                    "VERIFY_EXPECT_TOOLCHAIN",
20303                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20304                )
20305                .env(
20306                    "VERIFY_EXPECT_CACHE",
20307                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20308                )
20309                .output()
20310                .unwrap();
20311            assert!(
20312                output.status.success()
20313                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20314                "{}\n{}",
20315                String::from_utf8_lossy(&output.stdout),
20316                String::from_utf8_lossy(&output.stderr)
20317            );
20318            return;
20319        }
20320        let home = crate::run::home();
20321        let directory = home.join("workers/jobs/abcdef");
20322        std::fs::create_dir_all(&directory).unwrap();
20323        let job = crate::worker::Job {
20324            id: "abcdef".into(),
20325            sequence: 1,
20326            kind: "task".into(),
20327            key: "task:verification".into(),
20328            payload: serde_json::json!({}),
20329            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20330            version: Some(env!("CARGO_PKG_VERSION").into()),
20331            status: crate::worker::Status::Running,
20332            result: None,
20333            pid: Some(std::process::id()),
20334            created_at: 0,
20335            finished_at: None,
20336        };
20337        std::fs::write(
20338            directory.join("job.json"),
20339            serde_json::to_vec(&job).unwrap(),
20340        )
20341        .unwrap();
20342        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20343        let assert_worker = || {
20344            assert_eq!(
20345                crate::worker::current_job(&home).unwrap().unwrap().id,
20346                job.id
20347            );
20348            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20349            assert_eq!(
20350                crate::worker::helper_executable(&home).unwrap(),
20351                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20352            );
20353            assert!(
20354                crate::updater::Checker::new(&crate::config::Update {
20355                    mode: crate::config::UpdateMode::Notify,
20356                    interval: None,
20357                    auto_restart: false,
20358                })
20359                .is_none(),
20360                "the driver remains pinned, even while tests are standalone"
20361            );
20362        };
20363        assert_worker();
20364        let repo = home.join("repo");
20365        std::fs::create_dir_all(&repo).unwrap();
20366        init_repo(&repo);
20367        // Keep storage, seat/control context, containment and tool lookup.
20368        // No inherited values (or credentials) are printed on success.
20369        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20370            test -z "${MAGI_WORKER_GENERATION+x}" &&
20371            test -z "${MAGI_WORKER_HOME+x}" &&
20372            test -z "${MAGI_WORKER_EXE+x}" &&
20373            test "$MAGI_HOME/repo" -ef . &&
20374            test "$MAGI_RUN" = verification-parent &&
20375            test "$MAGI_NODE" = gate &&
20376            test "$MAGI_SEAT" = worker-fixture &&
20377            test -n "$MAGI_PROCESS_OWNER" &&
20378            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20379            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20380            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20381            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20382        // Negative control: an ordinary child inherits the contamination.
20383        assert!(
20384            !std::process::Command::new("sh")
20385                .args(["-c", probe])
20386                .current_dir(&repo)
20387                .status()
20388                .unwrap()
20389                .success()
20390        );
20391        let unrelated_home = tempfile::tempdir().unwrap();
20392        assert!(
20393            crate::worker::current_job(unrelated_home.path()).is_err(),
20394            "an inherited worker id cannot be resolved in a unit fixture's home"
20395        );
20396        let mut runner = runner_at(RunStatus::Gating);
20397        runner.state.repo = repo.clone();
20398        runner.state.config.disk.min_free_bytes = 0;
20399        runner.state.config.coordination = crate::config::test_coordination();
20400        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20401        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20402        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20403        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20404        let winner = Candidate {
20405            index: 0,
20406            label: 'A',
20407            agent: "fixture".into(),
20408            branch: "main".into(),
20409            worktree: repo.clone(),
20410            summary: String::new(),
20411            stat: String::new(),
20412            files: 0,
20413            commits: 0,
20414            empty: false,
20415            failed: None,
20416            verified_noop: None,
20417            duration_ms: 0,
20418            folded: false,
20419        };
20420        let (e2e, retried, pids) = run_e2e_with_retry(
20421            &mut runner.state,
20422            &["sh".into(), "-c".into()],
20423            &[format!("{probe} && printf initial")],
20424            &repo,
20425            Duration::from_secs(10),
20426            "initial",
20427        )
20428        .await;
20429        assert!(!retried && pids.is_empty());
20430        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20431        assert_eq!(e2e[0].output_tail, "initial");
20432        let gate = runner.run_gate(&winner).await.unwrap();
20433        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20434        // An interrupted repair must recheck e2e from saved state. Landing
20435        // reuses that state and must run both pre-gate and gate again.
20436        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20437            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20438            passed: false,
20439            outcomes: Vec::new(),
20440        });
20441        runner.state.save().unwrap();
20442        let mut resumed = RunState::load(&runner.state.id).unwrap();
20443        let wait = crate::repo_lock::WaitPolicy::default();
20444        assert!(matches!(
20445            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20446                .await
20447                .unwrap(),
20448            GateFix::Retry
20449        ));
20450        let saved = RunState::load(&resumed.id).unwrap();
20451        let repair = saved.gate_repair_e2e.unwrap();
20452        assert!(repair.passed);
20453        assert_eq!(repair.outcomes.len(), 1);
20454        assert!(repair.outcomes[0].ok());
20455        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20456            .await
20457            .unwrap();
20458        assert_eq!(landing.len(), 1);
20459        assert!(landing[0].ok(), "{landing:?}");
20460        assert_eq!(
20461            std::fs::read_to_string(repo.join("phases")).unwrap(),
20462            "pre\ngate\ne2e\npre\ngate\n"
20463        );
20464        assert_worker();
20465    }
20466
20467    #[cfg(unix)]
20468    #[tokio::test]
20469    async fn verification_keeps_the_panic_and_the_full_output() {
20470        crate::run::pin_test_home();
20471        let tmp = tempfile::tempdir().unwrap();
20472        let mut config = Config::default();
20473        config.disk.min_free_bytes = 0;
20474        let mut state = RunState::new(
20475            tmp.path().to_owned(),
20476            "main".into(),
20477            "base".into(),
20478            "verification evidence".into(),
20479            config,
20480        );
20481        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20482                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20483                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20484        let mut names = Vec::new();
20485        for _ in 0..2 {
20486            let (outcomes, _) = run_commands(
20487                &mut state,
20488                "gate",
20489                "gate",
20490                0,
20491                &["sh".into(), "-c".into()],
20492                &[script.into()],
20493                tmp.path(),
20494                Duration::from_secs(60),
20495            )
20496            .await;
20497            let tail = &outcomes[0].output_tail;
20498            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20499            assert!(tail.contains("boom assertion"));
20500            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20501            let line = tail.lines().next().unwrap();
20502            names.push(line.to_owned());
20503        }
20504        assert_ne!(names[0], names[1], "the second run must not overwrite");
20505        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20506        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20507        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20508
20509        // Storage failures are evidence too; preserve the command's verdict.
20510        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20511        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20512        let (outcomes, _) = run_commands(
20513            &mut state,
20514            "gate",
20515            "gate",
20516            0,
20517            &["sh".into(), "-c".into()],
20518            &["echo failure-output; exit 7".into()],
20519            tmp.path(),
20520            Duration::from_secs(10),
20521        )
20522        .await;
20523        assert_eq!(outcomes[0].code, Some(7));
20524        assert!(state.events.iter().any(|e| {
20525            e.node == "gate" && e.message.contains("could not save the full gate output:")
20526        }));
20527    }
20528
20529    #[cfg(unix)]
20530    #[tokio::test]
20531    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20532        crate::run::pin_test_home();
20533        let tmp = tempfile::tempdir().unwrap();
20534        let mut config = Config::default();
20535        config.disk.min_free_bytes = 0;
20536        let mut state = RunState::new(
20537            tmp.path().to_owned(),
20538            "main".into(),
20539            "base".into(),
20540            "verification evidence".into(),
20541            config,
20542        );
20543        let (outcomes, pids) = run_commands(
20544            &mut state,
20545            "gate",
20546            "gate",
20547            1,
20548            &["sh".into(), "-c".into()],
20549            &[
20550                "printf before-timeout; exec sleep 30".into(),
20551                "kill -TERM $$".into(),
20552                "exit 7".into(),
20553                "printf success".into(),
20554            ],
20555            tmp.path(),
20556            Duration::from_secs(1),
20557        )
20558        .await;
20559        wait_for_timed_out_children_to_die(&pids).await;
20560        assert_eq!(
20561            pids.len(),
20562            1,
20563            "unleased timeout retains its death-check PID"
20564        );
20565        assert_eq!(outcomes.len(), 4);
20566        assert_eq!(outcomes[0].code, None);
20567        assert!(outcomes[0].output_tail.contains("before-timeout"));
20568        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20569        assert_eq!(outcomes[1].code, None);
20570        assert!(outcomes[1].output_tail.contains("command terminated:"));
20571        assert!(!outcomes[1].output_tail.contains("timed out"));
20572        assert_eq!(outcomes[2].code, Some(7));
20573        assert!(!outcomes[2].output_tail.contains("timed out"));
20574        assert_eq!(outcomes[3].code, Some(0));
20575        assert_eq!(outcomes[3].output_tail, "success");
20576        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20577    }
20578
20579    /// The addendum's second gap: a `verify.gate` command running for real
20580    /// wall-clock time had nothing at all to show for it in `active` before
20581    /// `run_commands` learned to record it — a run could sit in `Gating` for
20582    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20583    /// what was actually happening. Proven with a genuinely still-running
20584    /// command, not just a before/after check on the final state: a poller
20585    /// task reads the same `run.json` `gate()` is writing, the same way the
20586    /// phone or `magi show` would, while the shell command is still blocked
20587    /// on its own release marker.
20588    #[tokio::test]
20589    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20590        crate::run::pin_test_home();
20591
20592        let tmp = tempfile::tempdir().expect("tempdir");
20593        let repo = tmp.path().join("repo");
20594        std::fs::create_dir_all(&repo).unwrap();
20595        init_repo(&repo);
20596
20597        let mut config = Config::default();
20598        config.disk.min_free_bytes = 0;
20599        config.verify.gate = vec![
20600            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20601             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20602                .to_owned(),
20603        ];
20604
20605        let mut state = RunState::new(
20606            repo.clone(),
20607            "main".to_owned(),
20608            "deadbeef".to_owned(),
20609            "task".to_owned(),
20610            config,
20611        );
20612        let run_id = state.id.clone();
20613        state.candidates = vec![Candidate {
20614            index: 0,
20615            label: 'A',
20616            agent: "alpha".to_owned(),
20617            branch: "does-not-exist".to_owned(),
20618            worktree: repo.clone(),
20619            summary: String::new(),
20620            stat: String::new(),
20621            files: 0,
20622            commits: 0,
20623            empty: false,
20624            failed: None,
20625            verified_noop: None,
20626            duration_ms: 0,
20627            folded: false,
20628        }];
20629        state.tally = Some(Tally {
20630            first_choice: BTreeMap::from([('A', 1)]),
20631            borda: BTreeMap::new(),
20632            winner: 'A',
20633            rankings: 1,
20634            unanimous_initial: true,
20635            deliberated: false,
20636            changed_votes: 0,
20637            unanimous_final: true,
20638            tie_break: None,
20639            judges: 0,
20640            present: 0,
20641            quorum: 0,
20642            met_quorum: true,
20643            uncontested: Some("only candidate A produced a change".to_owned()),
20644        });
20645        state.reviews = vec![ReviewRound {
20646            round: 1,
20647            head: "deadbeef".to_owned(),
20648            verified_head: None,
20649            verified_at: None,
20650            reviews: Vec::new(),
20651            e2e: Vec::new(),
20652            fix: None,
20653            blocking: 0,
20654            answered: 0,
20655            expected: 0,
20656            clean: true,
20657            verify_retried: false,
20658            e2e_deferred: false,
20659            e2e_defer_reason: None,
20660            progressed: false,
20661            vote_split: false,
20662            reconsideration: Vec::new(),
20663            verdict: None,
20664        }];
20665
20666        let mut runner = Runner {
20667            state,
20668            roles: ResolvedRoles {
20669                implementers: Vec::new(),
20670                judges: Vec::new(),
20671                reviewers: Vec::new(),
20672                fixer: None,
20673                conductor: conductor(),
20674                implementer_roster: Vec::new(),
20675                judge_roster: Vec::new(),
20676                reviewer_roster: Vec::new(),
20677            },
20678            sem: Arc::new(Semaphore::new(1)),
20679            pause: Pause::new(),
20680            interrupt: Pause::new(),
20681        };
20682
20683        let started_marker = repo.join("started.marker");
20684        let release_marker = repo.join("release.marker");
20685        let poller = tokio::spawn(async move {
20686            // Bounded so a regression that never records the task entry
20687            // fails this test in seconds instead of hanging the suite —
20688            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20689            // does_not_cut_it_short` uses for the same reason.
20690            for _ in 0..100 {
20691                if started_marker.exists()
20692                    && let Ok(s) = crate::run::RunState::load(&run_id)
20693                    && let Some(a) = s.active.get("gate")
20694                {
20695                    std::fs::write(&release_marker, b"go").expect("release marker");
20696                    return Some(a.clone());
20697                }
20698                tokio::time::sleep(Duration::from_millis(50)).await;
20699            }
20700            None
20701        });
20702
20703        runner.gate().await.expect("gate");
20704        let captured = poller.await.expect("poller task");
20705        let captured = captured.expect(
20706            "the poller never saw a `gate` task entry in run.json while the command was \
20707             still blocked on its own release marker",
20708        );
20709
20710        assert_eq!(captured.task.as_deref(), Some("gate"));
20711        assert_eq!(captured.node, "gate");
20712        assert_eq!(captured.index, Some(1));
20713        assert_eq!(captured.total, Some(1));
20714        assert!(
20715            captured
20716                .command
20717                .as_deref()
20718                .is_some_and(|c| c.contains("started.marker")),
20719            "{captured:?}"
20720        );
20721
20722        assert!(
20723            runner.state.active.is_empty(),
20724            "the entry must be cleared once the command actually finished: {:?}",
20725            runner.state.active
20726        );
20727        assert!(runner.state.gate_ran);
20728        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20729    }
20730
20731    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20732    /// record for `land`; one with only a Minor, or no reject, leaves none.
20733    #[tokio::test]
20734    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20735        use crate::verdict::{Finding, ReviewVote, Severity};
20736        crate::run::pin_test_home();
20737        let tmp = tempfile::tempdir().expect("tempdir");
20738        let repo = tmp.path().join("repo");
20739        std::fs::create_dir_all(&repo).unwrap();
20740        init_repo(&repo);
20741
20742        for (severity, vote, expect) in [
20743            (Severity::Major, ReviewVote::Reject, true),
20744            (Severity::Minor, ReviewVote::Reject, false),
20745            (Severity::Major, ReviewVote::Approve, false),
20746        ] {
20747            let mut round = review_round(false, 1, 1, 1, false, true);
20748            round.reviews = vec![ReviewRecord {
20749                reviewer: 1,
20750                agent: "alpha".to_owned(),
20751                summary: String::new(),
20752                findings: vec![Finding {
20753                    id: "R1-1-1".to_owned(),
20754                    severity,
20755                    file: None,
20756                    line: None,
20757                    title: "t".to_owned(),
20758                    detail: String::new(),
20759                }],
20760                vote: Some(vote),
20761                failed: None,
20762                duration_ms: 0,
20763                attempts: 0,
20764            }];
20765            let mut state = RunState::new(
20766                repo.clone(),
20767                "main".to_owned(),
20768                "deadbeef".to_owned(),
20769                "task".to_owned(),
20770                Config::default(),
20771            );
20772            state.reviews = vec![round];
20773            let mut runner = Runner {
20774                state,
20775                roles: ResolvedRoles {
20776                    implementers: Vec::new(),
20777                    judges: Vec::new(),
20778                    reviewers: Vec::new(),
20779                    fixer: None,
20780                    conductor: conductor(),
20781                    implementer_roster: Vec::new(),
20782                    judge_roster: Vec::new(),
20783                    reviewer_roster: Vec::new(),
20784                },
20785                sem: Arc::new(Semaphore::new(1)),
20786                pause: Pause::new(),
20787                interrupt: Pause::new(),
20788            };
20789            let shell = runner.state.config.shell();
20790            runner
20791                .stop_reviewing("round budget spent", &shell, &repo)
20792                .await
20793                .expect("stop_reviewing");
20794            assert_eq!(runner.state.status, RunStatus::Gating);
20795            assert_eq!(
20796                runner.state.contested_handoff.is_some(),
20797                expect,
20798                "{severity:?} + {vote:?}"
20799            );
20800        }
20801    }
20802
20803    /// The shape the incident this whole fix responds to actually had: the
20804    /// round budget spent, the last round's own e2e blocked on the shared
20805    /// build cache (held here by a live pid — this test process — exactly
20806    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20807    /// without forking a process). `stop_reviewing` must retry it — not
20808    /// silently leave the round looking untouched (the catch-up-only half of
20809    /// the bug), and not read the contention as a red `e2e` and block the
20810    /// run on it (the other half). Called directly, the same way
20811    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20812    /// above exercises `gate`, so this never needs a real cargo build to
20813    /// reach: the lease is never released, so `with_cache_lease` never gets
20814    /// past acquiring it into anything that would need a real workspace.
20815    #[tokio::test]
20816    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20817        crate::run::pin_test_home();
20818        let home = crate::run::home();
20819
20820        let tmp = tempfile::tempdir().expect("tempdir");
20821        let repo = tmp.path().join("repo");
20822        std::fs::create_dir_all(&repo).unwrap();
20823        init_repo(&repo);
20824        let head = crate::git::rev_parse(&repo, "HEAD")
20825            .await
20826            .expect("rev-parse");
20827        // Unique to this test, so holding its lease cannot collide with
20828        // another test sharing the same process-wide `home`.
20829        let cache_dir = tmp.path().join("target");
20830
20831        let mut config = Config::default();
20832        config.verify.e2e = vec![format!(
20833            "CARGO_TARGET_DIR='{}' test -f README.md",
20834            cache_dir.display()
20835        )];
20836        config.graph.review_rounds = 1;
20837        // Bounded so a regression that does start waiting fails the test in
20838        // seconds, not hangs it.
20839        config.graph.timeout_verify = Some(2);
20840        config.graph.timeout_resource_wait = 2;
20841
20842        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20843        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20844            .expect("no io error acquiring directly")
20845        {
20846            crate::cache::AcquireOutcome::Acquired(g) => g,
20847            crate::cache::AcquireOutcome::Busy(b) => {
20848                panic!("expected the direct acquire to win the lease first: {b:?}")
20849            }
20850        };
20851
20852        let mut state = RunState::new(
20853            repo.clone(),
20854            "main".to_owned(),
20855            head.clone(),
20856            "task".to_owned(),
20857            config,
20858        );
20859        state.candidates = vec![Candidate {
20860            index: 0,
20861            label: 'A',
20862            agent: "alpha".to_owned(),
20863            branch: "does-not-exist".to_owned(),
20864            worktree: repo.clone(),
20865            summary: String::new(),
20866            stat: String::new(),
20867            files: 0,
20868            commits: 0,
20869            empty: false,
20870            failed: None,
20871            verified_noop: None,
20872            duration_ms: 0,
20873            folded: false,
20874        }];
20875        state.tally = Some(Tally {
20876            first_choice: BTreeMap::from([('A', 1)]),
20877            borda: BTreeMap::new(),
20878            winner: 'A',
20879            rankings: 1,
20880            unanimous_initial: true,
20881            deliberated: false,
20882            changed_votes: 0,
20883            unanimous_final: true,
20884            tie_break: None,
20885            judges: 0,
20886            present: 0,
20887            quorum: 0,
20888            met_quorum: true,
20889            uncontested: Some("only candidate A produced a change".to_owned()),
20890        });
20891        // The round budget's last round, deferred: `needs_catchup_run`'s
20892        // other trigger. `stop_reviewing`'s retry machinery must treat this
20893        // exactly like a resource-blocked attempt once it actually runs.
20894        state.reviews = vec![ReviewRound {
20895            round: 1,
20896            head: head.clone(),
20897            verified_head: None,
20898            verified_at: None,
20899            reviews: Vec::new(),
20900            e2e: Vec::new(),
20901            fix: None,
20902            blocking: 1,
20903            answered: 1,
20904            expected: 1,
20905            clean: false,
20906            verify_retried: false,
20907            e2e_deferred: true,
20908            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20909            progressed: false,
20910            vote_split: false,
20911            reconsideration: Vec::new(),
20912            verdict: None,
20913        }];
20914
20915        let mut runner = Runner {
20916            state,
20917            roles: ResolvedRoles {
20918                implementers: Vec::new(),
20919                judges: Vec::new(),
20920                reviewers: Vec::new(),
20921                fixer: None,
20922                conductor: conductor(),
20923                implementer_roster: Vec::new(),
20924                judge_roster: Vec::new(),
20925                reviewer_roster: Vec::new(),
20926            },
20927            sem: Arc::new(Semaphore::new(1)),
20928            pause: Pause::new(),
20929            interrupt: Pause::new(),
20930        };
20931
20932        let shell = runner.state.config.shell();
20933        runner
20934            .stop_reviewing("round budget spent", &shell, &repo)
20935            .await
20936            .expect("stop_reviewing");
20937
20938        let last = runner.state.reviews.last().expect("round record");
20939        assert_eq!(
20940            last.e2e_status(),
20941            E2eStatus::ResourceBlocked,
20942            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20943             failed: {last:?}"
20944        );
20945        assert_eq!(
20946            last.verified_head.as_deref(),
20947            Some(head.as_str()),
20948            "which commit this attempt targeted is known even though nothing finished checking \
20949             it"
20950        );
20951        let first_attempt_at = last
20952            .verified_at
20953            .expect("when this attempt ran is known too");
20954        assert_ne!(
20955            runner.state.status,
20956            RunStatus::Blocked,
20957            "contention is evidence about the machine, not the patch — it must not settle the \
20958             run as blocked: {:?}",
20959            runner.state.status
20960        );
20961        assert!(
20962            !runner
20963                .state
20964                .events
20965                .iter()
20966                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20967            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20968            runner.state.events
20969        );
20970
20971        // The cache is still held: a later reentry must retry the same
20972        // round's verification again — not leave it looking exactly as
20973        // untouched as the first blocked attempt, which is indistinguishable
20974        // from never having tried again at all.
20975        runner
20976            .stop_reviewing("round budget spent", &shell, &repo)
20977            .await
20978            .expect("stop_reviewing retry");
20979        assert_eq!(
20980            runner.state.reviews.len(),
20981            1,
20982            "no new round was started: {:?}",
20983            runner.state.reviews
20984        );
20985        let last = runner.state.reviews.last().expect("round record");
20986        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20987        assert!(
20988            last.verified_at.expect("still known") > first_attempt_at,
20989            "a second reentry must be a fresh attempt, not a stale copy of the first"
20990        );
20991        assert_ne!(runner.state.status, RunStatus::Blocked);
20992
20993        held.release();
20994    }
20995
20996    #[cfg(unix)]
20997    #[tokio::test]
20998    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20999        ask_test_home();
21000        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21001            let tmp = tempfile::tempdir().unwrap();
21002            let repo = tmp.path().join("repo");
21003            std::fs::create_dir(&repo).unwrap();
21004            init_repo(&repo);
21005            let run_git = |args: &[&str]| {
21006                let out = std::process::Command::new("git")
21007                    .args(args)
21008                    .current_dir(&repo)
21009                    .output()
21010                    .unwrap();
21011                assert!(
21012                    out.status.success(),
21013                    "{args:?}: {}",
21014                    String::from_utf8_lossy(&out.stderr)
21015                );
21016            };
21017            let origin = tmp.path().join("origin.git");
21018            run_git(&["init", "--bare", origin.to_str().unwrap()]);
21019            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21020            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21021            run_git(&["add", "spec.rs"]);
21022            run_git(&["commit", "-m", "initial spec"]);
21023            run_git(&["push", "origin", "main"]);
21024            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21025            run_git(&["checkout", "-b", "fix/review"]);
21026            std::fs::write(
21027                repo.join("check.rs"),
21028                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21029            )
21030            .unwrap();
21031            run_git(&["add", "check.rs"]);
21032            run_git(&["commit", "-m", "use spec"]);
21033            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21034            run_git(&["checkout", "main"]);
21035            std::fs::write(
21036                repo.join("spec.rs"),
21037                "struct AgentSpec { max_turn_secs: u64 }\n",
21038            )
21039            .unwrap();
21040            run_git(&["add", "spec.rs"]);
21041            run_git(&["commit", "-m", "extend spec"]);
21042            run_git(&["push", "origin", "main"]);
21043            run_git(&["checkout", "fix/review"]);
21044
21045            let mut runner = runner_at(RunStatus::Reviewing);
21046            runner.state.repo = repo.clone();
21047            runner.state.base_commit = base;
21048            runner.state.pr_head = Some("fix/review".into());
21049            runner.state.config.graph.review_rounds = max_rounds;
21050            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21051            runner.state.config.graph.sessions = false;
21052            runner.state.config.disk.min_free_bytes = 0;
21053            runner.state.config.verify.e2e =
21054                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21055            runner.state.candidates = vec![Candidate {
21056                index: 0,
21057                label: 'A',
21058                agent: "fixture".into(),
21059                branch: "fix/review".into(),
21060                worktree: repo.clone(),
21061                summary: String::new(),
21062                stat: String::new(),
21063                files: 1,
21064                commits: 1,
21065                empty: false,
21066                failed: None,
21067                verified_noop: None,
21068                duration_ms: 0,
21069                folded: false,
21070            }];
21071            runner.state.tally = Some(Tally {
21072                first_choice: BTreeMap::new(),
21073                borda: BTreeMap::new(),
21074                winner: 'A',
21075                rankings: 0,
21076                unanimous_initial: true,
21077                deliberated: false,
21078                changed_votes: 0,
21079                unanimous_final: true,
21080                tie_break: None,
21081                judges: 0,
21082                present: 0,
21083                quorum: 0,
21084                met_quorum: true,
21085                uncontested: Some("review-only".into()),
21086            });
21087            let mut last = review_round(false, 0, 1, 1, false, false);
21088            last.head = reviewed.clone();
21089            last.e2e_deferred = true;
21090            last.e2e[0].resource_blocked = true;
21091            last.e2e[0].output_tail = "waiting for the shared build cache".into();
21092            if max_rounds == 1 {
21093                last.e2e.clear();
21094            }
21095            runner.state.reviews = vec![last];
21096            let calls = tmp.path().join("fix-calls");
21097            let mut fixer = spec("repair");
21098            fixer
21099                .env
21100                .insert("FIX_CALLS".into(), calls.display().to_string());
21101            fixer.command = vec!["sh".into(), "-c".into(), r#"
21102                echo fix >> "$FIX_CALLS"
21103                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21104                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21105                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21106                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21107            "#.into()];
21108            let mut reviewer = spec("reviewer");
21109            reviewer.command = vec![
21110                "sh".into(),
21111                "-c".into(),
21112                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21113                    .into(),
21114            ];
21115            runner.roles.fixer = Some(vec![fixer]);
21116            runner.roles.reviewers = vec![reviewer];
21117
21118            runner.sync_to_base().await.unwrap();
21119            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21120            assert_ne!(rebased, reviewed);
21121            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21122            if reentry {
21123                // Simulate a crash after catch-up evidence was saved but before
21124                // dispatching the fixer. No dedicated pending-repair field exists.
21125                let shell = runner.state.config.shell();
21126                runner
21127                    .stop_unresolved_review("deferred verification", &shell, &repo)
21128                    .await
21129                    .unwrap();
21130                assert!(runner.catchup_repairable());
21131                assert!(!calls.exists());
21132                runner.state = RunState::load(&runner.state.id).unwrap();
21133                assert!(runner.catchup_repairable());
21134            }
21135            runner.review_loop().await.unwrap();
21136            let first = &runner.state.reviews[0];
21137            assert_eq!(first.head, reviewed);
21138            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21139            assert!(
21140                first.verify_retried,
21141                "a reproduced build failure gets exactly one retry: {:?}",
21142                first.e2e
21143            );
21144            assert!(first.e2e[0].output_tail.contains("E0063"));
21145            assert!(!runner.catchup_repairable());
21146            if max_rounds == 2 {
21147                assert_eq!(
21148                    runner.state.status,
21149                    RunStatus::Gating,
21150                    "{}",
21151                    crate::report::run(&runner.state)
21152                );
21153                assert_eq!(runner.state.reviews.len(), 2);
21154                assert!(first.fix.as_ref().unwrap().committed);
21155                assert!(runner.state.reviews[1].clean);
21156                assert_eq!(
21157                    runner
21158                        .state
21159                        .reviews
21160                        .iter()
21161                        .filter(|r| r.fix.is_some())
21162                        .count(),
21163                    1
21164                );
21165                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21166                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21167            } else {
21168                assert_eq!(runner.state.status, RunStatus::Blocked);
21169                assert!(first.fix.is_none());
21170                assert!(!calls.exists());
21171                let recovery = crate::recovery::assess(
21172                    "prior",
21173                    Some(&runner.state),
21174                    &runner.landing_base(),
21175                    crate::recovery::Relation::Diverged,
21176                );
21177                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21178                let reasons = recovery.reasons.join("\n");
21179                assert!(
21180                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21181                    "{reasons}"
21182                );
21183                assert!(
21184                    reasons.contains("base sync to")
21185                        && reasons.contains("additional review is required"),
21186                    "{reasons}"
21187                );
21188            }
21189        }
21190    }
21191
21192    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21193    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21194    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21195    /// own top-of-function fast path (`review_conclusion`) correctly reads
21196    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21197    /// own `for` range is empty once the round budget is spent, so
21198    /// `review_loop` must retry the check itself rather than silently doing
21199    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21200    /// above exercises directly, but through `review_loop`'s own entry point
21201    /// this time, proving the wiring between the two rather than just the
21202    /// retry logic in isolation.
21203    #[tokio::test]
21204    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21205        crate::run::pin_test_home();
21206        let home = crate::run::home();
21207
21208        let tmp = tempfile::tempdir().expect("tempdir");
21209        let repo = tmp.path().join("repo");
21210        std::fs::create_dir_all(&repo).unwrap();
21211        init_repo(&repo);
21212        let head = crate::git::rev_parse(&repo, "HEAD")
21213            .await
21214            .expect("rev-parse");
21215        let cache_dir = tmp.path().join("target");
21216
21217        let mut config = Config::default();
21218        config.verify.e2e = vec![format!(
21219            "CARGO_TARGET_DIR='{}' test -f README.md",
21220            cache_dir.display()
21221        )];
21222        config.graph.review_rounds = 1;
21223        config.graph.timeout_verify = Some(2);
21224        config.graph.timeout_resource_wait = 2;
21225
21226        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21227        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21228            .expect("no io error acquiring directly")
21229        {
21230            crate::cache::AcquireOutcome::Acquired(g) => g,
21231            crate::cache::AcquireOutcome::Busy(b) => {
21232                panic!("expected the direct acquire to win the lease first: {b:?}")
21233            }
21234        };
21235
21236        let mut state = RunState::new(
21237            repo.clone(),
21238            "main".to_owned(),
21239            head.clone(),
21240            "task".to_owned(),
21241            config,
21242        );
21243        state.candidates = vec![Candidate {
21244            index: 0,
21245            label: 'A',
21246            agent: "alpha".to_owned(),
21247            branch: "does-not-exist".to_owned(),
21248            worktree: repo.clone(),
21249            summary: String::new(),
21250            stat: String::new(),
21251            files: 0,
21252            commits: 0,
21253            empty: false,
21254            failed: None,
21255            verified_noop: None,
21256            duration_ms: 0,
21257            folded: false,
21258        }];
21259        state.tally = Some(Tally {
21260            first_choice: BTreeMap::from([('A', 1)]),
21261            borda: BTreeMap::new(),
21262            winner: 'A',
21263            rankings: 1,
21264            unanimous_initial: true,
21265            deliberated: false,
21266            changed_votes: 0,
21267            unanimous_final: true,
21268            tie_break: None,
21269            judges: 0,
21270            present: 0,
21271            quorum: 0,
21272            met_quorum: true,
21273            uncontested: Some("only candidate A produced a change".to_owned()),
21274        });
21275        // The exact shape a prior process's `stop_reviewing` would have left
21276        // on disk: the round budget's last round, a real attempt already
21277        // made and already resource-blocked.
21278        state.reviews = vec![ReviewRound {
21279            round: 1,
21280            head: head.clone(),
21281            verified_head: Some(head.clone()),
21282            verified_at: Some(jiff::Timestamp::now()),
21283            reviews: Vec::new(),
21284            e2e: vec![CommandOutcome {
21285                command: format!(
21286                    "CARGO_TARGET_DIR='{}' test -f README.md",
21287                    cache_dir.display()
21288                ),
21289                code: None,
21290                output_tail: "waiting for the shared build cache".to_owned(),
21291                duration_ms: 0,
21292                resource_blocked: true,
21293            }],
21294            fix: None,
21295            blocking: 1,
21296            answered: 1,
21297            expected: 1,
21298            clean: false,
21299            verify_retried: false,
21300            e2e_deferred: false,
21301            e2e_defer_reason: None,
21302            progressed: false,
21303            vote_split: false,
21304            reconsideration: Vec::new(),
21305            verdict: None,
21306        }];
21307
21308        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21309        let mut runner = Runner {
21310            state,
21311            roles: ResolvedRoles {
21312                implementers: Vec::new(),
21313                judges: Vec::new(),
21314                reviewers: Vec::new(),
21315                fixer: None,
21316                conductor: conductor(),
21317                implementer_roster: Vec::new(),
21318                judge_roster: Vec::new(),
21319                reviewer_roster: Vec::new(),
21320            },
21321            sem: Arc::new(Semaphore::new(1)),
21322            pause: Pause::new(),
21323            interrupt: Pause::new(),
21324        };
21325
21326        // The lease is still held throughout, so this reentry's own retry is
21327        // also contended — proving `review_loop` actually tried again (not
21328        // that it happened to succeed) is what the timestamp comparison
21329        // below is for.
21330        runner.review_loop().await.expect("review_loop");
21331
21332        assert_eq!(
21333            runner.state.reviews.len(),
21334            1,
21335            "no new round was started on top of the unresolved one: {:?}",
21336            runner.state.reviews
21337        );
21338        let last = &runner.state.reviews[0];
21339        assert_eq!(
21340            last.e2e_status(),
21341            E2eStatus::ResourceBlocked,
21342            "still contended: {last:?}"
21343        );
21344        assert!(
21345            last.verified_at.expect("still known") > first_attempt_at,
21346            "review_loop must have actually retried the check, not left it exactly as found"
21347        );
21348        assert_ne!(
21349            runner.state.status,
21350            RunStatus::Blocked,
21351            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21352            runner.state.status
21353        );
21354
21355        held.release();
21356    }
21357
21358    #[tokio::test]
21359    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21360        crate::run::pin_test_home();
21361        for status in [RunStatus::Blocked, RunStatus::Gating] {
21362            let mut runner = runner_at(status);
21363            runner.state.config.graph.land = true;
21364            runner.state.gate_ran = true;
21365            runner.state.merge = Some(MergeOutcome {
21366                mode: MergeMode::Pr,
21367                ok: false,
21368                detail: "the fixer produced no commit".into(),
21369                empty: false,
21370            });
21371            let url = "https://example.invalid/o/r/pull/1";
21372            runner.state.pr = Some(crate::run::PrRecord {
21373                url: url.into(),
21374                number: 1,
21375                state: "open".into(),
21376                checks: "red".into(),
21377                round: 1,
21378                rounds: 1,
21379                red_at_merge: Vec::new(),
21380            });
21381            runner.state.save().unwrap();
21382            // The nonexistent repository makes observation fail before a forge
21383            // command can run. We still must reach land, rather than silently
21384            // return from the graph with its old gate and merge records.
21385            runner.execute_graph().await.unwrap();
21386            assert_eq!(runner.state.status, RunStatus::Blocked);
21387            assert!(runner.state.land_sync_pending);
21388            assert!(runner.state.events.iter().any(|event|
21389                event.node == "land" && event.message == format!("watching {url}")));
21390            assert_eq!(
21391                runner.state.merge.as_ref().unwrap().detail,
21392                "the fixer produced no commit"
21393            );
21394        }
21395    }
21396
21397    #[tokio::test]
21398    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21399        crate::run::pin_test_home();
21400        let tmp = tempfile::tempdir().expect("tempdir");
21401        let repo = tmp.path().join("repo");
21402        std::fs::create_dir_all(&repo).unwrap();
21403        init_repo(&repo);
21404
21405        let mut config = Config::default();
21406        config.merge.mode = MergeMode::Pr;
21407        config.graph.land = true;
21408        config.graph.land_approval = false;
21409
21410        let mut state = RunState::new(
21411            repo.clone(),
21412            "main".to_owned(),
21413            "deadbeef".to_owned(),
21414            "task".to_owned(),
21415            config,
21416        );
21417        state.candidates = vec![Candidate {
21418            index: 0,
21419            label: 'A',
21420            agent: "alpha".to_owned(),
21421            branch: "does-not-exist".to_owned(),
21422            worktree: repo.clone(),
21423            summary: String::new(),
21424            stat: String::new(),
21425            files: 0,
21426            commits: 0,
21427            empty: false,
21428            failed: None,
21429            verified_noop: None,
21430            duration_ms: 0,
21431            folded: false,
21432        }];
21433        state.tally = Some(Tally {
21434            first_choice: BTreeMap::from([('A', 1)]),
21435            borda: BTreeMap::new(),
21436            winner: 'A',
21437            rankings: 1,
21438            unanimous_initial: true,
21439            deliberated: false,
21440            changed_votes: 0,
21441            unanimous_final: true,
21442            tie_break: None,
21443            judges: 0,
21444            present: 0,
21445            quorum: 0,
21446            met_quorum: true,
21447            uncontested: Some("only candidate A produced a change".to_owned()),
21448        });
21449        state.reviews = vec![ReviewRound {
21450            round: 1,
21451            head: "deadbeef".to_owned(),
21452            verified_head: None,
21453            verified_at: None,
21454            reviews: Vec::new(),
21455            e2e: Vec::new(),
21456            fix: None,
21457            blocking: 0,
21458            answered: 0,
21459            expected: 0,
21460            clean: true,
21461            verify_retried: false,
21462            e2e_deferred: false,
21463            e2e_defer_reason: None,
21464            progressed: false,
21465            vote_split: false,
21466            reconsideration: Vec::new(),
21467            verdict: None,
21468        }];
21469        state.gate = vec![CommandOutcome {
21470            command: "test".to_owned(),
21471            code: Some(0),
21472            output_tail: String::new(),
21473            duration_ms: 0,
21474            resource_blocked: false,
21475        }];
21476        state.gate_ran = true;
21477        // A first pass through `merge` already pushed and opened this pull
21478        // request; `status` is `Landing` because a previous call into `land`
21479        // parked or was interrupted before it reached a terminal outcome.
21480        state.status = RunStatus::Landing;
21481        state.merge = Some(MergeOutcome {
21482            mode: MergeMode::Pr,
21483            ok: true,
21484            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21485            empty: false,
21486        });
21487
21488        // The Landing-resume shortcut calls `run_land` directly rather than
21489        // through `merge`, which is exactly the call site that used to skip
21490        // `settle_questions` - see the fixture below.
21491        ask_test_home();
21492        // A real resume loads a persisted run; driver ownership is checked
21493        // against that record before the Landing shortcut is entered.
21494        state.save().expect("save landing run");
21495        let store = ask::Questions::open();
21496        let q = ask_open_question(&store, &state.id);
21497
21498        let mut runner = Runner {
21499            state,
21500            roles: ResolvedRoles {
21501                implementers: Vec::new(),
21502                judges: Vec::new(),
21503                reviewers: Vec::new(),
21504                fixer: None,
21505                conductor: conductor(),
21506                implementer_roster: Vec::new(),
21507                judge_roster: Vec::new(),
21508                reviewer_roster: Vec::new(),
21509            },
21510            sem: Arc::new(Semaphore::new(1)),
21511            pause: Pause::new(),
21512            interrupt: Pause::new(),
21513        };
21514
21515        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21516        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21517        // exactly because `review_loop` would otherwise clobber the marker
21518        // first.
21519        runner.execute().await.expect("execute");
21520
21521        assert_eq!(
21522            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21523            Some("https://example.invalid/x/y/pull/1"),
21524            "reentry must not push again or open a second pull request over the \
21525             one `land` is already watching"
21526        );
21527        assert_ne!(
21528            runner.state.status,
21529            RunStatus::Landing,
21530            "land could not actually reach the fake pull request, so it must \
21531             have given up rather than left the run silently parked forever"
21532        );
21533        // `land` could not reach the fake pull request, so it gave up into
21534        // `Blocked` - still resumable, so the question must not have been
21535        // swept just because this branch now also calls `settle_questions`.
21536        assert_eq!(runner.state.status, RunStatus::Blocked);
21537        assert!(
21538            store.get(&q.id).unwrap().status.open(),
21539            "Blocked is still alive; settle_questions must have been a no-op here"
21540        );
21541    }
21542
21543    fn state_with_round(round: ReviewRound) -> RunState {
21544        let mut s = RunState::new(
21545            PathBuf::from("/repo"),
21546            "main".to_owned(),
21547            "abc1234".to_owned(),
21548            "add retries".to_owned(),
21549            Config::default(),
21550        );
21551        s.reviews = vec![round];
21552        s
21553    }
21554
21555    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21556        crate::verdict::Finding {
21557            id: id.to_owned(),
21558            severity,
21559            file: None,
21560            line: None,
21561            title: title.to_owned(),
21562            detail: String::new(),
21563        }
21564    }
21565
21566    #[test]
21567    fn pr_body_names_open_findings_and_declined_ones() {
21568        let round = ReviewRound {
21569            round: 2,
21570            head: "deadbee".to_owned(),
21571            verified_head: None,
21572            verified_at: None,
21573            reviews: vec![ReviewRecord {
21574                attempts: 0,
21575                reviewer: 1,
21576                agent: "alpha".to_owned(),
21577                summary: String::new(),
21578                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21579                vote: None,
21580                failed: None,
21581                duration_ms: 0,
21582            }],
21583            e2e: vec![CommandOutcome {
21584                command: "cargo test".to_owned(),
21585                code: Some(0),
21586                output_tail: String::new(),
21587                duration_ms: 0,
21588                resource_blocked: false,
21589            }],
21590            verify_retried: false,
21591            e2e_deferred: false,
21592            e2e_defer_reason: None,
21593            fix: Some(FixRecord {
21594                agent: "alpha".to_owned(),
21595                addressed: Vec::new(),
21596                rejected: vec![crate::verdict::Rejection {
21597                    id: "R1-1-1".to_owned(),
21598                    why: "not reachable from any caller".to_owned(),
21599                }],
21600                notes: String::new(),
21601                committed: true,
21602                failed: None,
21603                duration_ms: 0,
21604                continuation: None,
21605            }),
21606            blocking: 0,
21607            answered: 1,
21608            expected: 1,
21609            clean: false,
21610            progressed: true,
21611            vote_split: false,
21612            reconsideration: Vec::new(),
21613            verdict: None,
21614        };
21615        let state = state_with_round(round);
21616        let body = pr_message(&state, 'A').body;
21617
21618        assert!(body.contains("add retries"), "the task must still be there");
21619        assert!(body.contains("R2-1-1"), "{body}");
21620        assert!(!body.contains("unused import"), "{body}");
21621        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21622        assert!(
21623            !body.contains("not reachable from any caller"),
21624            "the reason it was declined: {body}"
21625        );
21626    }
21627
21628    #[test]
21629    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21630        let round = ReviewRound {
21631            round: 1,
21632            head: "deadbee".to_owned(),
21633            verified_head: None,
21634            verified_at: None,
21635            reviews: vec![ReviewRecord {
21636                attempts: 0,
21637                reviewer: 1,
21638                agent: "alpha".to_owned(),
21639                summary: String::new(),
21640                findings: Vec::new(),
21641                vote: None,
21642                failed: None,
21643                duration_ms: 0,
21644            }],
21645            e2e: Vec::new(),
21646            verify_retried: false,
21647            e2e_deferred: false,
21648            e2e_defer_reason: None,
21649            fix: None,
21650            blocking: 0,
21651            answered: 1,
21652            expected: 1,
21653            clean: true,
21654            progressed: false,
21655            vote_split: false,
21656            reconsideration: Vec::new(),
21657            verdict: None,
21658        };
21659        let state = state_with_round(round);
21660        let body = pr_message(&state, 'A').body;
21661        assert!(!body.contains("Open review findings"), "{body}");
21662        assert!(!body.contains("Declined"), "{body}");
21663    }
21664
21665    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21666        let mut state = RunState::new(
21667            PathBuf::from("/repo"),
21668            "main".to_owned(),
21669            "abc1234".to_owned(),
21670            instruction.to_owned(),
21671            Config::default(),
21672        );
21673        // These fixtures test execution flow, independent of host disk capacity.
21674        state.config.disk.min_free_bytes = 0;
21675        state.candidates.push(Candidate {
21676            index: 0,
21677            label: 'A',
21678            agent: "alpha".to_owned(),
21679            branch: "magi/x/A".to_owned(),
21680            worktree: PathBuf::from("/wt"),
21681            summary: summary.to_owned(),
21682            stat: String::new(),
21683            files: 1,
21684            commits: 1,
21685            empty: false,
21686            failed: None,
21687            verified_noop: None,
21688            folded: false,
21689            duration_ms: 0,
21690        });
21691        state
21692    }
21693
21694    fn review_state(subjects: &[&str]) -> RunState {
21695        let mut state = state_with_summary(
21696            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21697            "",
21698        );
21699        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21700        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21701        state
21702    }
21703
21704    /// A branch rebased onto a main that moved past the recorded
21705    /// `base_commit` is titled from its own first commit, never main's.
21706    #[tokio::test]
21707    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21708        ask_test_home();
21709        let tmp = tempfile::tempdir().unwrap();
21710        let repo = tmp.path().join("repo");
21711        std::fs::create_dir_all(&repo).unwrap();
21712        init_repo(&repo);
21713        let origin = tmp.path().join("origin.git");
21714        let g = |dir: &Path, args: &[&str]| {
21715            let out = std::process::Command::new("git")
21716                .args(args)
21717                .current_dir(dir)
21718                .quiet()
21719                .output()
21720                .expect("spawn git");
21721            assert!(
21722                out.status.success(),
21723                "git {args:?}: {}",
21724                String::from_utf8_lossy(&out.stderr)
21725            );
21726        };
21727        g(
21728            tmp.path(),
21729            &[
21730                "clone",
21731                "--bare",
21732                "-q",
21733                repo.to_str().unwrap(),
21734                origin.to_str().unwrap(),
21735            ],
21736        );
21737        g(
21738            &repo,
21739            &["remote", "add", "origin", origin.to_str().unwrap()],
21740        );
21741        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21742
21743        // Main moves on; the branch is built on top of the new main.
21744        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21745        g(&repo, &["add", "-A"]);
21746        g(
21747            &repo,
21748            &["commit", "-q", "-m", "chore(deps): update a crate"],
21749        );
21750        g(&repo, &["push", "-q", "origin", "main"]);
21751        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21752        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21753        g(&repo, &["add", "-A"]);
21754        g(
21755            &repo,
21756            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21757        );
21758        g(&repo, &["checkout", "-q", "main"]);
21759
21760        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21761        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21762        // Without a readable tracking ref the recorded base's merge base is used.
21763        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21764        assert_eq!(fallback, c1);
21765
21766        let mut state = review_state(&[
21767            "chore(deps): update a crate",
21768            "fix(daemon): apply a chosen action",
21769        ]);
21770        state.repo = repo.clone();
21771        state.base_branch = "main".to_owned();
21772        state.base_commit = c1;
21773        refresh_reviewed_commits(&mut state, "feat/own").await;
21774        assert_eq!(
21775            state.reviewed_commits,
21776            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21777        );
21778        assert_eq!(
21779            review_title(&state).as_deref(),
21780            Some("fix(daemon): apply a chosen action")
21781        );
21782        assert_eq!(
21783            leaked_subjects(&state, "feat/own").await,
21784            Some(vec!["chore(deps): update a crate".to_owned()])
21785        );
21786        // A stale tracking ref is still used when the fetch fails, but the
21787        // leak list is withheld.
21788        state.config.merge.remote = "nowhere".to_owned();
21789        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21790    }
21791
21792    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21793    /// with `origin/main` one commit behind it, and a summarizing agent that
21794    /// prints whatever `reply` holds.
21795    struct SummaryFixture {
21796        _tmp: tempfile::TempDir,
21797        repo: PathBuf,
21798        reply: PathBuf,
21799        runner: Runner,
21800    }
21801
21802    fn summary_fixture() -> SummaryFixture {
21803        ask_test_home();
21804        let tmp = tempfile::tempdir().unwrap();
21805        let repo = tmp.path().join("repo");
21806        std::fs::create_dir_all(&repo).unwrap();
21807        init_repo(&repo);
21808        let origin = tmp.path().join("origin.git");
21809        let g = |args: &[&str]| {
21810            let out = std::process::Command::new("git")
21811                .args(args)
21812                .current_dir(&repo)
21813                .quiet()
21814                .output()
21815                .expect("spawn git");
21816            assert!(
21817                out.status.success(),
21818                "git {args:?}: {}",
21819                String::from_utf8_lossy(&out.stderr)
21820            );
21821        };
21822        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21823        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21824        g(&["push", "-q", "origin", "main"]);
21825        g(&["checkout", "-q", "-b", "feat/own"]);
21826        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21827        g(&["add", "-A"]);
21828        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21829        let reply = tmp.path().join("reply.txt");
21830        std::fs::write(
21831            &reply,
21832            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21833        )
21834        .unwrap();
21835        let mut runner = runner_at(RunStatus::Gating);
21836        let mut state = review_state(&["feat(own): add the own file"]);
21837        state.repo = repo.clone();
21838        state.base_branch = "main".to_owned();
21839        state.candidates[0].branch = "feat/own".to_owned();
21840        state.candidates[0].worktree = repo.clone();
21841        let mut author = spec("alpha");
21842        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21843        author
21844            .env
21845            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21846        state.config.agents = vec![author];
21847        runner.state = state;
21848        SummaryFixture {
21849            _tmp: tmp,
21850            repo,
21851            reply,
21852            runner,
21853        }
21854    }
21855
21856    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21857        let winner = f.runner.state.candidates[0].clone();
21858        f.runner
21859            .guarded_pr_message(&winner, None, true)
21860            .await
21861            .unwrap()
21862            .expect("an English description never parks")
21863    }
21864
21865    fn clean_round_with_minor_findings() -> ReviewRound {
21866        let mut round = passing_round(2, None);
21867        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21868        round.reviews = vec![ReviewRecord {
21869            attempts: 0,
21870            reviewer: 1,
21871            agent: "alpha".into(),
21872            summary: "内部レビュー".into(),
21873            findings: vec![
21874                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21875                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21876            ],
21877            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21878            failed: None,
21879            duration_ms: 0,
21880        }];
21881        round
21882    }
21883
21884    #[tokio::test]
21885    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21886        for review_only in [false, true] {
21887            let mut f = summary_fixture();
21888            if !review_only {
21889                f.runner.state.candidates[0].agent = "alpha".into();
21890            }
21891            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21892            assert!(f.runner.state.reviews[0].clean);
21893            assert!(f.runner.state.reviews[0].fix.is_none());
21894            assert!(f.runner.state.open_findings().is_empty());
21895            assert_eq!(
21896                crate::pr_outcome::required_finding_ids(&f.runner.state),
21897                ["R2-1-1", "R2-1-2"]
21898            );
21899            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21900            let evidence = crate::pr_outcome::summary_evidence(
21901                &f.runner.state,
21902                crate::pr_outcome::FinalHead::At(&head),
21903            );
21904            assert!(
21905                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21906            );
21907            // The ordinary reply omits both IDs: no cached description is adopted.
21908            let omitted = posted(&mut f).await;
21909            assert!(omitted.body.contains("No description"));
21910            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21911            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21912            assert!(f.runner.state.change_summary.is_none());
21913            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21914            let described = posted(&mut f).await;
21915            assert!(described.body.contains("improve documentation for callers"));
21916            assert!(described.body.contains("remaining edge case"));
21917            assert!(described.body.contains("## Open review findings"));
21918            assert!(!described.body.contains("未修正"));
21919        }
21920    }
21921
21922    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21923        let mut f = summary_fixture();
21924        if !review_only {
21925            f.runner.state.candidates[0].agent = "alpha".into();
21926        }
21927        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21928        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21929        fixed
21930            .fix
21931            .as_mut()
21932            .unwrap()
21933            .rejected
21934            .push(crate::verdict::Rejection {
21935                id: "R1-1-1".into(),
21936                why: "呼び出されないコードのため辞退".into(),
21937            });
21938        let mut clean = clean_round_with_minor_findings();
21939        clean.verified_head = Some(head.clone());
21940        f.runner.state.reviews = vec![fixed, clean];
21941        ok_gate(&mut f.runner.state, Some(&head));
21942        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21943        f.runner.state.config.agents.push(scripted("beta", reply));
21944        f.runner.state.config.roles.language_judge =
21945            Some(crate::config::AgentChoice::One("beta".into()));
21946        f
21947    }
21948
21949    #[tokio::test]
21950    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21951        for review_only in [false, true] {
21952            let mut f = rewrite_fixture_with_findings(review_only,
21953                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21954            let m = posted(&mut f).await;
21955            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21956            let expected =
21957                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21958            assert!(m.body.contains(&expected));
21959            assert!(m.body.contains("## Review rounds"));
21960            assert!(
21961                m.body
21962                    .contains("Gate (`verify.gate`): passed on the final commit")
21963            );
21964            assert!(
21965                m.body
21966                    .contains("Tests (`verify.e2e`): passed on the final commit")
21967            );
21968            let c = f.runner.state.change_summary.as_ref().unwrap();
21969            assert!(
21970                m.body
21971                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21972            );
21973            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21974            assert!(crate::pr_outcome::covers_required_findings(
21975                &f.runner.state,
21976                &m.body
21977            ));
21978            assert!(m.body.contains("documentation remains incomplete"));
21979            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21980            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21981            assert!(f.runner.state.github_text.is_none());
21982        }
21983    }
21984
21985    #[tokio::test]
21986    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21987        for review_only in [false, true] {
21988            for reply in [
21989                "TITLE: fix: public description\nA short English description.",
21990                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21991            ] {
21992                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21993                let winner = f.runner.state.candidates[0].clone();
21994                let posted = f
21995                    .runner
21996                    .guarded_pr_message(&winner, None, true)
21997                    .await
21998                    .unwrap();
21999                assert!(posted.is_none());
22000                assert!(f.runner.state.parked);
22001                let withheld = f.runner.state.github_text.as_ref().unwrap();
22002                assert!(
22003                    withheld.rewrite.is_none(),
22004                    "an incomplete rewrite must not be selectable"
22005                );
22006                let q = ask::Questions::open()
22007                    .get(withheld.question.as_ref().unwrap())
22008                    .unwrap();
22009                assert!(
22010                    !q.choices
22011                        .contains(&crate::github_text::USE_REWRITE.to_owned())
22012                );
22013                assert!(
22014                    q.detail.contains("R1-1-1")
22015                        && q.detail.contains("R2-1-1")
22016                        && q.detail.contains("R2-1-2")
22017                );
22018            }
22019        }
22020    }
22021
22022    #[tokio::test]
22023    async fn japanese_reports_stay_local_in_both_public_run_modes() {
22024        for review_only in [false, true] {
22025            let mut f = summary_fixture();
22026            if !review_only {
22027                f.runner.state.candidates[0].agent = "alpha".into();
22028            }
22029            f.runner.state.instruction = "内部タスクの説明".into();
22030            f.runner.state.candidates[0].summary = "内部の候補報告".into();
22031            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22032            f.runner.state.reviews = vec![
22033                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22034                round_with_notes(2, Some("二回目の内部修正報告です。")),
22035                passing_round(3, Some(&head)),
22036            ];
22037            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22038            ok_gate(&mut f.runner.state, Some(&head));
22039            let captured = f.repo.join("public-prompt.txt");
22040            let author = &mut f.runner.state.config.agents[0];
22041            author.command = vec![
22042                "sh".into(),
22043                "-c".into(),
22044                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22045            ];
22046            author.env.insert(
22047                "PROMPT_FILE".into(),
22048                captured.to_string_lossy().into_owned(),
22049            );
22050            let m = posted(&mut f).await;
22051            assert!(m.body.contains("Adds `own.txt`"));
22052            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22053            assert!(m.body.contains("R1-1-1"));
22054            assert!(
22055                m.body
22056                    .contains("Gate (`verify.gate`): passed on the final commit")
22057            );
22058            assert!(
22059                m.body
22060                    .contains("Tests (`verify.e2e`): passed on the final commit")
22061            );
22062            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22063            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22064            let prompt = std::fs::read_to_string(captured).unwrap();
22065            assert!(prompt.contains(&format!(
22066                "git diff {} {head}",
22067                f.runner.state.change_summary.as_ref().unwrap().base
22068            )));
22069            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22070            assert_eq!(
22071                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22072                "二回目の内部修正報告です。"
22073            );
22074        }
22075    }
22076
22077    #[tokio::test]
22078    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22079        for review_only in [false, true] {
22080            let mut f = summary_fixture();
22081            if !review_only {
22082                f.runner.state.candidates[0].agent = "alpha".into();
22083            }
22084            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22085            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22086            std::fs::write(&f.reply, "no usable description").unwrap();
22087            let failed = posted(&mut f).await;
22088            assert!(
22089                failed
22090                    .body
22091                    .contains("No description of what this change does could be generated")
22092            );
22093            assert!(failed.body.is_ascii());
22094            assert!(f.runner.state.github_text.is_none());
22095            assert!(f.runner.state.change_summary.is_none());
22096            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22097            let evidence = crate::pr_outcome::summary_evidence(
22098                &f.runner.state,
22099                crate::pr_outcome::FinalHead::At(&head),
22100            );
22101            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22102            assert!(
22103                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22104                    .exists()
22105            );
22106            f.runner.state =
22107                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22108            std::fs::write(
22109                &f.reply,
22110                "TITLE: feat(own): add the own file\nAdds the committed own file.",
22111            )
22112            .unwrap();
22113            let retried = posted(&mut f).await;
22114            assert!(retried.body.contains("Adds the committed own file."));
22115            assert_eq!(
22116                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22117                "日本語の修正報告"
22118            );
22119        }
22120    }
22121
22122    #[tokio::test]
22123    async fn same_head_verification_and_findings_invalidate_public_prose() {
22124        let mut f = summary_fixture();
22125        posted(&mut f).await;
22126        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22127        let original = f
22128            .runner
22129            .state
22130            .change_summary
22131            .as_ref()
22132            .unwrap()
22133            .evidence
22134            .clone();
22135        ok_gate(&mut f.runner.state, Some(&head));
22136        std::fs::write(
22137            &f.reply,
22138            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22139        )
22140        .unwrap();
22141        let verified = posted(&mut f).await;
22142        assert!(verified.body.contains("now verified"));
22143        assert_ne!(
22144            f.runner.state.change_summary.as_ref().unwrap().evidence,
22145            original
22146        );
22147        let mut round = round_with_notes(1, Some("内部報告"));
22148        round.reviews.push(ReviewRecord {
22149            attempts: 0,
22150            reviewer: 1,
22151            agent: "alpha".into(),
22152            summary: "内部レビュー".into(),
22153            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22154            vote: None,
22155            failed: None,
22156            duration_ms: 0,
22157        });
22158        round
22159            .fix
22160            .as_mut()
22161            .unwrap()
22162            .rejected
22163            .push(crate::verdict::Rejection {
22164                id: "R1-1-1".into(),
22165                why: "呼び出し元から到達できない".into(),
22166            });
22167        f.runner.state.reviews = vec![round];
22168        // A description omitting the required ID is unusable, even with a title.
22169        let omitted = posted(&mut f).await;
22170        assert!(omitted.body.contains("No description"));
22171        assert!(omitted.body.contains("R1-1-1"));
22172        assert!(!omitted.body.contains("到達"));
22173        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22174        let described = posted(&mut f).await;
22175        assert!(described.body.contains("unreachable from callers"));
22176        assert!(described.body.contains("R1-1-1"));
22177        assert!(!described.body.contains("到達"));
22178        let evidence = f
22179            .runner
22180            .state
22181            .change_summary
22182            .as_ref()
22183            .unwrap()
22184            .evidence
22185            .clone();
22186        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22187        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22188        assert!(posted(&mut f).await.body.contains("rationale updated"));
22189        assert_ne!(
22190            f.runner.state.change_summary.as_ref().unwrap().evidence,
22191            evidence
22192        );
22193        // Old serialized summaries have no evidence fingerprint and must regenerate.
22194        f.runner
22195            .state
22196            .change_summary
22197            .as_mut()
22198            .unwrap()
22199            .evidence
22200            .clear();
22201        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22202        assert!(posted(&mut f).await.body.contains("current specification"));
22203    }
22204
22205    #[tokio::test]
22206    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22207        let mut f = summary_fixture();
22208        let m = posted(&mut f).await;
22209        assert_eq!(m.title, "feat(own): add the own file");
22210        assert!(
22211            m.body
22212                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22213            "{}",
22214            m.body
22215        );
22216        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22217        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22218        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22219        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22220        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22221        let kept = f.runner.state.change_summary.as_ref().unwrap();
22222        assert_eq!(
22223            (kept.head.as_str(), kept.base.as_str()),
22224            (head.as_str(), base.as_str())
22225        );
22226        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22227    }
22228
22229    #[tokio::test]
22230    async fn review_only_summary_receives_the_configured_github_language() {
22231        let mut f = summary_fixture();
22232        f.runner.state.config.graph.github_language = "ja".into();
22233        let captured = f.repo.join("summary-prompt.txt");
22234        let author = &mut f.runner.state.config.agents[0];
22235        author.command = vec![
22236            "sh".into(),
22237            "-c".into(),
22238            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22239        ];
22240        author.env.insert(
22241            "PROMPT_FILE".into(),
22242            captured.to_string_lossy().into_owned(),
22243        );
22244        std::fs::write(
22245            &f.reply,
22246            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22247        )
22248        .unwrap();
22249        let m = posted(&mut f).await;
22250        let prompt = std::fs::read_to_string(captured).unwrap();
22251        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22252        assert!(!prompt.contains("in English"));
22253        assert_eq!(m.title, "feat(own): ファイルを追加");
22254        assert!(m.body.contains("own.txt を追加します。"));
22255    }
22256
22257    #[tokio::test]
22258    async fn review_fixes_come_after_the_whole_change_summary() {
22259        let mut f = summary_fixture();
22260        f.runner.state.reviews = vec![
22261            round_with_notes(1, Some("handled the PATH case")),
22262            round_with_notes(2, None),
22263        ];
22264        let m = posted(&mut f).await;
22265        let summary = m.body.find("## Summary").expect("summary");
22266        let fixes = m.body.find("## Review rounds").expect("review rounds");
22267        assert!(summary < fixes, "{}", m.body);
22268        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22269        assert!(m.body.contains("## Final verification"), "{}", m.body);
22270    }
22271
22272    #[tokio::test]
22273    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22274        let mut f = summary_fixture();
22275        let first = posted(&mut f).await;
22276        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22277        // The same head reuses the record: a different reply is not read.
22278        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22279        let again = posted(&mut f).await;
22280        assert_eq!((again.title, again.body), (first.title, first.body));
22281
22282        let g = |args: &[&str]| {
22283            let out = std::process::Command::new("git")
22284                .args(args)
22285                .current_dir(&f.repo)
22286                .quiet()
22287                .output()
22288                .expect("spawn git");
22289            assert!(out.status.success(), "git {args:?}");
22290        };
22291        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22292        g(&["add", "-A"]);
22293        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22294        let second = posted(&mut f).await;
22295        let kept = f.runner.state.change_summary.clone().unwrap();
22296        assert_ne!(kept.head, first_head);
22297        assert!(second.body.contains("Not read yet."), "{}", second.body);
22298        assert!(!second.body.contains("so the first summary is read"));
22299        assert_eq!(second.title, "feat(own): other");
22300    }
22301
22302    #[tokio::test]
22303    async fn an_unavailable_summary_is_stated_and_never_invented() {
22304        let mut f = summary_fixture();
22305        std::fs::write(&f.reply, "no title line here").unwrap();
22306        let m = posted(&mut f).await;
22307        assert!(
22308            m.body
22309                .contains("No description of what this change does could be generated"),
22310            "{}",
22311            m.body
22312        );
22313        assert!(m.body.contains("internal reports remain in local records"));
22314        assert!(f.runner.state.change_summary.is_none());
22315        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22316        // A preview asks nobody and claims nothing.
22317        let winner = f.runner.state.candidates[0].clone();
22318        let preview = f
22319            .runner
22320            .guarded_pr_message(&winner, None, false)
22321            .await
22322            .unwrap()
22323            .unwrap();
22324        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22325    }
22326
22327    #[test]
22328    fn pr_message_review_single_commit_uses_its_subject() {
22329        let state = review_state(&["feat(nats): per-role user"]);
22330        let m = pr_message(&state, 'A');
22331        assert_eq!(m.title, "feat(nats): per-role user");
22332        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22333        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22334    }
22335
22336    #[test]
22337    fn pr_message_review_multi_commit_takes_the_oldest() {
22338        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22339        let m = pr_message(&state, 'A');
22340        assert_eq!(m.title, "feat: the change");
22341        for s in ["feat: the change", "fix: typo", "fix: again"] {
22342            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22343        }
22344    }
22345
22346    #[test]
22347    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22348        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22349            let mut state = review_state(&[first, "fix: later fixup"]);
22350            state.candidates[0].branch = "機能/ログイン".to_owned();
22351            let m = pr_message(&state, 'A');
22352            assert!(
22353                m.title.starts_with("chore: land candidate A of run"),
22354                "{}",
22355                m.title
22356            );
22357        }
22358    }
22359
22360    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22361        BranchFacts {
22362            commits: commits
22363                .iter()
22364                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22365                .collect(),
22366            stat: stat.to_owned(),
22367        }
22368    }
22369
22370    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22371        let mut r = review_round(true, 0, 1, 1, true, true);
22372        r.round = round;
22373        r.fix = notes.map(|n| FixRecord {
22374            agent: "fixer".to_owned(),
22375            addressed: Vec::new(),
22376            rejected: Vec::new(),
22377            notes: n.to_owned(),
22378            committed: true,
22379            failed: None,
22380            duration_ms: 0,
22381            continuation: None,
22382        });
22383        r
22384    }
22385
22386    #[test]
22387    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22388        let state = review_state(&["ignored"]);
22389        let f = facts(
22390            &[
22391                (
22392                    "fix(login): resolve PATH on macOS",
22393                    "Login shells skip rc files.",
22394                ),
22395                ("fix: address review", ""),
22396            ],
22397            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22398        );
22399        let m = pr_message_with(&state, 'A', Some(&f));
22400        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22401        assert!(
22402            m.body
22403                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22404            "{}",
22405            m.body
22406        );
22407        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22408        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22409        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22410        for banned in [
22411            "Review the work already",
22412            "no task statement",
22413            "Original task",
22414        ] {
22415            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22416        }
22417        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22418    }
22419
22420    #[test]
22421    fn pr_message_review_truncates_a_huge_first_commit_body() {
22422        let state = review_state(&["ignored"]);
22423        let f = facts(
22424            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22425            "s",
22426        );
22427        let m = pr_message_with(&state, 'A', Some(&f));
22428        assert!(m.body.len() < 30_000, "{}", m.body.len());
22429        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22430        assert!(
22431            m.body.contains("- ... 1 more commit(s)"),
22432            "{}",
22433            m.body.len()
22434        );
22435        assert!(m.body.ends_with("magi:candidate-a\n"));
22436    }
22437
22438    #[test]
22439    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22440        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22441        assert_eq!(m.title, "feat: x");
22442        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22443        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22444        assert!(!m.body.contains("no task statement"), "{}", m.body);
22445    }
22446
22447    #[test]
22448    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22449        let mut state = review_state(&["日本語の件名"]);
22450        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22451        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22452        state.candidates[0].branch = "機能/ログイン".to_owned();
22453        assert!(
22454            pr_message(&state, 'A')
22455                .title
22456                .starts_with("chore: land candidate A")
22457        );
22458    }
22459
22460    #[test]
22461    fn pr_message_review_fixes_survive_a_clean_final_round() {
22462        let mut state = review_state(&["feat: x"]);
22463        state.reviews = vec![
22464            round_with_notes(1, Some("handled the PATH case")),
22465            round_with_notes(2, None),
22466        ];
22467        let body = pr_message(&state, 'A').body;
22468        assert!(
22469            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22470            "{body}"
22471        );
22472        assert!(body.contains("### Round 1"), "{body}");
22473
22474        state.reviews = vec![
22475            round_with_notes(1, Some("first fix")),
22476            round_with_notes(2, Some("")),
22477            round_with_notes(3, Some("second fix")),
22478            round_with_notes(4, None),
22479        ];
22480        let body = pr_message(&state, 'A').body;
22481        assert!(
22482            body.contains("### Round 1") && !body.contains("first fix"),
22483            "{body}"
22484        );
22485        assert!(
22486            body.contains("### Round 3") && !body.contains("second fix"),
22487            "{body}"
22488        );
22489        assert!(body.contains("### Round 2"), "{body}");
22490    }
22491
22492    const FINAL: &str = "abcdef1234567890";
22493
22494    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22495        let mut r = round_with_notes(round, None);
22496        r.verified_head = verified.map(str::to_owned);
22497        r
22498    }
22499
22500    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22501        state.gate = vec![CommandOutcome {
22502            command: "gate".to_owned(),
22503            code: Some(0),
22504            output_tail: String::new(),
22505            duration_ms: 0,
22506            resource_blocked: false,
22507        }];
22508        state.gate_ran = true;
22509        state.gate_head = head.map(str::to_owned);
22510    }
22511
22512    fn body_at(state: &RunState) -> String {
22513        pr_message_raw(
22514            state,
22515            'A',
22516            None,
22517            crate::pr_outcome::FinalHead::At(FINAL),
22518            ChangeNote::NotAsked,
22519        )
22520        .body
22521    }
22522
22523    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22524
22525    #[test]
22526    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22527        let mut state = review_state(&["feat: x"]);
22528        state.reviews = vec![
22529            round_with_notes(1, Some(SANDBOXED)),
22530            passing_round(2, Some(FINAL)),
22531        ];
22532        ok_gate(&mut state, Some(FINAL));
22533        let body = body_at(&state);
22534        assert!(!body.contains("Handled the PATH case."), "{body}");
22535        assert!(!body.contains("could not commit"), "{body}");
22536        assert!(!body.contains("could not run"), "{body}");
22537        assert!(
22538            body.contains("The fix is committed on the branch"),
22539            "{body}"
22540        );
22541        assert!(!body.contains("sandbox"), "{body}");
22542        assert!(
22543            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22544            "{body}"
22545        );
22546        assert!(
22547            body.contains("Gate (`verify.gate`): passed on the final commit"),
22548            "{body}"
22549        );
22550        // The raw report is untouched in the record.
22551        assert!(
22552            state.reviews[0]
22553                .fix
22554                .as_ref()
22555                .unwrap()
22556                .notes
22557                .contains("could not commit")
22558        );
22559    }
22560
22561    #[test]
22562    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22563        let mut state = review_state(&["feat: x"]);
22564        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22565        r1.fix.as_mut().unwrap().committed = false;
22566        let mut r2 = passing_round(2, Some(FINAL));
22567        r2.e2e[0].code = Some(1);
22568        state.reviews = vec![r1, r2];
22569        let body = body_at(&state);
22570        assert!(!body.contains("could not commit"), "{body}");
22571        assert!(!body.contains("could not run"), "{body}");
22572        assert!(body.contains("This round produced no commit"), "{body}");
22573        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22574        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22575        assert!(!body.contains("passed"), "{body}");
22576    }
22577
22578    #[test]
22579    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22580        let mut state = review_state(&["feat: x"]);
22581        state.reviews = vec![
22582            round_with_notes(1, Some(SANDBOXED)),
22583            passing_round(2, Some("0123456789abcdef")),
22584        ];
22585        ok_gate(&mut state, Some("0123456789abcdef"));
22586        let body = body_at(&state);
22587        assert!(!body.contains("HTTP tests could not run"), "{body}");
22588        // The commit claim is still settled by the structured record.
22589        assert!(!body.contains("could not commit"), "{body}");
22590        assert!(
22591            body.contains("passed on an earlier commit (`0123456`)"),
22592            "{body}"
22593        );
22594        assert!(!body.contains("passed on the final commit"), "{body}");
22595
22596        // A gate recorded before `gate_head` existed is "not recorded".
22597        ok_gate(&mut state, None);
22598        let body = body_at(&state);
22599        assert!(
22600            body.contains("passed on a commit that was not recorded"),
22601            "{body}"
22602        );
22603    }
22604
22605    #[test]
22606    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22607        let state = state_with_summary(
22608            "add retries to the client",
22609            "TITLE: feat: retries\n\nDid it.",
22610        );
22611        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22612        assert_eq!(m.title, "feat: retries");
22613        assert!(
22614            m.body.contains("<summary>Original task</summary>"),
22615            "{}",
22616            m.body
22617        );
22618        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22619        assert!(
22620            m.body
22621                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22622            "{}",
22623            m.body
22624        );
22625    }
22626
22627    #[test]
22628    fn pr_message_review_bounds_a_long_english_subject() {
22629        let long = format!("feat: {}", "word ".repeat(100));
22630        let m = pr_message(&review_state(&[&long]), 'A');
22631        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22632        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22633    }
22634
22635    #[test]
22636    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22637        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22638        let m = pr_message(&state, 'A');
22639        assert_eq!(m.title, "add retries");
22640        assert!(m.body.contains("<summary>Original task</summary>"));
22641        assert!(!m.body.contains("Commits under review"));
22642        assert_eq!(landing_subject_source(&state), state.instruction);
22643    }
22644
22645    #[test]
22646    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22647        let state = review_state(&["feat: the change", "fix: typo"]);
22648        let source = landing_subject_source(&state);
22649        assert_eq!(land::merge_subject("", &source), "feat: the change");
22650        assert_eq!(
22651            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22652            "feat: the change"
22653        );
22654        // An operator's rename still wins.
22655        assert_eq!(
22656            land::merge_subject("feat: renamed by hand", &source),
22657            "feat: renamed by hand"
22658        );
22659        let blank = review_state(&["日本語"]);
22660        assert!(
22661            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22662        );
22663    }
22664
22665    #[test]
22666    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22667        let state = review_state(&["feat: the change"]);
22668        let source = landing_subject_source(&state);
22669        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22670        assert_eq!(
22671            land::merge_subject(landing_title(&state, old), &source),
22672            "feat: the change"
22673        );
22674        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22675        let task = state_with_summary("add retries", "");
22676        assert_eq!(landing_title(&task, old), old);
22677    }
22678
22679    #[test]
22680    fn pr_message_describes_the_change_not_the_task() {
22681        let state = state_with_summary(
22682            "今回やってほしいこと: results projector を直す",
22683            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22684        );
22685        let m = pr_message(&state, 'A');
22686        assert_eq!(m.title, "fix(web): batch the runs list reads");
22687        assert!(
22688            m.body.starts_with("## Summary\n\n- reads run.json once"),
22689            "{}",
22690            m.body
22691        );
22692        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22693        let task_at = m.body.find("今回やってほしいこと").unwrap();
22694        let details_at = m.body.find("<details>").unwrap();
22695        assert!(
22696            details_at < task_at,
22697            "the task lives inside <details>: {}",
22698            m.body
22699        );
22700        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22701        assert!(m.body.contains("magi:candidate-a"));
22702    }
22703
22704    #[test]
22705    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22706        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22707        let m = pr_message(&state, 'A');
22708        assert_eq!(m.title, "add retries");
22709        assert!(
22710            m.body.contains("## Summary\n\n- did some things"),
22711            "{}",
22712            m.body
22713        );
22714
22715        let none = RunState::new(
22716            PathBuf::from("/repo"),
22717            "main".to_owned(),
22718            "abc1234".to_owned(),
22719            "add retries".to_owned(),
22720            Config::default(),
22721        );
22722        let m = pr_message(&none, 'A');
22723        assert_eq!(m.title, "add retries");
22724        assert!(!m.body.contains("## Summary"), "{}", m.body);
22725    }
22726
22727    #[test]
22728    fn pr_message_refuses_the_candidate_commit_subject() {
22729        for bad in [
22730            "TITLE: magi: candidate A (uncommitted work)",
22731            "TITLE: chore: stuff (uncommitted work)",
22732            "TITLE:   ",
22733        ] {
22734            let state = state_with_summary("add retries", bad);
22735            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22736        }
22737    }
22738
22739    #[test]
22740    fn pr_message_bounds_a_very_long_task_and_title() {
22741        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22742        let state = state_with_summary(&long, "- nothing");
22743        let m = pr_message(&state, 'A');
22744        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22745        assert!(!m.title.contains('\n'));
22746
22747        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22748        let m = pr_message(&state, 'A');
22749        assert!(m.title.starts_with("feat: "));
22750        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22751        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22752    }
22753
22754    fn long_title_of(instruction: &str) -> String {
22755        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22756    }
22757
22758    #[test]
22759    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22760        let first = "Make the landing path keep a readable title for long tasks";
22761        let line = format!(
22762            "{first}. {}",
22763            "And then keep going with more words ".repeat(20)
22764        );
22765        let t = long_title_of(&line);
22766        assert_eq!(t, first);
22767        assert!(!t.starts_with("chore: land"));
22768    }
22769
22770    #[test]
22771    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22772        let line = "word ".repeat(200);
22773        let t = long_title_of(&line);
22774        assert!(t.ends_with("word..."), "{t}");
22775        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22776    }
22777
22778    #[test]
22779    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22780        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22781            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22782        }
22783    }
22784
22785    #[test]
22786    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22787        let mut state = state_with_summary("再試行を追加する", "- nothing");
22788        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22789        state.config.graph.github_language = "ja".into();
22790        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22791    }
22792
22793    #[test]
22794    fn pr_message_title_limit_is_exact() {
22795        let at = "a".repeat(PR_TITLE_MAX);
22796        assert_eq!(long_title_of(&at), at);
22797        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22798        assert!(over.ends_with("..."), "{over}");
22799        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22800    }
22801
22802    #[test]
22803    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22804        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22805        let t = long_title_of(&line);
22806        assert!(t.ends_with("..."), "{t}");
22807        assert!(t.is_ascii(), "{t}");
22808    }
22809
22810    #[test]
22811    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22812        let line = format!(
22813            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22814            "plus more filler words ".repeat(20)
22815        );
22816        let t = long_title_of(&line);
22817        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22818    }
22819
22820    #[test]
22821    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22822        let line = format!("\n\n# {}", "title words ".repeat(40));
22823        let state = state_with_summary(&line, "- nothing");
22824        let m = pr_message(&state, 'A');
22825        assert!(m.title.starts_with("title words"), "{}", m.title);
22826        assert_eq!(
22827            land::merge_subject(&m.title, &landing_subject_source(&state)),
22828            m.title
22829        );
22830        // An operator's rename wins untouched.
22831        assert_eq!(
22832            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22833            "feat: renamed by hand"
22834        );
22835    }
22836
22837    #[tokio::test]
22838    async fn github_text_rewrite_is_bounded_and_falls_back() {
22839        crate::run::pin_test_home();
22840        for (reply, accepted) in [
22841            (
22842                "TITLE: fix: retries\nAdd retries for failed requests.",
22843                true,
22844            ),
22845            (
22846                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22847                false,
22848            ),
22849            (
22850                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22851                true,
22852            ),
22853            (
22854                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22855                true,
22856            ),
22857            (
22858                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22859                true,
22860            ),
22861            (
22862                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22863                false,
22864            ),
22865            ("TITLE: fix: retries\n", false),
22866        ] {
22867            let dir = tempfile::tempdir().unwrap();
22868            let mut runner = runner_at(RunStatus::Gating);
22869            runner.state = state_with_summary(
22870                "add retries",
22871                "TITLE: fix: retries\n日本語の説明を書きます。",
22872            );
22873            runner.state.repo = dir.path().to_owned();
22874            runner.state.candidates[0].worktree = dir.path().to_owned();
22875            let mut author = spec("alpha");
22876            author.command = vec![
22877                "sh".into(),
22878                "-c".into(),
22879                "printf '%s' \"$REWRITE_REPLY\"".into(),
22880            ];
22881            author.env.insert("REWRITE_REPLY".into(), reply.into());
22882            runner.state.config.agents = vec![author];
22883            seed_public_summary(&mut runner);
22884            let winner = runner.state.candidates[0].clone();
22885            let first = runner
22886                .guarded_pr_message(&winner, None, true)
22887                .await
22888                .unwrap();
22889            let message = if accepted {
22890                first.expect("a passing rewrite never asks")
22891            } else {
22892                // Withheld: the run parks on a question and posts nothing yet.
22893                assert!(first.is_none());
22894                assert!(runner.state.parked);
22895                let store = ask::Questions::open();
22896                let asked: Vec<_> = store
22897                    .list()
22898                    .into_iter()
22899                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22900                    .collect();
22901                assert_eq!(asked.len(), 1);
22902                assert!(!asked[0].detail.contains("secret"));
22903                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22904                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22905                assert_eq!(
22906                    asked[0]
22907                        .choices
22908                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22909                    rewrite.is_some()
22910                );
22911                // Silence: the question lapses and the neutral text is used,
22912                // without a second rewrite call or a second question.
22913                store
22914                    .update(&asked[0].id, |q| {
22915                        q.abandon("test");
22916                        Ok(())
22917                    })
22918                    .unwrap();
22919                let again = runner
22920                    .guarded_pr_message(&winner, None, true)
22921                    .await
22922                    .unwrap()
22923                    .expect("an abandoned question falls back");
22924                let n = store
22925                    .list()
22926                    .into_iter()
22927                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22928                    .count();
22929                assert_eq!(n, 1, "asked once per rejected text");
22930                again
22931            };
22932            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22933            assert!(!message.body.contains("secret"));
22934            assert!(!message.body.contains("prefix"));
22935            assert!(!message.title.contains("secret"));
22936            assert_eq!(
22937                message.body.contains("Add retries for failed requests."),
22938                accepted
22939            );
22940            assert_eq!(
22941                runner.state.seats["github-text-rewrite"].turns, 1,
22942                "only one rewrite invocation"
22943            );
22944            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22945            if accepted {
22946                assert!(runner.state.github_text.is_none());
22947                assert!(!runner.state.parked);
22948                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22949                assert!(message.body.contains("## Final verification"));
22950                assert!(message.body.contains("Evidence: final diff"));
22951                assert!(
22952                    message
22953                        .body
22954                        .contains(&format!("magi:run/{}", runner.state.id))
22955                );
22956            }
22957            if !accepted {
22958                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22959                assert!(
22960                    message
22961                        .body
22962                        .contains(&format!("magi:run/{}", runner.state.id))
22963                );
22964            }
22965        }
22966    }
22967
22968    fn scripted(id: &str, reply: &str) -> AgentSpec {
22969        let mut a = spec(id);
22970        a.command = vec![
22971            "sh".into(),
22972            "-c".into(),
22973            "printf '%s' \"$REWRITE_REPLY\"".into(),
22974        ];
22975        a.env.insert("REWRITE_REPLY".into(), reply.into());
22976        a
22977    }
22978
22979    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22980    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22981
22982    /// Gate tests inject generated public output, not internal reports.
22983    fn seed_public_summary(runner: &mut Runner) {
22984        let repo = &runner.state.repo;
22985        let g = |args: &[&str]| {
22986            let out = std::process::Command::new("git")
22987                .args(args)
22988                .current_dir(repo)
22989                .quiet()
22990                .output()
22991                .unwrap();
22992            assert!(out.status.success(), "git {args:?}");
22993            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22994        };
22995        g(&["init", "-q", "-b", "main"]);
22996        g(&[
22997            "-c",
22998            "user.name=Test",
22999            "-c",
23000            "user.email=test@example.invalid",
23001            "commit",
23002            "-q",
23003            "--allow-empty",
23004            "-m",
23005            "feat: fixture",
23006        ]);
23007        let head = g(&["rev-parse", "HEAD"]);
23008        g(&["update-ref", "refs/remotes/origin/main", &head]);
23009        let evidence = crate::pr_outcome::summary_evidence(
23010            &runner.state,
23011            crate::pr_outcome::FinalHead::At(&head),
23012        );
23013        let generated = &runner.state.candidates[0].summary;
23014        runner.state.change_summary = Some(crate::run::ChangeSummary {
23015            base: head.clone(),
23016            head,
23017            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23018            title: summary_title(generated).unwrap(),
23019            body: summary_without_title(generated),
23020        });
23021    }
23022
23023    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23024        let mut runner = runner_at(RunStatus::Gating);
23025        runner.state = state_with_summary("add retries", JA_SUMMARY);
23026        if review_only {
23027            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23028        }
23029        runner.state.repo = dir.to_owned();
23030        runner.state.candidates[0].worktree = dir.to_owned();
23031        runner.state.config.agents = agents;
23032        seed_public_summary(&mut runner);
23033        runner
23034    }
23035
23036    fn gate_questions(runner: &Runner) -> usize {
23037        ask::Questions::open()
23038            .list()
23039            .into_iter()
23040            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23041            .count()
23042    }
23043
23044    #[tokio::test]
23045    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23046        crate::run::pin_test_home();
23047        let dir = tempfile::tempdir().unwrap();
23048        let mut runner = rewrite_runner(
23049            dir.path(),
23050            true,
23051            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23052        );
23053        runner.state.config.roles.language_judge =
23054            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23055        let winner = runner.state.candidates[0].clone();
23056        let msg = runner
23057            .guarded_pr_message(&winner, None, true)
23058            .await
23059            .unwrap()
23060            .expect("an English rewrite posts without asking");
23061        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23062        assert!(msg.body.contains("<details>kept</details>"));
23063        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23064        assert_eq!(gate_questions(&runner), 0);
23065        assert!(runner.state.github_text.is_none());
23066        assert!(!runner.state.seats.contains_key("impl-A"));
23067        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23068    }
23069
23070    #[tokio::test]
23071    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23072        crate::run::pin_test_home();
23073        let dir = tempfile::tempdir().unwrap();
23074        let mut runner = rewrite_runner(
23075            dir.path(),
23076            false,
23077            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23078        );
23079        runner.state.config.roles.language_judge =
23080            Some(crate::config::AgentChoice::One("beta".into()));
23081        runner.state.seats.insert(
23082            "impl-A".into(),
23083            SeatState::new("impl-A", "alpha", runner.state.seed),
23084        );
23085        let winner = runner.state.candidates[0].clone();
23086        let msg = runner
23087            .guarded_pr_message(&winner, None, true)
23088            .await
23089            .unwrap()
23090            .expect("rewritten");
23091        assert!(msg.body.contains("<details>kept</details>"));
23092        assert_eq!(runner.state.seats["impl-A"].turns, 0);
23093        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23094        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23095    }
23096
23097    #[tokio::test]
23098    async fn rewrite_reuses_a_resumable_implementer_session() {
23099        crate::run::pin_test_home();
23100        let dir = tempfile::tempdir().unwrap();
23101        let mut runner = rewrite_runner(
23102            dir.path(),
23103            false,
23104            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23105        );
23106        runner.state.config.roles.language_judge =
23107            Some(crate::config::AgentChoice::One("beta".into()));
23108        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23109        seat.turns = 1;
23110        runner.state.seats.insert("impl-A".into(), seat);
23111        let winner = runner.state.candidates[0].clone();
23112        runner
23113            .guarded_pr_message(&winner, None, true)
23114            .await
23115            .unwrap()
23116            .expect("rewritten");
23117        assert_eq!(runner.state.seats["impl-A"].turns, 2);
23118        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23119    }
23120
23121    #[tokio::test]
23122    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23123        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
23124        // session, was out of quota, and nothing else was asked.
23125        crate::run::pin_test_home();
23126        let dir = tempfile::tempdir().unwrap();
23127        let mut exhausted = spec("alpha");
23128        exhausted.command = vec!["false".into()];
23129        let mut runner = rewrite_runner(
23130            dir.path(),
23131            false,
23132            vec![exhausted, scripted("beta", EN_REWRITE)],
23133        );
23134        // The implementer's own agent is in the role too: it is not asked twice.
23135        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23136            "alpha".into(),
23137            "beta".into(),
23138        ]));
23139        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23140        seat.turns = 1;
23141        runner.state.seats.insert("impl-A".into(), seat);
23142        let winner = runner.state.candidates[0].clone();
23143        let msg = runner
23144            .guarded_pr_message(&winner, None, true)
23145            .await
23146            .unwrap()
23147            .expect("the role rewrites after the implementer fails");
23148        assert!(msg.body.contains("`src/net.rs`"));
23149        assert_eq!(
23150            runner.state.seats["impl-A"].turns, 2,
23151            "the implementer was tried once"
23152        );
23153        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23154        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23155    }
23156
23157    #[tokio::test]
23158    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23159        crate::run::pin_test_home();
23160        let dir = tempfile::tempdir().unwrap();
23161        let mut broken = spec("bad");
23162        broken.command = vec!["false".into()];
23163        let mut runner = rewrite_runner(
23164            dir.path(),
23165            true,
23166            vec![
23167                broken,
23168                scripted("noformat", "Add retries."),
23169                scripted("good", EN_REWRITE),
23170            ],
23171        );
23172        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23173            "bad".into(),
23174            "noformat".into(),
23175            "good".into(),
23176        ]));
23177        let winner = runner.state.candidates[0].clone();
23178        let msg = runner
23179            .guarded_pr_message(&winner, None, true)
23180            .await
23181            .unwrap()
23182            .expect("the third agent rewrites");
23183        assert!(msg.body.contains("`src/net.rs`"));
23184        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23185        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23186    }
23187
23188    #[tokio::test]
23189    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23190        crate::run::pin_test_home();
23191        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23192        for (agents, role, saved) in [
23193            (
23194                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23195                Some("beta"),
23196                true,
23197            ),
23198            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23199            (Vec::new(), None, false),
23200        ] {
23201            let dir = tempfile::tempdir().unwrap();
23202            let mut runner = rewrite_runner(dir.path(), true, agents);
23203            runner.state.config.roles.language_judge =
23204                role.map(|r| crate::config::AgentChoice::One(r.into()));
23205            let winner = runner.state.candidates[0].clone();
23206            let first = runner
23207                .guarded_pr_message(&winner, None, true)
23208                .await
23209                .unwrap();
23210            assert!(first.is_none(), "withheld text parks the run");
23211            assert!(runner.state.parked);
23212            assert_eq!(gate_questions(&runner), 1);
23213            let g = runner.state.github_text.as_ref().unwrap();
23214            assert_eq!(g.rewrite.is_some(), saved);
23215            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23216        }
23217    }
23218
23219    #[tokio::test]
23220    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23221        crate::run::pin_test_home();
23222        let dir = tempfile::tempdir().unwrap();
23223        let mut runner = runner_at(RunStatus::Gating);
23224        runner.state =
23225            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23226        runner.state.repo = dir.path().to_owned();
23227        runner.state.candidates[0].worktree = dir.path().to_owned();
23228        let mut author = spec("alpha");
23229        author.command = vec![
23230            "sh".into(),
23231            "-c".into(),
23232            "printf '%s' \"$REWRITE_REPLY\"".into(),
23233        ];
23234        author.env.insert(
23235            "REWRITE_REPLY".into(),
23236            format!(
23237                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23238                "English sentence. ".repeat(600)
23239            ),
23240        );
23241        runner.state.config.agents = vec![author];
23242        seed_public_summary(&mut runner);
23243        let winner = runner.state.candidates[0].clone();
23244        assert!(
23245            runner
23246                .guarded_pr_message(&winner, None, true)
23247                .await
23248                .unwrap()
23249                .is_none()
23250        );
23251        let record = runner.state.github_text.clone().unwrap();
23252        let rewrite = record.rewrite.clone().unwrap();
23253        assert_eq!(rewrite.failed, ["body-language"]);
23254        assert!(
23255            rewrite
23256                .body
23257                .ends_with(&format!("magi:run/{}", runner.state.id))
23258        );
23259        let store = ask::Questions::open();
23260        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23261        assert!(
23262            q.choices
23263                .contains(&crate::github_text::USE_REWRITE.to_owned())
23264        );
23265        for label in [
23266            "ORIGINAL",
23267            "FALLBACK",
23268            "REWRITE",
23269            "Rules still failed: body-language",
23270            "truncated",
23271        ] {
23272            assert!(q.detail.contains(label), "{label}");
23273        }
23274        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23275            &runner.state,
23276            &crate::github_text::artifact_name(&record.fingerprint),
23277        ))
23278        .unwrap();
23279        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23280        // Restart from serialized state, then retry after a bad owner title.
23281        runner.state =
23282            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23283        store
23284            .update(&q.id, |q| {
23285                q.say("まだ日本語")?;
23286                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23287            })
23288            .unwrap();
23289        assert!(
23290            runner
23291                .guarded_pr_message(&winner, None, true)
23292                .await
23293                .unwrap()
23294                .is_none()
23295        );
23296        let retry = runner.state.github_text.as_ref().unwrap();
23297        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23298        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23299        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23300        assert!(
23301            q.posting_comparison
23302                .as_ref()
23303                .unwrap()
23304                .matches_detail(&q.detail)
23305        );
23306        store
23307            .update(&q.id, |q| {
23308                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23309            })
23310            .unwrap();
23311        let message = runner
23312            .guarded_pr_message(&winner, None, true)
23313            .await
23314            .unwrap()
23315            .unwrap();
23316        assert_eq!(message.title, rewrite.title);
23317        assert_eq!(message.body, rewrite.body);
23318        assert_eq!(
23319            runner.state.github_text.as_ref().unwrap().choice,
23320            Some(GithubTextChoice::Rewrite)
23321        );
23322        let again = runner
23323            .guarded_pr_message(&winner, None, true)
23324            .await
23325            .unwrap()
23326            .unwrap();
23327        assert_eq!(again.title, message.title);
23328        assert_eq!(again.body, message.body);
23329        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23330        assert!(
23331            !runner
23332                .state
23333                .events
23334                .iter()
23335                .any(|e| e.message.contains(&rewrite.title)
23336                    || e.message.contains("English sentence."))
23337        );
23338    }
23339
23340    #[tokio::test]
23341    async fn historical_posting_reply_never_expires_into_fallback() {
23342        crate::run::pin_test_home();
23343        let dir = tempfile::tempdir().unwrap();
23344        let mut runner = runner_at(RunStatus::Gating);
23345        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23346        runner.state.repo = dir.path().to_owned();
23347        runner.state.candidates[0].worktree = dir.path().to_owned();
23348        seed_public_summary(&mut runner);
23349        let winner = runner.state.candidates[0].clone();
23350        assert!(
23351            runner
23352                .guarded_pr_message(&winner, None, true)
23353                .await
23354                .unwrap()
23355                .is_none()
23356        );
23357        let id = runner
23358            .state
23359            .github_text
23360            .as_ref()
23361            .unwrap()
23362            .question
23363            .clone()
23364            .unwrap();
23365        let store = ask::Questions::open();
23366        let mut q = store.get(&id).unwrap();
23367        q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23368        q.answer(ask::Answer::Choice(
23369            crate::github_text::POST_ORIGINAL.into(),
23370        ))
23371        .unwrap();
23372        q.answer_delivered = true;
23373        store.put(&mut q).unwrap();
23374        let source = std::fs::read(store.path_of(&id)).unwrap();
23375        let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23376        std::fs::create_dir_all(&migration).unwrap();
23377        std::fs::write(
23378            migration.join("state.json"),
23379            serde_json::to_vec(&serde_json::json!({
23380                "schema": 1, "sources": [id], "pending": [id]
23381            }))
23382            .unwrap(),
23383        )
23384        .unwrap();
23385        for _ in 0..2 {
23386            assert!(
23387                runner
23388                    .guarded_pr_message(&winner, None, true)
23389                    .await
23390                    .unwrap()
23391                    .is_none()
23392            );
23393            assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23394            assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23395        }
23396        let home = crate::run::home();
23397        crate::task_chat::sweep(
23398            &crate::queue::Queue::at(home.join("queue")),
23399            &crate::talk::Talks::at(home.join("talks")),
23400            &store,
23401            &home,
23402        );
23403        let outbox = crate::task_inquiry::Outbox::at(&home);
23404        outbox
23405            .update_decision(&id, |d| {
23406                d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23407                d.source_answer_at = q.answered_at;
23408                Ok(true)
23409            })
23410            .unwrap();
23411        outbox
23412            .settle(
23413                &id,
23414                "owner-chat",
23415                "Owner confirms no publication remains necessary",
23416                None,
23417            )
23418            .unwrap();
23419        assert!(
23420            runner
23421                .guarded_pr_message(&winner, None, true)
23422                .await
23423                .unwrap()
23424                .is_none()
23425        );
23426        assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23427        assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23428    }
23429
23430    #[tokio::test]
23431    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23432        crate::run::pin_test_home();
23433        let dir = tempfile::tempdir().unwrap();
23434        let mut runner = runner_at(RunStatus::Gating);
23435        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23436        runner.state.repo = dir.path().to_owned();
23437        runner.state.candidates[0].worktree = dir.path().to_owned();
23438        seed_public_summary(&mut runner);
23439        let winner = runner.state.candidates[0].clone();
23440        assert!(
23441            runner
23442                .guarded_pr_message(&winner, None, true)
23443                .await
23444                .unwrap()
23445                .is_none()
23446        );
23447        let record = runner.state.github_text.as_ref().unwrap();
23448        assert!(record.rewrite.is_none());
23449        // Records written before the additive field deserialize with no rewrite.
23450        let mut old = serde_json::to_value(record).unwrap();
23451        old.as_object_mut().unwrap().remove("rewrite");
23452        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23453        assert!(old.rewrite.is_none());
23454        let id = record.question.clone().unwrap();
23455        ask::Questions::open()
23456            .update(&id, |q| {
23457                q.choices.push(crate::github_text::USE_REWRITE.into());
23458                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23459            })
23460            .unwrap();
23461        let message = runner
23462            .guarded_pr_message(&winner, None, true)
23463            .await
23464            .unwrap()
23465            .unwrap();
23466        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23467        assert_eq!(
23468            runner.state.github_text.as_ref().unwrap().choice,
23469            Some(GithubTextChoice::Fallback)
23470        );
23471    }
23472
23473    #[tokio::test]
23474    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23475        crate::run::pin_test_home();
23476        let dir = tempfile::tempdir().unwrap();
23477        let mut runner = runner_at(RunStatus::Gating);
23478        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23479        runner.state.repo = dir.path().to_owned();
23480        runner.state.candidates[0].worktree = dir.path().to_owned();
23481        seed_public_summary(&mut runner);
23482        let winner = runner.state.candidates[0].clone();
23483        assert!(
23484            runner
23485                .guarded_pr_message(&winner, None, true)
23486                .await
23487                .unwrap()
23488                .is_none()
23489        );
23490        let store = ask::Questions::open();
23491        let find = |run: &str| {
23492            store
23493                .list()
23494                .into_iter()
23495                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23496                .unwrap()
23497        };
23498        let run = runner.state.id.clone();
23499        let q = find(&run);
23500        let fp = runner
23501            .state
23502            .github_text
23503            .as_ref()
23504            .unwrap()
23505            .fingerprint
23506            .clone();
23507        let name = crate::github_text::artifact_name(&fp);
23508        let path = crate::run::artifact_path(&runner.state, &name);
23509        assert!(q.detail.contains(&path.display().to_string()));
23510        assert!(q.detail.contains("title-language: title"));
23511        let saved = std::fs::read_to_string(&path).expect("artifact written");
23512        assert!(saved.contains("日本語のタイトル"));
23513        assert!(q.cwd.is_none());
23514        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23515        // A bad replacement is refused and asked about once more.
23516        let say = |id: &str, text: &str, choice: &str| {
23517            store
23518                .update(id, |q| {
23519                    q.thread.push(ask::Turn {
23520                        who: ask::Who::Operator,
23521                        body: text.to_owned(),
23522                        at: jiff::Timestamp::now(),
23523                        note: None,
23524                    });
23525                    q.answer(ask::Answer::Choice(choice.to_owned()))
23526                })
23527                .unwrap();
23528        };
23529        say(&q.id, "まだ日本語", "use my text");
23530        assert!(
23531            runner
23532                .guarded_pr_message(&winner, None, true)
23533                .await
23534                .unwrap()
23535                .is_none()
23536        );
23537        let second = find(&run);
23538        assert_ne!(second.id, q.id);
23539        assert!(second.detail.contains("did not pass"));
23540        // The retry still compares the original text; the rejected
23541        // replacement is named by category only.
23542        assert!(second.detail.contains("日本語のタイトル"));
23543        assert!(!second.detail.contains("まだ日本語"));
23544        assert!(second.detail.contains(&path.display().to_string()));
23545        say(&second.id, "fix: retry failed requests", "use my text");
23546        let message = runner
23547            .guarded_pr_message(&winner, None, true)
23548            .await
23549            .unwrap()
23550            .expect("a vetted replacement is posted");
23551        assert_eq!(message.title, "fix: retry failed requests");
23552        assert_eq!(
23553            runner
23554                .state
23555                .github_text
23556                .as_ref()
23557                .unwrap()
23558                .chosen_title
23559                .as_deref(),
23560            Some("fix: retry failed requests")
23561        );
23562        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23563        // Resumed again: the saved decision is applied, nothing is re-asked.
23564        let again = runner
23565            .guarded_pr_message(&winner, None, true)
23566            .await
23567            .unwrap()
23568            .unwrap();
23569        assert_eq!(again.title, message.title);
23570        assert_eq!(
23571            store
23572                .list()
23573                .into_iter()
23574                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23575                .count(),
23576            2
23577        );
23578    }
23579
23580    #[test]
23581    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23582        // What magi itself writes stays English under any configured language,
23583        // so a future localisation of these headings fails here. (The agents'
23584        // own text is also checked by the posting gate.)
23585        let mut state = state_with_summary(
23586            "add retries",
23587            "TITLE: fix(web): batch reads\n- reads run.json once",
23588        );
23589        state.config.graph.language = "ja".to_owned();
23590        let m = pr_message(&state, 'A');
23591        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23592        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23593
23594        // The task is the operator's own text: it goes in untouched, and the
23595        // fallback title (no summary) may be in its language too.
23596        let task = "今回やってほしいこと: results projector を直す";
23597        let mut state = state_with_summary(task, "- no title line");
23598        state.config.graph.language = "ja".to_owned();
23599        let m = pr_message(&state, 'A');
23600        assert_eq!(
23601            m.title,
23602            format!("chore: land candidate A of run {}", state.id)
23603        );
23604        assert!(
23605            m.body.contains(&format!(
23606                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23607            )),
23608            "{}",
23609            m.body
23610        );
23611    }
23612
23613    #[test]
23614    fn pr_message_scrubs_home_paths_and_addresses() {
23615        let state = state_with_summary(
23616            "fix it in /Users/someone/src/x",
23617            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23618        );
23619        let m = pr_message(&state, 'A');
23620        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23621            assert!(!m.body.contains(leak), "{}", m.body);
23622        }
23623        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23624    }
23625
23626    #[test]
23627    fn pr_message_survives_a_task_that_closes_details() {
23628        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23629        let m = pr_message(&state, 'A');
23630        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23631    }
23632
23633    #[test]
23634    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23635        let cmd = manual_merge_command(
23636            MergeStyle::Squash,
23637            Path::new("/repo"),
23638            "b",
23639            "fix: \"quoted\" $(x) `y`\n\nbody",
23640        );
23641        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23642    }
23643
23644    #[test]
23645    fn manual_merge_command_matches_the_configured_style() {
23646        let repo = Path::new("/repo");
23647        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23648
23649        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23650        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23651
23652        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23653        assert_eq!(
23654            squash,
23655            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23656             \"Merge magi run 0832 (candidate A)\""
23657        );
23658
23659        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23660        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23661    }
23662
23663    #[test]
23664    fn a_nudge_gets_a_quarter_of_the_budget() {
23665        // The judge and implement budgets magi ships with.
23666        assert_eq!(retry_budget(secs(1200), true), secs(300));
23667        assert_eq!(retry_budget(secs(3600), true), secs(900));
23668    }
23669
23670    #[test]
23671    fn a_resent_prompt_keeps_the_whole_budget() {
23672        // The seat kept no context, so the retry is the original job again and
23673        // shortening it would only guarantee a second failure.
23674        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23675        assert_eq!(retry_budget(secs(60), false), secs(60));
23676    }
23677
23678    #[test]
23679    fn the_floor_never_exceeds_the_original_budget() {
23680        // A short configured timeout must not be *raised* by the floor: the
23681        // operator asked for a bound, and a retry may not outlast the attempt
23682        // it is retrying.
23683        assert_eq!(retry_budget(secs(60), true), secs(60));
23684        assert_eq!(retry_budget(secs(480), true), secs(120));
23685        assert_eq!(retry_budget(secs(0), true), secs(0));
23686    }
23687
23688    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23689        agent::CommandEvidence {
23690            id: "item1".to_owned(),
23691            description: "cargo test".to_owned(),
23692            exit_code,
23693            result_summary: String::new(),
23694            source: "codex".to_owned(),
23695        }
23696    }
23697
23698    #[test]
23699    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23700        // No evidence is not the same fact as unconfirmed evidence: a
23701        // backend with no adapter, or a reply that ran no commands at all,
23702        // must not be misread as carrying a dangling job.
23703        assert!(!has_unconfirmed_command(&[]));
23704    }
23705
23706    #[test]
23707    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23708        // Deliberately not a check on the exit code's *value*: a fixer
23709        // legitimately runs something that fails mid-iteration before it
23710        // succeeds, and that must never by itself reopen a valid report.
23711        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23712        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23713        assert!(!has_unconfirmed_command(&[
23714            evidence(Some(0)),
23715            evidence(Some(101))
23716        ]));
23717    }
23718
23719    #[test]
23720    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23721        assert!(has_unconfirmed_command(&[
23722            evidence(Some(0)),
23723            evidence(None)
23724        ]));
23725    }
23726
23727    #[test]
23728    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23729        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23730        assert_eq!(
23731            verified_noop_claim(true, &[], text).as_deref(),
23732            Some("already fixed by b32cfc4, on main.")
23733        );
23734    }
23735
23736    #[test]
23737    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23738        // A timeout or a bad exit code reads as the ordinary loss it is,
23739        // whatever the reply's own prose claims.
23740        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23741        assert!(verified_noop_claim(false, &[], text).is_none());
23742    }
23743
23744    #[test]
23745    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23746        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23747        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23748        // A confirmed command alongside the marker is fine.
23749        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23750    }
23751
23752    #[test]
23753    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23754        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23755    }
23756
23757    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23758    /// pair, in order, labelled A, B, C, ...
23759    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23760        runner.state.candidates = shape
23761            .iter()
23762            .enumerate()
23763            .map(|(i, &(empty, verified))| Candidate {
23764                index: i,
23765                label: (b'A' + i as u8) as char,
23766                agent: "sonnet".to_owned(),
23767                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23768                worktree: PathBuf::from(format!("/wt/{i}")),
23769                summary: String::new(),
23770                stat: String::new(),
23771                files: 0,
23772                commits: 0,
23773                empty,
23774                failed: None,
23775                verified_noop: verified.map(str::to_owned),
23776                duration_ms: 0,
23777                folded: false,
23778            })
23779            .collect();
23780    }
23781
23782    #[test]
23783    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23784        ask_test_home();
23785        let mut runner = runner_at(RunStatus::Implementing);
23786        set_candidates(
23787            &mut runner,
23788            &[
23789                (true, Some("already on main at b32cfc4")),
23790                (true, Some("same fix, see the existing test")),
23791            ],
23792        );
23793
23794        runner
23795            .after_implement()
23796            .expect("a verified no-op is not an error");
23797
23798        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23799    }
23800
23801    #[test]
23802    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23803        ask_test_home();
23804        let mut runner = runner_at(RunStatus::Implementing);
23805        // Candidate A declares a verified no-op; candidate B simply wrote
23806        // nothing and said nothing about why. One candidate's claim is not
23807        // the whole run's agreement.
23808        set_candidates(
23809            &mut runner,
23810            &[(true, Some("already on main at b32cfc4")), (true, None)],
23811        );
23812
23813        let err = runner
23814            .after_implement()
23815            .expect_err("an unverified empty candidate must still fail the run");
23816
23817        assert!(
23818            err.to_string().contains("no candidate produced a change"),
23819            "{err}"
23820        );
23821        assert_eq!(runner.state.status, RunStatus::Failed);
23822    }
23823
23824    #[test]
23825    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23826        ask_test_home();
23827        let mut runner = runner_at(RunStatus::Implementing);
23828        set_candidates(&mut runner, &[(true, None), (true, None)]);
23829
23830        let err = runner
23831            .after_implement()
23832            .expect_err("no candidate declared anything; this is an ordinary failure");
23833
23834        assert!(
23835            err.to_string().contains("no candidate produced a change"),
23836            "{err}"
23837        );
23838        assert_eq!(runner.state.status, RunStatus::Failed);
23839    }
23840}
23841
23842/// A graft rollback whose restore could not be confirmed (see
23843/// [`Runner::graft`]).
23844#[derive(Debug)]
23845struct GraftRestoreFailed(String);
23846
23847impl std::fmt::Display for GraftRestoreFailed {
23848    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23849        write!(
23850            f,
23851            "could not restore the winner to its pre-graft commit: {}",
23852            self.0
23853        )
23854    }
23855}
23856
23857impl std::error::Error for GraftRestoreFailed {}
23858
23859/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23860/// not the agent: they never draw on an agent's own timeout, and running out of
23861/// them never reads as an agent failure.
23862#[cfg(test)]
23863mod resource_wait_tests {
23864    use super::*;
23865    use std::collections::BTreeMap;
23866
23867    fn sh(id: &str, script: &str) -> AgentSpec {
23868        AgentSpec {
23869            id: id.to_owned(),
23870            kind: crate::config::AgentKind::Command,
23871            model: None,
23872            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23873            extra_args: Vec::new(),
23874            env: BTreeMap::new(),
23875            prompt_delivery: None,
23876            max_turn_secs: None,
23877        }
23878    }
23879
23880    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23881        SeatJob {
23882            seat: SeatState::new("impl-A", &spec.id, 1),
23883            spec,
23884            cwd: dir.to_path_buf(),
23885            prompt: "do it".to_owned(),
23886            timeout,
23887            allow_write: true,
23888            sessions: false,
23889            artifacts: dir.join("art"),
23890            stem: "impl-A".to_owned(),
23891            handover: None,
23892        }
23893    }
23894
23895    fn state(resource_wait_secs: u64) -> RunState {
23896        let mut config = Config::default();
23897        config.graph.timeout_resource_wait = resource_wait_secs;
23898        RunState::new(
23899            PathBuf::from("/repo"),
23900            "main".to_owned(),
23901            "abc1234".to_owned(),
23902            "task".to_owned(),
23903            config,
23904        )
23905    }
23906
23907    fn hold(cache: &Path) -> crate::cache::Guard {
23908        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23909        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23910            crate::cache::AcquireOutcome::Acquired(g) => g,
23911            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23912        }
23913    }
23914
23915    async fn run(
23916        jobs: Vec<SeatJob>,
23917        sem: Arc<Semaphore>,
23918        cache: Option<&Path>,
23919        st: &mut RunState,
23920    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23921        let prompts = Prompts::default();
23922        let ctx = WaveCtx {
23923            run: "r",
23924            node: "implement",
23925            prompts: &prompts,
23926            cache,
23927            round: None,
23928            carry_seats: false,
23929            pauses: None,
23930        };
23931        wave(jobs, sem, &ctx, st, 0).await
23932    }
23933
23934    #[tokio::test]
23935    async fn verification_cache_wait_preserves_the_command_budget() {
23936        crate::run::pin_test_home();
23937        let tmp = tempfile::tempdir().unwrap();
23938        let cache = tmp.path().join("target");
23939        let guard = hold(&cache);
23940        let release = tokio::spawn(async move {
23941            tokio::time::sleep(Duration::from_millis(100)).await;
23942            guard.release();
23943        });
23944        let mut st = state(10);
23945        let budget = Duration::from_millis(50);
23946        let wait = repository_wait(&st, None);
23947        let (outcomes, _) = with_cache_lease(
23948            &mut st,
23949            Some(&cache),
23950            "gate",
23951            "gate",
23952            tmp.path(),
23953            "abc1234",
23954            budget,
23955            "test gate",
23956            &wait,
23957            |_, available| async move {
23958                assert_eq!(
23959                    available, budget,
23960                    "contention must not shorten verification"
23961                );
23962                (Vec::new(), false, Vec::new())
23963            },
23964        )
23965        .await
23966        .unwrap();
23967        release.await.unwrap();
23968        assert!(outcomes.is_empty(), "{outcomes:?}");
23969        assert!(
23970            st.events
23971                .iter()
23972                .any(|event| event.message.contains("waiting for"))
23973        );
23974        assert!(st.resource_wait.is_none());
23975        // The verification body no longer owns the lease after returning.
23976        hold(&cache).release();
23977    }
23978
23979    #[tokio::test]
23980    async fn verification_cache_wait_expires_without_starting_commands() {
23981        crate::run::pin_test_home();
23982        let tmp = tempfile::tempdir().unwrap();
23983        let cache = tmp.path().join("target");
23984        let _held = hold(&cache);
23985        let mut st = state(0);
23986        let wait = repository_wait(&st, None);
23987        let (outcomes, _) = with_cache_lease(
23988            &mut st,
23989            Some(&cache),
23990            "gate",
23991            "gate",
23992            tmp.path(),
23993            "abc1234",
23994            Duration::from_secs(60),
23995            "test gate",
23996            &wait,
23997            |_, _| async { panic!("commands must not start without the cache lease") },
23998        )
23999        .await
24000        .unwrap();
24001        assert_eq!(outcomes.len(), 1);
24002        assert!(outcomes[0].resource_blocked);
24003        assert!(st.resource_wait.is_none());
24004        assert!(matches!(
24005            st.resource_block.unwrap().cause,
24006            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24007        ));
24008    }
24009
24010    #[tokio::test]
24011    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24012        crate::run::pin_test_home();
24013        let tmp = tempfile::tempdir().unwrap();
24014        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24015        let cache = tmp.path().join("cache");
24016        let mut state = state(21600);
24017        let wait = repository_wait(&state, None);
24018        let (outcomes, _) = with_cache_lease(
24019            &mut state,
24020            Some(&cache),
24021            "gate",
24022            "gate",
24023            tmp.path(),
24024            "head",
24025            Duration::from_secs(60),
24026            "gate",
24027            &wait,
24028            |_, _| async { panic!("freshness failure must not start verification") },
24029        )
24030        .await
24031        .unwrap();
24032        assert_eq!(outcomes.len(), 1);
24033        assert!(outcomes[0].resource_blocked);
24034        let block = state.resource_block.unwrap();
24035        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24036        assert!(!block.description().contains("timed out"));
24037    }
24038
24039    #[tokio::test]
24040    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24041        crate::run::pin_test_home();
24042        let tmp = tempfile::tempdir().unwrap();
24043        let cache = tmp.path().join("cache");
24044        let _held = hold(&cache);
24045        let mut state = state(21600);
24046        let pause = Pause::new();
24047        let interrupt = Pause::new();
24048        let cause = ParkCause::Operator;
24049        pause.park_for(cause.clone());
24050        let prompts = Prompts::default();
24051        let ctx = WaveCtx {
24052            run: "test",
24053            node: "fix",
24054            prompts: &prompts,
24055            cache: Some(&cache),
24056            round: None,
24057            carry_seats: false,
24058            pauses: Some((&pause, &interrupt)),
24059        };
24060        let result = wave(
24061            vec![job(
24062                sh("writer", "exit 99"),
24063                tmp.path(),
24064                Duration::from_secs(1),
24065            )],
24066            Arc::new(Semaphore::new(1)),
24067            &ctx,
24068            &mut state,
24069            0,
24070        )
24071        .await;
24072        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24073        assert_eq!(result[0].1.turns, 0);
24074        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24075        assert_eq!(
24076            state.resource_block.unwrap().cause,
24077            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24078        );
24079        assert!(state.active.is_empty());
24080    }
24081
24082    #[tokio::test]
24083    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24084        crate::run::pin_test_home();
24085        let tmp = tempfile::tempdir().unwrap();
24086        let cache = tmp.path().join("target-a");
24087        let guard = hold(&cache);
24088        // Held for longer than the agent's whole 1s budget.
24089        tokio::spawn(async move {
24090            tokio::time::sleep(Duration::from_millis(1500)).await;
24091            guard.release();
24092        });
24093        let mut st = state(60);
24094        let results = run(
24095            vec![job(
24096                sh("alpha", "echo ok"),
24097                tmp.path(),
24098                Duration::from_secs(1),
24099            )],
24100            Arc::new(Semaphore::new(1)),
24101            Some(&cache),
24102            &mut st,
24103        )
24104        .await;
24105        assert!(
24106            matches!(results[0].2, AgentOutcome::Ok(_)),
24107            "the wait must not be charged to the agent"
24108        );
24109        assert!(st.resource_block.is_none());
24110        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24111        assert!(st.resource_wait.is_none() && st.active.is_empty());
24112    }
24113
24114    #[tokio::test]
24115    #[cfg(unix)]
24116    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24117        use std::os::fd::AsRawFd as _;
24118        crate::run::pin_test_home();
24119        let tmp = tempfile::tempdir().unwrap();
24120        crate::git::git(tmp.path(), &["init"]).await.unwrap();
24121        let common = tmp.path().join(".git");
24122        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24123        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24124        assert_eq!(
24125            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24126            0
24127        );
24128        let mut st = state(1);
24129        st.config.coordination = crate::config::test_coordination();
24130        let mut reader = job(
24131            sh("reader", "echo must-not-run"),
24132            tmp.path(),
24133            Duration::from_secs(30),
24134        );
24135        reader.allow_write = false;
24136        let results = run(
24137            vec![reader.clone()],
24138            Arc::new(Semaphore::new(1)),
24139            None,
24140            &mut st,
24141        )
24142        .await;
24143        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24144        assert_eq!(results[0].1.turns, 0);
24145        assert!(FailClass::of(&results[0].2).is_none());
24146        assert_eq!(
24147            st.resource_block.as_ref().unwrap().kind,
24148            "repository coordination"
24149        );
24150        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24151        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24152        drop(external);
24153        let editor = crate::repo_lock::SeatLease::for_checkout(
24154            tmp.path(),
24155            true,
24156            crate::config::test_coordination().as_ref(),
24157        )
24158        .await
24159        .unwrap();
24160        let pause = Pause::new();
24161        let interrupt = Pause::new();
24162        let prompts = Prompts::default();
24163        let ctx = WaveCtx {
24164            run: "r",
24165            node: "review",
24166            prompts: &prompts,
24167            cache: None,
24168            round: None,
24169            carry_seats: false,
24170            pauses: Some((&pause, &interrupt)),
24171        };
24172        let (results, ()) = tokio::join!(
24173            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24174            async {
24175                tokio::time::sleep(Duration::from_millis(50)).await;
24176                pause.park();
24177            }
24178        );
24179        assert!(
24180            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24181        );
24182        assert_eq!(results[0].1.turns, 0);
24183        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24184        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24185        drop(editor);
24186    }
24187
24188    #[tokio::test]
24189    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24190        crate::run::pin_test_home();
24191        let tmp = tempfile::tempdir().unwrap();
24192        let cache = tmp.path().join("target-b");
24193        let _held = hold(&cache);
24194        let mut st = state(1);
24195        let results = run(
24196            vec![job(
24197                sh("alpha", "echo ok"),
24198                tmp.path(),
24199                Duration::from_secs(1),
24200            )],
24201            Arc::new(Semaphore::new(1)),
24202            Some(&cache),
24203            &mut st,
24204        )
24205        .await;
24206        let (_, seat, out) = &results[0];
24207        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24208        assert!(wave_blocked(&results));
24209        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24210        assert_eq!(seat.turns, 0, "nothing was asked");
24211        assert!(
24212            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24213            "no agent was spawned"
24214        );
24215        assert!(FailClass::of(out).is_none(), "not a failure class");
24216        let block = st.resource_block.as_ref().expect("recorded for the resume");
24217        assert_eq!(block.kind, "shared build cache");
24218        assert!(matches!(
24219            block.cause,
24220            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24221        ));
24222        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24223        assert!(st.active.is_empty());
24224    }
24225
24226    #[tokio::test]
24227    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24228        crate::run::pin_test_home();
24229        let tmp = tempfile::tempdir().unwrap();
24230        let mut st = state(1);
24231        let results = run(
24232            vec![job(
24233                sh("alpha", "echo ok"),
24234                tmp.path(),
24235                Duration::from_secs(1),
24236            )],
24237            Arc::new(Semaphore::new(0)),
24238            None,
24239            &mut st,
24240        )
24241        .await;
24242        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24243        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24244        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24245    }
24246
24247    #[tokio::test]
24248    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24249        crate::run::pin_test_home();
24250        let tmp = tempfile::tempdir().unwrap();
24251        let mut st = state(60);
24252        let results = run(
24253            vec![job(
24254                sh("alpha", "sleep 30"),
24255                tmp.path(),
24256                Duration::from_secs(1),
24257            )],
24258            Arc::new(Semaphore::new(1)),
24259            None,
24260            &mut st,
24261        )
24262        .await;
24263        let out = &results[0].2;
24264        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24265        let class = FailClass::of(out).expect("a failure");
24266        assert_eq!(class, FailClass::Timeout);
24267        assert!(should_hand_over(None, &class), "existing handover policy");
24268        assert!(st.resource_block.is_none());
24269    }
24270}