Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize one bounded synchronization interval, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false, None).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true, None).await
1357    }
1358
1359    pub(crate) async fn authorize_base_recovery_control(
1360        home: &Path,
1361        task: &str,
1362        control: &mut crate::base_recovery_control::Transaction,
1363    ) -> Result<()> {
1364        let id = control.request.run.clone();
1365        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1366    }
1367
1368    async fn authorize_base_recovery_request(
1369        home: &Path,
1370        id: &str,
1371        task: &str,
1372        resumption: bool,
1373        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1374    ) -> Result<()> {
1375        let _owner = crate::pr_head::owner_lock(home, id)?;
1376        let mut state = RunState::load_under(id, home)?;
1377        crate::pr_head::ensure_owner(&state)?;
1378        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1379        anyhow::ensure!(
1380            queued.execution_runs().last().is_some_and(|run| run == id)
1381                && queued.status != crate::queue::TaskStatus::Done
1382                && !queued.fresh_start
1383                && queued.review_branch.is_none(),
1384            "run task ownership changed"
1385        );
1386        anyhow::ensure!(
1387            state
1388                .origin
1389                .as_ref()
1390                .and_then(|o| o.task.as_deref())
1391                .is_none_or(|owner| owner == task),
1392            "run belongs to another task"
1393        );
1394        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1395        anyhow::ensure!(
1396            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1397            "run is live or its ownership is unknown"
1398        );
1399        // Judge the previous driver's lease before acquiring our maintenance
1400        // lease. The owner lock excludes new driver claims across both steps.
1401        let _execution = crate::driver::maintenance(home, id)?;
1402        anyhow::ensure!(
1403            state.status.resumable()
1404                && state.released_to.is_none()
1405                && state.retired_to.is_none()
1406                && state.pr_taken_over_by.is_none(),
1407            "run ownership is final"
1408        );
1409        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1410        anyhow::ensure!(
1411            !state.pr_publication_pending
1412                && !crate::handover::landing_checkpoint(&state)
1413                && state.merge_decision.is_none(),
1414            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1415        );
1416        let winner = state.winner().context("run has no retained winner")?;
1417        anyhow::ensure!(
1418            git::is_clean(&winner.worktree).await?,
1419            "retained worktree has uncommitted changes"
1420        );
1421        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1422        anyhow::ensure!(
1423            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1424            "retained worktree does not match the winner branch"
1425        );
1426        if let Some(control) = control.as_ref() {
1427            anyhow::ensure!(
1428                head.starts_with(&control.request.head),
1429                "retained HEAD changed since the owner request"
1430            );
1431            anyhow::ensure!(
1432                state
1433                    .base_sync_recovery
1434                    .stop
1435                    .as_ref()
1436                    .is_none_or(|stop| stop.head == head),
1437                "retained HEAD differs from the recorded stop head; inspect it first"
1438            );
1439        }
1440        state.ensure_base_episode();
1441        state.record_base_sync_stop(&head);
1442        anyhow::ensure!(
1443            state
1444                .base_sync_recovery
1445                .stop
1446                .as_ref()
1447                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1448            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1449        );
1450        let fetched =
1451            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1452        anyhow::ensure!(
1453            fetched.ok(),
1454            "cannot authorize base recovery without a fetched base"
1455        );
1456        let approved_tip = git::rev_parse(
1457            &state.repo,
1458            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1459        )
1460        .await?;
1461        if let Some(control) = control.as_deref_mut() {
1462            let queue = crate::queue::Queue::at(home.join("queue"));
1463            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1464            if let Some(prior) = state
1465                .base_sync_recovery
1466                .authorizations
1467                .iter()
1468                .find(|a| a.request.as_deref() == Some(&control.key()))
1469            {
1470                anyhow::ensure!(
1471                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1472                    "recovery request was consumed or invalidated; it cannot queue again"
1473                );
1474                return control.enqueue(&queue);
1475            }
1476        }
1477        if resumption {
1478            anyhow::ensure!(
1479                crate::daemon::compatible_run_instruction(&state, &queued),
1480                "resumption instruction does not match the retained run"
1481            );
1482            let approval = queued
1483                .resumption
1484                .as_ref()
1485                .context("no operator resumption")?;
1486            anyhow::ensure!(
1487                approval.instruction == queued.instruction,
1488                "resumption approval does not match the current task instruction"
1489            );
1490            state.authorize_base_sync_resumption(task, approval)?;
1491        } else {
1492            state.authorize_base_sync(task)?;
1493        }
1494        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1495        state
1496            .base_sync_recovery
1497            .authorizations
1498            .last_mut()
1499            .unwrap()
1500            .scope = Some(crate::base_sync_recovery::Scope {
1501            tip: approved_tip,
1502            instruction: queued.instruction.clone(),
1503            episode,
1504        });
1505        if let Some(control) = control.as_deref_mut() {
1506            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1507            authorization.request = Some(control.key());
1508            authorization.generation = control.generation();
1509            authorization.decision = control.decision();
1510            authorization.question = control
1511                .request
1512                .decision
1513                .as_ref()
1514                .map(|_| control.request.request.clone());
1515        }
1516        state.event(
1517            "base-recovery",
1518            "owner authorized a bounded base synchronization interval; fresh verification required",
1519        );
1520        state.save_under(home)?;
1521        if let Some(control) = control {
1522            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1523        }
1524        Ok(())
1525    }
1526
1527    /// Reopen an existing run.
1528    pub fn resume(id: &str) -> Result<Self> {
1529        let state = RunState::load(id)?;
1530        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1531        if state.startup.as_ref().is_some_and(|s| !s.complete)
1532            || (state.status == RunStatus::Prep
1533                && state.instruction.trim().is_empty()
1534                && state.candidates.is_empty()
1535                && state.prep_complete == Some(false))
1536        {
1537            bail!(
1538                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1539            );
1540        }
1541        crate::pr_head::ensure_owner(&state)?;
1542        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1543            bail!(
1544                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1545            );
1546        }
1547        if let Some(to) = &state.released_to {
1548            bail!(
1549                "run {} cannot be resumed: its worktree was released to run {}",
1550                state.short(),
1551                crate::run::short_of(to)
1552            );
1553        }
1554        let roles = state.config.resolve_roles()?;
1555        if let Some(id) = &state.config.graph.selected_implementer
1556            && state.candidates.iter().any(|c| c.agent != *id)
1557        {
1558            bail!(
1559                "replacement implementer pin does not match the saved candidates; task must stay held"
1560            );
1561        }
1562        let max_parallel = state.config.graph.max_parallel.max(1);
1563        Ok(Self {
1564            state,
1565            roles,
1566            sem: Arc::new(Semaphore::new(max_parallel)),
1567            pause: Pause::new(),
1568            interrupt: Pause::new(),
1569        })
1570    }
1571
1572    /// Walk the graph to a terminal state, skipping nodes already recorded.
1573    ///
1574    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1575    /// the phone - ends here, so this is the one place a run that ended
1576    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1577    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1578    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1579        // Allocate before polling: boxing inside an async wrapper would still
1580        // retain its construction frame while the nested graph is polled.
1581        Box::pin(self.execute_inner())
1582    }
1583
1584    async fn execute_inner(&mut self) -> Result<()> {
1585        anyhow::ensure!(
1586            self.state.startup.as_ref().is_none_or(|s| s.complete),
1587            "startup incomplete; inspect retained evidence before explicit recovery"
1588        );
1589        if let Some(selection) = &self.state.pr_recovery
1590            && (selection.source_proof.is_some() || selection.source_run.is_some())
1591        {
1592            Box::pin(selection.verify_source(&self.state.repo)).await?;
1593        }
1594        // Queue claims cover daemon work; this also serializes manual drivers
1595        // consuming a repair answer, including the new review and decision.
1596        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1597            || self.state.land_approval.is_some()
1598            || self.state.merge_decision.is_some()
1599            || !self.state.merge_repairs.is_empty()
1600            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1601        {
1602            let claim = FixClaim::acquire(&self.state.dir())?;
1603            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1604            let disk = RunState::load(&self.state.id)?;
1605            let checkpoint = |s: &RunState| {
1606                (
1607                    s.merge_repairs.len(),
1608                    s.merge_repairs
1609                        .last()
1610                        .map(|r| (r.question.clone(), r.phase)),
1611                    s.land_approval
1612                        .as_ref()
1613                        .map(|a| (a.question.clone(), a.head.clone())),
1614                )
1615            };
1616            if checkpoint(&disk) != checkpoint(&self.state) {
1617                bail!(
1618                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1619                );
1620            }
1621            Some(claim)
1622        } else {
1623            None
1624        };
1625        let mut execution = self.claim_driver().await?;
1626        let runner = &mut *execution.runner;
1627        let wait = runner.repository_wait();
1628        // Keep the graph's large future off the callers' stack, including
1629        // standalone task attempts running on a default-sized test thread.
1630        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1631        let result = match result {
1632            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1633                record_repository_block(&mut runner.state, &e);
1634                runner.park_here()?;
1635                Ok(())
1636            }
1637            other => other,
1638        };
1639        execution.driver.finish(&mut runner.state).await?;
1640        let ended = if result.is_err() {
1641            Some({
1642                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1643                if let Err(error) = &result {
1644                    notice.message = format!(
1645                        "Run {} stopped with an error: {}",
1646                        runner.state.id,
1647                        error.to_string().lines().next().unwrap_or("unknown error")
1648                    );
1649                }
1650                notice
1651            })
1652        } else {
1653            crate::notices::run_ended(&runner.state)
1654        };
1655        if let Some(notice) = ended {
1656            crate::notices::raise(notice);
1657        }
1658        result
1659    }
1660
1661    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1662        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1663    }
1664
1665    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1666        let wait = self.repository_wait();
1667        coordinate_operation(&mut self.state, cwd, &wait).await
1668    }
1669
1670    /// Validate ownership and persist the driver claim in one critical section.
1671    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1672        let owner = match crate::pr_head::owner_lock_wait(
1673            &crate::run::home(),
1674            &self.state.id,
1675            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1676        )
1677        .await
1678        {
1679            Ok(owner) => owner,
1680            Err(error) => {
1681                let detail = format!("worktree maintenance or ownership update: {error:#}");
1682                // Admission did not own run.json: retain this resource outcome
1683                // in memory for queue settlement without overwriting a concurrent
1684                // retirement or publication transaction.
1685                self.state.resource_block = Some(crate::run::ResourceBlock {
1686                    cause: crate::run::ResourceBlockCause::Other,
1687                    node: "start".into(),
1688                    kind: "worktree maintenance".into(),
1689                    detail: detail.clone(),
1690                    at: Timestamp::now(),
1691                });
1692                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1693            }
1694        };
1695        let disk = RunState::load(&self.state.id)?;
1696        crate::pr_head::ensure_owner(&disk)?;
1697        let claimed =
1698            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1699        // Only proven-live ownership refuses here. Unknown records (legacy runs
1700        // with no claim marker or process identity) fall through to the
1701        // exclusive execution lease taken just below, which is what actually
1702        // keeps a second driver out.
1703        anyhow::ensure!(
1704            disk.liveness(claimed) != crate::run::Liveness::Live,
1705            "run already has a live driver; wait for its execution to end"
1706        );
1707        if let Some(to) = &disk.released_to {
1708            bail!(
1709                "run {} cannot continue: its worktree was released to run {}",
1710                self.state.short(),
1711                crate::run::short_of(to)
1712            );
1713        }
1714        anyhow::ensure!(
1715            serde_json::to_value(&disk.base_sync_recovery)?
1716                == serde_json::to_value(&self.state.base_sync_recovery)?,
1717            "base recovery changed since this run was loaded; reload the saved run"
1718        );
1719        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1720        let mut execution = DriverExecution {
1721            runner: self,
1722            driver,
1723        };
1724        // Declare after the execution guard: on any claim error this short
1725        // lock must drop before that guard reloads and releases ownership.
1726        let _owner = owner;
1727        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1728            .await?;
1729        let state = &mut execution.runner.state;
1730        // Moving again, so it is no longer parked. Set before the walk rather
1731        // than in `resume`, so every way of re-entering the graph clears it
1732        // and a card cannot claim a run is waiting to be resumed while the
1733        // agents are already working.
1734        state.parked = false;
1735        state.resource_wait = None;
1736        state.resource_block = None;
1737        // Any seat this state still lists as answering belongs to whatever
1738        // process last drove this run — this one included, if it crashed
1739        // mid-wave. Cleared and flushed immediately, before anything else
1740        // runs, so a resume can never show a seat as live when nothing is
1741        // asking it anything yet; the node that actually dispatches the next
1742        // wave repopulates it.
1743        state.clear_active();
1744        // Recorded in the same spot, and flushed together with the clear
1745        // above: this is the pid a reader checks (`RunState::liveness`) when
1746        // no daemon claim exists to answer "is a process still driving this
1747        // run" — a plain `magi run` / `magi review` typed into a terminal
1748        // claims nothing there. Always overwritten, never only-if-absent, so
1749        // a resumed run's stale pid from a previous, possibly-dead process
1750        // can never survive into this one's own report. Unlike
1751        // `clear_active`, this changes on every single `execute()` call, so
1752        // the save below is now unconditional rather than only-if-cleared.
1753        //
1754        // `driver_started_at` is recorded in the same breath, from this same
1755        // pid, so `liveness` can tell a live pid that is genuinely still us
1756        // apart from one the OS has since handed to an unrelated process —
1757        // see that field's own doc for why the pid alone is not enough.
1758        let pid = std::process::id();
1759        state.driver_pid = Some(pid);
1760        state.driver_started_at = crate::proc::process_started_at(pid);
1761        state.driver_exited = false;
1762        state.driver_claimed = Some(true);
1763        state.begin_pass();
1764        execution.driver.bind(state);
1765        state.save()?;
1766        execution.driver.record(state)?;
1767        Ok(execution)
1768    }
1769
1770    /// No conflict checkpoint proves dispatch was never reached. End that
1771    /// consumed generation so an owner can explicitly authorize a fresh one.
1772    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1773        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1774        if let Err(error) = &result
1775            && !error.is::<crate::repo_lock::ResourceBlocked>()
1776            && self
1777                .state
1778                .repair_grants
1779                .last()
1780                .is_some_and(|g| g.conflict.is_none())
1781        {
1782            self.state.repair_grants.last_mut().unwrap().phase =
1783                crate::repair_budget::Phase::Complete;
1784            self.state.status = RunStatus::Blocked;
1785            self.state.save()?;
1786            bail!(
1787                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1788                self.state.passes.last().map_or(0, |p| p.index)
1789            );
1790        }
1791        result
1792    }
1793
1794    /// A grant owns only these repair attempts. Confirmation reviews cannot
1795    /// create another fixer allowance, and every operation is restart-checkpointed.
1796    async fn execute_granted_repair(&mut self) -> Result<()> {
1797        use crate::repair_budget::{Kind, Phase};
1798        let index = self.state.repair_grants.len() - 1;
1799        let winner = self
1800            .state
1801            .winner()
1802            .cloned()
1803            .context("repair winner missing")?;
1804        loop {
1805            let grant = self.state.repair_grants[index].clone();
1806            match grant.phase {
1807                Phase::Pending => {
1808                    if grant.spent >= grant.receipt.request.rounds {
1809                        self.state.repair_grants[index].phase = Phase::Complete;
1810                        self.state.status = RunStatus::Blocked;
1811                        self.state.save()?;
1812                        return Ok(());
1813                    }
1814                    // Reserve before dispatch. A crash can consume an attempt,
1815                    // but can never duplicate it or reset saved fixer history.
1816                    self.state.repair_grants[index].spent += 1;
1817                    self.state.repair_grants[index].phase = Phase::Fixing;
1818                    self.state.save()?;
1819                    match grant.receipt.request.kind {
1820                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1821                        Kind::Gate => {
1822                            let failed = self.state.gate.clone();
1823                            let _ = self.gate_fix_round(&winner, &failed).await?;
1824                        }
1825                        Kind::Review => {
1826                            let mut record = self
1827                                .state
1828                                .reviews
1829                                .last()
1830                                .cloned()
1831                                .context("review failure missing")?;
1832                            let round = record.round;
1833                            let step = self.run_fix_round(&winner, &mut record).await?;
1834                            self.state.reviews[round - 1] = record;
1835                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1836                                self.state.repair_grants[index].phase = Phase::Complete;
1837                                self.state.status = RunStatus::Blocked;
1838                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1839                                self.state.save()?;
1840                                return Ok(());
1841                            }
1842                            if matches!(step, FixStep::Resource) {
1843                                self.stop_for_resource()?;
1844                                return Ok(());
1845                            }
1846                            self.state.save()?;
1847                        }
1848                    }
1849                    if self.resource_halt()? {
1850                        return Ok(());
1851                    }
1852                }
1853                Phase::Fixing => {
1854                    if grant.receipt.request.kind == Kind::BaseConflict {
1855                        self.repair_base_conflict(false).await?;
1856                    }
1857                    // Interrupted fixers may have left source changes or an
1858                    // unpushed commit. Preserve them before fresh assessment.
1859                    let _coordination = self.coordinate(&winner.worktree).await?;
1860                    let rescued = git::rescue_commit(
1861                        &winner.worktree,
1862                        "magi: preserve interrupted granted repair",
1863                    )
1864                    .await?;
1865                    self.state.note_withheld("repair-grant", &rescued.withheld);
1866                    // Fresh checks must supersede even a same-head failure.
1867                    // Archive rather than erase failure evidence or counters.
1868                    self.state
1869                        .verification_history
1870                        .push(crate::run::VerificationEvidence {
1871                            superseded_by: None,
1872                            archived_at: Some(Timestamp::now()),
1873                            fixes_spent: Some(self.state.gate_fixes.len()),
1874                            gate_head: self.state.gate_head.take(),
1875                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1876                            gate: std::mem::take(&mut self.state.gate),
1877                            e2e: self.state.head_e2e.take(),
1878                        });
1879                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1880                        self.state
1881                            .verification_history
1882                            .push(crate::run::VerificationEvidence {
1883                                superseded_by: None,
1884                                archived_at: Some(Timestamp::now()),
1885                                fixes_spent: Some(self.state.gate_fixes.len()),
1886                                gate_head: None,
1887                                gate_ran: false,
1888                                gate: Vec::new(),
1889                                e2e: Some(e2e),
1890                            });
1891                    }
1892                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1893                    self.state.repair_grants[index].head =
1894                        git::rev_parse(&winner.worktree, "HEAD").await?;
1895                    self.state.repair_grants[index].phase = Phase::Reviewing;
1896                    if grant.receipt.request.kind == Kind::BaseConflict {
1897                        self.state.base_sync_recovery.stop = None;
1898                        if let Some(sync) = &mut self.state.base_sync {
1899                            sync.tip = grant
1900                                .receipt
1901                                .request
1902                                .tip
1903                                .clone()
1904                                .context("repair base tip missing")?;
1905                            sync.conflict = None;
1906                            sync.behind = 0;
1907                        }
1908                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1909                    }
1910                    self.state.save()?;
1911                }
1912                Phase::Reviewing => {
1913                    self.review_loop_pass().await?;
1914                    if self.resource_halt()? {
1915                        return Ok(());
1916                    }
1917                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1918                    let clean = self.state.reviews.len() > grant.review_start
1919                        && self.state.reviews.last().is_some_and(|r| {
1920                            r.head == head
1921                                && r.clean
1922                                && r.expected > 0
1923                                && r.answered == r.expected
1924                                && matches!(
1925                                    r.e2e_status(),
1926                                    E2eStatus::Passed | E2eStatus::NotConfigured
1927                                )
1928                        });
1929                    if !clean {
1930                        // A confirmation review carries no additional fix budget.
1931                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1932                            == Kind::Review
1933                            && grant.spent < grant.receipt.request.rounds
1934                            && self.state.reviews.last().is_some_and(|r| {
1935                                r.fix.is_none()
1936                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1937                            }) {
1938                            Phase::Pending
1939                        } else {
1940                            Phase::Complete
1941                        };
1942                        self.state.status = RunStatus::Blocked;
1943                        self.state.event(
1944                            "repair-grant",
1945                            "fresh review did not clear the repaired head; no publication",
1946                        );
1947                        self.state.save()?;
1948                        if self.state.repair_grants[index].phase == Phase::Pending {
1949                            continue;
1950                        }
1951                        return Ok(());
1952                    }
1953                    self.state.repair_grants[index].phase = Phase::Gating;
1954                    self.state.save()?;
1955                }
1956                Phase::Gating => {
1957                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1958                    self.state.reconcile_verification_head(&head);
1959                    let wait = self.repository_wait();
1960                    if self
1961                        .state
1962                        .reviews
1963                        .last()
1964                        .is_none_or(|r| r.head != head || !r.clean)
1965                    {
1966                        self.state.status = RunStatus::Blocked;
1967                        self.state.repair_grants[index].phase = Phase::Complete;
1968                        self.state.event(
1969                            "repair-grant",
1970                            "head changed since fresh review; repair held",
1971                        );
1972                        self.state.save()?;
1973                        return Ok(());
1974                    }
1975                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1976                        return Ok(());
1977                    }
1978                    let outcomes = self.run_gate(&winner).await?;
1979                    if verify_inconclusive(&outcomes) {
1980                        return Ok(());
1981                    }
1982                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1983                    if after != head {
1984                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1985                        self.state.repair_grants[index].head = after;
1986                        self.state.repair_grants[index].phase = Phase::Reviewing;
1987                        self.state.status = RunStatus::Blocked;
1988                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1989                        self.state.save()?;
1990                        return Ok(());
1991                    }
1992                    let passed = outcomes.iter().all(CommandOutcome::ok);
1993                    // Follow-up grants owe the same guarded checkout delivery
1994                    // as the conflict repair that introduced the new head.
1995                    if passed
1996                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1997                    {
1998                        return Ok(());
1999                    }
2000                    self.state.repair_grants[index].succeeded = passed;
2001                    self.state.gate = outcomes;
2002                    self.state.gate_ran = true;
2003                    self.state.status = if passed {
2004                        RunStatus::Gating
2005                    } else {
2006                        RunStatus::Blocked
2007                    };
2008                    self.state.repair_grants[index].phase = if !passed
2009                        && grant.receipt.request.kind == Kind::Gate
2010                        && grant.spent < grant.receipt.request.rounds
2011                    {
2012                        Phase::Pending
2013                    } else {
2014                        Phase::Complete
2015                    };
2016                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2017                        // Persist publication debt with the successful checkpoint.
2018                        self.state.pr_publication_pending = true;
2019                    } else if passed {
2020                        self.state.status = RunStatus::Ready;
2021                    }
2022                    self.state.save()?;
2023                    if !passed {
2024                        continue;
2025                    }
2026                    // Existing PR ownership and push lease stay in place. The
2027                    // normal publisher owes the new reviewed/gated SHA, and
2028                    // land's fresh owner approval remains mandatory.
2029                    if self.state.config.merge.mode != MergeMode::Pr {
2030                        self.state.status = RunStatus::Ready;
2031                        self.state.event(
2032                            "repair-grant",
2033                            "repair verified locally; grant carries no merge authorization",
2034                        );
2035                        self.state.save()?;
2036                        return Ok(());
2037                    }
2038                    return self.merge().await;
2039                }
2040                Phase::Complete => return Ok(()),
2041            }
2042        }
2043    }
2044
2045    async fn execute_graph(&mut self) -> Result<()> {
2046        // Even a terminal run may reenter through an explicit recovery action.
2047        // Restore before prep shortcuts, saved conversations or graph nodes use
2048        // a disposable path; GC may already have removed its original objects.
2049        let archive_home = crate::run::home().canonicalize()?;
2050        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2051            let repo = self.state.repo.clone();
2052            let _coordination = self.coordinate(&repo).await?;
2053            ensure_setup_config(&mut self.state, &repo).await?;
2054            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
2055        }
2056        self.state.configure_recovery_resume()?;
2057        if self
2058            .state
2059            .repair_grants
2060            .last()
2061            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2062        {
2063            return self.execute_granted_repair().await;
2064        }
2065        if self
2066            .state
2067            .repair_grants
2068            .last()
2069            .is_some_and(|g| !g.succeeded)
2070        {
2071            self.state.status = RunStatus::Blocked;
2072            self.state.save()?;
2073            return Ok(());
2074        }
2075        // Preservation may invalidate an old empty classification. Resolve it
2076        // before any resume shortcut can reuse a verdict or publish a branch.
2077        self.persist_candidates().await?;
2078        // A saved base stop outranks stage markers, including legacy Voting.
2079        // Recovery never reenters implementation, deliberation or voting.
2080        if self.state.base_sync_blocked() {
2081            self.sync_to_base().await?;
2082            if !self.state.base_sync_blocked()
2083                && self.state.status != RunStatus::AlreadyInBase
2084                && !self.resource_halt()?
2085            {
2086                self.finish_after_tally().await?;
2087            }
2088            return Ok(());
2089        }
2090        if self
2091            .state
2092            .merge_decision
2093            .as_ref()
2094            .is_some_and(|decision| !decision.closed)
2095            && self.state.status != RunStatus::Landing
2096        {
2097            if crate::merge_decision::resumable_hold(&self.state) {
2098                self.state.merge = Some(MergeOutcome {
2099                    mode: MergeMode::Pr,
2100                    ok: false,
2101                    detail: self
2102                        .state
2103                        .merge_decision
2104                        .as_ref()
2105                        .unwrap()
2106                        .target
2107                        .pr
2108                        .clone(),
2109                    empty: false,
2110                });
2111                self.run_land().await?;
2112                self.settle_questions();
2113                return Ok(());
2114            }
2115            bail!(
2116                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2117            );
2118        }
2119        if self.state.config.graph.selected_implementer.is_some()
2120            && self.state.status == RunStatus::Blocked
2121            && self.state.tally.is_none()
2122            && self.state.candidate_preservation.is_empty()
2123            && self.state.interrupted_implementations.is_empty()
2124        {
2125            bail!(
2126                "selected implementation is held; explicitly release the owning task to retry it"
2127            );
2128        }
2129
2130        if self.state.merge_repairs.last().is_some_and(|r| {
2131            matches!(
2132                r.phase,
2133                crate::run::MergeRepairPhase::Requested
2134                    | crate::run::MergeRepairPhase::Fixing
2135                    | crate::run::MergeRepairPhase::Reviewing
2136            )
2137        }) {
2138            if self.repair_merge_review().await? {
2139                self.run_land().await?;
2140            }
2141            self.settle_questions();
2142            return Ok(());
2143        }
2144        // Publication already passed review and gate. Resume only that node:
2145        // syncing to a newer base here could change the verified commit.
2146        if self.state.pr_publication_pending {
2147            self.merge().await?;
2148            if self.resource_halt()? {
2149                return Ok(());
2150            }
2151            self.state.save()?;
2152            self.settle_questions();
2153            return Ok(());
2154        }
2155        // A run that already lost its quorum never resumes into the verdict
2156        // machinery: `deliberate` and `vote` would otherwise clobber the
2157        // stalled marker back to Voting and the run would keep going past a
2158        // verdict that is no longer trustworthy. Everything already recorded is
2159        // kept, so the run stays resumable (or foldable) for a human to pick up.
2160        //
2161        // On --resume the run gets one chance to repair itself: the seats a
2162        // rate limit took out are re-asked. If their quota has since reset and
2163        // the quorum is restored, the run picks up and finishes; otherwise it
2164        // stays stale and still-resumable for a later retry. If it does not
2165        // recover, the returned status stays `Stalled` and nothing was
2166        // clobbered (the recovery only mutates entries for the lost seats).
2167        if self.state.status == RunStatus::Stalled {
2168            if self.recover_stall().await? {
2169                self.finish_after_tally().await?;
2170            } else {
2171                // Still below quorum: persist the marker and stay resumable.
2172                self.state.save()?;
2173            }
2174            return Ok(());
2175        }
2176        // A run parked inside `land` - watching CI, mid fix-round, or
2177        // waiting on the owner's merge approval - resumes directly into it,
2178        // never back through `prep`. Everything before `merge` already
2179        // concluded; that is the only way `status` reaches `Landing` in the
2180        // first place. Re-walking `review_loop` first would also be actively
2181        // wrong: its own status recomputation (see its doc) treats any
2182        // clean round as reason to set `status` to `Gating`, which would
2183        // clobber this marker before `merge` ever ran, and this run would
2184        // never find its way back into `land` at all.
2185        if self.state.status == RunStatus::Landing
2186            || crate::handover::landing_checkpoint(&self.state)
2187        {
2188            self.state.land_sync_pending = true;
2189            self.state.save()?;
2190            self.run_land().await?;
2191            // `run_land` may have settled the run right here - CI came back
2192            // green and the PR merged, say - without ever passing back
2193            // through `merge`'s own trailing call. Whatever it left `status`
2194            // as is what this has to read.
2195            self.settle_questions();
2196            return Ok(());
2197        }
2198        // A run parked at `merge` on the owner's word about a withheld PR text
2199        // resumes straight into `merge`. Re-walking the tail would sync to the
2200        // base again (possibly rebasing, possibly spending the fixer) after the
2201        // gate had passed, and then push a tree nobody reviewed or gated.
2202        if self.state.github_text.is_some()
2203            && self.state.merge.is_none()
2204            && self.state.status == RunStatus::Gating
2205        {
2206            self.merge().await?;
2207            if self.resource_halt()? {
2208                return Ok(());
2209            }
2210            self.state.save()?;
2211            return Ok(());
2212        }
2213        if let Err(e) = self.prep().await {
2214            if e.is::<crate::repo_lock::ResourceBlocked>() {
2215                return Err(e);
2216            }
2217            self.state.status = RunStatus::Blocked;
2218            self.state
2219                .prep_failure
2220                .get_or_insert_with(|| format!("{e:#}"));
2221            self.state
2222                .event("prep", format!("preparation stopped: {e:#}"));
2223            self.state.save()?;
2224            return Err(e);
2225        }
2226        if self.park_here()? {
2227            return Ok(());
2228        }
2229        self.advise().await?;
2230        if self.park_here()? {
2231            return Ok(());
2232        }
2233        self.implement().await?;
2234        if self.park_here()? {
2235            return Ok(());
2236        }
2237        // `after_implement` already saved the state and settled any open
2238        // questions when it set this; nothing later in the graph has
2239        // anything to judge.
2240        if self.state.status == RunStatus::VerifiedNoop {
2241            return Ok(());
2242        }
2243        self.judge().await?;
2244        if self.park_here()? {
2245            return Ok(());
2246        }
2247        self.deliberate().await?;
2248        if self.park_here()? {
2249            return Ok(());
2250        }
2251        self.vote().await?;
2252        if self.park_here()? {
2253            return Ok(());
2254        }
2255        self.tally()?;
2256        // A verdict that lost its quorum is not trustworthy: do not review,
2257        // gate, or merge on it. Everything already done is kept, so the run
2258        // stays resumable (or foldable); the human can replace the agent that
2259        // ran out of quota and pick it up.
2260        if self.state.status == RunStatus::Stalled {
2261            // Persist the stalled marker now — the normal end-of-execute save
2262            // below is below this early return, and without it a resumed run
2263            // would reload a pre-tally status and keep going.
2264            self.state.save()?;
2265            return Ok(());
2266        }
2267        self.finish_after_tally().await?;
2268        Ok(())
2269    }
2270
2271    /// A wave could not start its agents because a shared resource stayed busy
2272    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2273    /// resumable. Unstarted seats have no result; completed peers retain
2274    /// their saved candidate state.
2275    fn stop_for_resource(&mut self) -> Result<()> {
2276        // Resource acquisition observes cancellation inside a node. Complete
2277        // the same durable park as every other node boundary before returning.
2278        if self.pause.parked() || self.interrupt.parked() {
2279            self.park_here()?;
2280            return Ok(());
2281        }
2282        if self.state.resource_block.is_some() {
2283            self.state.status = RunStatus::Blocked;
2284        }
2285        self.state.save()
2286    }
2287
2288    /// Stop walking the graph if a node ended on a resource block.
2289    fn resource_halt(&mut self) -> Result<bool> {
2290        if self.state.resource_block.is_none() {
2291            return Ok(false);
2292        }
2293        self.stop_for_resource()?;
2294        Ok(true)
2295    }
2296
2297    /// Park here if asked to, recording it in the run's own timeline.
2298    ///
2299    /// Returns whether the caller should stop walking the graph. The state is
2300    /// saved either way by the node that just finished; this adds the event so
2301    /// the operator's card says why a run that is neither finished nor moving
2302    /// is sitting where it is.
2303    fn park_here(&mut self) -> Result<bool> {
2304        let parking = self.pause.parked() || self.interrupt.parked();
2305        if !parking && self.state.resource_block.is_some() {
2306            self.stop_for_resource()?;
2307            return Ok(true);
2308        }
2309        // Either handle asking is enough - see `Pause`'s own doc for why
2310        // they are never the same one. `interrupt` is checked second so a
2311        // reason it carries is preferred in the message below over a plain
2312        // shutdown park racing it at the same boundary.
2313        if !self.pause.parked() && !self.interrupt.parked() {
2314            return Ok(false);
2315        }
2316        if self.state.parked {
2317            return Ok(true);
2318        }
2319        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2320        if let Some(block) = &mut self.state.resource_block
2321            && matches!(
2322                block.cause,
2323                crate::run::ResourceBlockCause::Cancelled { .. }
2324            )
2325        {
2326            block.cause = crate::run::ResourceBlockCause::Cancelled {
2327                park: cause.clone(),
2328            };
2329        }
2330        let why = match cause.as_ref().map(ParkCause::label) {
2331            Some(reason) => format!(
2332                "parked after `{}` ({reason}) — resume to carry on from here",
2333                self.state.status.as_str()
2334            ),
2335            None => format!(
2336                "parked after `{}` — resume to carry on from here",
2337                self.state.status.as_str()
2338            ),
2339        };
2340        self.state.event("park", why);
2341        self.state
2342            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2343        self.state.parked = true;
2344        self.state.save()?;
2345        Ok(true)
2346    }
2347
2348    /// Only a new explicit operator approval may retry failed preparation.
2349    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2350        if self.state.prep_failure.is_none() {
2351            return Ok(());
2352        }
2353        let failed_at = self
2354            .state
2355            .events
2356            .iter()
2357            .rev()
2358            .find(|event| event.node == "prep")
2359            .map_or(self.state.updated_at, |event| event.at);
2360        anyhow::ensure!(
2361            approval_at > failed_at,
2362            "preparation retry requires a new explicit release after the failure"
2363        );
2364        self.state.prep_failure = None;
2365        self.state.prep_complete = Some(false);
2366        self.state
2367            .event("prep-retry", "operator approved retry of preparation");
2368        Ok(())
2369    }
2370
2371    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2372    pub fn on_pause(&mut self, pause: Pause) {
2373        self.pause = pause;
2374    }
2375
2376    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2377    /// scheduler asking this one run - and no other - to park so a task
2378    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2379    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2380    /// handle.
2381    pub fn watch_interrupt(&mut self, pause: Pause) {
2382        self.interrupt = pause;
2383    }
2384
2385    /// Abandon this run's own open questions, once `status` has actually
2386    /// settled rather than merely paused.
2387    ///
2388    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2389    /// either back up with the candidates, the review round and the seat
2390    /// sessions already on disk, so a question an implementer asked mid-round
2391    /// may still get a real answer read by a real resume. Only the statuses
2392    /// `resumable` excludes are actually final: the run merged, it reached
2393    /// `Ready` with nothing left to do, it failed outright with no
2394    /// established point to continue from, or every candidate agreed, with
2395    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2396    /// every one of those the seat that asked is gone for good, exactly like
2397    /// the run being deleted under `magi run rm` - so the same cleanup
2398    /// applies, worded for what actually happened instead of "the run was
2399    /// deleted".
2400    ///
2401    /// Best-effort and silent on success: called from every place `status`
2402    /// can land on one of those three, including ones a resumed run revisits,
2403    /// so it must cost nothing when there was nothing open to begin with.
2404    fn settle_questions(&mut self) {
2405        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2406            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2407        }
2408    }
2409
2410    /// The tail of the graph after a trustworthy tally: fold losers, review,
2411    /// gate, merge, and persist.
2412    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2413        Box::pin(self.finish_after_tally_inner())
2414    }
2415
2416    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2417        // Before the losers are folded: their branches are what the graft
2418        // reads from.
2419        self.graft().await?;
2420        if self.resource_halt()? {
2421            return Ok(());
2422        }
2423        self.fold_losers().await?;
2424        // Before review starts, and again right before the gate: a run's
2425        // review rounds can themselves take long enough for the base to move
2426        // a second time, and the gate is the one node whose "green" gets
2427        // acted on.
2428        self.sync_to_base().await?;
2429        if self.hold_base_sync().await? || self.resource_halt()? {
2430            return Ok(());
2431        }
2432        if self.state.status == RunStatus::AlreadyInBase {
2433            return Ok(());
2434        }
2435        self.review_loop().await?;
2436        if self.resource_halt()? {
2437            return Ok(());
2438        }
2439        self.sync_to_base().await?;
2440        if self.hold_base_sync().await? || self.resource_halt()? {
2441            return Ok(());
2442        }
2443        if self.state.status == RunStatus::AlreadyInBase {
2444            return Ok(());
2445        }
2446        self.gate().await?;
2447        self.merge().await?;
2448        if self.resource_halt()? {
2449            return Ok(());
2450        }
2451        self.state.save()?;
2452        Ok(())
2453    }
2454
2455    // ---------------------------------------------------------------- prep
2456
2457    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2458    /// run with the step and its output in the event; it is never swallowed.
2459    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2460    /// resume starts over instead of finding candidates and missing seats.
2461    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2462        let result = match ensure_setup_config(&mut self.state, repo).await {
2463            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2464            Err(e) => Err(e),
2465        };
2466        if let Err(e) = result {
2467            git::worktree_remove_clean(repo, wt).await.ok();
2468            if in_prep {
2469                self.rollback_prep(repo).await;
2470            }
2471            self.state.status = RunStatus::Blocked;
2472            self.state.event(
2473                "setup",
2474                format!("worktree setup failed in {}: {e:#}", wt.display()),
2475            );
2476            self.state.save()?;
2477            return Err(e);
2478        }
2479        Ok(())
2480    }
2481
2482    /// Undo everything `prep` created, so a resume re-runs it whole.
2483    async fn rollback_prep(&mut self, repo: &Path) {
2484        let root = self.state.worktree_root();
2485        for c in std::mem::take(&mut self.state.candidates) {
2486            if matches!(
2487                git::worktree_remove_clean(repo, &c.worktree).await,
2488                Ok(true)
2489            ) {
2490                git::branch_delete(repo, &c.branch).await.ok();
2491            } else {
2492                self.state.candidates.push(c);
2493            }
2494        }
2495        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2496        for j in 1..=seats {
2497            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2498                .await
2499                .ok();
2500        }
2501        for k in 1..=self.state.config.graph.advisors {
2502            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2503                .await
2504                .ok();
2505        }
2506    }
2507
2508    async fn prep(&mut self) -> Result<()> {
2509        if let Some(reason) = &self.state.prep_failure {
2510            bail!(
2511                "candidate preparation previously failed: {reason}; correct the source before retrying"
2512            );
2513        }
2514        if self
2515            .state
2516            .prep_complete
2517            .unwrap_or(!self.state.candidates.is_empty())
2518        {
2519            return Ok(());
2520        }
2521        let repo = self.state.repo.clone();
2522        // Includes rollback, config, refs, checkout creation and setup. No
2523        // repository mutation may precede the real shared lock acquisition.
2524        let _coordination = self.coordinate(&repo).await?;
2525        let base = self.state.base_commit.clone();
2526        let plan = self.state.sources.plan(&repo, &base).await?;
2527        self.rollback_prep(&repo).await;
2528        if !self.state.candidates.is_empty() {
2529            bail!(
2530                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2531            );
2532        }
2533        self.state.prep_complete = Some(false);
2534        self.state.status = RunStatus::Prep;
2535        self.state.save()?;
2536        let repo = self.state.repo.clone();
2537        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2538        let root = self.state.worktree_root();
2539        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2540
2541        // The hook is the write-time half of the blindness contract; the
2542        // presentation filter in `blind` is the half that cannot be bypassed.
2543        let hooks_dir = self.state.dir().join("hooks");
2544        if self.state.config.blind.commit_msg_hook {
2545            std::fs::create_dir_all(&hooks_dir)
2546                .with_context(|| format!("create {}", hooks_dir.display()))?;
2547            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2548            let path = hooks_dir.join("commit-msg");
2549            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2550            make_executable(&path)?;
2551            // Ref-counted rather than a plain idempotent set: with more than
2552            // one run able to be in flight in the same repository at once
2553            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2554            // true?" check cannot tell "another run of mine still needs
2555            // this" from "nobody does", and the run that happens to finish
2556            // first would disable the hook out from under a sibling still
2557            // relying on it.
2558            git::acquire_worktree_config(&repo).await?;
2559            self.state.enabled_worktree_config = true;
2560        }
2561
2562        // `[worktree] setup` hides its products per worktree, which needs the
2563        // same per-worktree config the hook does. Held until fold, like it.
2564        ensure_setup_config(&mut self.state, &repo).await?;
2565
2566        for (index, (spec, label)) in self
2567            .roles
2568            .implementers
2569            .clone()
2570            .into_iter()
2571            .zip(labels)
2572            .enumerate()
2573        {
2574            let branch = self.state.branch_for(label);
2575            let worktree = root.join(format!("cand-{label}"));
2576            // Persist intent before Git can leave a partially created tree.
2577            // Candidate registration follows setup, so it cannot inventory
2578            // crashes or refusals during creation itself.
2579            self.state.event(
2580                "prep-intent",
2581                serde_json::json!({
2582                    "branch": branch,
2583                    "worktree": worktree,
2584                    "start": start,
2585                    "picks": plan.picks,
2586                })
2587                .to_string(),
2588            );
2589            self.state.save()?;
2590            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2591            if self.state.config.blind.commit_msg_hook {
2592                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2593            }
2594            git::local_exclude(&worktree, "/.magi/").await?;
2595            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2596                // Nothing of this prep may survive: a resume must start over,
2597                // not find half the candidates and skip the rest.
2598                git::worktree_remove_clean(&repo, &worktree).await.ok();
2599                git::branch_delete(&repo, &branch).await.ok();
2600                self.rollback_prep(&repo).await;
2601                self.state.status = RunStatus::Blocked;
2602                self.state
2603                    .event("prep", format!("worktree setup failed: {e:#}"));
2604                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2605                self.state.save()?;
2606                return Err(e);
2607            }
2608            self.state.candidates.push(Candidate {
2609                index,
2610                label,
2611                agent: spec.id.clone(),
2612                branch,
2613                worktree,
2614                summary: String::new(),
2615                stat: String::new(),
2616                files: 0,
2617                commits: 0,
2618                empty: false,
2619                failed: None,
2620                verified_noop: None,
2621                duration_ms: 0,
2622                folded: false,
2623            });
2624            self.state.save()?;
2625            for pick in &plan.picks {
2626                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2627                {
2628                    self.state.status = RunStatus::Blocked;
2629                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2630                    self.state
2631                        .event("prep", format!("cannot apply referenced commit: {e}"));
2632                    self.state.save()?;
2633                    return Err(e);
2634                }
2635            }
2636        }
2637
2638        for j in 1..=self.roles.judges.len() {
2639            let wt = root.join(format!("judge-{j}"));
2640            if !wt.exists() {
2641                git::worktree_add_detached(&repo, &wt, &base).await?;
2642                self.setup_seat_worktree(&repo, &wt, true).await?;
2643            }
2644        }
2645
2646        // Disposable, detached checkouts for the design-deliberation stage's
2647        // advisor seats — the same shape as the judges' above, at the same
2648        // base commit, since advisors also only ever read. Sized off the
2649        // configured count directly rather than a resolved roster: unlike
2650        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2651        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2652        // `prep` has no `ResolvedRoles` field to read a count from here.
2653        if self.state.config.graph.advise {
2654            for k in 1..=self.state.config.graph.advisors {
2655                let wt = root.join(format!("advisor-{k}"));
2656                if !wt.exists() {
2657                    git::worktree_add_detached(&repo, &wt, &base).await?;
2658                    self.setup_seat_worktree(&repo, &wt, true).await?;
2659                }
2660            }
2661        }
2662
2663        // A judge cannot tell it is looking at its own patch — the seats keep
2664        // separate conversations — but a panel that shares agents with the
2665        // field is less independent than it looks, and that is worth saying out
2666        // loud once per run rather than leaving it in the config.
2667        let authors: Vec<&str> = self
2668            .roles
2669            .implementers
2670            .iter()
2671            .map(|a| a.id.as_str())
2672            .collect();
2673        let overlap: Vec<String> = self
2674            .roles
2675            .judges
2676            .iter()
2677            .enumerate()
2678            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2679            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2680            .collect();
2681        if !overlap.is_empty() {
2682            let note = format!(
2683                "{} also authored a candidate; blind, but the panel is less \
2684                 independent than {} distinct agents would be",
2685                overlap.join(", "),
2686                self.roles.judges.len()
2687            );
2688            self.state.event("prep", note);
2689        }
2690
2691        self.state.event(
2692            "prep",
2693            format!(
2694                "{} candidates, {} judges, base {} ({})",
2695                self.state.candidates.len(),
2696                self.roles.judges.len(),
2697                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2698                self.state.base_branch
2699            ),
2700        );
2701        self.state.prep_complete = Some(true);
2702        self.state.status = RunStatus::Implementing;
2703        self.state.save()?;
2704        Ok(())
2705    }
2706
2707    // -------------------------------------------------------------- advise
2708
2709    /// The design-deliberation stage: independent, read-only advisor seats
2710    /// each sketch a design before any implementer touches the repository,
2711    /// and (when at least one produced a usable proposal) a synthesis seat
2712    /// blends them into a brief `implement` carries in every candidate's
2713    /// prompt.
2714    ///
2715    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2716    /// advisors` is the proposal count. Everything here is best-effort and
2717    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2718    /// that cannot reach quota, or a synthesis seat that produced nothing
2719    /// usable all leave `implement` exactly as it was before this stage
2720    /// existed — the task instruction alone — rather than failing the whole
2721    /// competition over an enrichment stage. Every outcome is still recorded
2722    /// as an event, so a run that got nothing from this stage says why.
2723    ///
2724    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2725    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2726    /// resumed run whose stage failed would re-run it, and re-spend the
2727    /// agent calls, on every reentry before `implement`.
2728    ///
2729    /// Also skipped once any candidate shows implementation progress — the
2730    /// exact predicate `implement` itself uses to decide a candidate is no
2731    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2732    /// is not enough: a run created by an older binary that predates this
2733    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2734    /// an already-`Implementing`-or-later run under this build would
2735    /// otherwise walk straight back through `prep` (a no-op once candidates
2736    /// exist) into this node and spawn every advisor seat against worktrees
2737    /// `prep` never recreated — after implementation has already started,
2738    /// which is exactly the invariant this stage exists to guarantee.
2739    async fn advise(&mut self) -> Result<()> {
2740        let implement_untouched = self
2741            .state
2742            .candidates
2743            .iter()
2744            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2745        if !self.state.config.graph.advise || self.state.advise_attempted {
2746            return Ok(());
2747        }
2748        if !implement_untouched {
2749            self.state.event(
2750                "advise",
2751                "skipping the design-deliberation stage: at least one \
2752                 candidate already shows implementation progress, so this \
2753                 run is past the point the stage exists to run before"
2754                    .to_owned(),
2755            );
2756            self.state.advise_attempted = true;
2757            self.state.save()?;
2758            return Ok(());
2759        }
2760        let run_id = self.state.id.clone();
2761        let prompts = self.state.config.prompts.clone();
2762        let instruction = self.state.instruction.clone();
2763        let language = self.state.config.graph.language.clone();
2764        let root = self.state.worktree_root();
2765        let n = self.state.config.graph.advisors;
2766        let where_recorded = self.state.dir().join("run.json");
2767
2768        let seats = match self.state.config.advisors() {
2769            Ok(seats) if !seats.is_empty() => seats,
2770            Ok(_) => {
2771                self.state.event(
2772                    "advise",
2773                    format!(
2774                        "[graph] advisors is 0; skipping the design-deliberation \
2775                         stage and continuing without a synthesis brief (see {})",
2776                        where_recorded.display()
2777                    ),
2778                );
2779                self.state.advise_attempted = true;
2780                self.state.save()?;
2781                return Ok(());
2782            }
2783            Err(e) => {
2784                self.state.event(
2785                    "advise",
2786                    format!(
2787                        "could not resolve advisor seats ({e:#}); continuing \
2788                         without a design-deliberation brief (see {})",
2789                        where_recorded.display()
2790                    ),
2791                );
2792                self.state.advise_attempted = true;
2793                self.state.save()?;
2794                return Ok(());
2795            }
2796        };
2797
2798        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2799        let artifacts = agent::artifacts_dir(&self.state.dir());
2800        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2801
2802        let mut jobs = Vec::new();
2803        for (i, spec) in seats.iter().cloned().enumerate() {
2804            let seat_key = format!("advisor-{}", i + 1);
2805            let seat = self.seat(&seat_key, &spec.id);
2806            jobs.push(SeatJob {
2807                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2808                spec,
2809                seat,
2810                cwd: worktrees[i % worktrees.len()].clone(),
2811                timeout,
2812                allow_write: false,
2813                sessions: false,
2814                artifacts: artifacts.clone(),
2815                stem: seat_key,
2816                handover: None,
2817            });
2818        }
2819
2820        self.state.event(
2821            "advise",
2822            format!(
2823                "{} advisor seat(s) sketching a design in parallel",
2824                jobs.len()
2825            ),
2826        );
2827        let mut quota_losses = Vec::new();
2828        let cache = self.state.config.cache_dir();
2829        let ctx = WaveCtx {
2830            carry_seats: false,
2831            pauses: Some((&self.pause, &self.interrupt)),
2832            run: &run_id,
2833            node: "advise",
2834            prompts: &prompts,
2835            cache: cache.as_deref(),
2836            round: None,
2837        };
2838        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2839        let results = ask_json_wave::<Proposal>(
2840            jobs,
2841            Arc::clone(&self.sem),
2842            self.state.config.graph.retries,
2843            &advisor_roster,
2844            &ctx,
2845            &mut quota_losses,
2846            &mut self.state,
2847            &|p: &Proposal| p.validate(),
2848        )
2849        .await;
2850        if self.state.resource_block.is_some() {
2851            return self.stop_for_resource();
2852        }
2853        self.state.quota.extend(quota_losses);
2854
2855        let mut records = Vec::with_capacity(results.len());
2856        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2857            let agent_id = seat.agent.clone();
2858            self.state.seats.insert(seat.key.clone(), seat);
2859            match res {
2860                Ok((proposal, out)) => {
2861                    self.state
2862                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2863                    records.push(advise::AdvisorRecord::proposed(
2864                        i + 1,
2865                        agent_id,
2866                        proposal,
2867                        out.duration_ms,
2868                    ));
2869                }
2870                Err(e) => {
2871                    self.state.event(
2872                        "advise",
2873                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2874                    );
2875                    records.push(advise::AdvisorRecord::failed(
2876                        i + 1,
2877                        agent_id,
2878                        e.to_string(),
2879                    ));
2880                }
2881            }
2882        }
2883
2884        let mut advice = advise::Advice {
2885            records,
2886            synthesis: None,
2887        };
2888        if advice.proposals().is_empty() {
2889            self.state.event(
2890                "advise",
2891                "no advisor produced a usable proposal; continuing without a \
2892                 synthesis brief"
2893                    .to_owned(),
2894            );
2895        } else {
2896            match self
2897                .synthesize_brief(
2898                    &advice,
2899                    &instruction,
2900                    &language,
2901                    &worktrees[0],
2902                    &artifacts,
2903                    &run_id,
2904                    &prompts,
2905                    cache.as_deref(),
2906                )
2907                .await
2908            {
2909                Ok(Some(text)) => {
2910                    self.state.event(
2911                        "advise",
2912                        "synthesized a design brief for the implementer".to_owned(),
2913                    );
2914                    advice.synthesis = Some(text);
2915                }
2916                Ok(None) => {
2917                    self.state.event(
2918                        "advise",
2919                        "the synthesis seat produced nothing usable; continuing \
2920                         without a design brief"
2921                            .to_owned(),
2922                    );
2923                }
2924                Err(e) => {
2925                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2926                        return Err(e);
2927                    }
2928                    self.state.event(
2929                        "advise",
2930                        format!("could not synthesize a design brief: {e:#}"),
2931                    );
2932                }
2933            }
2934        }
2935        advise::apply_reflection(&mut advice);
2936
2937        self.state.advice = Some(advice);
2938        self.state.advise_attempted = true;
2939        self.state.save()?;
2940        Ok(())
2941    }
2942
2943    /// The synthesis seat: reads every advisor's proposal and blends them
2944    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2945    /// out of [`Runner::advise`] only for readability — it is not called
2946    /// anywhere else.
2947    ///
2948    /// Picked the same way [`crate::talk`]'s standing conversation and
2949    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2950    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2951    /// order (a claude seat, else the first runnable agent in roster order)
2952    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2953    /// in [`crate::config`] for why a dedicated field exists here at all.
2954    #[allow(clippy::too_many_arguments)]
2955    async fn synthesize_brief(
2956        &mut self,
2957        advice: &advise::Advice,
2958        instruction: &str,
2959        language: &str,
2960        cwd: &Path,
2961        artifacts: &Path,
2962        run_id: &str,
2963        prompts: &Prompts,
2964        cache: Option<&Path>,
2965    ) -> Result<Option<String>> {
2966        let chain = agent::pick_chain(
2967            &self.state.config.agents,
2968            self.state.config.roles.synthesizer.as_ref(),
2969            &agent::installed,
2970            "synthesizer",
2971        )?;
2972        let proposals = advice.proposals();
2973        let mut prompt = prompt::with_overlay(
2974            prompt::synthesize_brief(instruction, &proposals, language),
2975            prompts.overlay("advise"),
2976        );
2977        if cache.is_some() {
2978            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2979            // below — see `prompt::build_cache_note`'s doc for why telling a
2980            // read-only seat to build through the shared cache is exactly how
2981            // a sandbox's write refusal gets misread as a defect.
2982            prompt.push('\n');
2983            prompt.push_str(&prompt::build_cache_note("advise", false));
2984        }
2985        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2986        // Each id is tried once, in order; a quota hit, error or unusable
2987        // answer moves to the next. The seat is single-turn (`sessions:
2988        // false`) and the prompt is the whole context, so a fallback agent
2989        // needs nothing carried over.
2990        let mut last = None;
2991        for (n, spec) in chain.iter().enumerate() {
2992            if n > 0 {
2993                self.state
2994                    .event("advise", format!("synthesis falling back to {}", spec.id));
2995            }
2996            let mut seat = self.seat("advise-synthesis", &spec.id);
2997            let wait = self.repository_wait();
2998            let outcome = wait
2999                .scope(agent::invoke(
3000                    spec,
3001                    &mut seat,
3002                    &Invocation {
3003                        coordination: self.state.config.coordination.as_ref(),
3004                        fetched_base: None,
3005                        cwd,
3006                        prompt: &prompt,
3007                        timeout,
3008                        allow_write: false,
3009                        sessions: false,
3010                        artifacts,
3011                        stem: &if n == 0 {
3012                            "advise-synthesis".to_owned()
3013                        } else {
3014                            format!("advise-synthesis-{}", spec.id)
3015                        },
3016                        run: run_id,
3017                        node: "advise",
3018                        cache_dir: None,
3019                        attachments: &[],
3020                        writable: &[],
3021                    },
3022                ))
3023                .await;
3024            if outcome.is_ok() {
3025                self.state.seats.insert(seat.key.clone(), seat);
3026            }
3027            let advance = agent::chain_advances(&outcome);
3028            last = Some(outcome);
3029            if !advance {
3030                break;
3031            }
3032        }
3033        // Exhausted: the last attempt's result is what a single failed seat
3034        // would have produced.
3035        let out = last.expect("a chain holds at least one agent")?;
3036        if !out.usable() {
3037            return Ok(None);
3038        }
3039        let text =
3040            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3041        Ok((!text.trim().is_empty()).then_some(text))
3042    }
3043
3044    // ----------------------------------------------------------- implement
3045
3046    async fn implement(&mut self) -> Result<()> {
3047        self.persist_candidates().await?;
3048        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3049        // agent files with `magi task add` name the run that paid for it. The
3050        // prompt overlay is cloned alongside it because the waves borrow it
3051        // while `self` is mutably borrowed by the node's own bookkeeping.
3052        let run_id = self.state.id.clone();
3053        let prompts = self.state.config.prompts.clone();
3054        let todo: Vec<usize> = self
3055            .state
3056            .candidates
3057            .iter()
3058            .enumerate()
3059            .filter(|(_, c)| {
3060                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3061                    && c.failed.is_none()
3062                    && !c.empty
3063            })
3064            .map(|(i, _)| i)
3065            .collect();
3066        if todo.is_empty() {
3067            return self.after_implement();
3068        }
3069        self.state.status = RunStatus::Implementing;
3070
3071        let language = self.state.config.graph.language.clone();
3072        let github_language = self.state.config.graph.github_language.clone();
3073        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3074        let sessions = self.state.config.graph.sessions;
3075        let artifacts = agent::artifacts_dir(&self.state.dir());
3076        // The design-deliberation stage's blended brief, when `advise` found
3077        // one — carried into every implementer's prompt the same way
3078        // regardless of which candidate it is.
3079        let brief = self
3080            .state
3081            .advice
3082            .as_ref()
3083            .and_then(|a| a.synthesis.as_deref())
3084            .map(str::to_owned);
3085        let attachments = self.state.attachments.clone();
3086
3087        let mut jobs = Vec::new();
3088        for &i in &todo {
3089            let (index, label, worktree) = {
3090                let c = &self.state.candidates[i];
3091                (c.index, c.label, c.worktree.clone())
3092            };
3093            let spec = self.roles.implementers[index].clone();
3094            let seat_key = format!("impl-{label}");
3095            let seat = self.seat(&seat_key, &spec.id);
3096            let instruction = seeded_instruction(&self.state);
3097            jobs.push(SeatJob {
3098                spec,
3099                seat,
3100                prompt: prompt::implement_in(
3101                    &instruction,
3102                    &worktree.to_string_lossy(),
3103                    &language,
3104                    &github_language,
3105                    brief.as_deref(),
3106                    &attachments,
3107                ),
3108                cwd: worktree,
3109                timeout,
3110                allow_write: true,
3111                sessions,
3112                artifacts: artifacts.clone(),
3113                stem: format!("impl-{label}"),
3114                handover: None,
3115            });
3116        }
3117
3118        self.state.event(
3119            "implement",
3120            format!("{} candidates in parallel", jobs.len()),
3121        );
3122        // Kept so a seat whose CLI hung up can be asked again from the same
3123        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3124        // can update a seat's own entry once a fallback agent takes it over —
3125        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3126        // whichever agent actually answered, not the one that quota'd out.
3127        let mut sent = jobs.clone();
3128        let cache = self.state.config.cache_dir();
3129        let ctx = WaveCtx {
3130            carry_seats: false,
3131            pauses: Some((&self.pause, &self.interrupt)),
3132            run: &run_id,
3133            node: "implement",
3134            prompts: &prompts,
3135            cache: cache.as_deref(),
3136            round: None,
3137        };
3138        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3139        if !wave_blocked(&results) {
3140            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3141                .await;
3142            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3143                .await?;
3144            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3145                .await;
3146        }
3147        // Do not finalize failed candidates when the entire wave needs machine
3148        // recovery: leaving them pending resumes this node with its work intact.
3149        if !results.is_empty()
3150            && results
3151                .iter()
3152                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3153        {
3154            let losses: Vec<_> = results
3155                .iter()
3156                .map(|(_, seat, out)| {
3157                    let AgentOutcome::Quota(o) = out else {
3158                        unreachable!()
3159                    };
3160                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3161                })
3162                .collect();
3163            self.state.quota.extend(losses.clone());
3164            for (_, seat, _) in results {
3165                self.state.seats.insert(seat.key.clone(), seat);
3166            }
3167            for &i in &todo {
3168                let label = self.state.candidates[i].label;
3169                self.state.interrupted_implementations.insert(label);
3170                self.state
3171                    .candidate_preservation
3172                    .insert(label, "machine outage awaiting persistence".into());
3173            }
3174            // Preserve partial edits too, while keeping every interrupted seat
3175            // pending even when its tree already contains an inherited patch.
3176            block_machine_outage(&mut self.state, "implement", &losses);
3177            self.state.save()?;
3178            self.persist_candidates().await?;
3179            for &i in &todo {
3180                self.state
3181                    .interrupted_implementations
3182                    .insert(self.state.candidates[i].label);
3183            }
3184            return self.stop_for_resource();
3185        }
3186        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3187
3188        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3189            && results
3190                .iter()
3191                .any(|(_, _, out)| FailClass::of(out).is_some());
3192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3193            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3194                // Executed turns interrupted by disk pressure already carry
3195                // a preservation barrier from wave. Admission-only waits do
3196                // not. Keep the interrupted seat's session for recovery too.
3197                if self
3198                    .state
3199                    .candidate_preservation
3200                    .contains_key(&self.state.candidates[i].label)
3201                {
3202                    self.state.seats.insert(seat.key.clone(), seat);
3203                }
3204                continue;
3205            }
3206            let seat_key = seat.key.clone();
3207            // A quota fallback (`resume_seat_handovers`) may have handed this
3208            // seat to a different agent than the one `prep` recorded on the
3209            // candidate; the stats tables and any later fixer-defaults-to-
3210            // winner's-author lookup must credit whoever actually answered —
3211            // unless every fallback also quota'd out, in which case nobody
3212            // actually answered and crediting the last agent tried would
3213            // erase every earlier agent's own quota loss from the stats
3214            // tables instead of just this one seat's.
3215            let agent = seat.agent.clone();
3216            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3217            self.state.seats.insert(seat.key.clone(), seat);
3218            let label = self.state.candidates[i].label;
3219            self.state.interrupted_implementations.remove(&label);
3220
3221            let (summary, duration, failed, verified_claim) = match out {
3222                AgentOutcome::Ok(o) => {
3223                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3224                    let failed = (!o.usable()).then(|| {
3225                        if o.timed_out {
3226                            "agent timed out".to_owned()
3227                        } else {
3228                            format!(
3229                                "agent exited with {}",
3230                                crate::notices::exit_status(o.exit_code)
3231                            )
3232                        }
3233                    });
3234                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3235                    (text, o.duration_ms, failed, verified_claim)
3236                }
3237                // Left un-resumed by `resume_undelivered` (a dirty tree
3238                // already rescues the work, or there was no session left to
3239                // resume into) — reported like the ordinary failure it is,
3240                // never as if `o.text` (the CLI's raw error JSON) were an
3241                // answer.
3242                AgentOutcome::Dropped(o) => {
3243                    let why = o
3244                        .dropped
3245                        .as_ref()
3246                        .map(|d| d.why.as_str())
3247                        .unwrap_or("the CLI ended the stream without delivering its answer");
3248                    (
3249                        String::new(),
3250                        o.duration_ms,
3251                        Some(format!("the CLI dropped the stream ({why})")),
3252                        None,
3253                    )
3254                }
3255                AgentOutcome::Quota(o) => {
3256                    self.state.quota.push(QuotaLoss::from_report(
3257                        seat_key,
3258                        "implement",
3259                        o.quota.as_ref(),
3260                    ));
3261                    (
3262                        String::new(),
3263                        o.duration_ms,
3264                        Some("rate limited (quota); produced no change".to_owned()),
3265                        None,
3266                    )
3267                }
3268                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3269                    (String::new(), 0, Some(e), None)
3270                }
3271            };
3272
3273            // Save every completed turn before any fallible Git operation. A failed
3274            // rescue must not erase the reply or prevent collection of other seats.
3275            let c = &mut self.state.candidates[i];
3276            if !exhausted_the_fallback_chain {
3277                c.agent = agent;
3278            }
3279            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3280            c.duration_ms = duration;
3281            c.failed = failed;
3282            c.verified_noop = verified_claim;
3283            self.state
3284                .candidate_preservation
3285                .insert(label, "awaiting persistence".to_owned());
3286        }
3287        self.state.save()?;
3288        self.persist_candidates().await?;
3289
3290        if resource_blocked {
3291            return self.stop_for_resource();
3292        }
3293        if selected_failed {
3294            self.state.status = RunStatus::Blocked;
3295            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3296            self.state.save()?;
3297            bail!(
3298                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3299            );
3300        }
3301        self.after_implement()
3302    }
3303
3304    /// Retry only persistence after a completed implementation turn.
3305    async fn persist_candidates(&mut self) -> Result<()> {
3306        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3307        for label in labels {
3308            let i = self
3309                .state
3310                .candidates
3311                .iter()
3312                .position(|c| c.label == label)
3313                .context("pending candidate is missing")?;
3314            let c = self.state.candidates[i].clone();
3315            let interrupted = self.state.interrupted_implementations.contains(&label);
3316            let result = async {
3317                let wait = crate::repo_lock::WaitPolicy::new(
3318                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3319                    || false,
3320                );
3321                let _coordination =
3322                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3323                // Best effort evidence requires no index writes. A successful snapshot
3324                // never authorizes removal of a tree whose rescue failed.
3325                if let Err(e) = git::snapshot_uncommitted(
3326                    &c.worktree,
3327                    &self
3328                        .state
3329                        .dir()
3330                        .join("artifacts")
3331                        .join(format!("candidate-{label}-recovery")),
3332                )
3333                .await
3334                {
3335                    self.state.event(
3336                        "preserve",
3337                        format!("candidate {label} snapshot incomplete: {e:#}"),
3338                    );
3339                }
3340                let rescued = git::rescue_commit(
3341                    &c.worktree,
3342                    &format!("magi: candidate {label} (uncommitted work)"),
3343                )
3344                .await?;
3345                self.state.note_withheld("implement", &rescued.withheld);
3346                let base = &self.state.base_commit;
3347                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3348                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3349                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3350                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3351                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3352                if c.empty && commits > 0 && !patch.trim().is_empty()
3353                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3354                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3355                        || self.state.tally.is_some())
3356                {
3357                    // Keep the old classification and pending marker so every
3358                    // retry stops too. Retain decisions as recovery evidence;
3359                    // only a supported recovery operation may replace them.
3360                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3361                }
3362                let c = &mut self.state.candidates[i];
3363                c.commits = commits;
3364                c.stat = stat;
3365                c.files = files;
3366                let no_changes = commits == 0 || patch.trim().is_empty();
3367                c.empty = no_changes && !interrupted;
3368                if interrupted && no_changes {
3369                    c.failed = None;
3370                    c.verified_noop = None;
3371                }
3372                if !c.empty {
3373                    c.failed = None;
3374                    c.verified_noop = None;
3375                }
3376                self.state.event(
3377                    "preserve",
3378                    format!("candidate {label}: {files} files, {commits} commits"),
3379                );
3380                Ok::<_, anyhow::Error>(())
3381            }
3382            .await;
3383            match result {
3384                Ok(()) => {
3385                    self.state.candidate_preservation.remove(&label);
3386                    if !interrupted || self.state.candidates[i].files > 0 {
3387                        self.state.interrupted_implementations.remove(&label);
3388                    }
3389                }
3390                Err(e) => {
3391                    let reason = format!("candidate {label} preservation failed: {e:#}");
3392                    self.state.event("preserve", &reason);
3393                    self.state.candidate_preservation.insert(label, reason);
3394                }
3395            }
3396            if self.state.candidate_preservation.is_empty()
3397                && self.state.status == RunStatus::Blocked
3398                && self.state.candidates.iter().all(|c| c.failed.is_none())
3399            {
3400                // Persist the recovered phase with the cleared pending marker;
3401                // a crash between saves must not turn this into a selected-seat
3402                // failure hold on the next resume.
3403                self.state.status = RunStatus::Implementing;
3404                self.state.event(
3405                    "preserve",
3406                    "candidate persistence recovered; implementation can continue",
3407                );
3408            }
3409            self.state.save()?;
3410        }
3411        if !self.state.candidate_preservation.is_empty() {
3412            self.state.status = RunStatus::Blocked;
3413            self.state.save()?;
3414            bail!(
3415                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3416            );
3417        }
3418        Ok(())
3419    }
3420
3421    /// Ask again, once, for work a CLI did and then failed to hand over.
3422    ///
3423    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3424    /// status with an empty response and a usage report showing output tokens,
3425    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3426    /// seven minutes and 14,267 output tokens that arrived as an empty
3427    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3428    ///
3429    /// Two conditions, and both matter:
3430    ///
3431    /// - **Only when the tree is untouched.** Often the agent has already
3432    ///   written its files and only the closing message was lost; the rescue
3433    ///   commit below picks that up and there is nothing to ask for. Re-asking
3434    ///   then would pay for a second implementation of work already on disk.
3435    /// - **Once.** A CLI that drops one stream can drop the next, and this
3436    ///   node is the most expensive in the graph.
3437    ///
3438    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3439    /// dropped reply still carried its `conversation_id`, so the seat is asked
3440    /// to finish what it was doing rather than sent the whole task again. It
3441    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3442    /// node's - for the same reason a re-ranked judge does: restating finished
3443    /// work is not the work.
3444    ///
3445    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3446    /// same way until it resets, while an abandoned conversation is still
3447    /// there to be picked up.
3448    async fn resume_undelivered(
3449        &mut self,
3450        results: &mut [(usize, SeatState, AgentOutcome)],
3451        sent: &[SeatJob],
3452        prompts: &Prompts,
3453        run_id: &str,
3454    ) {
3455        for (wi, seat, out) in results.iter_mut() {
3456            let Some(dropped) = (match &*out {
3457                AgentOutcome::Dropped(o) => o.dropped.clone(),
3458                _ => None,
3459            }) else {
3460                continue;
3461            };
3462            let Some(job) = sent.get(*wi) else { continue };
3463            // Already on disk? Then only the closing message was lost.
3464            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3465                self.state.event(
3466                    "implement",
3467                    format!(
3468                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3469                         work is in the tree",
3470                        seat.key, dropped.output_tokens, dropped.why
3471                    ),
3472                );
3473                continue;
3474            }
3475            // The re-ask only makes sense as a resume: `resume_after_drop`
3476            // says nothing about the task, trusting the seat to still hold it.
3477            // Without a session to resume — sessions disabled, or this CLI's
3478            // drop shape happened not to carry a session id — that prompt
3479            // would open a brand-new conversation with no context at all,
3480            // which is worse than leaving this as the ordinary failure it
3481            // already is.
3482            if !has_context(&job.spec, seat, job.sessions) {
3483                self.state.event(
3484                    "implement",
3485                    format!(
3486                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3487                         is no session left to resume",
3488                        seat.key, dropped.output_tokens, dropped.why
3489                    ),
3490                );
3491                continue;
3492            }
3493            self.state.event(
3494                "implement",
3495                format!(
3496                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3497                     conversation",
3498                    seat.key, dropped.output_tokens, dropped.why
3499                ),
3500            );
3501            let mut retry = job.clone();
3502            retry.seat = seat.clone();
3503            retry.prompt = prompt::resume_after_drop(&dropped.why);
3504            retry.timeout = retry_budget(job.timeout, true);
3505            retry.stem = format!("{}-resume", job.stem);
3506            let cache = self.state.config.cache_dir();
3507            let ctx = WaveCtx {
3508                carry_seats: false,
3509                pauses: Some((&self.pause, &self.interrupt)),
3510                run: run_id,
3511                node: "implement",
3512                prompts,
3513                cache: cache.as_deref(),
3514                round: None,
3515            };
3516            let (resumed_seat, resumed) =
3517                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3518            *seat = resumed_seat;
3519            *out = resumed;
3520        }
3521    }
3522
3523    /// Fall an implement seat through to the next untried agent in the
3524    /// implementer roster when it lost to quota — or, since the handover was
3525    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3526    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3527    /// quota path itself being unchanged — instead of leaving the
3528    /// seat's loss final the moment one agent's account runs dry.
3529    ///
3530    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3531    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3532    /// truncates to the single slot rotation picked, so a solo task whose one
3533    /// implementer hits quota mid-run used to have nothing else to try. This
3534    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3535    /// unrotated roster — which is the only place the *other* candidates in
3536    /// the machine's roster still exist once `implementers` has been cut down
3537    /// to size.
3538    ///
3539    /// Walks forward from just past the seat's own original position in the
3540    /// roster, never wrapping back to the front: a later candidate slot (say
3541    /// `beta`, the roster's second entry) must fall through to the *next*
3542    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3543    /// almost certainly a different candidate's own agent already — and once
3544    /// the roster's tail is exhausted there is nothing left to fall through
3545    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3546    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3547    /// let this retry that id forever. The loop keeps falling through until
3548    /// an attempt lands something other than `Quota` or the roster's tail
3549    /// runs out of untried ids, at which point the seat is left exactly as
3550    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3551    /// `QuotaLoss` recorded, the candidate failed/empty.
3552    ///
3553    /// `sent` is taken mutably and updated with the fallback agent's spec:
3554    /// `resume_unconfirmed_commands`, which runs after this and also reads
3555    /// `sent`, must see whichever agent actually ended up answering the seat
3556    /// — reading the stale, original spec there would check session
3557    /// eligibility against the wrong CLI and could hand a fallback agent's
3558    /// session id to the agent that just lost the seat to quota.
3559    ///
3560    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3561    /// — `self.seat` only reuses state when the agent id is unchanged, so
3562    /// handing it a different id already gets this for free. Reusing the old
3563    /// seat would resume a different CLI's session as if it were a
3564    /// continuation of this one.
3565    ///
3566    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3567    /// a quota loss cuts an agent off mid-turn, so anything already in the
3568    /// tree is unfinished work, not a completed candidate a re-ask would pay
3569    /// for twice. A dirty tree is rescued into a commit first (the same
3570    /// neutral-identity rescue `implement`'s own outcome loop gives every
3571    /// candidate) so the next agent starts clean.
3572    ///
3573    /// The new agent gets the implementer's full prompt and full
3574    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3575    /// it has no session and no context, and is implementing the task from
3576    /// nothing, unlike a resumed drop which is only restating work already
3577    /// done.
3578    ///
3579    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3580    /// `implement` event, never into `self.state.quota` — that is what
3581    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3582    /// go unspent, and a seat that ultimately recovered on its second or
3583    /// third agent is not the stalled panel that check exists to catch. Only
3584    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3585    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3586    /// outcome loop already has — this helper never pushes to it itself.
3587    async fn resume_seat_handovers(
3588        &mut self,
3589        results: &mut [(usize, SeatState, AgentOutcome)],
3590        sent: &mut [SeatJob],
3591        prompts: &Prompts,
3592        run_id: &str,
3593    ) -> Result<()> {
3594        let instruction = seeded_instruction(&self.state);
3595        let language = self.state.config.graph.language.clone();
3596        let github_language = self.state.config.graph.github_language.clone();
3597        let brief = self
3598            .state
3599            .advice
3600            .as_ref()
3601            .and_then(|a| a.synthesis.as_deref())
3602            .map(str::to_owned);
3603        let attachments = self.state.attachments.clone();
3604        for (wi, seat, out) in results.iter_mut() {
3605            // Who holds the other candidate seats of this wave right now
3606            // (earlier handovers already written back to `sent`).
3607            let others: BTreeSet<String> = sent
3608                .iter()
3609                .enumerate()
3610                .filter(|(j, _)| j != wi)
3611                .map(|(_, j)| j.spec.id.clone())
3612                .collect();
3613            let Some(job) = sent.get_mut(*wi) else {
3614                continue;
3615            };
3616            // Where the seat's own original agent sits in the roster — the
3617            // fallback walk starts just past here, never at the front, so a
3618            // later candidate slot's quota loss does not fall back onto an
3619            // earlier slot's own agent.
3620            let start = self
3621                .roles
3622                .implementer_roster
3623                .iter()
3624                .position(|s| s.id == job.spec.id)
3625                .unwrap_or(0);
3626            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3627            let mut fallback_attempt = 0usize;
3628            let mut prev: Option<FailClass> = None;
3629            while let Some(cur) = FailClass::of(&*out) {
3630                if !should_hand_over(prev.as_ref(), &cur) {
3631                    break;
3632                }
3633                let Some(next) =
3634                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3635                        .cloned()
3636                else {
3637                    break;
3638                };
3639                tried.insert(next.id.clone());
3640                fallback_attempt += 1;
3641
3642                let coordination = self.coordinate(&job.cwd).await?;
3643                if let Ok(r) = git::rescue_commit(
3644                    &job.cwd,
3645                    &format!(
3646                        "magi: candidate {} (uncommitted work before {} fallback)",
3647                        seat.key,
3648                        if cur == FailClass::Quota {
3649                            "quota"
3650                        } else {
3651                            "handover"
3652                        }
3653                    ),
3654                )
3655                .await
3656                {
3657                    self.state.note_withheld("implement", &r.withheld);
3658                }
3659                drop(coordination);
3660
3661                record_handover(
3662                    &mut self.state,
3663                    "implement",
3664                    &seat.key,
3665                    &seat.agent,
3666                    &next.id,
3667                    &cur,
3668                    &fail_reason(&*out),
3669                );
3670                prev = Some(cur.clone());
3671
3672                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3673                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3674                // Kept in sync on `sent` itself, not just the local retry: a
3675                // later helper (`resume_unconfirmed_commands`) reads `sent`
3676                // after this one returns and must see whichever agent is now
3677                // occupying the seat, not the one that just quota'd out —
3678                // otherwise it would judge session/continuation eligibility
3679                // by the wrong CLI and could resend a fallback's session id
3680                // to the agent that lost it the seat in the first place.
3681                job.spec = next.clone();
3682                let mut retry = job.clone();
3683                retry.seat = new_seat;
3684                retry.prompt = prompt::implement_in(
3685                    &instruction,
3686                    &job.cwd.to_string_lossy(),
3687                    &language,
3688                    &github_language,
3689                    brief.as_deref(),
3690                    &attachments,
3691                );
3692                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3693                let cache = self.state.config.cache_dir();
3694                let ctx = WaveCtx {
3695                    carry_seats: false,
3696                    pauses: Some((&self.pause, &self.interrupt)),
3697                    run: run_id,
3698                    node: "implement",
3699                    prompts,
3700                    cache: cache.as_deref(),
3701                    round: None,
3702                };
3703                let (fallback_seat, fallback_out) = run_one(
3704                    retry,
3705                    Arc::clone(&self.sem),
3706                    &ctx,
3707                    &mut self.state,
3708                    fallback_attempt,
3709                )
3710                .await;
3711                *seat = fallback_seat;
3712                *out = fallback_out;
3713            }
3714        }
3715        Ok(())
3716    }
3717
3718    /// Ask an implement seat's own CLI to confirm what it started, once, when
3719    /// its reply reported a command whose completion status it never
3720    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3721    /// that does and does not mean.
3722    ///
3723    /// The completion contract this task asks for, extended to `implement`
3724    /// with the same signal `continue_fix_report` reads for the fixer,
3725    /// rather than a keyword search over the reply or a hard requirement on
3726    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3727    /// a candidate's CLI turn ended cleanly while a test run it had started
3728    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3729    /// named in it at all is untouched by this: `commands` is empty, so
3730    /// there is nothing to be unconfirmed.
3731    ///
3732    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3733    /// this is not about recovering edits that might already be on disk, it
3734    /// is about a result the seat itself never vouched for, which resuming
3735    /// asks for regardless of what the tree already holds. Bounded to one
3736    /// attempt for the same reason `resume_undelivered` is — this is the
3737    /// most expensive node in the graph — and a seat that still cannot
3738    /// confirm on that attempt is left as whatever its (possibly still
3739    /// unconfirmed) reply says; this does not invent a new "failed" reason
3740    /// for a candidate that otherwise produced a real, committed change.
3741    async fn resume_unconfirmed_commands(
3742        &mut self,
3743        results: &mut [(usize, SeatState, AgentOutcome)],
3744        sent: &[SeatJob],
3745        prompts: &Prompts,
3746        run_id: &str,
3747    ) {
3748        for (wi, seat, out) in results.iter_mut() {
3749            let AgentOutcome::Ok(o) = &*out else {
3750                continue;
3751            };
3752            if !has_unconfirmed_command(&o.commands) {
3753                continue;
3754            }
3755            let Some(job) = sent.get(*wi) else { continue };
3756            if !has_context(&job.spec, seat, job.sessions) {
3757                self.state.event(
3758                    "implement",
3759                    format!(
3760                        "{}: the reply named a command whose own CLI never confirmed the exit \
3761                         status of, but there is no session left to resume",
3762                        seat.key
3763                    ),
3764                );
3765                continue;
3766            }
3767            self.state.event(
3768                "implement",
3769                format!(
3770                    "{}: the reply named a command whose own CLI never confirmed the exit \
3771                     status of; resuming the conversation",
3772                    seat.key
3773                ),
3774            );
3775            let mut retry = job.clone();
3776            retry.seat = seat.clone();
3777            retry.prompt = prompt::resume_incomplete(
3778                "a command in your last reply had no confirmed exit status",
3779            );
3780            retry.timeout = retry_budget(job.timeout, true);
3781            retry.stem = format!("{}-confirm", job.stem);
3782            let cache = self.state.config.cache_dir();
3783            let ctx = WaveCtx {
3784                carry_seats: false,
3785                pauses: Some((&self.pause, &self.interrupt)),
3786                run: run_id,
3787                node: "implement",
3788                prompts,
3789                cache: cache.as_deref(),
3790                round: None,
3791            };
3792            let (resumed_seat, resumed) =
3793                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3794            *seat = resumed_seat;
3795            *out = resumed;
3796        }
3797    }
3798
3799    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3800    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3801    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3802    /// that motivated this.
3803    ///
3804    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3805    /// own nudge loop already covers for judge/review/vote seats, and not a
3806    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3807    /// implement seats: here the CLI turn genuinely finished while the node's
3808    /// own work — the fixer's account of what it did — had not. Gated purely
3809    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3810    /// reply, never on any wording in it, so a fixer whose valid, first-try
3811    /// `FixReport` happens to mention having waited on a background test is
3812    /// never resumed — the `Ok(report)` branch at the call site returns
3813    /// before this is ever invoked.
3814    ///
3815    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3816    /// attempt ([`retry_budget`]), nothing attempted once the session is
3817    /// gone, and a quota hit ends the loop immediately rather than retrying a
3818    /// rate limit that fails the same way again.
3819    async fn continue_fix_report(
3820        &mut self,
3821        mut seat: SeatState,
3822        parse_err: String,
3823        job: &SeatJob,
3824        prompts: &Prompts,
3825        run_id: &str,
3826        round: usize,
3827    ) -> (
3828        SeatState,
3829        Option<FixReport>,
3830        Option<String>,
3831        ContinuationRecord,
3832    ) {
3833        let mut last_err = parse_err;
3834        let mut cumulative_wait_ms = 0u64;
3835        let mut attempts = 0usize;
3836        loop {
3837            if !has_context(&job.spec, &seat, job.sessions) {
3838                self.state.event(
3839                    "fix",
3840                    format!(
3841                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3842                         session left to resume into"
3843                    ),
3844                );
3845                let outcome = if attempts == 0 {
3846                    ContinuationOutcome::NoSession
3847                } else {
3848                    ContinuationOutcome::Exhausted
3849                };
3850                return (
3851                    seat,
3852                    None,
3853                    Some(format!("unparsable fix report: {last_err}")),
3854                    ContinuationRecord {
3855                        attempts,
3856                        cumulative_wait_ms,
3857                        outcome,
3858                    },
3859                );
3860            }
3861            if attempts >= MAX_FIX_CONTINUATIONS {
3862                self.state.event(
3863                    "fix",
3864                    format!(
3865                        "round {round}: fixer's reply still had no adoption report after \
3866                         {attempts} continuation(s) ({last_err}); giving up"
3867                    ),
3868                );
3869                return (
3870                    seat,
3871                    None,
3872                    Some(format!(
3873                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3874                    )),
3875                    ContinuationRecord {
3876                        attempts,
3877                        cumulative_wait_ms,
3878                        outcome: ContinuationOutcome::Exhausted,
3879                    },
3880                );
3881            }
3882            attempts += 1;
3883            self.state.event(
3884                "fix",
3885                format!(
3886                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3887                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3888                ),
3889            );
3890            let mut retry = job.clone();
3891            retry.seat = seat.clone();
3892            retry.prompt = prompt::resume_incomplete(&last_err);
3893            retry.timeout = retry_budget(job.timeout, true);
3894            retry.stem = format!("{}-continue{attempts}", job.stem);
3895            let cache = self.state.config.cache_dir();
3896            let ctx = WaveCtx {
3897                carry_seats: false,
3898                pauses: Some((&self.pause, &self.interrupt)),
3899                run: run_id,
3900                node: "fix",
3901                prompts,
3902                cache: cache.as_deref(),
3903                round: Some(round),
3904            };
3905            let (resumed_seat, resumed_out) = run_one(
3906                retry,
3907                Arc::clone(&self.sem),
3908                &ctx,
3909                &mut self.state,
3910                attempts,
3911            )
3912            .await;
3913            seat = resumed_seat;
3914            match resumed_out {
3915                AgentOutcome::Ok(o) => {
3916                    cumulative_wait_ms += o.duration_ms;
3917                    match verdict::extract_json::<FixReport>(&o.text) {
3918                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3919                            self.state.event(
3920                                "fix",
3921                                format!(
3922                                    "round {round}: fixer's adoption report recovered after \
3923                                     {attempts} continuation(s)"
3924                                ),
3925                            );
3926                            return (
3927                                seat,
3928                                Some(report),
3929                                None,
3930                                ContinuationRecord {
3931                                    attempts,
3932                                    cumulative_wait_ms,
3933                                    outcome: ContinuationOutcome::Resumed,
3934                                },
3935                            );
3936                        }
3937                        // The report parsed, but this same reply's own
3938                        // CommandEvidence — the identical record `state.jobs`
3939                        // renders — names a command whose CLI never
3940                        // confirmed an exit status. Read together, that is
3941                        // not a resolved answer: keep nudging rather than
3942                        // accept a report standing next to a command the
3943                        // seat's own CLI cannot vouch for.
3944                        Ok(_) => {
3945                            last_err = "the reply parsed, but it reported a command whose own CLI \
3946                                 never confirmed an exit status"
3947                                .to_owned();
3948                        }
3949                        Err(e) => last_err = e.to_string(),
3950                    }
3951                }
3952                AgentOutcome::Quota(o) => {
3953                    cumulative_wait_ms += o.duration_ms;
3954                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3955                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3956                    self.state.quota.push(loss);
3957                    self.state.event(
3958                        "fix",
3959                        format!(
3960                            "round {round}: continuation rate limited (quota); not retrying now"
3961                        ),
3962                    );
3963                    return (
3964                        seat,
3965                        None,
3966                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3967                        ContinuationRecord {
3968                            attempts,
3969                            cumulative_wait_ms,
3970                            outcome: ContinuationOutcome::QuotaLost,
3971                        },
3972                    );
3973                }
3974                AgentOutcome::Dropped(o) => {
3975                    cumulative_wait_ms += o.duration_ms;
3976                    let why = o
3977                        .dropped
3978                        .as_ref()
3979                        .map(|d| d.why.as_str())
3980                        .unwrap_or("the CLI ended the stream without delivering its answer");
3981                    last_err = format!("the CLI dropped the stream ({why})");
3982                }
3983                AgentOutcome::Failed(e) => last_err = e,
3984                // The continuation was never started; the report stays lost
3985                // and the caller records that, without any handover.
3986                AgentOutcome::ResourceBlocked(e) => {
3987                    return (
3988                        seat,
3989                        None,
3990                        Some(format!("the continuation was not started: {e}")),
3991                        ContinuationRecord {
3992                            attempts: attempts.saturating_sub(1),
3993                            cumulative_wait_ms,
3994                            outcome: ContinuationOutcome::NoSession,
3995                        },
3996                    );
3997                }
3998            }
3999        }
4000    }
4001
4002    fn after_implement(&mut self) -> Result<()> {
4003        if !self.state.candidate_preservation.is_empty() {
4004            bail!("candidate preservation incomplete; judging is unsafe");
4005        }
4006        // Scan every candidate patch once the set is complete.
4007        if self.state.leaks.is_empty() {
4008            let cfg = self.state.config.blind.clone();
4009            let mut leaks = Vec::new();
4010            for c in &self.state.candidates {
4011                let Some(patch) =
4012                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4013                else {
4014                    continue;
4015                };
4016                leaks.extend(blind::scan(
4017                    &format!("candidate {} patch", c.label),
4018                    &patch,
4019                    &cfg.vendor_tokens,
4020                ));
4021            }
4022            if !leaks.is_empty() {
4023                let summary = leaks
4024                    .iter()
4025                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4026                    .collect::<Vec<_>>()
4027                    .join(", ");
4028                match cfg.on_leak {
4029                    LeakPolicy::Fail => {
4030                        self.state.status = RunStatus::Failed;
4031                        self.state
4032                            .event("blind", format!("vendor text in a patch: {summary}"));
4033                        self.state.leaks = leaks;
4034                        self.state.save()?;
4035                        self.settle_questions();
4036                        bail!(
4037                            "blind.on_leak = \"fail\" and vendor text reached a \
4038                             judged patch: {summary}"
4039                        );
4040                    }
4041                    LeakPolicy::Redact => self.state.event(
4042                        "blind",
4043                        format!("redacting vendor text for judging: {summary}"),
4044                    ),
4045                    LeakPolicy::Warn => self.state.event(
4046                        "blind",
4047                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4048                    ),
4049                }
4050                self.state.leaks = leaks;
4051            }
4052        }
4053
4054        if self.state.viable().is_empty() {
4055            if self.state.all_candidates_verified_noop() {
4056                // Every candidate agreed, with evidence the adoption guard
4057                // accepted, that nothing belongs in this worktree. That is
4058                // not the same fact as a candidate that simply failed to
4059                // write anything, and settling it as an ordinary `Failed`
4060                // (see `SCHEMA`'s doc for schema 10) is what let two of
4061                // task 391f's attempts burn a retry each re-discovering the
4062                // same already-landed fix. Terminal either way, so `judge`
4063                // must never run over an empty candidate set — unlike the
4064                // `Failed` branch below this returns `Ok`, not an error:
4065                // nothing here failed.
4066                self.state.status = RunStatus::VerifiedNoop;
4067                self.state.save()?;
4068                self.settle_questions();
4069                return Ok(());
4070            }
4071            self.state.status = RunStatus::Failed;
4072            self.state.save()?;
4073            self.settle_questions();
4074            bail!("no candidate produced a change; nothing to judge");
4075        }
4076        self.state.status = RunStatus::Judging;
4077        self.state.save()?;
4078        Ok(())
4079    }
4080
4081    // --------------------------------------------------------------- judge
4082
4083    async fn judge(&mut self) -> Result<()> {
4084        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4085        // agent files with `magi task add` name the run that paid for it. The
4086        // prompt overlay is cloned alongside it because the waves borrow it
4087        // while `self` is mutably borrowed by the node's own bookkeeping.
4088        let run_id = self.state.id.clone();
4089        let prompts = self.state.config.prompts.clone();
4090        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4091            return Ok(());
4092        }
4093        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4094        if viable.len() == 1 {
4095            // Recorded so this is a one-time event: `judgements` stays empty
4096            // either way, which without this flag is indistinguishable from
4097            // "not yet judged" on the next reentry — and status is left
4098            // untouched, so a later node's conclusion (e.g. `Blocked` after
4099            // the review budget ran out) survives a resume instead of being
4100            // clobbered back to `Judging` by this node running again.
4101            self.state.judge_skipped = true;
4102            self.state.event(
4103                "judge",
4104                format!(
4105                    "only candidate {} produced a change; judging skipped",
4106                    viable[0].label
4107                ),
4108            );
4109            self.state.save()?;
4110            return Ok(());
4111        }
4112        self.state.status = RunStatus::Judging;
4113
4114        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4115        let language = self.state.config.graph.language.clone();
4116        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4117        let sessions = self.state.config.graph.sessions;
4118        let artifacts = agent::artifacts_dir(&self.state.dir());
4119        let root = self.state.worktree_root();
4120        let base_short = short(&self.state.base_commit);
4121
4122        let mut jobs = Vec::new();
4123        let mut orders = Vec::new();
4124        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4125            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4126            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4127            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4128            let seat_key = format!("judge-{}", j + 1);
4129            let seat = self.seat(&seat_key, &spec.id);
4130            jobs.push(SeatJob {
4131                prompt: prompt::judge(
4132                    &self.state.instruction,
4133                    &views,
4134                    self.roles.judges.len(),
4135                    &base_short,
4136                    &language,
4137                ),
4138                spec,
4139                seat,
4140                cwd: root.join(format!("judge-{}", j + 1)),
4141                timeout,
4142                allow_write: false,
4143                sessions,
4144                artifacts: artifacts.clone(),
4145                stem: format!("judge-{}", j + 1),
4146                handover: None,
4147            });
4148        }
4149
4150        self.state.event(
4151            "judge",
4152            format!(
4153                "{} judges ranking {} candidates blind",
4154                jobs.len(),
4155                viable.len()
4156            ),
4157        );
4158        let labels_for_check = labels.clone();
4159        let mut quota_losses = Vec::new();
4160        let cache = self.state.config.cache_dir();
4161        let ctx = WaveCtx {
4162            carry_seats: false,
4163            pauses: Some((&self.pause, &self.interrupt)),
4164            run: &run_id,
4165            node: "judge",
4166            prompts: &prompts,
4167            cache: cache.as_deref(),
4168            round: None,
4169        };
4170        let results = ask_json_wave::<Ranking>(
4171            jobs,
4172            Arc::clone(&self.sem),
4173            self.state.config.graph.retries,
4174            &self.roles.judge_roster,
4175            &ctx,
4176            &mut quota_losses,
4177            &mut self.state,
4178            &move |r: &Ranking| r.validate(&labels_for_check),
4179        )
4180        .await;
4181        if self.state.resource_block.is_some() {
4182            return self.stop_for_resource();
4183        }
4184        self.state.quota.extend(quota_losses);
4185
4186        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4187            let agent_id = seat.agent.clone();
4188            self.state.seats.insert(seat.key.clone(), seat);
4189            let mut record = Judgement {
4190                judge: j + 1,
4191                seat: format!("judge-{}", j + 1),
4192                agent: agent_id,
4193                ranking: Vec::new(),
4194                reasons: BTreeMap::new(),
4195                confidence: None,
4196                strengths: BTreeMap::new(),
4197                order: orders[j].clone(),
4198                failed: None,
4199                duration_ms: 0,
4200            };
4201            match res {
4202                Ok((ranking, out)) => {
4203                    record.ranking = ranking.normalized();
4204                    record.reasons = ranking.reasons;
4205                    record.strengths = ranking.strengths;
4206                    record.confidence = ranking.confidence;
4207                    record.duration_ms = out.duration_ms;
4208                    self.state.event(
4209                        "judge",
4210                        format!(
4211                            "judge {} ranked {}",
4212                            j + 1,
4213                            record.ranking.iter().collect::<String>()
4214                        ),
4215                    );
4216                }
4217                Err(e) => {
4218                    record.failed = Some(e.to_string());
4219                    self.state
4220                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4221                }
4222            }
4223            self.state.judgements.push(record);
4224            self.state.save()?;
4225        }
4226        Ok(())
4227    }
4228
4229    // ---------------------------------------------------------- deliberate
4230
4231    async fn deliberate(&mut self) -> Result<()> {
4232        if self.hold_base_sync().await? {
4233            return Ok(());
4234        }
4235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4236        // agent files with `magi task add` name the run that paid for it. The
4237        // prompt overlay is cloned alongside it because the waves borrow it
4238        // while `self` is mutably borrowed by the node's own bookkeeping.
4239        let run_id = self.state.id.clone();
4240        let prompts = self.state.config.prompts.clone();
4241        let rounds = self.state.config.graph.deliberate_rounds;
4242        // Resume unfinished rounds after an outage, but never reopen a tally.
4243        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4244            return Ok(());
4245        }
4246        let tops: Vec<char> = self
4247            .state
4248            .judgements
4249            .iter()
4250            .filter_map(|j| j.ranking.first().copied())
4251            .collect();
4252        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4253            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4254                self.state.event(
4255                    "deliberate",
4256                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4257                );
4258            }
4259            self.state.status = RunStatus::Voting;
4260            self.state.save()?;
4261            return Ok(());
4262        }
4263
4264        self.state.status = RunStatus::Deliberating;
4265        self.state.event(
4266            "deliberate",
4267            format!(
4268                "split: first choices were {} — opening {rounds} round(s)",
4269                tops.iter().collect::<String>()
4270            ),
4271        );
4272
4273        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4274        let language = self.state.config.graph.language.clone();
4275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4276        let sessions = self.state.config.graph.sessions;
4277        let artifacts = agent::artifacts_dir(&self.state.dir());
4278        let root = self.state.worktree_root();
4279        let base_short = short(&self.state.base_commit);
4280
4281        // Judges argue in sequence so that a turn can answer the one before it;
4282        // that is the difference between deliberation and three parallel
4283        // monologues.
4284        for round in self.state.deliberation.len() + 1..=rounds {
4285            let mut turns: Vec<DeliberationTurn> = Vec::new();
4286            let mut round_losses = Vec::new();
4287            let mut asked = 0;
4288            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4289                if self.state.judgements[j].failed.is_some() {
4290                    continue;
4291                }
4292                let seat_key = format!("judge-{}", j + 1);
4293                let spec = self.occupant(&seat_key, spec);
4294                let mut seat = self.seat(&seat_key, &spec.id);
4295                let transcript = self.transcript(&turns, j);
4296                let build = |context: Option<&str>| {
4297                    prompt::deliberate(
4298                        &self.state.instruction,
4299                        context,
4300                        &transcript,
4301                        round,
4302                        rounds,
4303                        &language,
4304                    )
4305                };
4306                let block = self.candidate_block(&viable, &base_short);
4307                let full = build(Some(&block));
4308                let text = if has_context(&spec, &seat, sessions) {
4309                    build(None)
4310                } else {
4311                    full.clone()
4312                };
4313                let job = SeatJob {
4314                    spec,
4315                    seat: seat.clone(),
4316                    prompt: text,
4317                    cwd: root.join(format!("judge-{}", j + 1)),
4318                    timeout,
4319                    allow_write: false,
4320                    sessions,
4321                    artifacts: artifacts.clone(),
4322                    stem: format!("delib-{round}-judge-{}", j + 1),
4323                    handover: Some(full),
4324                };
4325                let cache = self.state.config.cache_dir();
4326                let ctx = WaveCtx {
4327                    carry_seats: false,
4328                    pauses: Some((&self.pause, &self.interrupt)),
4329                    run: &run_id,
4330                    node: "deliberate",
4331                    prompts: &prompts,
4332                    cache: cache.as_deref(),
4333                    round: None,
4334                };
4335                // A turn is never nudged (`retries` 0): a failed seat is
4336                // handed to the next roster agent, which gets the full
4337                // context. An empty answer is a turn, not a failure.
4338                let mut losses = Vec::new();
4339                asked += 1;
4340                let mut results = ask_wave_with::<String>(
4341                    vec![job],
4342                    Arc::clone(&self.sem),
4343                    0,
4344                    &self.roles.judge_roster,
4345                    &ctx,
4346                    &mut losses,
4347                    &mut self.state,
4348                    &|text: &str| {
4349                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4350                    },
4351                )
4352                .await;
4353                if self.state.resource_block.is_some() {
4354                    return self.stop_for_resource();
4355                }
4356                round_losses.extend(losses.iter().cloned());
4357                self.state.quota.extend(losses);
4358                let (updated, res, _) = results.pop().expect("one job in, one result out");
4359                seat = updated;
4360                let agent_id = seat.agent.clone();
4361                self.state.seats.insert(seat.key.clone(), seat);
4362                let body = match res {
4363                    Ok((body, _)) => body,
4364                    // Skip the seat; a CLI's raw error JSON is never read as
4365                    // this judge's position.
4366                    Err(e) => {
4367                        self.state
4368                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4369                        continue;
4370                    }
4371                };
4372                let tentative = verdict::extract_json::<Position>(&body)
4373                    .ok()
4374                    .and_then(|p| p.tentative)
4375                    .and_then(|s| s.trim().chars().next())
4376                    .map(|c| c.to_ascii_uppercase());
4377                self.state.event(
4378                    "deliberate",
4379                    format!(
4380                        "round {round}: judge {} now favours {}",
4381                        j + 1,
4382                        tentative.map_or("—".to_owned(), |c| c.to_string())
4383                    ),
4384                );
4385                turns.push(DeliberationTurn {
4386                    judge: j + 1,
4387                    agent: agent_id,
4388                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4389                    tentative,
4390                });
4391            }
4392            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4393                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4394                return self.stop_for_resource();
4395            }
4396            self.state
4397                .deliberation
4398                .push(DeliberationRound { round, turns });
4399            self.state.save()?;
4400        }
4401
4402        self.state.status = RunStatus::Voting;
4403        self.state.save()?;
4404        Ok(())
4405    }
4406
4407    // ---------------------------------------------------------------- vote
4408
4409    async fn vote(&mut self) -> Result<()> {
4410        if self.hold_base_sync().await? {
4411            return Ok(());
4412        }
4413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4414        // agent files with `magi task add` name the run that paid for it. The
4415        // prompt overlay is cloned alongside it because the waves borrow it
4416        // while `self` is mutably borrowed by the node's own bookkeeping.
4417        let run_id = self.state.id.clone();
4418        let prompts = self.state.config.prompts.clone();
4419        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4420            return Ok(());
4421        }
4422        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4423        if viable.len() == 1 {
4424            return Ok(());
4425        }
4426        self.state.status = RunStatus::Voting;
4427
4428        let language = self.state.config.graph.language.clone();
4429        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4430        let sessions = self.state.config.graph.sessions;
4431        let artifacts = agent::artifacts_dir(&self.state.dir());
4432        let root = self.state.worktree_root();
4433        let base_short = short(&self.state.base_commit);
4434        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4435
4436        let mut jobs = Vec::new();
4437        let mut seats_at = Vec::new();
4438        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4439            if self
4440                .state
4441                .judgements
4442                .get(j)
4443                .is_some_and(|r| r.failed.is_some())
4444            {
4445                continue;
4446            }
4447            let seat_key = format!("judge-{}", j + 1);
4448            let spec = self.occupant(&seat_key, spec);
4449            let seat = self.seat(&seat_key, &spec.id);
4450            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4451            let text = if has_context(&spec, &seat, sessions) {
4452                prompt::final_vote(&viable, &language)
4453            } else {
4454                full.clone()
4455            };
4456            jobs.push(SeatJob {
4457                spec,
4458                seat,
4459                prompt: text,
4460                cwd: root.join(format!("judge-{}", j + 1)),
4461                timeout,
4462                allow_write: false,
4463                sessions,
4464                artifacts: artifacts.clone(),
4465                stem: format!("vote-judge-{}", j + 1),
4466                handover: Some(full),
4467            });
4468            seats_at.push(j);
4469        }
4470
4471        self.state.event(
4472            "vote",
4473            format!(
4474                "collecting {} final votes one by one, privately",
4475                jobs.len()
4476            ),
4477        );
4478        let allowed = viable.clone();
4479        let mut quota_losses = Vec::new();
4480        let cache = self.state.config.cache_dir();
4481        let ctx = WaveCtx {
4482            carry_seats: false,
4483            pauses: Some((&self.pause, &self.interrupt)),
4484            run: &run_id,
4485            node: "vote",
4486            prompts: &prompts,
4487            cache: cache.as_deref(),
4488            round: None,
4489        };
4490        let results = ask_json_wave::<FinalVote>(
4491            jobs,
4492            Arc::clone(&self.sem),
4493            self.state.config.graph.retries,
4494            &self.roles.judge_roster,
4495            &ctx,
4496            &mut quota_losses,
4497            &mut self.state,
4498            &move |v: &FinalVote| match v.label() {
4499                Some(c) if allowed.contains(&c) => Ok(()),
4500                other => bail!("vote {other:?} is not one of {allowed:?}"),
4501            },
4502        )
4503        .await;
4504        if self.state.resource_block.is_some() {
4505            return self.stop_for_resource();
4506        }
4507        self.state.quota.extend(quota_losses);
4508
4509        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4510            let agent_id = seat.agent.clone();
4511            self.state.seats.insert(seat.key.clone(), seat);
4512            let initial = self
4513                .state
4514                .judgements
4515                .get(j)
4516                .and_then(|r| r.ranking.first().copied());
4517            let mut record = VoteRecord {
4518                judge: j + 1,
4519                agent: agent_id,
4520                vote: None,
4521                reason: String::new(),
4522                changed: false,
4523            };
4524            match res {
4525                Ok((v, _)) => {
4526                    record.vote = v.label();
4527                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4528                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4529                    self.state.event(
4530                        "vote",
4531                        format!(
4532                            "judge {} voted {}{}",
4533                            j + 1,
4534                            record.vote.unwrap_or('?'),
4535                            if record.changed { " (changed)" } else { "" }
4536                        ),
4537                    );
4538                }
4539                Err(e) => {
4540                    self.state
4541                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4542                }
4543            }
4544            self.state.votes.push(record);
4545            self.state.save()?;
4546        }
4547        Ok(())
4548    }
4549
4550    // --------------------------------------------------------------- tally
4551
4552    fn tally(&mut self) -> Result<()> {
4553        if self.state.tally.is_some() {
4554            return Ok(());
4555        }
4556        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4557        let tops: Vec<char> = self
4558            .state
4559            .judgements
4560            .iter()
4561            .filter_map(|j| j.ranking.first().copied())
4562            .collect();
4563        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4564
4565        // A judge whose private vote failed still counted once, in the initial
4566        // ranking; using it beats discarding a whole seat.
4567        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4568        let mut cast: Vec<char> = Vec::new();
4569        for (i, j) in self.state.judgements.iter().enumerate() {
4570            let vote = self
4571                .state
4572                .votes
4573                .iter()
4574                .find(|v| v.judge == i + 1)
4575                .and_then(|v| v.vote)
4576                .or_else(|| j.ranking.first().copied());
4577            if let Some(v) = vote {
4578                *first_choice.entry(v).or_insert(0) += 1;
4579                cast.push(v);
4580            }
4581        }
4582
4583        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4584        for j in &self.state.judgements {
4585            let n = j.ranking.len();
4586            for (pos, label) in j.ranking.iter().enumerate() {
4587                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4588            }
4589        }
4590
4591        let best = first_choice.values().copied().max().unwrap_or(0);
4592        let mut leaders: Vec<char> = first_choice
4593            .iter()
4594            .filter(|(_, v)| **v == best)
4595            .map(|(k, _)| *k)
4596            .collect();
4597        let mut tie_break = None;
4598        if leaders.len() > 1 {
4599            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4600            let borda_leaders: Vec<char> = leaders
4601                .iter()
4602                .copied()
4603                .filter(|l| borda[l] == top_borda)
4604                .collect();
4605            tie_break = Some(if borda_leaders.len() == 1 {
4606                format!(
4607                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4608                    leaders.len()
4609                )
4610            } else {
4611                format!(
4612                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4613                    leaders.len()
4614                )
4615            });
4616            leaders = borda_leaders;
4617            leaders.sort_unstable();
4618        }
4619        let winner = *leaders
4620            .first()
4621            .or(viable.first())
4622            .context("no candidate to declare a winner from")?;
4623
4624        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4625        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4626        let deliberated = !self.state.deliberation.is_empty();
4627
4628        // Whose verdict is this? A rate-limited seat is absent even if it
4629        // ranked before the limit hit, so presence is measured against the
4630        // recorded losses, not just "did a ranking ever appear".
4631        let quota_seats: std::collections::BTreeSet<&str> =
4632            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4633        let mut present = 0usize;
4634        for (i, j) in self.state.judgements.iter().enumerate() {
4635            if quota_seats.contains(j.seat.as_str()) {
4636                continue;
4637            }
4638            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4639            let voted = self
4640                .state
4641                .votes
4642                .iter()
4643                .any(|v| v.judge == i + 1 && v.vote.is_some());
4644            if ranked || voted {
4645                present += 1;
4646            }
4647        }
4648        // Strict majority of the configured panel. A bare majority is real
4649        // signal we can act on, while a minority verdict must never stand in
4650        // for a healthy one. A one-candidate run needs no panel at all, and
4651        // `judges` stays `0` rather than the roster size a panel that never
4652        // sat would otherwise be credited with.
4653        let needs_quorum = viable.len() > 1;
4654        let judges_total = if needs_quorum {
4655            self.roles.judges.len()
4656        } else {
4657            0
4658        };
4659        let quorum = if needs_quorum {
4660            judges_total / 2 + 1
4661        } else {
4662            0
4663        };
4664        let met_quorum = !needs_quorum || present >= quorum;
4665        let uncontested = (!needs_quorum).then(|| {
4666            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4667        });
4668
4669        self.state.event(
4670            "tally",
4671            match &uncontested {
4672                Some(reason) => format!("winner {winner} — {reason}"),
4673                None => format!(
4674                    "winner {winner} — votes {} | initial {} | {} changed | \
4675                     {present}/{judges_total} judges{}",
4676                    first_choice
4677                        .iter()
4678                        .map(|(k, v)| format!("{k}:{v}"))
4679                        .collect::<Vec<_>>()
4680                        .join(" "),
4681                    if unanimous_initial {
4682                        "unanimous"
4683                    } else {
4684                        "split"
4685                    },
4686                    changed_votes,
4687                    if met_quorum {
4688                        String::new()
4689                    } else {
4690                        format!(" — below quorum ({quorum} required)")
4691                    },
4692                ),
4693            },
4694        );
4695        if !met_quorum {
4696            self.state.event(
4697                "stall",
4698                format!(
4699                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4700                     the run stops here, resumable"
4701                ),
4702            );
4703        }
4704        self.state.tally = Some(Tally {
4705            first_choice,
4706            borda,
4707            winner,
4708            rankings: tops.len(),
4709            unanimous_initial,
4710            deliberated,
4711            changed_votes,
4712            unanimous_final,
4713            tie_break,
4714            judges: judges_total,
4715            present,
4716            quorum,
4717            met_quorum,
4718            uncontested,
4719        });
4720        self.state.status = if met_quorum {
4721            RunStatus::Reviewing
4722        } else {
4723            RunStatus::Stalled
4724        };
4725        self.state.save()?;
4726        Ok(())
4727    }
4728
4729    // ------------------------------------------------------------- recover
4730
4731    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4732    /// resumed toward completion once the transient cause clears.
4733    ///
4734    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4735    /// it toward the quorum, which is exactly the set of seats whose absence
4736    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4737    /// not depend on which node happened to hit the limit), or an ordinary
4738    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4739    /// seat is never disturbed.
4740    ///
4741    /// A seat that now answers with a usable ranking is "recovered": its
4742    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4743    /// `tally` counts it present again), and its vote re-collected. A seat that
4744    /// still fails keeps its loss and stays absent.
4745    ///
4746    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4747    /// to review/gate/merge), `false` when it is still below quorum (the run
4748    /// stays `Stalled`, still resumable for a later retry).
4749    #[allow(clippy::too_many_lines)]
4750    async fn recover_stall(&mut self) -> Result<bool> {
4751        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4752        // agent files with `magi task add` name the run that paid for it. The
4753        // prompt overlay is cloned alongside it because the waves borrow it
4754        // while `self` is mutably borrowed by the node's own bookkeeping.
4755        let run_id = self.state.id.clone();
4756        let prompts = self.state.config.prompts.clone();
4757        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4758        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4759        // is what keeps a non-quota collapse — or a quota loss recorded at the
4760        // deliberate node — from being a permanent dead-end on `--resume`.
4761        let quota_seats: BTreeSet<&str> =
4762            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4763        let absent: Vec<String> = self
4764            .state
4765            .judgements
4766            .iter()
4767            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4768            .map(|j| j.seat.clone())
4769            .collect();
4770        if absent.is_empty() {
4771            return Ok(false);
4772        }
4773        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4774        if viable.len() <= 1 {
4775            return Ok(false);
4776        }
4777        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4778        let language = self.state.config.graph.language.clone();
4779        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4780        let sessions = self.state.config.graph.sessions;
4781        let artifacts = agent::artifacts_dir(&self.state.dir());
4782        let root = self.state.worktree_root();
4783        let base_short = short(&self.state.base_commit);
4784        let candidates: Vec<Candidate> = viable.clone();
4785
4786        // Map each absent seat key to its 0-based position in `roles.judges`.
4787        let mut positions: Vec<usize> = absent
4788            .iter()
4789            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4790            .collect();
4791        if positions.is_empty() {
4792            return Ok(false);
4793        }
4794        positions.sort_unstable();
4795        positions.dedup();
4796
4797        // Re-rank the lost seats, one blind prompt each.
4798        let mut judge_jobs = Vec::new();
4799        for &j in &positions {
4800            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4801            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4802            let seat_key = format!("judge-{}", j + 1);
4803            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4804            let seat = self.seat(&seat_key, &spec.id);
4805            judge_jobs.push(SeatJob {
4806                spec,
4807                seat,
4808                prompt: prompt::judge(
4809                    &self.state.instruction,
4810                    &views,
4811                    self.roles.judges.len(),
4812                    &base_short,
4813                    &language,
4814                ),
4815                cwd: root.join(seat_key),
4816                timeout,
4817                allow_write: false,
4818                sessions,
4819                artifacts: artifacts.clone(),
4820                stem: format!("judge-{}-recover", j + 1),
4821                handover: None,
4822            });
4823        }
4824
4825        let labels_for_check = labels.clone();
4826        let mut judge_losses = Vec::new();
4827        let retries = self.state.config.graph.retries;
4828        let cache = self.state.config.cache_dir();
4829        let ctx = WaveCtx {
4830            carry_seats: false,
4831            pauses: Some((&self.pause, &self.interrupt)),
4832            run: &run_id,
4833            node: "judge",
4834            prompts: &prompts,
4835            cache: cache.as_deref(),
4836            round: None,
4837        };
4838        let results = ask_json_wave::<Ranking>(
4839            judge_jobs,
4840            Arc::clone(&self.sem),
4841            retries,
4842            &self.roles.judge_roster,
4843            &ctx,
4844            &mut judge_losses,
4845            &mut self.state,
4846            &move |r: &Ranking| r.validate(&labels_for_check),
4847        )
4848        .await;
4849        if self.state.resource_block.is_some() {
4850            self.stop_for_resource()?;
4851            return Ok(false);
4852        }
4853
4854        // Refresh the judgement of every seat that ranked again.
4855        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4856        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4857            let agent_id = seat.agent.clone();
4858            self.state.seats.insert(seat.key.clone(), seat);
4859            let record = &mut self.state.judgements[j];
4860            match res {
4861                Ok((ranking, out)) => {
4862                    record.agent = agent_id;
4863                    record.ranking = ranking.normalized();
4864                    record.reasons = ranking.reasons;
4865                    record.strengths = ranking.strengths;
4866                    record.confidence = ranking.confidence;
4867                    record.failed = None;
4868                    record.duration_ms = out.duration_ms;
4869                    recovered.insert(j);
4870                    self.state.event(
4871                        "recover",
4872                        format!("judge {} ranked again after the limit", j + 1),
4873                    );
4874                }
4875                Err(e) => {
4876                    self.state
4877                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4878                }
4879            }
4880        }
4881
4882        // Re-ask the votes of the seats that recovered a ranking.
4883        let mut vote_jobs = Vec::new();
4884        let mut vote_pos: Vec<usize> = Vec::new();
4885        for &j in &recovered {
4886            let seat_key = format!("judge-{}", j + 1);
4887            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4888            let seat = self.seat(&seat_key, &spec.id);
4889            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4890            let text = if has_context(&spec, &seat, sessions) {
4891                prompt::final_vote(&labels, &language)
4892            } else {
4893                full.clone()
4894            };
4895            vote_jobs.push(SeatJob {
4896                spec,
4897                seat,
4898                prompt: text,
4899                cwd: root.join(seat_key),
4900                timeout,
4901                allow_write: false,
4902                sessions,
4903                artifacts: artifacts.clone(),
4904                stem: format!("vote-judge-{}-recover", j + 1),
4905                handover: Some(full),
4906            });
4907            vote_pos.push(j);
4908        }
4909        let allowed = labels.clone();
4910        let mut vote_losses = Vec::new();
4911        let vote_retries = self.state.config.graph.retries;
4912        let vote_cache = self.state.config.cache_dir();
4913        let ctx = WaveCtx {
4914            carry_seats: false,
4915            pauses: Some((&self.pause, &self.interrupt)),
4916            run: &run_id,
4917            node: "vote",
4918            prompts: &prompts,
4919            cache: vote_cache.as_deref(),
4920            round: None,
4921        };
4922        let votes = ask_json_wave::<FinalVote>(
4923            vote_jobs,
4924            Arc::clone(&self.sem),
4925            vote_retries,
4926            &self.roles.judge_roster,
4927            &ctx,
4928            &mut vote_losses,
4929            &mut self.state,
4930            &move |v: &FinalVote| match v.label() {
4931                Some(c) if allowed.contains(&c) => Ok(()),
4932                other => bail!("vote {other:?} is not one of {allowed:?}"),
4933            },
4934        )
4935        .await;
4936        if self.state.resource_block.is_some() {
4937            self.stop_for_resource()?;
4938            return Ok(false);
4939        }
4940        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4941            let agent_id = seat.agent.clone();
4942            self.state.seats.insert(seat.key.clone(), seat);
4943            match res {
4944                Ok((v, _)) => {
4945                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4946                        rec.vote = v.label();
4947                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4948                    } else {
4949                        self.state.votes.push(VoteRecord {
4950                            judge: j + 1,
4951                            agent: agent_id,
4952                            vote: v.label(),
4953                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4954                            changed: false,
4955                        });
4956                    }
4957                    self.state.event(
4958                        "recover",
4959                        format!("judge {} voted again after the limit", j + 1),
4960                    );
4961                }
4962                Err(e) => {
4963                    self.state
4964                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4965                }
4966            }
4967        }
4968
4969        // A seat that ranked again is present even if its re-vote failed —
4970        // `tally` falls back to the initial ranking's first choice — so clear
4971        // its quota loss. Seats that still fail keep theirs and stay absent.
4972        let recovered_keys: BTreeSet<String> = recovered
4973            .iter()
4974            .map(|&j| format!("judge-{}", j + 1))
4975            .collect();
4976        self.state
4977            .quota
4978            .retain(|q| !recovered_keys.contains(&q.seat));
4979        // A seat that hit the limit again is a fresh loss, not the old one:
4980        // replace the stale entry so the history stays one-per-seat and the
4981        // daemon can tell this attempt's loss from a previous session's.
4982        for loss in judge_losses.into_iter().chain(vote_losses) {
4983            if recovered_keys.contains(&loss.seat) {
4984                continue;
4985            }
4986            self.state.quota.retain(|q| q.seat != loss.seat);
4987            self.state.quota.push(loss);
4988        }
4989
4990        // Recompute the verdict from the refreshed panel.
4991        self.state.tally = None;
4992        self.tally()?;
4993        Ok(self
4994            .state
4995            .tally
4996            .as_ref()
4997            .map(|t| t.met_quorum)
4998            .unwrap_or(false))
4999    }
5000
5001    // ---------------------------------------------------------------- graft
5002
5003    /// Port the strengths the judges attributed to the losing candidates onto
5004    /// the winner, once, between the tally and the review loop.
5005    ///
5006    /// Best-effort by construction: it returns nothing and fails nothing. A
5007    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5008    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5009    /// the run carries on as if the phase were off. See [`graft`] for the
5010    /// decisions and `AGENTS.md` for the invariants.
5011    ///
5012    /// The one exception is a rollback that could not be confirmed: the
5013    /// rejected commit may still be the winner's tip, so the run is blocked
5014    /// (resumable; the record stays `Running`, which makes a resume retry the
5015    /// restore) instead of reviewing, gating and merging it.
5016    async fn graft(&mut self) -> Result<()> {
5017        let Err(e) = self.graft_inner().await else {
5018            return Ok(());
5019        };
5020        if e.is::<crate::repo_lock::ResourceBlocked>() {
5021            return Err(e);
5022        }
5023        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5024            self.state.status = RunStatus::Blocked;
5025            self.state.event("graft", format!("blocked: {e:#}"));
5026            self.state.save()?;
5027            return Err(e);
5028        }
5029        // An error after the `Running` record was saved may have left the
5030        // fixer's work in the tree: the record must not stay unsettled
5031        // (a later resume would reset past the review fixes), so confirm the
5032        // restore now, or block.
5033        if let Some(rec) = self.state.graft.clone()
5034            && !rec.outcome.settled()
5035        {
5036            if let Some(winner) = self.state.winner().cloned() {
5037                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5038                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5039                        return Err(re);
5040                    }
5041                    self.state.status = RunStatus::Blocked;
5042                    self.state.event("graft", format!("blocked: {re:#}"));
5043                    self.state.save()?;
5044                    return Err(re);
5045                }
5046            }
5047            if let Some(g) = self.state.graft.as_mut() {
5048                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5049            }
5050        }
5051        self.state.event("graft", format!("skipped: {e:#}"));
5052        if let Err(e) = self.state.save() {
5053            tracing::warn!("save after graft: {e:#}");
5054        }
5055        Ok(())
5056    }
5057
5058    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5059    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5060    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5061        let _coordination = self.coordinate(wt).await?;
5062        crate::recovery::restore_repair(wt, pre)
5063            .await
5064            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5065    }
5066
5067    /// A skip is a run event, recorded once however often the node is re-entered.
5068    fn graft_skipped(&mut self, why: &str) {
5069        let message = format!("skipped: {why}");
5070        if !self
5071            .state
5072            .events
5073            .iter()
5074            .any(|e| e.node == "graft" && e.message == message)
5075        {
5076            self.state.event("graft", message);
5077        }
5078    }
5079
5080    async fn graft_inner(&mut self) -> Result<()> {
5081        // A round that was started and never finished (crash, kill): the
5082        // fixer may have left anything in the tree, so put the winner back.
5083        if let Some(rec) = self.state.graft.clone()
5084            && !rec.outcome.settled()
5085        {
5086            // Reviews exist only if the graft ended without its record being
5087            // settled (a swallowed error): the tree has moved on since, and
5088            // a reset to `pre` would discard the review fixes.
5089            if self.state.reviews.is_empty()
5090                && let Some(winner) = self.state.winner().cloned()
5091            {
5092                self.graft_restore(&winner.worktree, &rec.pre).await?;
5093            }
5094            if rec.outcome == GraftOutcome::WaitingForRestore {
5095                self.state.graft = None;
5096                self.state.save()?;
5097            } else {
5098                if let Some(g) = self.state.graft.as_mut() {
5099                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5100                }
5101                self.state.event(
5102                    "graft",
5103                    "rolled back: the previous round was interrupted; the winner is untouched",
5104                );
5105                self.state.save()?;
5106                return Ok(());
5107            }
5108        }
5109        let plan = match graft::plan(&self.state) {
5110            Ok(plan) => plan,
5111            Err(skip) => {
5112                if skip != graft::Skip::AlreadyDone {
5113                    self.graft_skipped(skip.reason());
5114                    self.state.save()?;
5115                }
5116                return Ok(());
5117            }
5118        };
5119        let Some(winner) = self.state.winner().cloned() else {
5120            return Ok(());
5121        };
5122        let wt = winner.worktree.clone();
5123        if !git::is_clean(&wt).await? {
5124            self.graft_skipped("the winner's worktree has uncommitted changes");
5125            self.state.save()?;
5126            return Ok(());
5127        }
5128        let pre = git::rev_parse(&wt, "HEAD").await?;
5129        // Saved before the fixer is called: a stop from here on is repaired
5130        // by the branch above on the next entry.
5131        self.state.graft = Some(GraftRecord {
5132            pre: pre.clone(),
5133            post: None,
5134            lines: None,
5135            limit: None,
5136            agent: None,
5137            outcome: GraftOutcome::Running,
5138            requested: graft::requested(&plan),
5139            adoption: None,
5140            adoption_error: None,
5141        });
5142        self.state.event(
5143            "graft",
5144            format!(
5145                "porting strengths of candidate(s) {} onto {}",
5146                plan.strengths
5147                    .iter()
5148                    .map(|(l, _, _)| l.to_string())
5149                    .collect::<Vec<_>>()
5150                    .join(", "),
5151                plan.winner
5152            ),
5153        );
5154        self.state.save()?;
5155
5156        let fix_prompt = prompt::graft(
5157            &self.state.instruction,
5158            plan.winner,
5159            &plan.strengths,
5160            &self.state.config.graph.language,
5161        );
5162        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5163        let sessions = self.state.config.graph.sessions;
5164        let artifacts = agent::artifacts_dir(&self.state.dir());
5165        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5166        let (_, seat, out) = self
5167            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5168                SeatJob {
5169                    prompt: fix_prompt.clone(),
5170                    spec,
5171                    seat,
5172                    cwd: wt.clone(),
5173                    timeout,
5174                    allow_write: true,
5175                    sessions,
5176                    artifacts: artifacts.clone(),
5177                    stem: "graft".to_owned(),
5178                    handover: None,
5179                }
5180            })
5181            .await;
5182        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5183            // Preserve the need to restore earlier fallback edits without
5184            // confusing an unstarted seat with an interrupted invocation.
5185            self.state.graft.as_mut().expect("graft record").outcome =
5186                GraftOutcome::WaitingForRestore;
5187            return self.stop_for_resource();
5188        }
5189        let mut why: Option<String> = None;
5190        match &out {
5191            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5192                why = Some("the fixer produced nothing usable".to_owned());
5193            }
5194            AgentOutcome::Ok(_) => {}
5195            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5196            AgentOutcome::Quota(o) => {
5197                self.state.quota.push(QuotaLoss::from_report(
5198                    seat.key.clone(),
5199                    "graft",
5200                    o.quota.as_ref(),
5201                ));
5202                why = Some("rate limited (quota)".to_owned());
5203            }
5204            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5205                why = Some(format!(
5206                    "the fixer failed: {}",
5207                    e.lines()
5208                        .next()
5209                        .unwrap_or("")
5210                        .chars()
5211                        .take(160)
5212                        .collect::<String>()
5213                ));
5214            }
5215        }
5216        let agent_id = seat.agent.clone();
5217        self.state.seats.insert(seat.key.clone(), seat);
5218
5219        let mut post = None;
5220        let mut lines = None;
5221        let _coordination = self.coordinate(&wt).await?;
5222        if why.is_none() {
5223            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5224            if let Ok(r) = &rescued {
5225                self.state.note_withheld("graft", &r.withheld);
5226            }
5227            let head = git::rev_parse(&wt, "HEAD").await?;
5228            // What is measured must be what gets reviewed and gated: a failed
5229            // rescue, or anything still uncommitted (withheld lockfiles
5230            // aside, which are never part of the tree), rejects the graft.
5231            let dirty = match &rescued {
5232                Ok(r) => git::status_porcelain(&wt)
5233                    .await
5234                    .map(|st| {
5235                        st.lines().any(|l| {
5236                            !r.withheld
5237                                .iter()
5238                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5239                        })
5240                    })
5241                    .unwrap_or(true),
5242                Err(_) => true,
5243            };
5244            if dirty {
5245                why = Some("the fixer left changes that could not be committed".to_owned());
5246            } else if head == pre {
5247                why = Some("the fixer changed nothing".to_owned());
5248            } else {
5249                lines = git::changed_lines(&wt, &pre, &head).await;
5250                post = Some(head);
5251            }
5252        }
5253        let (adoption, adoption_error) = match &out {
5254            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5255            _ => (None, None),
5256        };
5257        let requested = graft::requested(&plan);
5258        let outcome = match why {
5259            None => {
5260                self.state.event(
5261                    "graft",
5262                    format!(
5263                        "applied: {}",
5264                        lines.map_or_else(
5265                            || "changed lines unavailable".to_owned(),
5266                            |n| format!("{n} changed lines")
5267                        )
5268                    ),
5269                );
5270                GraftOutcome::Applied
5271            }
5272            Some(why) => {
5273                // Not settled until the restore is confirmed: the record
5274                // stays `Running`, so a resume retries it.
5275                crate::recovery::restore_repair(&wt, &pre)
5276                    .await
5277                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5278                self.state.event(
5279                    "graft",
5280                    format!("rolled back: {why}; the winner is untouched"),
5281                );
5282                GraftOutcome::RolledBack(why)
5283            }
5284        };
5285        self.state.graft = Some(GraftRecord {
5286            pre,
5287            post,
5288            lines,
5289            limit: None,
5290            agent: Some(agent_id),
5291            outcome,
5292            requested,
5293            adoption,
5294            adoption_error,
5295        });
5296        self.state.save()?;
5297        Ok(())
5298    }
5299
5300    // ----------------------------------------------------------------- fold
5301
5302    async fn fold_losers(&mut self) -> Result<()> {
5303        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5304            return Ok(());
5305        };
5306        if !self.state.candidate_preservation.is_empty() {
5307            bail!("candidate preservation incomplete; refusing cleanup");
5308        }
5309        let repo = self.state.repo.clone();
5310        let _coordination = self.coordinate(&repo).await?;
5311        let mut folded = Vec::new();
5312        for i in 0..self.state.candidates.len() {
5313            let c = &self.state.candidates[i];
5314            if c.label == winner || c.folded {
5315                continue;
5316            }
5317            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5318            remove_preserved_worktree(&repo, &wt).await?;
5319            if git::branch_exists(&repo, &branch).await?
5320                && !git::branch_delete(&repo, &branch).await?
5321            {
5322                bail!("candidate branch removal refused");
5323            }
5324            self.state.candidates[i].folded = true;
5325            self.state.save()?;
5326            folded.push(label.to_string());
5327        }
5328        // The judges are finished; their checkouts are pure cost from here.
5329        let root = self.state.worktree_root();
5330        for j in 1..=self.roles.judges.len() {
5331            let wt = root.join(format!("judge-{j}"));
5332            if wt.exists() {
5333                remove_preserved_worktree(&repo, &wt).await?;
5334            }
5335        }
5336        // The design-deliberation stage is finished by the time a tally
5337        // exists — same reasoning as the judges above.
5338        if self.state.config.graph.advise {
5339            for k in 1..=self.state.config.graph.advisors {
5340                let wt = root.join(format!("advisor-{k}"));
5341                if wt.exists() {
5342                    remove_preserved_worktree(&repo, &wt).await?;
5343                }
5344            }
5345        }
5346        if !folded.is_empty() {
5347            self.state
5348                .event("fold", format!("folded candidates {}", folded.join(", ")));
5349            self.state.save()?;
5350        }
5351        Ok(())
5352    }
5353
5354    // ------------------------------------------------------------ base sync
5355
5356    async fn hold_base_sync(&mut self) -> Result<bool> {
5357        self.hold_base_sync_as(None).await
5358    }
5359
5360    async fn hold_base_sync_kind(
5361        &mut self,
5362        kind: crate::base_sync_recovery::StopKind,
5363    ) -> Result<()> {
5364        self.hold_base_sync_as(Some(kind)).await?;
5365        Ok(())
5366    }
5367
5368    async fn hold_base_sync_as(
5369        &mut self,
5370        kind: Option<crate::base_sync_recovery::StopKind>,
5371    ) -> Result<bool> {
5372        if !self.state.base_sync_blocked() {
5373            self.state.base_sync_recovery.stop = None;
5374            return Ok(false);
5375        }
5376        let head = match self.state.winner() {
5377            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5378                .await
5379                .unwrap_or_default(),
5380            None => String::new(),
5381        };
5382        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5383            && !authorization.consumed
5384            && authorization.stop.head != head
5385        {
5386            authorization.refuse(format!(
5387                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5388                authorization.stop.head
5389            ));
5390        }
5391        self.state.record_base_sync_stop(&head);
5392        if let Some(kind) = kind {
5393            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5394            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5395                stop.kind = kind;
5396            }
5397        }
5398        self.state.save()?;
5399        Ok(true)
5400    }
5401
5402    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5403    /// anything verifies it.
5404    ///
5405    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5406    /// read whatever is checked out in the winner's worktree. Left alone that
5407    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5408    /// it when the run *branched* - and a run takes long enough that the base
5409    /// has usually moved by the time it gets here. A gate that ran there
5410    /// answers "green on the commit this run started from", not "green on
5411    /// what is about to land", and the difference showed up three times in
5412    /// one day as a green run whose merge would have reverted a file another
5413    /// pull request had already landed.
5414    ///
5415    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5416    /// `land::Step::Rebase` calls, rather than a second implementation of the
5417    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5418    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5419    /// doc warns about ("two answers to a question nobody notices until a
5420    /// diff is wrong").
5421    ///
5422    /// Foreign-checkout reviews instead merge the base into the owned shadow
5423    /// worktree, preserving the ancestry required for fast-forward delivery.
5424    /// A conflicting merge is aborted and held for operator resolution.
5425    ///
5426    /// A conflict is not the end of the road: the standing rebase is handed
5427    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5428    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5429    /// land). Once it finishes, review and the gate run as usual on the
5430    /// rebased tree, which is where a breakage the new base caused is caught
5431    /// by the ordinary gate-fix round. magi resolves nothing itself.
5432    ///
5433    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5434    /// the current durable recovery interval, bounds how often the base is
5435    /// synchronized during active recovery (lifetime attempts remain evidence).
5436    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5437    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5438    /// what was tried (rounds spent, paths still conflicted) and the branch
5439    /// and worktree stay exactly as they were - untouched, for a person to
5440    /// look at - which is also what makes re-entering this function
5441    /// afterwards a no-op instead of a second attempt at the same wall. A
5442    /// push failure ends the same way.
5443    async fn sync_to_base(&mut self) -> Result<()> {
5444        if self.state.status == RunStatus::AlreadyInBase {
5445            return Ok(());
5446        }
5447        self.hold_base_sync().await?;
5448        let conflicted = self
5449            .state
5450            .base_sync
5451            .as_ref()
5452            .is_some_and(|s| s.conflict.is_some());
5453        let Some(winner) = self.state.winner().cloned() else {
5454            return Ok(());
5455        };
5456
5457        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5458
5459        let repo = self.state.repo.clone();
5460        let remote = self.state.config.merge.remote.clone();
5461        let base_branch = self.state.base_branch.clone();
5462        let tracking = format!("{remote}/{base_branch}");
5463
5464        let fetched_base =
5465            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5466        // No network, or the remote never had this branch: the run already
5467        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5468        // otherwise), and one that got this far is not blocked by a fetch
5469        // that fails now. It just has no newer tip to compare against.
5470        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5471            return Ok(());
5472        };
5473
5474        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5475        self.state.ensure_base_episode();
5476        let mut pending = self
5477            .state
5478            .base_sync_recovery
5479            .operations
5480            .last()
5481            .filter(|o| !o.finished)
5482            .cloned();
5483        let scratch = self.state.dir().join("base-sync");
5484        if git::rebase_in_progress(&scratch).await {
5485            // Older resource stops predate operation reservations. Import the
5486            // standing Git operation, never retarget it to today's main.
5487            if pending.is_none()
5488                && self.state.base_sync_recovery.operations.is_empty()
5489                && !conflicted
5490                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5491                    .await
5492                    .ok()
5493                    .as_deref()
5494                    == Some(head.as_str())
5495                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5496                && git::is_clean(&winner.worktree).await?
5497            {
5498                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5499                if self.state.base_sync.is_none() {
5500                    self.state.base_sync = Some(BaseSync {
5501                        tip: saved_tip.clone(),
5502                        behind: 0,
5503                        attempts: 0,
5504                        conflict: None,
5505                        already_in: None,
5506                    });
5507                }
5508                self.state.reserve_base_operation(&head, &saved_tip);
5509                self.state.event(
5510                    "base-recovery",
5511                    "imported standing legacy rebase as a spent synchronization reservation",
5512                );
5513                self.state.save()?;
5514                pending = self.state.base_sync_recovery.operations.last().cloned();
5515            }
5516            let expected = pending.as_ref().map(|op| op.tip.as_str());
5517            if expected.is_none()
5518                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5519            {
5520                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5521                if self.state.base_sync.is_none() {
5522                    self.state.base_sync = Some(BaseSync {
5523                        tip: tip.clone(),
5524                        behind: 0,
5525                        attempts: 0,
5526                        conflict: None,
5527                        already_in: None,
5528                    });
5529                }
5530                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5531                self.state.event("land", why);
5532                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5533                    .await?;
5534                return Ok(());
5535            }
5536        }
5537        if let Some(op) = &pending {
5538            // A replay may add the pinned base's commits, never foreign work.
5539            let mut replay_proven =
5540                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5541            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5542                let wait = repository_wait(&self.state, None);
5543                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5544                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5545                    Ok(true) => {
5546                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5547                        self.state.save()?;
5548                        git::worktree_remove(&repo, &scratch).await.ok();
5549                        replay_proven = true;
5550                    }
5551                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5552                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5553                }
5554            }
5555            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5556                if self.state.foreign_checkout.is_some() {
5557                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5558                        .await
5559                        .ok()
5560                        .as_deref()
5561                        == Some(op.head.as_str())
5562                        && git::rev_parse(&repo, &format!("{head}^2"))
5563                            .await
5564                            .ok()
5565                            .as_deref()
5566                            == Some(op.tip.as_str());
5567                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5568                    .await
5569                    .is_ok_and(|m| m.is_empty())
5570                    && let Ok(added) =
5571                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5572                {
5573                    replay_proven = true;
5574                    for commit in added {
5575                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5576                            replay_proven = false;
5577                            break;
5578                        }
5579                    }
5580                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5581                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5582                }
5583            }
5584            // Never retarget an operation across a crash, even if main moved.
5585            // sync_to_head resets and cleans with -x, so even ignored untracked
5586            // files must stop recovery independently of the old tracked tree.
5587            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5588                || (head != op.head
5589                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5590                        .await?
5591                        .is_empty())
5592                || (!git::is_clean(&winner.worktree).await?
5593                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5594                        .await
5595                        .is_ok_and(|o| o.ok()))
5596                || !replay_proven
5597            {
5598                let why = format!(
5599                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5600                    op.id
5601                );
5602                let sync = self
5603                    .state
5604                    .base_sync
5605                    .as_mut()
5606                    .context("reserved synchronization has no lifetime accounting")?;
5607                sync.conflict = Some(why.clone());
5608                self.state.event("land", why);
5609                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5610                    .await?;
5611                return Ok(());
5612            }
5613            if head != op.head {
5614                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5615            }
5616            tip = op.tip.clone();
5617        }
5618        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5619        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5620        if pending.is_none()
5621            && !conflicted
5622            && fetched_base
5623            && self
5624                .state
5625                .base_sync_recovery
5626                .episodes
5627                .last()
5628                .is_some_and(|e| e.boundary.is_some())
5629            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5630            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5631            && owner.execution_runs().last() == Some(&self.state.id)
5632            && owner.status != crate::queue::TaskStatus::Done
5633            && !owner.fresh_start
5634            && owner.review_branch.is_none()
5635            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5636            && let Some(approval) = owner.resumption.as_ref()
5637            && approval.instruction == owner.instruction
5638            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5639            && git::is_clean(&winner.worktree).await?
5640            && self.state.renew_base_episode(&task, approval, &head, &tip)
5641        {
5642            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5643            self.state.save()?;
5644        }
5645
5646        // Before any rebase, and before a recorded conflict is honoured: a
5647        // branch whose change reached the base under other commit ids has
5648        // nothing to rebase and nothing to conflict with, and a run that
5649        // already stopped on that phantom conflict recovers here on resume.
5650        // `behind == 0` with head == tip is a branch the base has since taken
5651        // in whole, whether or not a conflict was ever recorded: the ancestry
5652        // proof must still run (`classify` ignores a head still on the start
5653        // commit).
5654        if (behind > 0 || conflicted || head == tip)
5655            && self
5656                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5657                .await?
5658        {
5659            return Ok(());
5660        }
5661        if conflicted && self.state.base_recovery_pending() {
5662            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5663            let owner =
5664                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5665            let owner_valid = owner.as_ref().is_ok_and(|t| {
5666                t.execution_runs().last() == Some(&self.state.id)
5667                    && t.status != crate::queue::TaskStatus::Done
5668                    && !t.fresh_start
5669                    && t.review_branch.is_none()
5670            });
5671            let invalidation = if !owner_valid {
5672                Some("latest task ownership changed")
5673            } else if !owner.as_ref().is_ok_and(|t| {
5674                authorization
5675                    .generation
5676                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5677            }) {
5678                Some("owner approval generation changed")
5679            } else if authorization.question.as_ref().is_some_and(|question| {
5680                !crate::base_recovery_control::decision_current(
5681                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5682                    &self.state,
5683                    &authorization.task,
5684                    question,
5685                )
5686            }) {
5687                Some("recovery decision ownership, routing or stop evidence changed")
5688            } else if authorization.request.is_some()
5689                && !owner
5690                    .as_ref()
5691                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5692            {
5693                Some("owner decision changed")
5694            } else if let Some(scope) = &authorization.scope {
5695                if scope.tip != tip {
5696                    Some("fetched base changed")
5697                } else if !self
5698                    .state
5699                    .base_sync_recovery
5700                    .episodes
5701                    .last()
5702                    .is_some_and(|e| e.id == scope.episode)
5703                {
5704                    Some("active synchronization interval changed")
5705                } else if !owner
5706                    .as_ref()
5707                    .is_ok_and(|t| t.instruction == scope.instruction)
5708                {
5709                    Some("task instruction changed")
5710                } else {
5711                    None
5712                }
5713            } else {
5714                None
5715            };
5716            if let Some(reason) = invalidation {
5717                let authorization = self
5718                    .state
5719                    .base_sync_recovery
5720                    .authorizations
5721                    .last_mut()
5722                    .unwrap();
5723                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5724                self.state.save()?;
5725                return Ok(());
5726            }
5727            if !fetched_base
5728                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5729                || !git::is_clean(&winner.worktree).await?
5730                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5731            {
5732                return Ok(());
5733            }
5734        }
5735        let recovery = conflicted && self.state.consume_base_recovery(&head);
5736        if conflicted {
5737            // Consumption (including a refused changed HEAD) is durable before Git.
5738            self.state.save()?;
5739            if !recovery {
5740                return Ok(());
5741            }
5742        }
5743
5744        if behind == 0 {
5745            // A fixer-finished rebase moves the branch ref before the
5746            // winner's worktree is told (`sync_to_head` below). A run that
5747            // died in between resumes here with `behind == 0` and a tree still
5748            // holding the pre-rebase files, which review and the gate would
5749            // then read. That state is exactly: HEAD moved off the tip the
5750            // rebase started from, yet the tree is still identical to that
5751            // tip. A tree with edits of its own differs from it, so nothing
5752            // is thrown away.
5753            if let Some(from) = self
5754                .state
5755                .rebase_fixes
5756                .iter()
5757                .rev()
5758                .find_map(|r| r.from.clone())
5759                && from != head
5760                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5761                    .await
5762                    .is_ok_and(|o| o.ok())
5763            {
5764                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5765            }
5766            // An earlier attempt may have rebased the branch locally and died
5767            // before pushing it (only the fresh-rebase arm below pushes).
5768            // Publish it now, so the plain push at PR time is not refused as
5769            // a non-fast-forward. A run already holding a recorded conflict
5770            // never reaches here; that case is out of scope.
5771            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5772            if let Some(why) = &conflict {
5773                self.state.status = RunStatus::Blocked;
5774                self.state.event("land", why.clone());
5775            }
5776            self.state.base_sync = Some(BaseSync {
5777                tip,
5778                behind: 0,
5779                attempts,
5780                conflict,
5781                already_in: None,
5782            });
5783            if pending.is_some() {
5784                self.state.finish_base_operation(Some(&head));
5785            }
5786            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5787                .await?;
5788            self.state.save()?;
5789            return Ok(());
5790        }
5791
5792        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5793            let why = format!(
5794                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5795                 synchronization(s); synchronizing again would only race it",
5796                winner.branch
5797            );
5798            self.state.status = RunStatus::Blocked;
5799            self.state.base_sync = Some(BaseSync {
5800                tip,
5801                behind,
5802                attempts,
5803                conflict: Some(why.clone()),
5804                already_in: None,
5805            });
5806            self.state.event("land", why);
5807            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5808                .await?;
5809            self.state.save()?;
5810            return Ok(());
5811        }
5812
5813        if self.state.foreign_checkout.is_some() {
5814            // Delivery to the operator's checkout is fast-forward only. A
5815            // rebase here would discard its saved head from our ancestry.
5816            let wait = repository_wait(&self.state, None);
5817            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5818            if self.state.base_sync.is_none() {
5819                self.state.base_sync = Some(BaseSync {
5820                    tip: tip.clone(),
5821                    behind,
5822                    attempts: 0,
5823                    conflict: None,
5824                    already_in: None,
5825                });
5826            }
5827            self.state.reserve_base_operation(&head, &tip);
5828            self.state.save()?;
5829            let merged = git::git_raw(
5830                &winner.worktree,
5831                &[
5832                    "-c",
5833                    "core.hooksPath=/dev/null",
5834                    "merge",
5835                    "--no-ff",
5836                    "--no-autostash",
5837                    "--no-edit",
5838                    "-m",
5839                    &format!("Merge {tracking} before verifying"),
5840                    &tip,
5841                ],
5842            )
5843            .await?;
5844            let conflict = if merged.ok() {
5845                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5846                self.state.event(
5847                    "land",
5848                    format!("merged {tracking} into {} before verifying", winner.branch),
5849                );
5850                None
5851            } else {
5852                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5853                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5854                }
5855                let why = format!(
5856                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5857                    winner.branch,
5858                    merged.stderr.chars().take(600).collect::<String>()
5859                );
5860                self.state.status = RunStatus::Blocked;
5861                self.state.event("land", &why);
5862                Some(why)
5863            };
5864            let synced = conflict.is_none();
5865            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5866            self.state
5867                .finish_base_operation(synced.then_some(fresh.as_str()));
5868            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5869            self.state.base_sync = Some(BaseSync {
5870                tip,
5871                behind: if synced { 0 } else { behind },
5872                attempts,
5873                conflict,
5874                already_in: None,
5875            });
5876            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5877                .await?;
5878            self.state.save()?;
5879            return Ok(());
5880        }
5881
5882        self.state.event(
5883            "land",
5884            format!(
5885                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5886                winner.branch
5887            ),
5888        );
5889        self.state.save()?;
5890
5891        // The remote's copy of the branch, read now and only if the fetch
5892        // really succeeded (a stale tracking ref must never pin a lease). It is
5893        // pushed over after a rebase only when it is a commit this branch
5894        // already contains, by ancestry or by patch (an earlier rebase of ours
5895        // that never reached the remote): anything else is somebody else's work.
5896        let branch_tracking = format!("{remote}/{}", winner.branch);
5897        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5898        let remote_tip = if self.state.task_pr_branch().is_none()
5899            && matches!(&fetched_branch, Ok(o) if o.ok())
5900        {
5901            git::rev_parse(&repo, &branch_tracking).await.ok()
5902        } else {
5903            None
5904        };
5905        // A remote tip this branch does not contain is somebody else's work:
5906        // rebasing would leave a local tip that can never be pushed. Stop
5907        // before touching anything and say so.
5908        if let Some(theirs) = &remote_tip
5909            && !git::is_ancestor(&repo, theirs, &head).await
5910            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5911                .await
5912                .is_ok_and(|missing| missing.is_empty())
5913        {
5914            let why = format!(
5915                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5916                 them",
5917                short(theirs),
5918                winner.branch
5919            );
5920            self.state.status = RunStatus::Blocked;
5921            self.state.base_sync = Some(BaseSync {
5922                tip,
5923                behind,
5924                attempts,
5925                conflict: Some(why.clone()),
5926                already_in: None,
5927            });
5928            self.state.event("land", why);
5929            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5930                .await?;
5931            self.state.save()?;
5932            return Ok(());
5933        }
5934
5935        if self.state.base_sync.is_none() {
5936            self.state.base_sync = Some(BaseSync {
5937                tip: tip.clone(),
5938                behind,
5939                attempts: 0,
5940                conflict: None,
5941                already_in: None,
5942            });
5943        }
5944        self.state.reserve_base_operation(&head, &tip);
5945        self.state.save()?;
5946        let scratch = self.state.dir().join("base-sync");
5947        let rebased =
5948            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5949                .await
5950            {
5951                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5952                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5953                Err(e) => Err(e),
5954            };
5955        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5956        let stop_kind = match &rebased {
5957            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5958            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5959            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5960        };
5961        match rebased {
5962            Ok(None) => {
5963                // The branch ref moved, but a worktree that already had it
5964                // checked out (the winner's) was not told; sync its index and
5965                // files before anything reads them.
5966                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5967                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5968                self.state.finish_base_operation(Some(&fresh));
5969                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5970                let mut conflict = None;
5971                if let Some(pinned) = &remote_tip {
5972                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5973                    match pushed {
5974                        Ok(o) if o.ok() => self.state.event(
5975                            "land",
5976                            format!("pushed rebased {} to {remote}", winner.branch),
5977                        ),
5978                        Ok(o) => {
5979                            conflict = Some(format!(
5980                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5981                                winner.branch,
5982                                short(pinned),
5983                                o.stderr.chars().take(600).collect::<String>()
5984                            ));
5985                        }
5986                        Err(e) => {
5987                            conflict = Some(format!(
5988                                "rebased {} locally but could not push it: {e:#}",
5989                                winner.branch
5990                            ));
5991                        }
5992                    }
5993                }
5994                if let Some(why) = &conflict {
5995                    self.state.status = RunStatus::Blocked;
5996                    self.state.event("land", why.clone());
5997                }
5998                self.state.base_sync = Some(BaseSync {
5999                    tip: tip.clone(),
6000                    behind: 0,
6001                    attempts,
6002                    conflict,
6003                    already_in: None,
6004                });
6005                self.state
6006                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6007            }
6008            Ok(Some(conflict)) => {
6009                self.state.finish_base_operation(None);
6010                let why = format!(
6011                    "{} conflicts with {tracking} and did not rebase: {}",
6012                    winner.branch,
6013                    conflict.chars().take(600).collect::<String>()
6014                );
6015                self.state.status = RunStatus::Blocked;
6016                self.state.base_sync = Some(BaseSync {
6017                    tip,
6018                    behind,
6019                    attempts,
6020                    conflict: Some(why.clone()),
6021                    already_in: None,
6022                });
6023                self.state.event("land", why);
6024            }
6025            // A refused Git write or a busy repository: nothing is wrong with
6026            // the branch. The scratch rebase stays standing and the run
6027            // resumes into it (`base_sync` must not claim a conflict).
6028            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6029                self.state.save()?;
6030                return Ok(());
6031            }
6032            Err(e) => {
6033                self.state.finish_base_operation(None);
6034                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6035                self.state.status = RunStatus::Blocked;
6036                self.state.base_sync = Some(BaseSync {
6037                    tip,
6038                    behind,
6039                    attempts,
6040                    conflict: Some(why.clone()),
6041                    already_in: None,
6042                });
6043                self.state.event("land", why);
6044            }
6045        }
6046        if !self.state.base_sync_blocked() {
6047            self.state.base_sync_recovery.stop = None;
6048        }
6049        self.hold_base_sync_kind(stop_kind).await?;
6050        self.state.save()?;
6051        Ok(())
6052    }
6053
6054    /// Push a branch an earlier attempt rebased locally but never published,
6055    /// pinned to the remote tip read right after a successful fetch. Returns
6056    /// the reason when the run must stop; `None` when there was nothing to do
6057    /// (no remote copy, the same tip, or a remote copy this branch already
6058    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6059    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6060        // Task competitions publish only after the PR text gate, to their
6061        // public head; never resurrect a candidate branch on the remote.
6062        if self.state.task_pr_branch().is_some() {
6063            return None;
6064        }
6065        let repo = self.state.repo.clone();
6066        let remote = self.state.config.merge.remote.clone();
6067        let fetched = git::fetch(&repo, &remote, branch).await;
6068        if !matches!(&fetched, Ok(o) if o.ok()) {
6069            return None;
6070        }
6071        let branch_tracking = format!("{remote}/{branch}");
6072        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6073        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6074            return None;
6075        }
6076        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6077            .await
6078            .is_ok_and(|missing| missing.is_empty())
6079        {
6080            return Some(format!(
6081                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6082                 them",
6083                short(&theirs)
6084            ));
6085        }
6086        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6087            Ok(o) if o.ok() => {
6088                self.state
6089                    .event("land", format!("pushed rebased {branch} to {remote}"));
6090                None
6091            }
6092            Ok(o) => Some(format!(
6093                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6094                 someone may have pushed): {}",
6095                short(&theirs),
6096                o.stderr.chars().take(600).collect::<String>()
6097            )),
6098            Err(e) => Some(format!(
6099                "{branch} is rebased locally but could not be pushed: {e:#}"
6100            )),
6101        }
6102    }
6103
6104    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6105    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6106    /// whether it did.
6107    ///
6108    /// Checked only when the base is ahead of the branch. A failing check is
6109    /// "not proven" - the ordinary rebase path then decides - never a reason to
6110    /// stop the run.
6111    ///
6112    /// The remote copy of the branch is held to the same standard as the local
6113    /// one: if it carries a tip this worktree does not, that tip must itself be
6114    /// proven in the base, or nothing is settled (a pull request would
6115    /// otherwise be closed over commits nobody checked). The pull request is
6116    /// closed *before* the terminal status is saved; if that fails for a
6117    /// reason other than a refusal (no network, a `gh` error) the run is left
6118    /// `Blocked` with the reason as its conflict, which a resume retries -
6119    /// the same recovery a phantom conflict gets.
6120    async fn settle_already_in(
6121        &mut self,
6122        branch: &str,
6123        tip: &str,
6124        head: &str,
6125        attempts: usize,
6126        behind: usize,
6127    ) -> Result<bool> {
6128        let repo = self.state.repo.clone();
6129        let remote = self.state.config.merge.remote.clone();
6130        let start = self.state.base_commit.clone();
6131        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6132            Ok(Some(e)) => e,
6133            Ok(None) => return Ok(false),
6134            Err(e) => {
6135                tracing::warn!("already-in-base check for {branch}: {e:#}");
6136                return Ok(false);
6137            }
6138        };
6139        let mut verified = vec![head.to_owned()];
6140        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6141        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6142        if matches!(&fetched, Ok(o) if o.ok())
6143            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6144            && theirs != head
6145        {
6146            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6147                Ok(Some(_)) => verified.push(theirs),
6148                _ => return Ok(false),
6149            }
6150        }
6151        let base_branch = self.state.base_branch.clone();
6152        let message = format!(
6153            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6154             land",
6155            evidence.names(),
6156            evidence.proof.as_str()
6157        );
6158        let closed =
6159            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6160                .await;
6161        match closed {
6162            Ok(Ok(url)) => self
6163                .state
6164                .event("land", format!("closed {url}: superseded on {base_branch}")),
6165            Ok(Err(why)) => self
6166                .state
6167                .event("land", format!("did not close a pull request: {why}")),
6168            Err(e) => {
6169                let why = format!(
6170                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6171                     not be closed ({e:#}); resume to retry"
6172                );
6173                self.state.status = RunStatus::Blocked;
6174                self.state.base_sync = Some(BaseSync {
6175                    tip: tip.to_owned(),
6176                    behind,
6177                    attempts,
6178                    conflict: Some(why.clone()),
6179                    already_in: None,
6180                });
6181                self.state.event("land", why);
6182                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6183                    .await?;
6184                self.state.save()?;
6185                return Ok(true);
6186            }
6187        }
6188        self.state.status = RunStatus::AlreadyInBase;
6189        self.state.base_sync_recovery.stop = None;
6190        self.state.base_sync = Some(BaseSync {
6191            tip: tip.to_owned(),
6192            behind,
6193            attempts,
6194            conflict: None,
6195            already_in: Some(evidence),
6196        });
6197        self.state.event("land", message);
6198        self.state.save()?;
6199        self.settle_questions();
6200        Ok(true)
6201    }
6202
6203    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6204    /// last landed the winner on, once it has run, else the commit the run
6205    /// branched from.
6206    ///
6207    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6208    /// and `vote` all happen before there is a winner to rebase, so they
6209    /// compare every candidate against the branch point on purpose, and a
6210    /// base that moves after they are already done cannot change an answer
6211    /// they already gave.
6212    fn landing_base(&self) -> String {
6213        self.state
6214            .base_sync
6215            .as_ref()
6216            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6217    }
6218
6219    // ------------------------------------------------------- operator fix
6220
6221    /// Route specific, already-recorded review findings to a fixer for a
6222    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6223    /// entry point.
6224    ///
6225    /// Distinct from `review_loop`'s own fix step in three ways: it never
6226    /// runs a reviewer wave, it never spends review-round budget, and what
6227    /// happened is recorded as an [`OperatorFixRequest`] appended to
6228    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6229    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6230    /// and vote must never be rewritten to look like a manufactured blocking
6231    /// verdict.
6232    ///
6233    /// Only meaningful once review has actually concluded: `Ready` (handed
6234    /// off with findings still open, or simply concluded clean while minor
6235    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6236    /// gate failed). Everything else is refused: a run still in progress
6237    /// should simply be resumed, and a `Merged` run's branch has already
6238    /// landed — reopening *this* run's own record cannot change that, so the
6239    /// answer there is a fresh `magi review <branch>`.
6240    ///
6241    /// A real commit here re-verifies through a fresh, ordinary review-only
6242    /// run on the same branch ([`Self::review`]) rather than reopening this
6243    /// run's own `review_loop`: once any round in this run's history went
6244    /// clean, `review_conclusion` treats that as permanent by design (the
6245    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6246    /// way to force one more genuine reviewer wave out of *this* run without
6247    /// either rewriting history or weakening that guarantee for every other
6248    /// caller. A review-only run costs nothing extra — no implementation, no
6249    /// judging, no vote — and exercises the exact same review → verify →
6250    /// gate → (human) merge path, unmodified.
6251    pub async fn fix_selected(
6252        &mut self,
6253        ids: &[String],
6254        reason: &str,
6255        allow_stale: bool,
6256    ) -> Result<()> {
6257        let reason = reason.trim();
6258        if reason.is_empty() {
6259            bail!("a fix request needs a reason — that is the operator's own record of why");
6260        }
6261        if ids.is_empty() {
6262            bail!("no finding id given");
6263        }
6264        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6265            bail!(
6266                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6267                 has already concluded — can be given a targeted fix. A run still \
6268                 in progress should simply be resumed; a `merged` run's branch has \
6269                 already landed, so its answer is a fresh `magi review <branch>`, \
6270                 not reopening this run's own record",
6271                self.state.id,
6272                self.state.status.as_str()
6273            );
6274        }
6275        // Reassignment shares this claim through its atomic task save. Check
6276        // the canonical run identity inside it, before preflight or any await.
6277        let _claim = FixClaim::acquire(&self.state.dir())?;
6278        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6279        let Some(winner) = self.state.winner().cloned() else {
6280            bail!("run {} has no winning candidate to fix", self.state.id);
6281        };
6282        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6283            bail!(
6284                "branch `{}` no longer exists; this run cannot be extended",
6285                winner.branch
6286            );
6287        }
6288        let home = crate::run::home();
6289        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6290            bail!(
6291                "run {} is currently being worked on by another magi process",
6292                self.state.id
6293            );
6294        }
6295        // Resolve every id before spending anything — an unknown id refuses
6296        // the whole request rather than silently dropping it — and dedup
6297        // while keeping the operator's own order.
6298        let mut seen = BTreeSet::new();
6299        let mut findings = Vec::new();
6300        let mut missing = Vec::new();
6301        for id in ids {
6302            if !seen.insert(id.clone()) {
6303                continue;
6304            }
6305            match self.state.finding(id) {
6306                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6307                    id: f.id.clone(),
6308                    severity: f.severity,
6309                    reviewer_vote: rec.vote,
6310                    round: round.round,
6311                    round_head: round.head.clone(),
6312                    reviewer: rec.reviewer,
6313                    agent: rec.agent.clone(),
6314                    file: f.file.clone(),
6315                    line: f.line,
6316                    title: f.title.clone(),
6317                    detail: f.detail.clone(),
6318                    outcome: OperatorFixOutcome::Pending,
6319                }),
6320                None => missing.push(id.clone()),
6321            }
6322        }
6323        if !missing.is_empty() {
6324            bail!(
6325                "unknown finding id(s): {}; nothing was changed",
6326                missing.join(", ")
6327            );
6328        }
6329
6330        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6331        let stale_details: Vec<(String, String)> = findings
6332            .iter()
6333            .filter(|f| f.round_head != head_at_request)
6334            .map(|f| (f.id.clone(), f.round_head.clone()))
6335            .collect();
6336        let stale = !stale_details.is_empty();
6337        if stale && !allow_stale {
6338            bail!(
6339                "the branch has moved since some finding(s) were raised — {} — now \
6340                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6341                stale_details
6342                    .iter()
6343                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6344                    .collect::<Vec<_>>()
6345                    .join(", "),
6346                short(&head_at_request)
6347            );
6348        }
6349
6350        let request = OperatorFixRequest {
6351            requested_at: Timestamp::now(),
6352            reason: reason.to_owned(),
6353            findings,
6354            head_at_request: head_at_request.clone(),
6355            allow_stale,
6356            stale,
6357            fix: None,
6358            result_head: None,
6359            follow_up_review_run: None,
6360        };
6361        self.state.event(
6362            "fix",
6363            format!(
6364                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6365                request.findings.len(),
6366                request
6367                    .findings
6368                    .iter()
6369                    .map(|f| f.id.as_str())
6370                    .collect::<Vec<_>>()
6371                    .join(", "),
6372            ),
6373        );
6374        // Recorded now, before any worktree work or the fixer call itself —
6375        // and re-saved at each checkpoint below: a crash at any point after
6376        // this (mid fixer call, mid follow-up review) must not lose the fact
6377        // that this was requested, for which findings, and why. Everything
6378        // past this point reads and writes through `request_index` rather
6379        // than a local variable, since `request` itself is moved here.
6380        self.state.operator_fixes.push(request);
6381        self.state.save()?;
6382        let request_index = self.state.operator_fixes.len() - 1;
6383
6384        // A fresh, dedicated worktree for this one call, never the winner's
6385        // own worktree in place: that one may already be gone (folded away),
6386        // and reusing it in place would leave the branch checked out there
6387        // when the follow-up review below tries to check it out again. Freed
6388        // immediately after, either way — but only once confirmed clean:
6389        // `worktree_remove` is a `git worktree remove --force`, which would
6390        // otherwise discard uncommitted work left there by the operator or
6391        // another process before this had a chance to even look at it.
6392        let repo = self.state.repo.clone();
6393        let coordination = self.coordinate(&repo).await?;
6394        if winner.worktree.exists() {
6395            // Lockfiles a rescue commit withheld stay untracked on purpose and
6396            // are already recorded; they are not the operator's work to protect.
6397            let dirty = git::git(
6398                &winner.worktree,
6399                &["status", "--porcelain", "--untracked-files=all"],
6400            )
6401            .await?;
6402            let only_withheld = dirty.lines().all(|l| {
6403                l.strip_prefix("?? ")
6404                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6405            });
6406            if !only_withheld {
6407                bail!(
6408                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6409                     discard them first",
6410                    winner.worktree.display()
6411                );
6412            }
6413            git::worktree_remove(&self.state.repo, &winner.worktree)
6414                .await
6415                .ok();
6416        }
6417        let fix_worktree = self.state.worktree_root().join("operator-fix");
6418        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6419        git::git(
6420            &self.state.repo,
6421            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6422        )
6423        .await
6424        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6425        if !git::is_clean(&fix_worktree).await? {
6426            git::worktree_remove(&self.state.repo, &fix_worktree)
6427                .await
6428                .ok();
6429            bail!(
6430                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6431                winner.branch
6432            );
6433        }
6434        let repo_for_setup = self.state.repo.clone();
6435        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6436            Ok(()) => {
6437                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6438            }
6439            Err(e) => Err(e),
6440        };
6441        if let Err(e) = setup {
6442            git::worktree_remove(&self.state.repo, &fix_worktree)
6443                .await
6444                .ok();
6445            return Err(e);
6446        }
6447
6448        // This explicit operator request retries availability independently
6449        // of execute(); the previous call has already saved and cleaned up.
6450        self.state.resource_block = None;
6451
6452        let run_id = self.state.id.clone();
6453        let prompts = self.state.config.prompts.clone();
6454        let language = self.state.config.graph.language.clone();
6455        let sessions = self.state.config.graph.sessions;
6456        let artifacts = agent::artifacts_dir(&self.state.dir());
6457        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6458            .findings
6459            .iter()
6460            .map(|f| Finding {
6461                id: f.id.clone(),
6462                severity: f.severity,
6463                file: f.file.clone(),
6464                line: f.line,
6465                title: f.title.clone(),
6466                detail: f.detail.clone(),
6467            })
6468            .collect();
6469        let fix_prompt = prompt::operator_fix_in(
6470            &self.state.instruction,
6471            &finding_list,
6472            reason,
6473            &stale_details,
6474            &head_at_request,
6475            &language,
6476            &self.state.config.graph.github_language,
6477        );
6478        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6479        drop(coordination);
6480        let (job, seat, out) = self
6481            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6482                prompt: fix_prompt.clone(),
6483                spec,
6484                seat,
6485                cwd: fix_worktree.clone(),
6486                timeout,
6487                allow_write: true,
6488                sessions,
6489                artifacts: artifacts.clone(),
6490                stem: "operator-fix".to_owned(),
6491                handover: None,
6492            })
6493            .await;
6494        let agent_id = seat.agent.clone();
6495
6496        let mut fix = FixRecord {
6497            agent: agent_id,
6498            addressed: Vec::new(),
6499            rejected: Vec::new(),
6500            notes: String::new(),
6501            committed: false,
6502            failed: None,
6503            duration_ms: 0,
6504            continuation: None,
6505        };
6506        let mut final_seat = seat.clone();
6507        match out {
6508            AgentOutcome::Ok(o) => {
6509                fix.duration_ms = o.duration_ms;
6510                let parsed = verdict::extract_json::<FixReport>(&o.text);
6511                let incomplete_reason = match &parsed {
6512                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6513                        "the reply parsed, but it reported a command whose own CLI \
6514                         never confirmed an exit status"
6515                            .to_owned(),
6516                    ),
6517                    Ok(_) => None,
6518                    Err(e) => Some(e.to_string()),
6519                };
6520                match incomplete_reason {
6521                    None => {
6522                        let report = parsed.expect("checked Ok above");
6523                        fix.addressed = report.addressed;
6524                        fix.rejected = report.rejected;
6525                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6526                    }
6527                    Some(reason) => {
6528                        let (resumed_seat, resolved, failure, cont) = self
6529                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6530                            .await;
6531                        fix.duration_ms += cont.cumulative_wait_ms;
6532                        fix.continuation = Some(cont);
6533                        final_seat = resumed_seat;
6534                        match resolved {
6535                            Some(report) => {
6536                                fix.addressed = report.addressed;
6537                                fix.rejected = report.rejected;
6538                                fix.notes =
6539                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6540                            }
6541                            None => fix.failed = failure,
6542                        }
6543                    }
6544                }
6545            }
6546            AgentOutcome::Dropped(o) => {
6547                fix.duration_ms = o.duration_ms;
6548                let why = o
6549                    .dropped
6550                    .as_ref()
6551                    .map(|d| d.why.as_str())
6552                    .unwrap_or("the CLI ended the stream without delivering its answer");
6553                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6554            }
6555            AgentOutcome::Quota(o) => {
6556                self.state.quota.push(QuotaLoss::from_report(
6557                    final_seat.key.clone(),
6558                    "fix",
6559                    o.quota.as_ref(),
6560                ));
6561                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6562            }
6563            AgentOutcome::Failed(e) => fix.failed = Some(e),
6564            AgentOutcome::ResourceBlocked(e) => {
6565                fix.failed = Some(format!("the fixer was not started: {e}"));
6566            }
6567        }
6568        // Even a deferred call must rescue edits, release the worktree and
6569        // finalize its record below so an explicit `magi fix` can retry.
6570        if fix.continuation.is_none() {
6571            fix.continuation = Some(ContinuationRecord::not_needed());
6572        }
6573        self.state.seats.insert(final_seat.key.clone(), final_seat);
6574
6575        let rescue_message = format!(
6576            "magi: operator-selected fix ({}) (uncommitted work)",
6577            self.state.operator_fixes[request_index]
6578                .findings
6579                .iter()
6580                .map(|f| f.id.as_str())
6581                .collect::<Vec<_>>()
6582                .join(", ")
6583        );
6584        let repo = self.state.repo.clone();
6585        let coordination = self.coordinate(&repo).await?;
6586        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6587            self.state.note_withheld("fix", &r.withheld);
6588        }
6589        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6590        fix.committed = after != head_at_request;
6591        git::worktree_remove(&self.state.repo, &fix_worktree)
6592            .await
6593            .ok();
6594        drop(coordination);
6595
6596        self.state.event(
6597            "fix",
6598            match &fix.failed {
6599                Some(reason) => format!(
6600                    "operator fix: adoption report was lost ({reason}); {}",
6601                    if fix.committed {
6602                        "committed"
6603                    } else {
6604                        "NO new commit"
6605                    }
6606                ),
6607                None => format!(
6608                    "operator fix: {} addressed, {} rejected, {}",
6609                    fix.addressed.len(),
6610                    fix.rejected.len(),
6611                    if fix.committed {
6612                        "committed"
6613                    } else {
6614                        "NO new commit"
6615                    }
6616                ),
6617            },
6618        );
6619
6620        // Every selected finding gets an outcome — never left `Pending` once
6621        // the fixer's own turn is over. A report that never came back at all
6622        // marks every one of them `Unreported`, not silently "not addressed":
6623        // quota, a dropped stream, or an exhausted continuation are gaps in
6624        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6625        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6626        for f in &mut self.state.operator_fixes[request_index].findings {
6627            f.outcome = if fix.failed.is_some() {
6628                OperatorFixOutcome::Unreported
6629            } else if fix.addressed.contains(&f.id) {
6630                OperatorFixOutcome::Addressed
6631            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6632                OperatorFixOutcome::Rejected { why: r.why.clone() }
6633            } else {
6634                OperatorFixOutcome::Unreported
6635            };
6636        }
6637
6638        let committed = fix.committed;
6639        if committed {
6640            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6641        }
6642        self.state.operator_fixes[request_index].fix = Some(fix);
6643        // Saved again now that the fixer's own outcome is final, on top of
6644        // the save right after the request was first pushed above.
6645        self.state.save()?;
6646        if self.state.resource_block.is_some() {
6647            return self.stop_for_resource();
6648        }
6649
6650        if committed {
6651            self.state.event(
6652                "fix",
6653                format!(
6654                    "operator fix committed {}; opening a follow-up review-only run",
6655                    short(&after)
6656                ),
6657            );
6658            // The operator asked for the fix, and the follow-up serves whatever
6659            // task the run it follows served.
6660            let origin =
6661                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6662            match Self::review(
6663                &self.state.repo,
6664                &winner.branch,
6665                self.state.config.clone(),
6666                origin,
6667            )
6668            .await
6669            {
6670                Ok(mut follow_up) => {
6671                    follow_up.state.event(
6672                        "start",
6673                        format!(
6674                            "requested by an operator fix on run {} for finding(s) {}",
6675                            self.state.id,
6676                            self.state.operator_fixes[request_index]
6677                                .findings
6678                                .iter()
6679                                .map(|f| f.id.as_str())
6680                                .collect::<Vec<_>>()
6681                                .join(", "),
6682                        ),
6683                    );
6684                    follow_up.state.save()?;
6685                    let follow_up_id = follow_up.state.id.clone();
6686                    // The follow-up is a run like any other: it belongs to the
6687                    // task of the run it follows, or to one filed for it.
6688                    let adopted = match crate::direct::adopt(
6689                        &follow_up.state,
6690                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6691                    ) {
6692                        Ok(a) => a,
6693                        Err(e) => {
6694                            // An ownerless run must not spend agent calls; it
6695                            // stays saved, and `magi run --resume` adopts it.
6696                            self.state.event(
6697                                "fix",
6698                                format!(
6699                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6700                                ),
6701                            );
6702                            self.state.save()?;
6703                            return Ok(());
6704                        }
6705                    };
6706                    let executed = follow_up.execute().await;
6707                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6708                    if let Some(a) = adopted {
6709                        a.finish(&follow_up.state, executed);
6710                    }
6711                    if let Some(e) = failure {
6712                        self.state.event(
6713                            "fix",
6714                            format!(
6715                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6716                            ),
6717                        );
6718                    }
6719                    self.state.operator_fixes[request_index].follow_up_review_run =
6720                        Some(follow_up_id);
6721                }
6722                Err(e) => {
6723                    self.state.event(
6724                        "fix",
6725                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6726                    );
6727                }
6728            }
6729            self.state.save()?;
6730        }
6731
6732        Ok(())
6733    }
6734
6735    // --------------------------------------------------------------- review
6736
6737    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6738    /// names, in order, each at most once, moving on only when the call
6739    /// advances (an error, a quota hit or nothing usable - the same decision
6740    /// point `agent::chain_advances` is for the other chained roles).
6741    ///
6742    /// Each agent gets its own seat from `Runner::seat`: the same agent
6743    /// continues its conversation (the winner's own implementer seat when it
6744    /// is the winner's author, now that the competition is over), another
6745    /// takes a fresh one so the full prompt is sent again. A handover is
6746    /// recorded under `node`, which is what makes the fallback stick for the
6747    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6748    /// is returned, so an exhausted chain reads exactly like a single failed
6749    /// fixer: one quota loss, the same wording, the same refund. An earlier
6750    /// attempt's edits are left in the tree and judged, with the final
6751    /// attempt's, by what git says afterwards.
6752    ///
6753    /// Returns the job that produced the outcome, for `continue_fix_report`.
6754    async fn ask_fixer(
6755        &mut self,
6756        winner: &Candidate,
6757        node: &'static str,
6758        round: Option<usize>,
6759        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6760    ) -> (SeatJob, SeatState, AgentOutcome) {
6761        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6762        self.ask_fixer_among(attempts, node, round, None, build)
6763            .await
6764    }
6765
6766    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6767    /// worktree and a commit: before the next agent is asked after one that
6768    /// did not satisfy the call, the worktree is put back on that commit, so
6769    /// a failed agent's partial edits are not carried into the next one.
6770    async fn ask_fixer_among(
6771        &mut self,
6772        attempts: Vec<(AgentSpec, String)>,
6773        node: &'static str,
6774        round: Option<usize>,
6775        restore: Option<(&Path, &str)>,
6776        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6777    ) -> (SeatJob, SeatState, AgentOutcome) {
6778        let run_id = self.state.id.clone();
6779        let prompts = self.state.config.prompts.clone();
6780        let cache = self.state.config.cache_dir();
6781        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6782        let mut last = None;
6783        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6784            let seat = self.seat(&key, &spec.id);
6785            let mut job = build(spec.clone(), seat);
6786            if i > 0 {
6787                job.stem = format!("{}-{}", job.stem, spec.id);
6788            }
6789            let ctx = WaveCtx {
6790                carry_seats: false,
6791                pauses: Some((&self.pause, &self.interrupt)),
6792                run: &run_id,
6793                node,
6794                prompts: &prompts,
6795                cache: cache.as_deref(),
6796                round,
6797            };
6798            let (seat, out) =
6799                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6800            // A resource block never advances the chain: the chosen agent
6801            // was not asked, so it must be the one asked again on resume.
6802            let advances = match &out {
6803                AgentOutcome::Ok(o) => agent::output_advances(o),
6804                AgentOutcome::ResourceBlocked(_) => false,
6805                _ => true,
6806            };
6807            if let Some(next) = ids.get(i + 1)
6808                && advances
6809            {
6810                self.state.seats.insert(seat.key.clone(), seat.clone());
6811                let class =
6812                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6813                let reason = match &out {
6814                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6815                    other => fail_reason(other),
6816                };
6817                // A failed agent's edits must not reach the next one, and a
6818                // restore that did not work ends the chain: the call fails
6819                // and the caller's own (confirmed) restore decides.
6820                if let Some((wt, rev)) = restore
6821                    && let Err(e) = self.graft_restore(wt, rev).await
6822                {
6823                    let why =
6824                        format!("could not restore the worktree before the next fixer: {e:#}");
6825                    tracing::warn!("{why}");
6826                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6827                        AgentOutcome::ResourceBlocked(why)
6828                    } else {
6829                        AgentOutcome::Failed(why)
6830                    };
6831                    last = Some((job, seat, out));
6832                    break;
6833                }
6834                record_handover(
6835                    &mut self.state,
6836                    node,
6837                    &seat.key,
6838                    &spec.id,
6839                    next,
6840                    &class,
6841                    &reason,
6842                );
6843                continue;
6844            }
6845            last = Some((job, seat, out));
6846            break;
6847        }
6848        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6849        if let AgentOutcome::Quota(output) = &out
6850            && node != "graft"
6851        {
6852            // Graft is best effort: its caller records the loss and restores
6853            // the pre-graft tree, then proceeds to review and gate.
6854            // Every fallback was tried. Machine outages cannot be repaired by
6855            // spending a fix round; use each caller's resumable resource path.
6856            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6857            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6858            self.state.quota.push(loss);
6859            self.state.seats.insert(seat.key.clone(), seat.clone());
6860            self.state.status = RunStatus::Blocked;
6861            return (
6862                job,
6863                seat,
6864                AgentOutcome::ResourceBlocked(
6865                    "fixer unavailable; restore agent availability and resume".into(),
6866                ),
6867            );
6868        }
6869        (job, seat, out)
6870    }
6871
6872    /// The one repair a verification-only failure earns (see
6873    /// [`crate::recovery`]), before the first review round of a run that took
6874    /// a branch over.
6875    ///
6876    /// Saved as started *before* the fixer is called, so a resume never
6877    /// repairs twice; a start found on re-entry is rolled back to the commit
6878    /// it began from. A failed or unusable fixer leaves the tree as it was and
6879    /// the review goes on over it, so the first round's own verification
6880    /// reports the failure again. The failure is never carried into a round as
6881    /// a result: verification is re-run on whatever commit this leaves.
6882    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6883        use crate::recovery::{Mode, Repair};
6884        if !self.state.reviews.is_empty() {
6885            return Ok(());
6886        }
6887        let Some(mut rec) = self.state.recovery.clone() else {
6888            return Ok(());
6889        };
6890        if rec.mode != Mode::VerifyRepair {
6891            return Ok(());
6892        }
6893        let wt = winner.worktree.clone();
6894        let mut head = git::rev_parse(&wt, "HEAD").await?;
6895        match rec.repair.clone() {
6896            Repair::Done { .. } => return Ok(()),
6897            Repair::WaitingForRestore { before } => {
6898                self.graft_restore(&wt, &before).await?;
6899                head = before;
6900                rec.repair = Repair::NotStarted;
6901                self.state.recovery = Some(rec.clone());
6902                self.state.save()?;
6903            }
6904            Repair::Started { before } => {
6905                // Interrupted: the unfinished edits are not trusted.
6906                // An unconfirmed rollback blocks the run with `Started` kept,
6907                // so a resume retries it; the tree is never reviewed as is.
6908                if let Err(e) = self.graft_restore(&wt, &before).await {
6909                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6910                        return Err(e);
6911                    }
6912                    return self
6913                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6914                        .await;
6915                }
6916                rec.repair = Repair::Done {
6917                    before: before.clone(),
6918                    after: before,
6919                    committed: false,
6920                    failed: Some("interrupted; rolled back".to_owned()),
6921                };
6922                self.state
6923                    .event("recovery", "the repair was interrupted and rolled back");
6924                self.state.recovery = Some(rec);
6925                self.state.save()?;
6926                return Ok(());
6927            }
6928            Repair::NotStarted => {}
6929        }
6930        if crate::recovery::recheck(&mut rec, &head) {
6931            self.state.event("recovery", rec.summary());
6932            self.state.recovery = Some(rec);
6933            self.state.save()?;
6934            return Ok(());
6935        }
6936        rec.repair = Repair::Started {
6937            before: head.clone(),
6938        };
6939        self.state.recovery = Some(rec.clone());
6940        self.state.save()?;
6941
6942        let language = self.state.config.graph.language.clone();
6943        let sessions = self.state.config.graph.sessions;
6944        let artifacts = agent::artifacts_dir(&self.state.dir());
6945        let fix_prompt = prompt::recovery_fix(
6946            &self.state.instruction,
6947            &rec,
6948            self.state.config.graph.review_rounds,
6949            &language,
6950        );
6951        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6952        self.state.event(
6953            "recovery",
6954            format!(
6955                "repair: the fixer starts from the failed verification on {}",
6956                short(&head)
6957            ),
6958        );
6959        let (_job, seat, out) = self
6960            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6961                prompt: fix_prompt.clone(),
6962                spec,
6963                seat,
6964                cwd: wt.clone(),
6965                timeout,
6966                allow_write: true,
6967                sessions,
6968                artifacts: artifacts.clone(),
6969                stem: "fix-recovery".to_owned(),
6970                handover: None,
6971            })
6972            .await;
6973        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6974            rec.repair = Repair::WaitingForRestore { before: head };
6975            self.state.recovery = Some(rec);
6976            return self.stop_for_resource();
6977        }
6978        let failed = match &out {
6979            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6980            AgentOutcome::Quota(o) => {
6981                self.state.quota.push(QuotaLoss::from_report(
6982                    seat.key.clone(),
6983                    "fix",
6984                    o.quota.as_ref(),
6985                ));
6986                Some("rate limited (quota); the fixer could not run".to_owned())
6987            }
6988            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6989            AgentOutcome::Failed(e) => Some(e.clone()),
6990        };
6991        self.state.seats.insert(seat.key.clone(), seat);
6992        let coordination = self.coordinate(&wt).await?;
6993        let failed = match failed {
6994            // The agent's partial edits are not kept.
6995            Some(why) => {
6996                if let Err(e) = git::restore_to(&wt, &head).await {
6997                    return self
6998                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6999                        .await;
7000                }
7001                Some(why)
7002            }
7003            None => {
7004                if let Ok(r) = git::rescue_commit(
7005                    &wt,
7006                    "magi: recovery repair of the failed verification (uncommitted work)",
7007                )
7008                .await
7009                {
7010                    self.state.note_withheld("fix", &r.withheld);
7011                }
7012                None
7013            }
7014        };
7015        let after = git::rev_parse(&wt, "HEAD").await?;
7016        drop(coordination);
7017        self.state.event(
7018            "recovery",
7019            match &failed {
7020                Some(why) => format!("repair: nothing kept ({why})"),
7021                None if after != head => {
7022                    format!("repair: committed {}", short(&after))
7023                }
7024                None => "repair: the fixer made no commit".to_owned(),
7025            },
7026        );
7027        rec.repair = Repair::Done {
7028            before: head.clone(),
7029            committed: after != head,
7030            after,
7031            failed,
7032        };
7033        self.state.recovery = Some(rec);
7034        self.state.save()?;
7035        Ok(())
7036    }
7037
7038    /// A repair whose rollback could not be confirmed: the run is blocked and
7039    /// resumable, and the unconfirmed state is never reviewed.
7040    async fn stop_recovery(
7041        &mut self,
7042        mut rec: crate::recovery::RecoveryRecord,
7043        head: &str,
7044        why: String,
7045    ) -> Result<()> {
7046        // `Started` stays recorded: a resume retries the restore.
7047        rec.repair = crate::recovery::Repair::Started {
7048            before: head.to_owned(),
7049        };
7050        self.state.recovery = Some(rec);
7051        self.state
7052            .event("recovery", format!("repair could not be undone: {why}"));
7053        self.state.status = RunStatus::Blocked;
7054        self.state.save()?;
7055        bail!("recovery repair could not be rolled back: {why}")
7056    }
7057
7058    async fn review_loop(&mut self) -> Result<()> {
7059        if self.hold_base_sync().await? {
7060            return Ok(());
7061        }
7062        loop {
7063            if self
7064                .state
7065                .base_sync
7066                .as_ref()
7067                .is_some_and(|s| s.conflict.is_some())
7068            {
7069                return Ok(());
7070            }
7071            if self.catchup_repairable() {
7072                let record = self.state.reviews.last().expect("repairable round").clone();
7073                let winner = self
7074                    .state
7075                    .winner()
7076                    .cloned()
7077                    .context("catch-up repair has no winner")?;
7078                if !self.fix_review_round(&winner, record).await? {
7079                    return Ok(());
7080                }
7081            }
7082            self.review_loop_pass().await?;
7083            if !self.catchup_repairable() {
7084                return Ok(());
7085            }
7086        }
7087    }
7088
7089    /// Derive an owed catch-up repair from durable round evidence, including
7090    /// reentry after a crash before the fixer was dispatched.
7091    fn catchup_repairable(&self) -> bool {
7092        self.state.reviews.len() < self.state.config.graph.review_rounds
7093            && self.state.reviews.last().is_some_and(|r| {
7094                !r.clean
7095                    && r.blocking == 0
7096                    && r.fix.is_none()
7097                    && r.e2e_status() == E2eStatus::Failed
7098            })
7099    }
7100
7101    async fn review_loop_pass(&mut self) -> Result<()> {
7102        // A base that would not rebase is a person's decision, not a review
7103        // round: nothing here would change the answer, and reviewers and a
7104        // fixer would be spending real budget on a tree that cannot land
7105        // regardless of what they find.
7106        if self
7107            .state
7108            .base_sync
7109            .as_ref()
7110            .is_some_and(|s| s.conflict.is_some())
7111        {
7112            return Ok(());
7113        }
7114        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7115        // agent files with `magi task add` name the run that paid for it. The
7116        // prompt overlay is cloned alongside it because the waves borrow it
7117        // while `self` is mutably borrowed by the node's own bookkeeping.
7118        let run_id = self.state.id.clone();
7119        let prompts = self.state.config.prompts.clone();
7120        let Some(winner) = self.state.winner().cloned() else {
7121            return Ok(());
7122        };
7123        let granted_review = self
7124            .state
7125            .repair_grants
7126            .last()
7127            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7128            .cloned();
7129        let fresh_review = granted_review
7130            .as_ref()
7131            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7132        let max_rounds = granted_review
7133            .as_ref()
7134            .map_or(self.state.config.graph.review_rounds, |g| {
7135                g.review_start.saturating_add(1)
7136            });
7137        // A clean round, an exhausted round budget, or a stalled tree (see
7138        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7139        // they are recorded — recomputed here, not read off `status`, so a
7140        // reentry into a run that already stopped restates the identical
7141        // verdict instead of silently handing back whatever an earlier node
7142        // in this same walk clobbered `status` to (a solo-candidate
7143        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7144        // below runs an empty range once the budget is spent, and would
7145        // otherwise fall through without touching `status` at all.
7146        if let Some(status) =
7147            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7148        {
7149            // A reentry after a crash between the last round's save and
7150            // `stop_reviewing` reaches the hand-off here, not there.
7151            if status == RunStatus::Gating {
7152                self.record_contested_handoff();
7153            }
7154            self.state.status = status;
7155            self.state.save()?;
7156            return Ok(());
7157        }
7158        self.state.status = RunStatus::Reviewing;
7159        // A last recorded round whose own verification never resolved
7160        // (`ResourceBlocked` — the shared build cache, not the patch) is
7161        // never a concluded round, whatever the round budget says: starting
7162        // a fresh round on top of it would spend a whole new reviewer wave
7163        // re-reading an unchanged patch instead of just retrying the one
7164        // check that actually needs it, and once the budget is spent the
7165        // loop below has nothing left to do at all (its range is empty).
7166        // Retry that check directly instead, exactly the same retry
7167        // `stop_reviewing` already does for its own catch-up case.
7168        if !fresh_review
7169            && self.state.reviews.last().is_some_and(|r| {
7170                r.e2e_status() == E2eStatus::ResourceBlocked
7171                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7172            })
7173        {
7174            let shell = self.state.config.shell();
7175            return self
7176                .stop_unresolved_review(
7177                    "the last round's own verification never resolved",
7178                    &shell,
7179                    &winner.worktree,
7180                )
7181                .await;
7182        }
7183
7184        if granted_review.is_none() {
7185            self.recovery_repair(&winner).await?;
7186        }
7187        if self.resource_halt()? {
7188            return Ok(());
7189        }
7190
7191        let repo = self.state.repo.clone();
7192        let root = self.state.worktree_root();
7193        let language = self.state.config.graph.language.clone();
7194        let sessions = self.state.config.graph.sessions;
7195        let artifacts = self.merge_repair_artifacts();
7196        let base = self.landing_base();
7197        let base_short = short(&base);
7198        let reviewers = self.roles.reviewers.clone();
7199        let shell = self.state.config.shell();
7200
7201        for round in (self.state.reviews.len() + 1)..=max_rounds {
7202            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7203            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7204            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7205            // The prior round's own record, already persisted — never a
7206            // hand-carried variable of just its failing output: that is
7207            // exactly what let a round's e2e result drift out of sync with
7208            // which commit it was actually about (see `SCHEMA`'s doc for
7209            // schema 8). Judged against `head`, the commit reviewers are
7210            // about to look at now, so the summary always reads as "an
7211            // earlier head" here — this round's own patch has not been
7212            // checked yet.
7213            let prev_verification = self
7214                .state
7215                .reviews
7216                .last()
7217                .and_then(|r| r.verification_summary(&head));
7218            // Only the first round of a run that took a branch over: later
7219            // rounds have this run's own history.
7220            let prior_section = match (&self.state.recovery, round) {
7221                (Some(rec), 1) if self.state.reviews.is_empty() => {
7222                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7223                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7224                            Some(crate::recovery::head_of(
7225                                &git::diff(&winner.worktree, reviewed, "HEAD")
7226                                    .await
7227                                    .unwrap_or_default(),
7228                                crate::recovery::SINCE_DIFF_MAX,
7229                            ))
7230                        }
7231                        _ => None,
7232                    };
7233                    Some(prompt::recovery_review(rec, since.as_deref()))
7234                }
7235                _ => None,
7236            };
7237
7238            // Each reviewer gets its own detached checkout of exactly this
7239            // commit: nobody can perturb the winner's tree, and the fixer can
7240            // keep working without racing a reviewer.
7241            let mut jobs = Vec::new();
7242            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7243                self.coordinate(&repo).await?
7244            } else {
7245                None
7246            };
7247            for (r, spec) in reviewers.iter().cloned().enumerate() {
7248                let wt = root.join(format!("review-{}", r + 1));
7249                // `clean -fdx` in a reset wipes setup's products, so every
7250                // round sets the seat up again.
7251                if wt.exists() {
7252                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7253                } else {
7254                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7255                        None
7256                    } else {
7257                        self.coordinate(&repo).await?
7258                    };
7259                    git::worktree_add_detached(&repo, &wt, &head).await?;
7260                }
7261                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7262                    None
7263                } else {
7264                    self.coordinate(&repo).await?
7265                };
7266                self.setup_seat_worktree(&repo, &wt, false).await?;
7267                let seat_key = format!("review-{}", r + 1);
7268                // The seat starts the round on whoever answered it last, not
7269                // on the agent the spec names, so a failure is not re-paid.
7270                let spec = pick_start_spec(
7271                    &self.roles.reviewer_roster,
7272                    spec,
7273                    self.state.seat_history.get(&seat_key),
7274                );
7275                let seat = self.seat(&seat_key, &spec.id);
7276                jobs.push(SeatJob {
7277                    prompt: prompt::review(&prompt::ReviewCtx {
7278                        instruction: &self.state.instruction,
7279                        branch: &winner.branch,
7280                        base_short: &base_short,
7281                        stat: &stat,
7282                        patch: &patch,
7283                        verification: prev_verification.as_ref(),
7284                        prior: prior_section.as_deref(),
7285                        reviewers: reviewers.len(),
7286                        round,
7287                        rounds: max_rounds,
7288                        // A review-only run has no rankings, so nothing
7289                        // competed for this patch and the reviewer is told so.
7290                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7291                        lens: Lens::for_seat(r),
7292                        language: &language,
7293                        github_language: &self.state.config.graph.github_language,
7294                    }),
7295                    spec,
7296                    seat,
7297                    cwd: wt,
7298                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7299                    allow_write: false,
7300                    sessions,
7301                    artifacts: artifacts.clone(),
7302                    stem: format!("review-{round}-{}", r + 1),
7303                    handover: None,
7304                });
7305            }
7306
7307            drop(coordination);
7308            self.state.event(
7309                "review",
7310                format!(
7311                    "round {round}: {} reviewers on {}",
7312                    jobs.len(),
7313                    short(&head)
7314                ),
7315            );
7316            let mut quota_losses = Vec::new();
7317            let review_retries = self.state.config.graph.retries;
7318            let review_cache = self.state.config.cache_dir();
7319            let ctx = WaveCtx {
7320                carry_seats: true,
7321                pauses: Some((&self.pause, &self.interrupt)),
7322                run: &run_id,
7323                node: "review",
7324                prompts: &prompts,
7325                cache: review_cache.as_deref(),
7326                round: Some(round),
7327            };
7328            let results = ask_json_wave::<Review>(
7329                jobs,
7330                Arc::clone(&self.sem),
7331                review_retries,
7332                &self.roles.reviewer_roster,
7333                &ctx,
7334                &mut quota_losses,
7335                &mut self.state,
7336                &|_: &Review| Ok(()),
7337            )
7338            .await;
7339            if self.state.resource_block.is_some() {
7340                return self.stop_for_resource();
7341            }
7342            // Counted before the move below: how many of *this* round's
7343            // reviewer seats were lost to their own rate limit, as opposed to
7344            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7345            let round_quota_missing = quota_losses.len();
7346            self.state.quota.extend(quota_losses);
7347
7348            let mut records = Vec::new();
7349            let mut all_findings = Vec::new();
7350            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7351                let agent_id = seat.agent.clone();
7352                self.state.seats.insert(seat.key.clone(), seat);
7353                let mut record = ReviewRecord {
7354                    reviewer: r + 1,
7355                    agent: agent_id,
7356                    summary: String::new(),
7357                    findings: Vec::new(),
7358                    vote: None,
7359                    failed: None,
7360                    duration_ms: 0,
7361                    // Set for both outcomes: `failed: Some(_)` with
7362                    // `attempts > 0` is a seat every retry still lost, not a
7363                    // recovered one — only `failed: None` with `attempts > 0`
7364                    // reads as "answered after a nudge" (see this field's own
7365                    // doc).
7366                    attempts,
7367                };
7368                match res {
7369                    Ok((review, out)) => {
7370                        // Sanitized here, at the point every other piece of
7371                        // agent prose in this file is (candidate summaries,
7372                        // deliberation turns, vote reasons): a reviewer's own
7373                        // words are the one thing about it that could name
7374                        // it, and reconsideration below broadcasts this same
7375                        // summary and these same findings to every other
7376                        // seat on the panel.
7377                        record.summary =
7378                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7379                        record.vote = Some(review.vote);
7380                        record.duration_ms = out.duration_ms;
7381                        for (n, mut f) in review.findings.into_iter().enumerate() {
7382                            // ids are magi's, never the agent's: the fixer's
7383                            // adoption report is keyed by them.
7384                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7385                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7386                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7387                            // `file` is agent-supplied prose too, never
7388                            // checked against the real tree — the same
7389                            // exposure `title`/`detail` above have, just in
7390                            // a field easy to forget because it looks like a
7391                            // path rather than free text.
7392                            f.file = f
7393                                .file
7394                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7395                            all_findings.push(f.clone());
7396                            record.findings.push(f);
7397                        }
7398                        self.state.event(
7399                            "review",
7400                            format!(
7401                                "round {round}: reviewer {} voted {} with {} finding(s)",
7402                                r + 1,
7403                                review.vote.label(),
7404                                record.findings.len()
7405                            ),
7406                        );
7407                    }
7408                    Err(e) => {
7409                        record.failed = Some(e.to_string());
7410                        self.state.event(
7411                            "review",
7412                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7413                        );
7414                    }
7415                }
7416                records.push(record);
7417            }
7418
7419            // Tally the round's votes and, if they split, spend the one
7420            // round of reconsideration the split -> deliberate -> revote
7421            // shape `judge`/`vote` use for the panel, sized down to what a
7422            // read-only review round can afford: one round, and a revote
7423            // rather than an argument, because the panel already wrote its
7424            // reasoning down as findings the first time around.
7425            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7426            let vote_split =
7427                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7428            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7429            if vote_split {
7430                self.state.event(
7431                    "review",
7432                    format!(
7433                        "round {round}: votes split ({}) — one round of reconsideration",
7434                        initial_votes
7435                            .iter()
7436                            .map(|v| v.label())
7437                            .collect::<Vec<_>>()
7438                            .join(", ")
7439                    ),
7440                );
7441                // Seats read every seat's findings and votes, still numbered
7442                // and never named — the same anonymity `review` itself keeps.
7443                let panel: Vec<ReviewSeatReport<'_>> = records
7444                    .iter()
7445                    .filter_map(|r| {
7446                        r.vote.map(|vote| ReviewSeatReport {
7447                            reviewer: r.reviewer,
7448                            vote,
7449                            summary: &r.summary,
7450                            findings: &r.findings,
7451                        })
7452                    })
7453                    .collect();
7454
7455                let mut jobs = Vec::new();
7456                let mut seats_at = Vec::new();
7457                for (r, spec) in reviewers.iter().cloned().enumerate() {
7458                    // A seat with no initial vote has nothing to reconsider
7459                    // from and stays absent, the same as it stayed absent
7460                    // from `panel` above.
7461                    if records[r].vote.is_none() {
7462                        continue;
7463                    }
7464                    let wt = root.join(format!("review-{}", r + 1));
7465                    let seat_key = format!("review-{}", r + 1);
7466                    let spec = self.occupant(&seat_key, spec);
7467                    let seat = self.seat(&seat_key, &spec.id);
7468                    // A seat with no live session has already forgotten the
7469                    // initial review's prompt — restate the patch it is
7470                    // voting on, the same as `deliberate`/`vote` do for a
7471                    // judge in the same position.
7472                    // The panel already carries this seat's own review and
7473                    // vote, so restating the patch makes the prompt whole for
7474                    // a seat handed to another agent.
7475                    let build = |with_patch: bool| {
7476                        prompt::review_reconsider(&ReviewReconsiderCtx {
7477                            instruction: &self.state.instruction,
7478                            reviewer: r + 1,
7479                            lens: Lens::for_seat(r),
7480                            panel: &panel,
7481                            patch: with_patch.then_some(ReviewPatch {
7482                                branch: &winner.branch,
7483                                base_short: &base_short,
7484                                stat: &stat,
7485                                patch: &patch,
7486                            }),
7487                            round,
7488                            rounds: max_rounds,
7489                            language: &language,
7490                        })
7491                    };
7492                    let full = build(true);
7493                    let prompt = if has_context(&spec, &seat, sessions) {
7494                        build(false)
7495                    } else {
7496                        full.clone()
7497                    };
7498                    jobs.push(SeatJob {
7499                        prompt,
7500                        spec,
7501                        seat,
7502                        cwd: wt,
7503                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7504                        allow_write: false,
7505                        sessions,
7506                        artifacts: artifacts.clone(),
7507                        stem: format!("review-{round}-reconsider-{}", r + 1),
7508                        handover: Some(full),
7509                    });
7510                    seats_at.push(r);
7511                }
7512
7513                let mut recon_quota_losses = Vec::new();
7514                let recon_cache = self.state.config.cache_dir();
7515                let recon_ctx = WaveCtx {
7516                    carry_seats: true,
7517                    pauses: Some((&self.pause, &self.interrupt)),
7518                    run: &run_id,
7519                    node: "review",
7520                    prompts: &prompts,
7521                    cache: recon_cache.as_deref(),
7522                    round: Some(round),
7523                };
7524                let recon_results = ask_json_wave::<ReviewRevote>(
7525                    jobs,
7526                    Arc::clone(&self.sem),
7527                    review_retries,
7528                    &self.roles.reviewer_roster,
7529                    &recon_ctx,
7530                    &mut recon_quota_losses,
7531                    &mut self.state,
7532                    &|_: &ReviewRevote| Ok(()),
7533                )
7534                .await;
7535                if self.state.resource_block.is_some() {
7536                    return self.stop_for_resource();
7537                }
7538                self.state.quota.extend(recon_quota_losses);
7539
7540                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7541                    let agent_id = seat.agent.clone();
7542                    self.state.seats.insert(seat.key.clone(), seat);
7543                    let mut rec = ReviewRevoteRecord {
7544                        reviewer: r + 1,
7545                        agent: agent_id,
7546                        vote: None,
7547                        reason: String::new(),
7548                        failed: None,
7549                    };
7550                    match res {
7551                        Ok((rv, _)) => {
7552                            rec.vote = Some(rv.vote);
7553                            rec.reason =
7554                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7555                            self.state.event(
7556                                "review",
7557                                format!(
7558                                    "round {round}: reviewer {} revoted {}",
7559                                    r + 1,
7560                                    rv.vote.label()
7561                                ),
7562                            );
7563                        }
7564                        Err(e) => {
7565                            rec.failed = Some(e.to_string());
7566                            self.state.event(
7567                                "review",
7568                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7569                            );
7570                        }
7571                    }
7572                    reconsideration.push(rec);
7573                }
7574            } else if initial_votes.len() > 1 {
7575                self.state.event(
7576                    "review",
7577                    format!(
7578                        "round {round}: votes agreed ({}) — no reconsideration",
7579                        initial_votes[0].label()
7580                    ),
7581                );
7582            }
7583
7584            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7585            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7586            // A round that already has a blocking finding and a round left to
7587            // try is going back to the fixer no matter what `verify.e2e`
7588            // says, so running it first only spends the loop's slowest step
7589            // (minutes, for a Rust repo's full test suite) on a head about
7590            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7591            // runs once a round has no blocking findings left (see
7592            // `round_is_clean`, which a deferred — empty — `e2e` can never
7593            // satisfy since `blocking` is nonzero whenever this branch is
7594            // taken), and `stop_reviewing` forces a real run before it will
7595            // ever read a deferred round as green.
7596            let defer_e2e =
7597                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7598            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7599                let reason =
7600                    format!("{blocking} blocking finding(s) already required a fix this round");
7601                self.state.event(
7602                    "verify",
7603                    format!(
7604                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7605                         {}); it will run once a round has none left",
7606                        short(&head)
7607                    ),
7608                );
7609                (Vec::new(), false, true, Some(reason))
7610            } else {
7611                let e2e_commands = self.state.config.verify.e2e.clone();
7612                let cache_dir = self.state.config.cache_dir();
7613                let context = format!("round {round}");
7614                let wait = self.repository_wait();
7615                let (e2e, verify_retried) = with_cache_lease(
7616                    &mut self.state,
7617                    cache_dir.as_deref(),
7618                    "e2e",
7619                    "e2e",
7620                    &winner.worktree,
7621                    &head,
7622                    verify_timeout,
7623                    &context,
7624                    &wait,
7625                    |state, budget| {
7626                        let shell = shell.clone();
7627                        let e2e_commands = e2e_commands.clone();
7628                        let worktree = winner.worktree.clone();
7629                        let context = context.clone();
7630                        async move {
7631                            run_e2e_with_retry(
7632                                state,
7633                                &shell,
7634                                &e2e_commands,
7635                                &worktree,
7636                                budget,
7637                                &context,
7638                            )
7639                            .await
7640                        }
7641                    },
7642                )
7643                .await?;
7644                (e2e, verify_retried, false, None)
7645            };
7646
7647            let expected = records.len();
7648            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7649            let incomplete = answered < expected;
7650            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7651            let policy = self.state.config.graph.incomplete_review;
7652            let clean = round_is_clean(
7653                blocking,
7654                e2e_ok,
7655                answered,
7656                expected,
7657                round_quota_missing,
7658                policy,
7659            );
7660
7661            let mut round_record = ReviewRound {
7662                round,
7663                head: head.clone(),
7664                verified_head: None,
7665                verified_at: None,
7666                reviews: records,
7667                e2e,
7668                verify_retried,
7669                e2e_deferred,
7670                e2e_defer_reason,
7671                fix: None,
7672                blocking,
7673                answered,
7674                expected,
7675                clean,
7676                progressed: false,
7677                vote_split,
7678                reconsideration,
7679                verdict: None,
7680            };
7681            // The final vote per seat is its revote where reconsideration
7682            // ran and answered, its initial vote otherwise — the same
7683            // fallback `tally` uses for a judge whose private vote failed.
7684            round_record.verdict = ReviewVote::worst(
7685                round_record
7686                    .final_votes()
7687                    .into_iter()
7688                    .map(|(_, _, vote)| vote),
7689            );
7690            // Which commit and when magi actually attempted to check —
7691            // known the moment a command was dispatched against `head`,
7692            // whether or not it finished: a resource-blocked attempt still
7693            // targeted a specific commit at a specific time, and leaving
7694            // that unrecorded is exactly what made `verification_summary`
7695            // report a fresh attempt as "commit unknown ... recorded before
7696            // this was tracked", indistinguishable from a genuinely old,
7697            // untracked record. Only a deferred or unconfigured round never
7698            // ran at all and has nothing to record — see
7699            // `ReviewRound::verified_head`'s own doc.
7700            if !matches!(
7701                round_record.e2e_status(),
7702                E2eStatus::Deferred | E2eStatus::NotConfigured
7703            ) {
7704                round_record.verified_head = Some(head.clone());
7705                round_record.verified_at = Some(Timestamp::now());
7706            }
7707
7708            if incomplete {
7709                let missing: Vec<String> = round_record
7710                    .reviews
7711                    .iter()
7712                    .filter(|r| r.failed.is_some())
7713                    .map(|r| format!("review-{}", r.reviewer))
7714                    .collect();
7715                self.state.event(
7716                    "review",
7717                    format!(
7718                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7719                        missing.join(", ")
7720                    ),
7721                );
7722            }
7723
7724            if clean {
7725                self.state.event(
7726                    "review",
7727                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7728                        format!(
7729                            "round {round}: clean (warn policy, incomplete panel) — no \
7730                             blocking findings from the seats that answered, verification green"
7731                        )
7732                    } else if incomplete {
7733                        format!(
7734                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7735                             quorum) — no blocking findings from the seats that answered, \
7736                             verification green",
7737                            expected - answered
7738                        )
7739                    } else {
7740                        format!("round {round}: clean — no blocking findings, verification green")
7741                    },
7742                );
7743                self.state.reviews.push(round_record);
7744                self.state.status = RunStatus::Gating;
7745                self.state.save()?;
7746                return Ok(());
7747            }
7748
7749            // Nothing was raised and verification passed, but not every seat
7750            // answered and `round_is_clean` still refused to call it clean —
7751            // either a seat is missing for a reason other than its own quota
7752            // (a crash, a timeout, unparsable output — worth another try), or
7753            // every seat that could have answered lost its quota and nobody
7754            // is left to decide on: re-review rather than send the fixer
7755            // after a round with nothing to fix.
7756            if incomplete && blocking == 0 && e2e_ok {
7757                self.state.reviews.push(round_record);
7758                self.state.save()?;
7759                if round == max_rounds {
7760                    self.state.status = RunStatus::Blocked;
7761                    self.state.event(
7762                        "review",
7763                        format!(
7764                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7765                             refusing to call it clean",
7766                            expected - answered
7767                        ),
7768                    );
7769                    return Ok(());
7770                }
7771                continue;
7772            }
7773
7774            // Nothing for the fixer to act on (`blocking == 0`) and the only
7775            // reason this round is not clean is that magi itself never got
7776            // a command to run — the shared build cache, not the patch (see
7777            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7778            // the fixer would invite a change to appease contention that has
7779            // nothing to do with the diff, and would leave this attempt
7780            // sitting in the next round's prompt as if it were about an
7781            // earlier, superseded commit rather than what it actually is:
7782            // the same head, still waiting to be checked. Wait for it the
7783            // same way the final round's own contention is already handled,
7784            // whatever round this happens to be.
7785            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7786                self.state.reviews.push(round_record);
7787                return self
7788                    .stop_unresolved_review(
7789                        "the round's own verification could not run",
7790                        &shell,
7791                        &winner.worktree,
7792                    )
7793                    .await;
7794            }
7795
7796            if round == max_rounds {
7797                self.state.reviews.push(round_record);
7798                return self
7799                    .stop_reviewing(
7800                        &format!(
7801                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7802                        ),
7803                        &shell,
7804                        &winner.worktree,
7805                    )
7806                    .await;
7807            }
7808
7809            if !self.fix_review_round(&winner, round_record).await? {
7810                return Ok(());
7811            }
7812        }
7813        Ok(())
7814    }
7815
7816    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7817    /// False means admission or continuation policy stopped the review loop.
7818    async fn fix_review_round(
7819        &mut self,
7820        winner: &Candidate,
7821        mut round_record: ReviewRound,
7822    ) -> Result<bool> {
7823        let round = round_record.round;
7824        let shell = self.state.config.shell();
7825        let step = self.run_fix_round(winner, &mut round_record).await?;
7826        if matches!(step, FixStep::Resource) {
7827            self.stop_for_resource()?;
7828            return Ok(false);
7829        }
7830        if self.state.reviews.len() < round {
7831            self.state.reviews.push(round_record);
7832        } else {
7833            self.state.reviews[round - 1] = round_record;
7834        }
7835        self.state.save()?;
7836
7837        // The fixer's own report never came back this round, even after
7838        // `continue_fix_report`'s own budget was spent on it — not an
7839        // ordinary "no report" (dropped stream, quota, plain failure),
7840        // which already reads that way and is left to the existing round
7841        // budget. Stopping here, rather than opening another round, is
7842        // what keeps a next reviewer/fixer wave from ever being
7843        // dispatched onto `winner.worktree` while whatever the seat's
7844        // last call may still have running there is unaccounted for: no
7845        // process liveness check exists (and none is being added — see
7846        // AGENTS.md/this task's own scope), so the only way to honour
7847        // "nothing starts before a valid report returns" is to not start
7848        // anything further on this worktree from this run at all.
7849        if matches!(step, FixStep::Recorded { report_lost: true }) {
7850            self.stop_reviewing(
7851                "the fixer's adoption report never came back, even after resuming its \
7852                         own seat; refusing to start another round against the same worktree \
7853                         while that is unresolved",
7854                &shell,
7855                &winner.worktree,
7856            )
7857            .await?;
7858            return Ok(false);
7859        }
7860
7861        let streak = self
7862            .state
7863            .reviews
7864            .iter()
7865            .rev()
7866            .take_while(|r| !r.progressed)
7867            .count();
7868        if streak >= STAGNANT_LIMIT {
7869            self.stop_reviewing(
7870                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7871                &shell,
7872                &winner.worktree,
7873            )
7874            .await?;
7875            return Ok(false);
7876        }
7877        Ok(true)
7878    }
7879
7880    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7881    async fn run_fix_round(
7882        &mut self,
7883        winner: &Candidate,
7884        round_record: &mut ReviewRound,
7885    ) -> Result<FixStep> {
7886        let round = round_record.round;
7887        let max_rounds = self.state.config.graph.review_rounds;
7888        let run_id = self.state.id.clone();
7889        let prompts = self.state.config.prompts.clone();
7890        let language = self.state.config.graph.language.clone();
7891        let sessions = self.state.config.graph.sessions;
7892        let artifacts = self.merge_repair_artifacts();
7893        let base = self.landing_base();
7894        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7895        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7896        let this_round_verification = round_record.verification_summary(&head);
7897        let all_findings: Vec<_> = round_record
7898            .reviews
7899            .iter()
7900            .flat_map(|r| r.findings.iter().cloned())
7901            .collect();
7902        // Fix. The winner's own implementer seat continues its conversation:
7903        // the competition is over, so context is pure benefit now.
7904        let blocking_findings: Vec<_> = all_findings
7905            .iter()
7906            .filter(|f| f.severity.blocks())
7907            .cloned()
7908            .collect();
7909        let fix_prompt = prompt::fix_in(
7910            &self.state.instruction,
7911            &blocking_findings,
7912            this_round_verification.as_ref(),
7913            round,
7914            max_rounds,
7915            &language,
7916            &self.state.config.graph.github_language,
7917        );
7918        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7919        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7920        let (job, seat, out) = self
7921            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7922                prompt: fix_prompt.clone(),
7923                spec,
7924                seat,
7925                cwd: winner.worktree.clone(),
7926                timeout,
7927                allow_write: true,
7928                sessions,
7929                artifacts: artifacts.clone(),
7930                stem: format!("fix-{round}"),
7931                handover: None,
7932            })
7933            .await;
7934        // Not asked: the round is not recorded (so a resume repeats it),
7935        // no fixer was handed over to, and the seat is left as it was.
7936        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7937            return Ok(FixStep::Resource);
7938        }
7939        let agent_id = seat.agent.clone();
7940
7941        let mut fix = FixRecord {
7942            agent: agent_id,
7943            addressed: Vec::new(),
7944            rejected: Vec::new(),
7945            notes: String::new(),
7946            committed: false,
7947            failed: None,
7948            duration_ms: 0,
7949            continuation: None,
7950        };
7951        let mut continuation = ContinuationRecord::not_needed();
7952        let mut final_seat = seat.clone();
7953        match out {
7954            AgentOutcome::Ok(o) => {
7955                fix.duration_ms = o.duration_ms;
7956                let parsed = verdict::extract_json::<FixReport>(&o.text);
7957                // A parsed report standing next to a command this same
7958                // reply's own CLI never confirmed the exit status of is
7959                // not a resolved answer — the identical `CommandEvidence`
7960                // `state.jobs` renders, read here instead of only on
7961                // display, per the completion judgment and the shown
7962                // record needing to agree.
7963                let incomplete_reason = match &parsed {
7964                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7965                        "the reply parsed, but it reported a command whose own CLI never \
7966                             confirmed an exit status"
7967                            .to_owned(),
7968                    ),
7969                    Ok(_) => None,
7970                    Err(e) => Some(e.to_string()),
7971                };
7972                match incomplete_reason {
7973                    None => {
7974                        let report = parsed.expect("checked Ok above");
7975                        fix.addressed = report.addressed;
7976                        fix.rejected = report.rejected;
7977                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7978                    }
7979                    Some(reason) => {
7980                        let (resumed_seat, resolved, failure, cont) = self
7981                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7982                            .await;
7983                        fix.duration_ms += cont.cumulative_wait_ms;
7984                        continuation = cont;
7985                        final_seat = resumed_seat;
7986                        match resolved {
7987                            Some(report) => {
7988                                fix.addressed = report.addressed;
7989                                fix.rejected = report.rejected;
7990                                fix.notes =
7991                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7992                            }
7993                            None => fix.failed = failure,
7994                        }
7995                    }
7996                }
7997            }
7998            // The CLI's raw error JSON is not a fix report to parse.
7999            AgentOutcome::Dropped(o) => {
8000                fix.duration_ms = o.duration_ms;
8001                let why = o
8002                    .dropped
8003                    .as_ref()
8004                    .map(|d| d.why.as_str())
8005                    .unwrap_or("the CLI ended the stream without delivering its answer");
8006                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8007            }
8008            AgentOutcome::Quota(o) => {
8009                self.state.quota.push(QuotaLoss::from_report(
8010                    final_seat.key.clone(),
8011                    "fix",
8012                    o.quota.as_ref(),
8013                ));
8014                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8015            }
8016            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8017        }
8018        if self.state.resource_block.is_some() {
8019            self.state.seats.insert(final_seat.key.clone(), final_seat);
8020            return Ok(FixStep::Resource);
8021        }
8022        fix.continuation = Some(continuation);
8023        self.state.seats.insert(final_seat.key.clone(), final_seat);
8024        let coordination = self.coordinate(&winner.worktree).await?;
8025        if let Ok(r) = git::rescue_commit(
8026            &winner.worktree,
8027            &format!("magi: review round {round} fixes (uncommitted work)"),
8028        )
8029        .await
8030        {
8031            self.state.note_withheld("fix", &r.withheld);
8032        }
8033        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8034        fix.committed = after != before;
8035        // Judged by what `git` says moved against base, never by the
8036        // fixer's own `addressed`/`rejected` count — see
8037        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8038        // `patch` snapshot above: swallowing this error would default
8039        // `diff_after` to empty, which almost always differs from a
8040        // non-empty `patch` and reads as "progressed" — exactly backwards
8041        // for a `git` failure the stagnation check cannot see through.
8042        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8043        drop(coordination);
8044        let progressed = diff_after != patch;
8045        let commit_note = if fix.committed {
8046            "committed"
8047        } else {
8048            "NO new commit"
8049        };
8050        let tree_note = if progressed {
8051            "changed vs base"
8052        } else {
8053            "unchanged vs base"
8054        };
8055        self.state.event(
8056            "fix",
8057            match &fix.failed {
8058                // Distinct on purpose from "0 addressed, 0 rejected": the
8059                // fixer's own diff still landed (blocking counts do keep
8060                // falling round over round), only its adoption report did
8061                // not come back, so this must never read like every
8062                // finding was reviewed and declined.
8063                Some(reason) => {
8064                    format!(
8065                        "round {round}: fixer's adoption report was lost ({reason}); \
8066                             {commit_note}, tree {tree_note}"
8067                    )
8068                }
8069                None => format!(
8070                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8071                         {tree_note}{}",
8072                    fix.addressed.len(),
8073                    fix.rejected.len(),
8074                    if continuation.outcome == ContinuationOutcome::Resumed {
8075                        format!(
8076                            " (adoption report recovered after {} continuation(s))",
8077                            continuation.attempts
8078                        )
8079                    } else {
8080                        String::new()
8081                    },
8082                ),
8083            },
8084        );
8085        round_record.fix = Some(fix);
8086        round_record.progressed = progressed;
8087        Ok(FixStep::Recorded {
8088            report_lost: matches!(
8089                continuation.outcome,
8090                ContinuationOutcome::Exhausted
8091                    | ContinuationOutcome::QuotaLost
8092                    | ContinuationOutcome::NoSession
8093            ),
8094        })
8095    }
8096
8097    /// Decide, from the last recorded round's own verification, whether
8098    /// stopping the review loop is a hand-off or a genuine block.
8099    ///
8100    /// Called once the loop has given up trying — the round budget is spent,
8101    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8102    /// findings still open, never while a round is still clean or the
8103    /// incomplete-panel case handled inline above. Gate and e2e are facts
8104    /// about the tree; a lingering review finding is an opinion, and this
8105    /// workload's own `magi stats` puts reviewer precision low enough
8106    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8107    /// must not by itself stand between a green, verified change and the
8108    /// human who decides what to do with it. A red e2e is not an opinion, so
8109    /// that case still blocks, with the failing command and a tail of its
8110    /// output recorded here rather than left in `run.json` for someone to go
8111    /// find.
8112    ///
8113    /// A round that deferred its own e2e (see [`Config::graph`]'s
8114    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8115    /// only because nothing ran, and treating an empty list as a passing one
8116    /// here is exactly the "deferred painted green" bug this function exists
8117    /// to not have. When the last round's own verification never resolved —
8118    /// deferred on purpose, or a real attempt the shared build cache blocked
8119    /// — this makes (or retries) the real run, on the actual worktree this
8120    /// loop is about to stop touching, before deciding anything. A
8121    /// resource-blocked attempt is likewise never read as either green or
8122    /// red: it is evidence about the machine, not the patch (see
8123    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8124    /// blocked cache leaves this call without deciding rather than guessing
8125    /// — the caller retries on a later reentry.
8126    /// Record, once, that the review loop handed off over a blocking finding
8127    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8128    /// `land` asks the owner even with `land_approval` off. Called from every
8129    /// path that concludes `Gating`; a reentry keeps the first record.
8130    fn record_contested_handoff(&mut self) {
8131        if self.state.contested_handoff.is_some() {
8132            return;
8133        }
8134        let Some(contested) = self
8135            .state
8136            .reviews
8137            .last()
8138            .and_then(ReviewRound::contested_handoff)
8139        else {
8140            return;
8141        };
8142        self.state.event(
8143            "review",
8144            format!(
8145                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8146                 wait for the owner's approval",
8147                contested.findings.len(),
8148                contested.rejecters.len()
8149            ),
8150        );
8151        self.state.contested_handoff = Some(contested);
8152    }
8153
8154    async fn stop_unresolved_review(
8155        &mut self,
8156        why: &str,
8157        shell: &[String],
8158        worktree: &Path,
8159    ) -> Result<()> {
8160        self.stop_reviewing_inner(why, shell, worktree, true).await
8161    }
8162
8163    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8164        self.stop_reviewing_inner(why, shell, worktree, false).await
8165    }
8166
8167    async fn stop_reviewing_inner(
8168        &mut self,
8169        why: &str,
8170        shell: &[String],
8171        worktree: &Path,
8172        repairable: bool,
8173    ) -> Result<()> {
8174        let round_idx = self.state.reviews.len() - 1;
8175        // A deferred round and a resource-blocked one are the same shape
8176        // here: neither has a real result yet, and both get one more
8177        // attempt. Read off `e2e_status` — the single source for this —
8178        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8179        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8180        // still retries instead of being read as a settled result the
8181        // instant it stops being empty.
8182        let needs_catchup_run = matches!(
8183            self.state.reviews[round_idx].e2e_status(),
8184            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8185        );
8186        if needs_catchup_run {
8187            let round = self.state.reviews[round_idx].round;
8188            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8189            let commands = self.state.config.verify.e2e.clone();
8190            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8191            let cache_dir = self.state.config.cache_dir();
8192            let context = format!(
8193                "round {round}: verification unresolved, catching up before the final decision"
8194            );
8195            let wait = self.repository_wait();
8196            let (outcomes, verify_retried) = with_cache_lease(
8197                &mut self.state,
8198                cache_dir.as_deref(),
8199                "e2e",
8200                "e2e",
8201                worktree,
8202                &attempted_head,
8203                timeout,
8204                &context,
8205                &wait,
8206                |state, budget| {
8207                    let shell = shell.to_vec();
8208                    let commands = commands.clone();
8209                    let context = context.clone();
8210                    async move {
8211                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8212                            .await
8213                    }
8214                },
8215            )
8216            .await?;
8217            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8218                self.state.record_base_progress(&attempted_head);
8219            }
8220            let last = &mut self.state.reviews[round_idx];
8221            last.e2e = outcomes;
8222            last.verify_retried = verify_retried;
8223            // Always the commit and time this attempt actually targeted,
8224            // whether or not it happens to equal the reviewed `head` and
8225            // whether or not a command finished — see
8226            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8227            // attempt is recorded too, so a later reader sees "attempted
8228            // again at T2" rather than silence.
8229            last.verified_head = Some(attempted_head);
8230            last.verified_at = Some(Timestamp::now());
8231            if verify_inconclusive(&last.e2e) {
8232                // Still not a real result: `e2e_deferred` is left exactly
8233                // as it was, so `needs_catchup_run` above reads
8234                // `ResourceBlocked` (via `e2e_status`, which checks
8235                // `resource_blocked` before `e2e_deferred`) and retries
8236                // again on the next reentry, rather than recording
8237                // contention as a red e2e and blocking the run on it.
8238                self.state.save()?;
8239                return Ok(());
8240            }
8241            last.e2e_deferred = false;
8242            // A complete panel may have been saved as unclean solely because
8243            // verification could not run. Refresh that verdict with the new
8244            // outcomes before a granted repair consumes the round's result.
8245            // Incomplete panels retain their original policy/quota decision.
8246            if last.answered == last.expected {
8247                last.clean = round_is_clean(
8248                    last.blocking,
8249                    last.e2e.iter().all(CommandOutcome::ok),
8250                    last.answered,
8251                    last.expected,
8252                    0,
8253                    self.state.config.graph.incomplete_review,
8254                );
8255            }
8256        }
8257        let last = &self.state.reviews[round_idx];
8258        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8259
8260        match last.e2e_status() {
8261            E2eStatus::Failed => {
8262                let red: Vec<String> = last
8263                    .e2e
8264                    .iter()
8265                    .filter(|o| !o.ok())
8266                    .map(|o| {
8267                        format!(
8268                            "`{}` -> {:?}\n{}",
8269                            o.command,
8270                            o.code,
8271                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8272                        )
8273                    })
8274                    .collect();
8275                let can_repair = last.blocking == 0 && last.fix.is_none();
8276                self.state.reviews[round_idx].clean = false;
8277                self.state
8278                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8279                self.state.status = RunStatus::Blocked;
8280                if repairable
8281                    && can_repair
8282                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8283                {
8284                    self.state.status = RunStatus::Reviewing;
8285                }
8286            }
8287            // `needs_catchup_run` above already retried once this call; if
8288            // it is still blocked, this is magi's own admission it could
8289            // not get a command to run, never a verdict on the patch — the
8290            // run is left exactly where a later reentry can retry again.
8291            E2eStatus::ResourceBlocked => {
8292                self.state.event(
8293                    "review",
8294                    format!(
8295                        "{why}; e2e could not run (resource unavailable); not \
8296                         deciding yet"
8297                    ),
8298                );
8299            }
8300            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8301                self.state.event(
8302                    "review",
8303                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8304                );
8305                self.record_contested_handoff();
8306                self.state.status = RunStatus::Gating;
8307            }
8308        }
8309        self.state.save()?;
8310        Ok(())
8311    }
8312
8313    // ----------------------------------------------------------------- gate
8314
8315    async fn gate(&mut self) -> Result<()> {
8316        if self.hold_base_sync().await? {
8317            return Ok(());
8318        }
8319        // Judged by the review record itself, not by `status`: a solo
8320        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8321        // reentry (see `judge`), and trusting it here is exactly how a run
8322        // that exhausted its review budget got gated and merged a second
8323        // time around. `review_conclusion` recomputes the review loop's own
8324        // verdict from the round records themselves — `Gating` for a clean
8325        // round or a hand-off (see `stop_reviewing`), anything else means the
8326        // loop is still going or genuinely blocked.
8327        // A base the winner could not be replayed onto is a decision, not a
8328        // round: there is no landing tree to gate. Read as its own record for
8329        // the same reason the review verdict is.
8330        if self.state.status == RunStatus::Failed
8331            || self
8332                .state
8333                .base_sync
8334                .as_ref()
8335                .is_some_and(|s| s.conflict.is_some())
8336            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8337                != Some(RunStatus::Gating)
8338        {
8339            return Ok(());
8340        }
8341        let Some(winner) = self.state.winner().cloned() else {
8342            return Ok(());
8343        };
8344        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8345        self.state.reconcile_verification_head(&head);
8346        self.state.save()?;
8347        let wait = self.repository_wait();
8348        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8349            return Ok(());
8350        }
8351        if self.state.gate_ran {
8352            // `review_loop` derives its conclusion from the clean review
8353            // record on every reentry and therefore puts a completed run back
8354            // in `Gating`. A recorded gate is a stronger, terminal fact:
8355            // retain its original command output (or lack of any, for a repo
8356            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8357            // doc) and restore `Blocked` on a real failure rather than
8358            // pretending the command is still running or running it a second
8359            // time. `gate_ran == false` remains the only shape — unattempted,
8360            // or a resource-blocked retry — that may still need to execute a
8361            // command.
8362            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8363                self.state.status = RunStatus::Blocked;
8364                self.state.save()?;
8365            }
8366            return Ok(());
8367        }
8368        self.state.status = RunStatus::Gating;
8369        let mut outcomes = self.run_gate(&winner).await?;
8370        let mut attempted_head = self.state.gate_head.clone();
8371        loop {
8372            // A resource-blocked outcome means the gate command never actually
8373            // ran - the shared build cache could not be acquired or confirmed
8374            // fresh in time - which is evidence about the machine, not about
8375            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8376            // Recording it as a red gate would mark a run `Blocked` on nothing
8377            // but contention magi has already logged; leaving `self.state.gate`
8378            // empty and `self.state.gate_ran` false instead keeps the shape
8379            // this function already treats as "still needs to run" (see the
8380            // early-return above), so the next call retries the command
8381            // rather than concluding anything.
8382            if verify_inconclusive(&outcomes) {
8383                self.state.save()?;
8384                return Ok(());
8385            }
8386            if outcomes.iter().all(CommandOutcome::ok) {
8387                break;
8388            }
8389            let fixes_before = self.state.gate_fixes.len();
8390            match self.gate_fix_round(&winner, &outcomes).await? {
8391                GateFix::Retry => {
8392                    outcomes = self.run_gate(&winner).await?;
8393                    attempted_head = self.state.gate_head.clone();
8394                }
8395                GateFix::Stop => {
8396                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8397                    if self.state.gate_fixes.len() > fixes_before
8398                        && attempted_head.as_deref() != Some(current.as_str())
8399                        && verify_current_head(&mut self.state, &winner, &wait).await?
8400                    {
8401                        // A new SHA owes a gate even when a spent fixer changed only
8402                        // commit metadata. This does not grant another repair round.
8403                        outcomes = self.run_gate(&winner).await?;
8404                        attempted_head = self.state.gate_head.clone();
8405                    } else {
8406                        break;
8407                    }
8408                }
8409                GateFix::Defer => {
8410                    self.state.save()?;
8411                    return Ok(());
8412                }
8413            }
8414        }
8415        let passed = outcomes.iter().all(CommandOutcome::ok);
8416        self.state.gate = outcomes;
8417        self.state.gate_ran = true;
8418        self.state.gate_head = attempted_head;
8419        if !self.state.gate.is_empty()
8420            && !verify_inconclusive(&self.state.gate)
8421            && let Some(head) = self.state.gate_head.clone()
8422        {
8423            self.state.record_base_progress(&head);
8424        }
8425        self.state.save()?;
8426        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8427            return Ok(());
8428        }
8429        if !passed {
8430            self.state.status = RunStatus::Blocked;
8431            let spent = self.state.gate_fixes.len();
8432            self.state.event(
8433                "gate",
8434                if spent == 0 {
8435                    "gate failed; not merging".to_owned()
8436                } else {
8437                    format!("gate failed after {spent} gate-fix round(s); not merging")
8438                },
8439            );
8440        }
8441        self.state.save()?;
8442        Ok(())
8443    }
8444
8445    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8446    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8447    /// after review is clean and never on a candidate awaiting judging.
8448    ///
8449    /// Never fails the run: a non-zero exit or timeout is a warning and a
8450    /// recorded outcome, and the gate remains the single arbiter. Nothing
8451    /// configured means nothing happens - no event, no commit. `commit_all`
8452    /// commits any leftover change under the neutral identity and returns
8453    /// `false` when the tree is clean, so no empty commit is ever made.
8454    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8455        let commands = state.config.verify.pre_gate.clone();
8456        if commands.is_empty() {
8457            return;
8458        }
8459        let shell = state.config.shell();
8460        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8461        let (outcomes, _) = run_commands(
8462            state,
8463            "pre_gate",
8464            "pre_gate",
8465            0,
8466            &shell,
8467            &commands,
8468            &winner.worktree,
8469            timeout,
8470        )
8471        .await;
8472        for o in &outcomes {
8473            if !o.ok() {
8474                tracing::warn!(
8475                    "pre_gate `{}` failed ({:?}); the gate decides",
8476                    o.command,
8477                    o.code
8478                );
8479            }
8480            state.event(
8481                "pre_gate",
8482                format!(
8483                    "`{}` -> {}",
8484                    o.command,
8485                    if o.ok() {
8486                        "pass".to_owned()
8487                    } else {
8488                        format!(
8489                            "FAIL ({:?})\n{}",
8490                            o.code,
8491                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8492                        )
8493                    }
8494                ),
8495            );
8496        }
8497        state.pre_gate = outcomes;
8498        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8499            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8500                Ok(head) => {
8501                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8502                    state.pre_gate_commit = Some(head);
8503                }
8504                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8505            },
8506            Ok(false) => {}
8507            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8508        }
8509        if let Err(e) = state.save() {
8510            tracing::warn!("could not persist the pre_gate record: {e:#}");
8511        }
8512    }
8513
8514    /// Run `verify.gate` once against the winner's current tree, logging one
8515    /// event per command. Empty when nothing is configured.
8516    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8517        let wait = self.repository_wait();
8518        run_landing_gate(&mut self.state, winner, &wait).await
8519    }
8520
8521    /// One bounded fix round for a failing gate.
8522    ///
8523    /// The fixer is told the failure came from the gate itself, not from a
8524    /// reviewer, and is shown the failed commands, their exit codes and a tail
8525    /// of their output - whatever `[verify].gate` holds, nothing here knows
8526    /// what those commands run. Only a normal non-zero exit that printed
8527    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8528    /// command or a full disk says nothing about the code, and a fixer sent
8529    /// after it can only appease the machine. The round is judged by what git
8530    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8531    /// again before the gate does, so a fix cannot trade a green gate for a
8532    /// red e2e unnoticed.
8533    async fn gate_fix_round(
8534        &mut self,
8535        winner: &Candidate,
8536        outcomes: &[CommandOutcome],
8537    ) -> Result<GateFix> {
8538        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8539            g.phase == crate::repair_budget::Phase::Fixing
8540                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8541        }) {
8542            self.state.gate_fixes.len().saturating_add(1)
8543        } else {
8544            self.state.config.graph.gate_fix_rounds
8545        };
8546        let spent = self.state.gate_fixes.len();
8547        if spent >= cap {
8548            if cap > 0 {
8549                self.state.event(
8550                    "gate",
8551                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8552                );
8553            }
8554            return Ok(GateFix::Stop);
8555        }
8556        if !gate_fixable(outcomes) {
8557            self.state.event(
8558                "gate",
8559                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8560                 command or similar); not spending a fix round on it",
8561            );
8562            return Ok(GateFix::Stop);
8563        }
8564        let min_free = self.state.config.disk.min_free_bytes;
8565        if min_free > 0 {
8566            match crate::disk::free_bytes(&winner.worktree) {
8567                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8568                Ok(free) => {
8569                    self.state.event(
8570                        "gate",
8571                        format!(
8572                            "only {free} bytes free ({min_free} required by `[disk] \
8573                             min_free_bytes`); not spending a fix round on a failure the disk \
8574                             may explain"
8575                        ),
8576                    );
8577                    return Ok(GateFix::Stop);
8578                }
8579                Err(e) => {
8580                    self.state.event(
8581                        "gate",
8582                        format!("free disk space could not be measured ({e:#}); no fix round"),
8583                    );
8584                    return Ok(GateFix::Stop);
8585                }
8586            }
8587        }
8588
8589        let attempt = spent + 1;
8590        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8591        let base = self.landing_base();
8592        let fix_prompt = prompt::gate_fix_in(
8593            &self.state.instruction,
8594            &failed,
8595            attempt,
8596            cap,
8597            &self.state.config.graph.language,
8598            &self.state.config.graph.github_language,
8599        );
8600        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8601        let sessions = self.state.config.graph.sessions;
8602        let artifacts = self.merge_repair_artifacts();
8603        self.state.event(
8604            "gate",
8605            format!("gate failed; gate-fix round {attempt} of {cap}"),
8606        );
8607        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8608        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8609        // Persist before invoking the fixer: it may commit and then be
8610        // interrupted before its result (or the following e2e) is saved.
8611        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8612            head: before.clone(),
8613            passed: false,
8614            outcomes: Vec::new(),
8615        });
8616        self.state.save()?;
8617        let (_, seat, out) = self
8618            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8619                prompt: fix_prompt.clone(),
8620                spec,
8621                seat,
8622                cwd: winner.worktree.clone(),
8623                timeout,
8624                allow_write: true,
8625                sessions,
8626                artifacts: artifacts.clone(),
8627                stem: format!("gate-fix-{attempt}"),
8628                handover: None,
8629            })
8630            .await;
8631        // The fixer was never asked: no round is spent (nothing is recorded)
8632        // and a later reentry asks the same agent again.
8633        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8634            return Ok(GateFix::Defer);
8635        }
8636        let mut record = GateFixRecord {
8637            agent: seat.agent.clone(),
8638            failed,
8639            notes: String::new(),
8640            committed: false,
8641            error: None,
8642        };
8643        match out {
8644            AgentOutcome::Ok(o) => {
8645                // A missing report is not a failed fix: the round is judged
8646                // by the tree below, and the report only carries prose.
8647                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8648                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8649                }
8650            }
8651            AgentOutcome::Dropped(_) => {
8652                record.error = Some("the CLI dropped the stream".to_owned());
8653            }
8654            AgentOutcome::Quota(o) => {
8655                self.state.quota.push(QuotaLoss::from_report(
8656                    seat.key.clone(),
8657                    "gate-fix",
8658                    o.quota.as_ref(),
8659                ));
8660                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8661            }
8662            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8663        }
8664        self.state.seats.insert(seat.key.clone(), seat);
8665        let coordination = self.coordinate(&winner.worktree).await?;
8666        if let Ok(r) = git::rescue_commit(
8667            &winner.worktree,
8668            &format!("magi: gate fix {attempt} (uncommitted work)"),
8669        )
8670        .await
8671        {
8672            self.state.note_withheld("gate-fix", &r.withheld);
8673        }
8674        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8675        record.committed = after != before;
8676        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8677        let note = record.error.clone();
8678        self.state.gate_fixes.push(record);
8679        self.state.save()?;
8680        if !changed {
8681            self.state.event(
8682                "gate",
8683                match note {
8684                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8685                    None => format!("gate-fix round {attempt}: the tree did not change"),
8686                },
8687            );
8688            return Ok(GateFix::Stop);
8689        }
8690        self.state.event(
8691            "gate",
8692            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8693        );
8694
8695        drop(coordination);
8696        let wait = self.repository_wait();
8697        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8698    }
8699
8700    // ---------------------------------------------------------------- merge
8701
8702    /// One read-only rewrite by the summary's author; a text the gate still
8703    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8704    /// applies on their word or on silence. `None` means the run is parked on
8705    /// that question and nothing may be pushed or opened yet.
8706    async fn guarded_pr_message(
8707        &mut self,
8708        winner: &Candidate,
8709        facts: Option<&BranchFacts>,
8710        posting: bool,
8711    ) -> Result<Option<PrMessage>> {
8712        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8713        let head = match final_head.as_deref() {
8714            Some(h) => crate::pr_outcome::FinalHead::At(h),
8715            None => crate::pr_outcome::FinalHead::Unknown,
8716        };
8717        // Only a pull request that is about to be posted pays for a summary.
8718        let summarized = if posting {
8719            Some(self.summarize_change(winner, final_head.as_deref()).await)
8720        } else {
8721            None
8722        };
8723        // A busy shared resource is not a permanent cause: publishing an
8724        // "unavailable" description would stick (a later head move never
8725        // edits the body). Stop before anything is pushed or opened; the
8726        // run is `Blocked` and a resume asks the summary again.
8727        if self.state.resource_block.is_some() {
8728            self.state.event(
8729                "pr-summary",
8730                "a shared resource was busy; nothing was published, resume to summarize again",
8731            );
8732            return Ok(None);
8733        }
8734        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8735            anyhow::bail!(
8736                "the final commit moved during public description generation; retry before publishing"
8737            );
8738        }
8739        let note = match (&summarized, &self.state.change_summary) {
8740            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8741            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8742            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8743            (None, _) => ChangeNote::NotAsked,
8744        };
8745        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8746        if !posting {
8747            let identity = crate::scrub::Identity::current();
8748            return Ok(Some(PrMessage {
8749                title: crate::scrub::scrub(&pr.title, &identity),
8750                body: crate::scrub::scrub(&pr.body, &identity),
8751            }));
8752        }
8753        // This very text was already rejected and asked about: no second
8754        // judge call, rewrite or question, just carry the record on.
8755        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8756        if let Some(known) = self
8757            .state
8758            .github_text
8759            .clone()
8760            .filter(|g| g.fingerprint == raw_fp)
8761        {
8762            return self.settle_withheld(winner, &pr, known).await;
8763        }
8764        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8765        let mut violations =
8766            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8767        let identity = crate::scrub::Identity::current();
8768        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8769            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8770            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8771        {
8772            violations.push(crate::github_text::Violation::SensitiveData);
8773        }
8774        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8775        // only a language violation starts one; sensitive data alone goes
8776        // straight to the owner, with the redacted text as the fallback.
8777        let has_language = violations
8778            .iter()
8779            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8780        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8781            self.state.event(
8782                "github-text",
8783                if has_language {
8784                    format!("description rejected: {violations:?}; requesting one rewrite")
8785                } else {
8786                    format!("description rejected: {violations:?}; asking the owner")
8787                },
8788            );
8789            let mut rewritten = false;
8790            let mut rejected_rewrite = None;
8791            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8792            {
8793                let output = text;
8794                if let Some(title) = summary_title(&output) {
8795                    // Judge the redacted candidate that will actually be posted.
8796                    // Sensitive data alone must not discard a useful rewrite.
8797                    let raw_body = summary_without_title(&output);
8798                    let clean_title = crate::scrub::scrub(&title, &identity);
8799                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8800                    let redacted = clean_title != title || body != raw_body;
8801                    let title = clean_title;
8802                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8803                    if !complete {
8804                        self.state.event(
8805                            "github-text",
8806                            "description rewrite omitted required findings",
8807                        );
8808                    }
8809                    if !body.trim().is_empty() {
8810                        body.push_str("\n\n");
8811                        body.push_str(&public_pr_evidence(&self.state, head));
8812                    }
8813                    let marker = format!("magi:run/{}", self.state.id);
8814                    if !body.trim().is_empty() && !body.contains(&marker) {
8815                        body.push_str(&format!("\n\n{marker}"));
8816                    }
8817                    let rewrite = PrMessage {
8818                        title: title.clone(),
8819                        body: body.clone(),
8820                    };
8821                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8822                    let mut failed = crate::github_text::check_for(
8823                        &self.state.config.graph,
8824                        &title,
8825                        &body,
8826                        redecision,
8827                    );
8828                    if (!crate::github_text::shareable(&title)
8829                        || !crate::github_text::shareable(&body))
8830                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8831                    {
8832                        failed.push(crate::github_text::Violation::SensitiveData);
8833                    }
8834                    if complete
8835                        && !title.trim().is_empty()
8836                        && !body.trim().is_empty()
8837                        && !failed.is_empty()
8838                    {
8839                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8840                            title: title.clone(),
8841                            body: body.clone(),
8842                            failed: failed
8843                                .iter()
8844                                .map(|v| crate::github_text::category(*v).to_owned())
8845                                .collect(),
8846                        });
8847                    }
8848                    if complete
8849                        && !title.trim().is_empty()
8850                        && !raw_body.trim().is_empty()
8851                        && failed.is_empty()
8852                    {
8853                        decision = redecision;
8854                        pr = PrMessage { title, body };
8855                        rewritten = true;
8856                        self.state
8857                            .event("github-text", "description rewrite passed");
8858                        if redacted {
8859                            self.state.event(
8860                                "github-text",
8861                                "sensitive data removed from description rewrite",
8862                            );
8863                        }
8864                    }
8865                }
8866            }
8867            if !rewritten {
8868                self.state.event(
8869                    "github-text",
8870                    "description rewrite unavailable or rejected; asking the owner",
8871                );
8872                if let Some(w) =
8873                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8874                {
8875                    // A different text than the one asked about before: that
8876                    // question no longer describes anything.
8877                    self.retire_github_text_question("the withheld text changed");
8878                    let record = crate::run::GithubTextAsk {
8879                        rewrite: rejected_rewrite,
8880                        fingerprint: raw_fp,
8881                        title: w.title,
8882                        body: w.body,
8883                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8884                        question: None,
8885                        asks: 0,
8886                        resolved: false,
8887                        chosen_title: None,
8888                        choice: None,
8889                        publication: None,
8890                    };
8891                    return self.settle_withheld(winner, &pr, record).await;
8892                }
8893            }
8894        }
8895        let (title, body) =
8896            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8897        Ok(Some(PrMessage { title, body }))
8898    }
8899
8900    /// Describe the whole change from its final diff in either run mode.
8901    ///
8902    /// The summary is tied to the final commit and to the merge base of the
8903    /// resolved remote base; a record for any other pair is dropped and the
8904    /// agent asked again, so a description never outlives the diff it was
8905    /// written from. The seat is read-only and the prompt tells it to read the
8906    /// diff and the code around it. `[roles] synthesizer` selects the agents
8907    /// (unset: the default pick), each asked at most once, advancing on an
8908    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8909    /// carries the reason it is unavailable; nothing is invented then.
8910    async fn summarize_change(
8911        &mut self,
8912        winner: &Candidate,
8913        head: Option<&str>,
8914    ) -> std::result::Result<(), String> {
8915        let Some(head) = head else {
8916            return Err("the final commit could not be read".to_owned());
8917        };
8918        let remote = self.state.config.merge.remote.clone();
8919        let tracking = format!("{remote}/{}", self.state.base_branch);
8920        if !git::rev_exists(&winner.worktree, &tracking).await {
8921            return Err(format!("the remote base `{tracking}` could not be read"));
8922        }
8923        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8924            Ok(b) if !b.is_empty() => b,
8925            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8926        };
8927        let evidence = crate::pr_outcome::summary_evidence(
8928            &self.state,
8929            crate::pr_outcome::FinalHead::At(head),
8930        );
8931        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8932        if self
8933            .state
8934            .change_summary
8935            .as_ref()
8936            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8937        {
8938            return Ok(());
8939        }
8940        if self.state.change_summary.take().is_some() {
8941            self.state.event(
8942                "pr-summary",
8943                "the final diff or evidence changed; the earlier change summary was discarded",
8944            );
8945        }
8946        // Save the source snapshot even if no generation agent is available.
8947        if let Err(e) = write_artifact(
8948            &self.state,
8949            &format!("public-description-{fingerprint}.md"),
8950            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8951        ) {
8952            tracing::warn!("could not save public description evidence: {e:#}");
8953        }
8954        let chain = match agent::pick_chain(
8955            &self.state.config.agents,
8956            self.state.config.roles.synthesizer.as_ref(),
8957            &agent::installed,
8958            "synthesizer",
8959        ) {
8960            Ok(chain) => chain,
8961            Err(e) => {
8962                tracing::warn!("no agent for the change summary: {e:#}");
8963                self.state.event("pr-summary", "no summary agent available");
8964                return Err("no agent was available to summarize it".to_owned());
8965            }
8966        };
8967        let subjects = git::subjects(&winner.worktree, &base, head)
8968            .await
8969            .unwrap_or_default();
8970        let stat = git::diff_stat(&winner.worktree, &base, head)
8971            .await
8972            .unwrap_or_default();
8973        let stat: String = stat.chars().take(6_000).collect();
8974        let mut prompt = crate::prompt::change_summary_in(
8975            &base,
8976            head,
8977            &subjects,
8978            stat.trim(),
8979            &self.state.config.graph.github_language,
8980        );
8981        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8982        let sessions = self.state.config.graph.sessions;
8983        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8984        let prompts = self.state.config.prompts.clone();
8985        let cache = self.state.config.cache_dir();
8986        let run = self.state.id.clone();
8987        for (i, spec) in chain.into_iter().enumerate() {
8988            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8989            let github_spec = match agent::for_github_text(
8990                &spec,
8991                &self.state.config.graph.github_language,
8992                &winner.worktree,
8993            ) {
8994                Ok(spec) => spec,
8995                Err(e) => {
8996                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8997                    continue;
8998                }
8999            };
9000            let job = SeatJob {
9001                spec: github_spec,
9002                seat,
9003                cwd: winner.worktree.clone(),
9004                prompt: prompt.clone(),
9005                timeout,
9006                allow_write: false,
9007                sessions,
9008                artifacts: agent::artifacts_dir(&self.state.dir()),
9009                stem: if i == 0 {
9010                    "change-summary".to_owned()
9011                } else {
9012                    format!("change-summary-{}", spec.id)
9013                },
9014                handover: None,
9015            };
9016            let ctx = WaveCtx {
9017                run: &run,
9018                node: "pr-summary",
9019                prompts: &prompts,
9020                cache: cache.as_deref(),
9021                round: None,
9022                carry_seats: false,
9023                pauses: Some((&self.pause, &self.interrupt)),
9024            };
9025            let (seat, outcome) =
9026                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9027            self.state.seats.insert(seat.key.clone(), seat);
9028            let text = match outcome {
9029                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9030                AgentOutcome::ResourceBlocked(why) => {
9031                    // No agent was asked: the run stops resumably before
9032                    // anything is published (see `guarded_pr_message`).
9033                    if self.state.resource_block.is_none() {
9034                        self.state.resource_block = Some(crate::run::ResourceBlock {
9035                            cause: crate::run::ResourceBlockCause::Other,
9036                            node: "pr-summary".to_owned(),
9037                            kind: "shared resource".to_owned(),
9038                            detail: why,
9039                            at: Timestamp::now(),
9040                        });
9041                    }
9042                    return Err("a shared resource was busy".to_owned());
9043                }
9044                _ => continue,
9045            };
9046            let blind = &self.state.config.blind;
9047            let Some(title) = summary_title(&text) else {
9048                continue;
9049            };
9050            let body: String = summary_without_title(&text)
9051                .lines()
9052                .filter(|l| !l.contains("magi:run/"))
9053                .collect::<Vec<_>>()
9054                .join("\n");
9055            let body = blind::sanitize_prose(body.trim(), blind);
9056            let title = blind::sanitize_prose(&title, blind);
9057            if body.trim().is_empty() || title.trim().is_empty() {
9058                continue;
9059            }
9060            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9061                continue;
9062            }
9063            // The branch must still be the one the agent was shown.
9064            if git::rev_parse(&winner.worktree, "HEAD")
9065                .await
9066                .ok()
9067                .as_deref()
9068                != Some(head)
9069            {
9070                return Err("the branch moved while the summary was written".to_owned());
9071            }
9072            if git::merge_base(&winner.worktree, &tracking, head)
9073                .await
9074                .ok()
9075                .as_deref()
9076                != Some(base.as_str())
9077                || crate::pr_outcome::summary_evidence(
9078                    &self.state,
9079                    crate::pr_outcome::FinalHead::At(head),
9080                ) != evidence
9081            {
9082                return Err("the summary evidence changed while it was written".to_owned());
9083            }
9084            self.state.change_summary = Some(crate::run::ChangeSummary {
9085                head: head.to_owned(),
9086                base,
9087                evidence: fingerprint,
9088                title,
9089                body,
9090            });
9091            self.state.event(
9092                "pr-summary",
9093                format!("change summary written by {} for the final diff", spec.id),
9094            );
9095            if let Err(e) = self.state.save() {
9096                tracing::warn!("could not save the change summary: {e:#}");
9097            }
9098            return Ok(());
9099        }
9100        self.state
9101            .event("pr-summary", "no agent produced a change summary");
9102        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9103    }
9104
9105    /// One read-only rewrite request for a withheld description. The
9106    /// implementer's own conversation is tried first when it exists and can be
9107    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9108    /// writes it in a separate conversation, skipping the implementer's agent.
9109    /// The prompt carries the whole text, so no agent needs any memory. Each
9110    /// agent is asked at most once and the chain advances on an error, quota,
9111    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9112    /// that has run out of quota must not end the rewrite. Nothing is recorded
9113    /// as a handover.
9114    async fn rewrite_pr_text(
9115        &mut self,
9116        winner: &Candidate,
9117        pr: &PrMessage,
9118        violations: &[crate::github_text::Violation],
9119    ) -> Option<String> {
9120        let prompt = format!(
9121            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9122            pr.title, pr.body
9123        );
9124        let sessions = self.state.config.graph.sessions;
9125        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9126        // (spec, the resumed implementer's seat). A fallback seat is created
9127        // only when its turn comes, so an unused fallback leaves no record.
9128        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9129        let key = format!("impl-{}", winner.label);
9130        if let Some(spec) = self
9131            .state
9132            .config
9133            .agents
9134            .iter()
9135            .find(|a| a.id == winner.agent)
9136            .cloned()
9137            && let Some(existing) = self.state.seats.get(&key)
9138            && existing.agent == spec.id
9139            && agent::has_session(spec.kind, existing, sessions)
9140        {
9141            attempts.push((spec, Some(existing.clone())));
9142        }
9143        match agent::pick_chain(
9144            &self.state.config.agents,
9145            self.state.config.roles.language_judge.as_ref(),
9146            &agent::installed,
9147            "language_judge",
9148        ) {
9149            Ok(chain) => {
9150                for spec in chain {
9151                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9152                        continue;
9153                    }
9154                    attempts.push((spec, None));
9155                }
9156            }
9157            Err(e) if attempts.is_empty() => {
9158                tracing::warn!("no agent for the description rewrite: {e:#}");
9159                self.state
9160                    .event("github-text", "no rewrite agent available");
9161                return None;
9162            }
9163            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9164        }
9165        let prompts = self.state.config.prompts.clone();
9166        let cache = self.state.config.cache_dir();
9167        let run = self.state.id.clone();
9168        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9169            let resumed = resumed_seat.is_some();
9170            let seat = resumed_seat.unwrap_or_else(|| {
9171                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9172            });
9173            let github_spec = match agent::for_github_text(
9174                &spec,
9175                &self.state.config.graph.github_language,
9176                &winner.worktree,
9177            ) {
9178                Ok(spec) => spec,
9179                Err(e) => {
9180                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9181                    continue;
9182                }
9183            };
9184            let job = SeatJob {
9185                spec: github_spec,
9186                seat,
9187                cwd: winner.worktree.clone(),
9188                prompt: prompt.clone(),
9189                timeout: retry_budget(timeout_full, resumed),
9190                allow_write: false,
9191                sessions,
9192                artifacts: agent::artifacts_dir(&self.state.dir()),
9193                stem: if i == 0 {
9194                    "github-text-rewrite".to_owned()
9195                } else {
9196                    format!("github-text-rewrite-{}", spec.id)
9197                },
9198                handover: None,
9199            };
9200            let ctx = WaveCtx {
9201                run: &run,
9202                node: "github-text",
9203                prompts: &prompts,
9204                cache: cache.as_deref(),
9205                round: None,
9206                carry_seats: false,
9207                pauses: Some((&self.pause, &self.interrupt)),
9208            };
9209            let (seat, outcome) =
9210                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9211            self.state.seats.insert(seat.key.clone(), seat);
9212            match outcome {
9213                AgentOutcome::Ok(o)
9214                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9215                {
9216                    return Some(o.text);
9217                }
9218                AgentOutcome::ResourceBlocked(_) => return None,
9219                _ => {}
9220            }
9221        }
9222        None
9223    }
9224
9225    /// Abandon the standing posting-gate question, if any.
9226    fn retire_github_text_question(&mut self, why: &str) {
9227        let Some(id) = self
9228            .state
9229            .github_text
9230            .as_ref()
9231            .and_then(|g| g.question.clone())
9232        else {
9233            return;
9234        };
9235        let _ = ask::Questions::open().update(&id, |q| {
9236            q.abandon(why);
9237            Ok(())
9238        });
9239    }
9240
9241    /// Carry a withheld text to its end: file the question, park while it is
9242    /// open, and on an answer, silence or timeout pick the text to post. The
9243    /// decision is saved before it is used, so it applies once.
9244    async fn settle_withheld(
9245        &mut self,
9246        winner: &Candidate,
9247        pr: &PrMessage,
9248        mut g: crate::run::GithubTextAsk,
9249    ) -> Result<Option<PrMessage>> {
9250        use crate::github_text as gt;
9251        let store = ask::Questions::open();
9252        let timeout = self.state.config.graph.answer_timeout;
9253        if !g.resolved {
9254            let mut standing = g
9255                .question
9256                .as_ref()
9257                .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9258            if let Some(q) = standing.as_ref()
9259                && q.status.open()
9260                && !store.held_reply(&q.id)
9261                && gt::expired(q, timeout)
9262            {
9263                let why = format!("no answer within {}s of asking", timeout.max(1));
9264                standing = store
9265                    .update(&q.id, |q| {
9266                        q.abandon(&why);
9267                        Ok(())
9268                    })
9269                    .ok()
9270                    .map(|(q, ())| q);
9271            }
9272            let mut ask_again: Option<Option<Vec<String>>> = None;
9273            match standing {
9274                // Recorded but never written (a stop between the two saves),
9275                // or removed: ask again without spending the retry.
9276                None if g.asks <= 1 => {
9277                    g.asks = 0;
9278                    ask_again = Some(None);
9279                }
9280                None => {
9281                    g.resolved = true;
9282                    g.chosen_title = None;
9283                    g.choice = Some(GithubTextChoice::Fallback);
9284                }
9285                Some(q) if q.status.open() => {
9286                    self.park_on_github_text(&g)?;
9287                    return Ok(None);
9288                }
9289                Some(q) => match gt::read_reply(&q) {
9290                    gt::Reply::Fallback => {
9291                        g.resolved = true;
9292                        g.chosen_title = None;
9293                        g.choice = Some(GithubTextChoice::Fallback);
9294                    }
9295                    gt::Reply::Original => {
9296                        g.resolved = true;
9297                        g.chosen_title = None;
9298                        g.choice = Some(GithubTextChoice::Original);
9299                    }
9300                    gt::Reply::Rewrite => {
9301                        g.resolved = true;
9302                        g.chosen_title = None;
9303                        g.choice = Some(if g.rewrite.is_some() {
9304                            GithubTextChoice::Rewrite
9305                        } else {
9306                            GithubTextChoice::Fallback
9307                        });
9308                    }
9309                    gt::Reply::Title(text) => {
9310                        let first = text
9311                            .lines()
9312                            .map(str::trim)
9313                            .find(|l| !l.is_empty())
9314                            .unwrap_or("");
9315                        let decision = if first.is_empty() {
9316                            None
9317                        } else {
9318                            let candidate = PrMessage {
9319                                title: first.to_owned(),
9320                                body: String::new(),
9321                            };
9322                            self.judge_pr_language(&winner.worktree, &candidate).await
9323                        };
9324                        match gt::vet_title(
9325                            &text,
9326                            decision,
9327                            self.state.config.graph.github_english(),
9328                        ) {
9329                            Ok(title) => {
9330                                g.resolved = true;
9331                                g.chosen_title = Some(title);
9332                                g.choice = Some(GithubTextChoice::MyTitle);
9333                            }
9334                            Err(cats) if g.asks < 2 => {
9335                                ask_again = Some(Some(
9336                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9337                                ))
9338                            }
9339                            Err(_) => {
9340                                g.resolved = true;
9341                                g.chosen_title = None;
9342                                g.choice = Some(GithubTextChoice::Fallback);
9343                            }
9344                        }
9345                    }
9346                },
9347            }
9348            if let Some(rejected) = ask_again {
9349                // A retry (after a rejected replacement title) is still about
9350                // the original text: the same ORIGINAL / FALLBACK comparison.
9351                let w = gt::Withheld {
9352                    title: g.title,
9353                    body: g.body,
9354                    categories: g.categories.clone(),
9355                };
9356                // Named by the original text's fingerprint alone, so a resume
9357                // rewrites the same file and the detail stays identical.
9358                let artifact = match crate::run::write_artifact(
9359                    &self.state,
9360                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9361                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9362                ) {
9363                    Ok(p) => Some(p),
9364                    Err(e) => {
9365                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9366                        None
9367                    }
9368                };
9369                let (detail, comparison) = gt::question_content(
9370                    &w,
9371                    &pr.title,
9372                    &pr.body,
9373                    &self.state.id,
9374                    rejected.as_deref(),
9375                    artifact.as_deref(),
9376                    g.rewrite.as_ref(),
9377                );
9378                let mut q = ask::Question::new(
9379                    self.state.id.clone(),
9380                    gt::ASK_NODE.to_owned(),
9381                    gt::ASK_SEAT.to_owned(),
9382                    gt::question_summary(&self.state.config.graph.language, &w),
9383                    detail,
9384                    gt::question_choices(&w, g.rewrite.as_ref()),
9385                );
9386                q.posting_comparison = Some(comparison);
9387                q.answer_timeout = timeout;
9388                // Recorded before it is written: a stop in between leaves a
9389                // record naming a question the resume then files again.
9390                g.question = Some(q.id.clone());
9391                g.asks += 1;
9392                self.state.github_text = Some(g.clone());
9393                self.state.event(
9394                    "github-text",
9395                    format!("asking the owner about the withheld text ({})", q.short()),
9396                );
9397                self.state.save()?;
9398                store
9399                    .put_executable(&mut q)
9400                    .context("file the posting-gate question")?;
9401                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9402                    tracing::warn!(
9403                        "could not notify about posting-gate question {}: {e:#}",
9404                        q.short()
9405                    );
9406                }
9407                self.park_on_github_text(&g)?;
9408                return Ok(None);
9409            }
9410        }
9411        let resolved = g.chosen_title.clone();
9412        let original = g.choice == Some(GithubTextChoice::Original);
9413        // Field names only: the event log never records the text.
9414        self.state.event(
9415            "github-text",
9416            if original {
9417                format!(
9418                    "posting the original text for: {}",
9419                    match (g.title, g.body) {
9420                        (true, true) => "title, description",
9421                        (true, false) => "title",
9422                        _ => "description",
9423                    }
9424                )
9425            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9426                "posting the saved rewrite title and description".to_owned()
9427            } else if resolved.is_some() {
9428                "posting the owner's replacement title".to_owned()
9429            } else {
9430                "posting the neutral text for the withheld fields".to_owned()
9431            },
9432        );
9433        self.state.github_text = Some(g.clone());
9434        self.state.save()?;
9435        if g.choice == Some(GithubTextChoice::Rewrite)
9436            && let Some(r) = g.rewrite.as_ref()
9437        {
9438            return Ok(Some(PrMessage {
9439                title: r.title.clone(),
9440                body: r.body.clone(),
9441            }));
9442        }
9443        let w = gt::Withheld {
9444            title: g.title,
9445            body: g.body,
9446            categories: g.categories.clone(),
9447        };
9448        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9449        if let Some(t) = resolved {
9450            title = t;
9451        }
9452        // Every withheld field is now neutral or vetted English.
9453        let english = gt::LanguageDecision {
9454            title_english: true,
9455            body_english: true,
9456            body_complete: true,
9457        };
9458        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9459        if original {
9460            // The owner's explicit choice overrides the gate for the fields it
9461            // was made for; the others keep what the gate produced.
9462            if g.title {
9463                title = pr.title.clone();
9464            }
9465            if g.body {
9466                body = pr.body.clone();
9467            }
9468        }
9469        Ok(Some(PrMessage { title, body }))
9470    }
9471
9472    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9473        self.state.github_text = Some(g.clone());
9474        self.state.end_pass_parked(ParkCause::Question);
9475        self.state.parked = true;
9476        self.state.event(
9477            "github-text",
9478            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9479        );
9480        self.state.save()
9481    }
9482
9483    /// Ask `[roles] language_judge` about this text and record which source
9484    /// decided; the text itself is never recorded.
9485    async fn judge_pr_language(
9486        &mut self,
9487        cwd: &Path,
9488        pr: &PrMessage,
9489    ) -> Option<crate::github_text::LanguageDecision> {
9490        self.state.config.roles.language_judge.as_ref()?;
9491        if !self.state.config.graph.language_gate() {
9492            return None;
9493        }
9494        let decision =
9495            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9496        self.state.event(
9497            "github-text",
9498            if decision.is_some() {
9499                "language decided by the language judge"
9500            } else {
9501                "language judge unavailable; using built-in heuristics"
9502            },
9503        );
9504        decision
9505    }
9506
9507    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9508        Box::pin(self.merge_inner())
9509    }
9510
9511    async fn merge_inner(&mut self) -> Result<()> {
9512        if self.resource_halt()? || self.hold_base_sync().await? {
9513            return Ok(());
9514        }
9515        // Same reasoning as `gate`: ask the review and gate records directly
9516        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9517        // skip can rewrite on reentry to something that no longer says
9518        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9519        // so a hand-off (open findings, green verification) reaches merge
9520        // exactly like a genuinely clean round does.
9521        //
9522        // A run resumed mid-`land` never reaches here at all: `execute`
9523        // recognises `RunStatus::Landing` before it even calls `prep`, and
9524        // routes straight to `run_land` instead. That has to happen a level
9525        // up from this function, not with a check in here, because
9526        // `review_loop`'s own status recomputation (see its doc) runs
9527        // *before* `merge` on every reentry and would otherwise overwrite
9528        // the `Landing` marker with `Gating` before this node ever saw it.
9529        if self
9530            .state
9531            .base_sync
9532            .as_ref()
9533            .is_some_and(|s| s.conflict.is_some())
9534            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9535                != Some(RunStatus::Gating)
9536            // `gate_ran == false` is not "passed" - `gate` leaves it false
9537            // both before it has ever run and when its last attempt was
9538            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9539            // permission to merge on nothing but the review record. Only a
9540            // gate that actually ran may proceed; an intentional local
9541            // review can also finish with verification explicitly not configured.
9542            // `RunState::gate_status` computes that distinction.
9543            || !self.state.gate_allows_progress()
9544        {
9545            return Ok(());
9546        }
9547        // This node's own record, not `status`: `status == Ready` is not
9548        // unique to the harmless `MergeMode::None` path this line was
9549        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9550        // run's PR was closed without merging — and on that run `mode` is
9551        // still `Pr`, so a reentry that fell through here would push and
9552        // open a second pull request. `self.state.merge` is set exactly once
9553        // this node (or `land`) has already produced a verdict, under every
9554        // mode, which is what "already done" actually means here.
9555        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9556            return Ok(());
9557        }
9558        let Some(winner) = self.state.winner().cloned() else {
9559            return Ok(());
9560        };
9561        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9562        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9563            self.state.status = RunStatus::Blocked;
9564            self.state.event(
9565                "gate",
9566                "winner changed after gate; fresh verification required",
9567            );
9568            self.state.save()?;
9569            return Ok(());
9570        }
9571        let wait = self.repository_wait();
9572        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9573            return Ok(());
9574        }
9575        let repo = self.state.repo.clone();
9576        let base = self.state.base_branch.clone();
9577        let mode = self.state.config.merge.mode;
9578        let style = self.state.config.merge.style;
9579        let facts = if is_review_run(&self.state) {
9580            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9581            let start = review_base(
9582                &repo,
9583                &self.state.config.merge.remote,
9584                &base,
9585                &self.state.base_commit,
9586                &winner.branch,
9587            )
9588            .await;
9589            branch_facts(&repo, &start, &winner.branch).await
9590        } else {
9591            None
9592        };
9593        // `None` when the base could not be freshly read: then an adopted
9594        // pull request's title is left alone.
9595        let leaked = if is_review_run(&self.state) {
9596            leaked_subjects(&self.state, &winner.branch).await
9597        } else {
9598            Some(Vec::new())
9599        };
9600        // `None`: parked on the owner's word about a withheld title or body.
9601        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9602        // the resume arrives here again.
9603        let Some(pr) = self
9604            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9605            .await?
9606        else {
9607            return Ok(());
9608        };
9609        let message = pr.commit_message();
9610
9611        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9612        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9613            return Ok(());
9614        }
9615        let outcome = match mode {
9616            MergeMode::None => MergeOutcome {
9617                mode,
9618                ok: true,
9619                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9620                empty: false,
9621            },
9622            MergeMode::Pr | MergeMode::Local
9623                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9624            {
9625                MergeOutcome {
9626                    mode,
9627                    ok: false,
9628                    detail: empty_candidate_detail(&self.state, &base),
9629                    empty: true,
9630                }
9631            }
9632            MergeMode::Local => {
9633                let on = git::current_branch(&repo).await?;
9634                if on.as_deref() != Some(base.as_str()) {
9635                    MergeOutcome {
9636                        mode,
9637                        ok: false,
9638                        detail: format!(
9639                            "{} has {} checked out, not the base branch {base}",
9640                            repo.display(),
9641                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9642                        ),
9643                        empty: false,
9644                    }
9645                } else if !git::is_clean(&repo).await? {
9646                    MergeOutcome {
9647                        mode,
9648                        ok: false,
9649                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9650                        empty: false,
9651                    }
9652                } else {
9653                    let out = match style {
9654                        MergeStyle::Merge => {
9655                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9656                        }
9657                        MergeStyle::Squash => {
9658                            git::merge_squash(&repo, &winner.branch, &message).await?
9659                        }
9660                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9661                    };
9662                    MergeOutcome {
9663                        mode,
9664                        ok: out.ok(),
9665                        detail: if out.ok() { out.stdout } else { out.stderr },
9666                        empty: false,
9667                    }
9668                }
9669            }
9670            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9671                self.state.pr_publication_pending = true;
9672                self.state.save()?;
9673                let out =
9674                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9675                self.state.pr_publication_pending = out.is_err();
9676                MergeOutcome {
9677                    mode,
9678                    ok: out.is_ok(),
9679                    detail: match out {
9680                        Ok(url) => url,
9681                        Err(e) => format!("{e:#}"),
9682                    },
9683                    empty: false,
9684                }
9685            }
9686            MergeMode::Pr => {
9687                let remote = self.state.config.merge.remote.clone();
9688                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9689                let pushed = if self.state.foreign_checkout.is_some() {
9690                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9691                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9692                } else {
9693                    git::push(&winner.worktree, &remote, &winner.branch).await?
9694                };
9695                if !pushed.ok() {
9696                    MergeOutcome {
9697                        mode,
9698                        ok: false,
9699                        detail: pushed.stderr,
9700                        empty: false,
9701                    }
9702                } else {
9703                    if self.state.foreign_checkout.is_some() {
9704                        self.state.pr_pushed_head = Some(delivery_head.clone());
9705                    }
9706                    // A retry or resume of a run whose branch already has an
9707                    // open pull request adopts it rather than failing on a
9708                    // duplicate. Only this winner branch into this base:
9709                    // `branch_for` derives the name from the run id, so a
9710                    // different run's pull request never matches.
9711                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9712                    let out = match pr_merge_plan(found) {
9713                        PrPlan::Create => {
9714                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9715                                .await
9716                        }
9717                        PrPlan::Adopt { url, title } => {
9718                            self.state
9719                                .event("merge", format!("Pr: adopted open pull request {url}"));
9720                            if self.state.explicit_github_text_selection() {
9721                                // An explicit publication selection applies the prepared
9722                                // payload; acceptance alone is not publication proof.
9723                                land::set_pr_text_verbatim(
9724                                    &winner.worktree,
9725                                    &url,
9726                                    &pr.title,
9727                                    &pr.body,
9728                                )
9729                                .await
9730                                .map(|()| url)
9731                            } else {
9732                                if title != pr.title
9733                                    && (!is_review_run(&self.state)
9734                                        || leaked
9735                                            .as_deref()
9736                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9737                                    && let Err(e) =
9738                                        if self.state.github_text.as_ref().is_some_and(|g| {
9739                                            g.resolved
9740                                                && g.title
9741                                                && g.choice == Some(GithubTextChoice::Original)
9742                                        }) {
9743                                            // The owner chose to post this title unmodified.
9744                                            land::set_pr_title_verbatim(
9745                                                &winner.worktree,
9746                                                &url,
9747                                                &pr.title,
9748                                            )
9749                                            .await
9750                                        } else {
9751                                            land::set_pr_title(
9752                                                &mut self.state,
9753                                                &winner.worktree,
9754                                                &url,
9755                                                &pr.title,
9756                                            )
9757                                            .await
9758                                        }
9759                                {
9760                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9761                                    self.state
9762                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9763                                }
9764                                Ok(url)
9765                            }
9766                        }
9767                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9768                    };
9769                    match out {
9770                        Ok(url) => MergeOutcome {
9771                            mode,
9772                            ok: true,
9773                            detail: url,
9774                            empty: false,
9775                        },
9776                        Err(e) => MergeOutcome {
9777                            mode,
9778                            ok: false,
9779                            detail: e.to_string(),
9780                            empty: false,
9781                        },
9782                    }
9783                }
9784            }
9785        };
9786
9787        self.state.status = match (mode, outcome.ok) {
9788            (MergeMode::None, _) => RunStatus::Ready,
9789            (_, true) if outcome.empty => RunStatus::Ready,
9790            (MergeMode::Pr, true) => RunStatus::Ready,
9791            (_, true) => RunStatus::Merged,
9792            (_, false) => RunStatus::Blocked,
9793        };
9794        self.state.event(
9795            "merge",
9796            format!(
9797                "{:?}: {}",
9798                mode,
9799                outcome.detail.lines().next().unwrap_or("")
9800            ),
9801        );
9802        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9803            self.state.record_github_text_publication(&outcome.detail);
9804        }
9805        self.state.merge = Some(outcome);
9806        self.state.save()?;
9807
9808        // The PR is open and the run would historically stop here, leaving the
9809        // operator to watch checks, feed review comments back to a fixer, and
9810        // merge. That was done by hand six times in one session before this
9811        // existed. Opt-in, because merging is the one irreversible thing magi
9812        // can do to a repository.
9813        if self.state.config.graph.land
9814            && mode == MergeMode::Pr
9815            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9816        {
9817            self.run_land().await?;
9818        }
9819        // `run_land` may have left `status` at `Landing` - still waiting on
9820        // CI or the owner's approval, not actually settled - so this has to
9821        // read whatever `status` ended up as here, not the `Merged` this
9822        // function set a few lines up.
9823        self.settle_questions();
9824        Ok(())
9825    }
9826
9827    /// Enter `land`.
9828    ///
9829    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9830    /// a resumed run's re-entry. `land::land` itself is what serialises the
9831    /// two git-mutating moments inside the loop — the rebase push and
9832    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9833    /// to hold a lock across the whole call, and doing so would serialise
9834    /// this run's CI wait against a *different* run's land-approval resume
9835    /// in the same repository, which is exactly the "must not wait on
9836    /// another task" property the daemon's slot-freeing exists to give.
9837    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9838        Box::pin(self.run_land_inner())
9839    }
9840
9841    async fn run_land_inner(&mut self) -> Result<()> {
9842        crate::pr_head::ensure_owner(&self.state)?;
9843        let url = self
9844            .state
9845            .pr
9846            .as_ref()
9847            .map(|pr| pr.url.clone())
9848            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9849            .unwrap_or_default();
9850        let url = url.lines().next().unwrap_or("").trim().to_owned();
9851        if !url.starts_with("http") {
9852            return Ok(());
9853        }
9854        // A land failure is not a lost run: the work is on a branch and the
9855        // pull request is open, which is exactly where a human takes over.
9856        loop {
9857            match land::land(&mut self.state, &url).await {
9858                Ok(_)
9859                    if self
9860                        .state
9861                        .merge_repairs
9862                        .last()
9863                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9864                {
9865                    if self.repair_merge_review().await? {
9866                        continue;
9867                    }
9868                }
9869                Ok(_) if self.state.resource_block.is_some() => {
9870                    return self.stop_for_resource();
9871                }
9872                Ok(pr) if self.state.parked => {
9873                    // `land` already saved the parked marker; nothing here
9874                    // overrides `status` back to a terminal value while an
9875                    // approval is still outstanding.
9876                    let _ = pr;
9877                }
9878                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9879                    self.state.status = RunStatus::Superseded;
9880                    self.state.save()?;
9881                }
9882                Ok(pr) => {
9883                    self.state.status = match pr.state {
9884                        land::PrLifecycle::Merged => RunStatus::Merged,
9885                        _ => RunStatus::Blocked,
9886                    };
9887                    // Downstream of a confirmed merge only - see
9888                    // `bump::should_release_bump`'s own doc for why this one
9889                    // check covers all three of `land`'s success paths.
9890                    // Best-effort: the run already landed, so a failure here
9891                    // (the decision call, `gh`, `cargo`) is recorded and never
9892                    // turns a landed run into a failed one.
9893                    if bump::should_release_bump(self.state.status)
9894                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9895                    {
9896                        // The event is the run's own record. Not-eligible cases
9897                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9898                        // `Err` is a bump that was tried and failed:
9899                        // `after_merge` itself raises the operator notice for
9900                        // that, whether or not a release PR exists yet.
9901                        self.state
9902                            .event("bump", format!("release bump skipped: {e:#}"));
9903                    }
9904                    // Independent of the bump, and best-effort in the same way:
9905                    // findings the merge left open become follow-up tasks.
9906                    if self.state.status == RunStatus::Merged {
9907                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9908                    }
9909                    self.state.save()?;
9910                }
9911                Err(e) => {
9912                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9913                        return Err(e);
9914                    }
9915                    self.state.status = RunStatus::Blocked;
9916                    self.state.event("land", format!("gave up: {e}"));
9917                    self.state.save()?;
9918                }
9919            }
9920            break;
9921        }
9922        Ok(())
9923    }
9924
9925    /// One repair cycle stays under the existing driver/task claim. An
9926    /// interrupted fixer is rolled back and held rather than run twice.
9927    async fn repair_merge_review(&mut self) -> Result<bool> {
9928        use crate::run::MergeRepairPhase as Phase;
9929        crate::pr_head::ensure_owner(&self.state)?;
9930        let i = self.state.merge_repairs.len() - 1;
9931        let request = self.state.merge_repairs[i].clone();
9932        if request.phase == Phase::Fixing {
9933            let restored = match self.state.winner().cloned() {
9934                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9935                None => Err(anyhow::anyhow!("repair has no winning branch")),
9936            };
9937            let why = match restored {
9938                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9939                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9940                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9941            };
9942            self.hold_merge_repair(i, why)?;
9943            return Ok(false);
9944        }
9945        // A completed repair gate is a publication checkpoint. On reentry,
9946        // reject external changes before the ordinary gate path can replace
9947        // that evidence by verifying a new head. Requested repairs still
9948        // start their own review generation below.
9949        if request.phase == Phase::Reviewing && self.state.gate_ran {
9950            let winner = self
9951                .state
9952                .winner()
9953                .context("repair has no winning branch")?;
9954            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9955            if self.state.gate_head.as_deref() != Some(head.as_str())
9956                || !git::is_clean(&winner.worktree).await?
9957            {
9958                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9959                return Ok(false);
9960            }
9961        }
9962        let failure = if i >= self.state.config.graph.land_rounds
9963            || self.state.config.graph.review_rounds == 0
9964        {
9965            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9966        } else {
9967            None
9968        };
9969        if let Some(why) = failure {
9970            self.hold_merge_repair(i, why)?;
9971            return Ok(false);
9972        }
9973        if request.phase == Phase::Requested {
9974            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9975                Ok(pr) => pr,
9976                Err(e) => {
9977                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9978                    return Ok(false);
9979                }
9980            };
9981            self.state.merge_repairs[i].phase = Phase::Fixing;
9982            self.state.land_approval = None;
9983            self.state.land_armed_head = None;
9984            self.state.save()?;
9985            // Serialize the exact records, including finding ids/details,
9986            // reconsideration reasons, final votes and verification output.
9987            let evidence = serde_json::to_string_pretty(&request)?;
9988            let reason = format!(
9989                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9990            );
9991            let budget = self.state.config.graph.land_rounds;
9992            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9993            match result {
9994                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9995                Ok(land::Fixed::Deferred) => {
9996                    // An unavailable fixer did not spend this repair request.
9997                    // Restore partial fallback work before retrying it, as for
9998                    // an interrupted repair, without closing the request.
9999                    let worktree = self
10000                        .state
10001                        .winner()
10002                        .context("repair has no winning branch")?
10003                        .worktree
10004                        .clone();
10005                    self.graft_restore(&worktree, &request.head).await?;
10006                    self.state.merge_repairs[i].phase = Phase::Requested;
10007                    self.stop_for_resource()?;
10008                    return Ok(false);
10009                }
10010                Ok(land::Fixed::Failed(why)) => {
10011                    self.hold_merge_repair(
10012                        i,
10013                        format!(
10014                            "repair failed: {why}; inspect the branch and run an explicit review"
10015                        ),
10016                    )?;
10017                    return Ok(false);
10018                }
10019                Err(e) => {
10020                    self.hold_merge_repair(
10021                        i,
10022                        format!(
10023                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10024                        ),
10025                    )?;
10026                    return Ok(false);
10027                }
10028            }
10029            // Even no change gets a real new reviewer wave. The previous
10030            // evidence remains in the request; a former clean/concluded round
10031            // cannot short-circuit the new review or gate.
10032            self.begin_merge_repair_review(i)?;
10033        }
10034        self.review_loop().await?;
10035        self.gate().await?;
10036        if self.resource_halt()? || self.state.parked {
10037            return Ok(false);
10038        }
10039        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10040            != Some(RunStatus::Gating)
10041            || !self.state.gate_allows_progress()
10042        {
10043            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10044            return Ok(false);
10045        }
10046        // Review and gate may have added commits after the initial repair.
10047        // Publish exactly their final head to the existing PR, with a lease.
10048        let winner = self
10049            .state
10050            .winner()
10051            .cloned()
10052            .context("repair has no winning branch")?;
10053        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10054        if self.state.gate_head.as_deref() != Some(after.as_str())
10055            || !git::is_clean(&winner.worktree).await?
10056        {
10057            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10058            return Ok(false);
10059        }
10060        let expected = self
10061            .state
10062            .pr_pushed_head
10063            .as_deref()
10064            .unwrap_or(&request.head)
10065            .to_owned();
10066        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10067        // Push the checked commit itself: a local ref moving after the check
10068        // must not change which commit the forge receives.
10069        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10070            return Ok(false);
10071        }
10072        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10073        let refspec = format!("{after}:refs/heads/{public}");
10074        let push = git::git_raw(
10075            &winner.worktree,
10076            &["push", &lease, &self.state.config.merge.remote, &refspec],
10077        )
10078        .await;
10079        match push {
10080            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10081            Ok(out) => {
10082                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10083                return Ok(false);
10084            }
10085            Err(e) => {
10086                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10087                return Ok(false);
10088            }
10089        }
10090        self.state.merge_repairs[i].phase = Phase::Complete;
10091        self.state.status = RunStatus::Landing;
10092        self.state.event(
10093            "repair",
10094            "new review and gate completed; fresh owner merge decision required",
10095        );
10096        self.state.save()?;
10097        Ok(true)
10098    }
10099
10100    fn merge_repair_artifacts(&self) -> PathBuf {
10101        let artifacts = agent::artifacts_dir(&self.state.dir());
10102        match self
10103            .state
10104            .merge_repairs
10105            .last()
10106            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10107        {
10108            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10109            None => artifacts,
10110        }
10111    }
10112
10113    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10114        if let Some(e2e) = self.state.head_e2e.take() {
10115            self.state
10116                .verification_history
10117                .push(crate::run::VerificationEvidence {
10118                    superseded_by: None,
10119                    archived_at: Some(Timestamp::now()),
10120                    fixes_spent: Some(self.state.gate_fixes.len()),
10121                    gate_head: None,
10122                    gate_ran: false,
10123                    gate: Vec::new(),
10124                    e2e: Some(e2e),
10125                });
10126        }
10127        self.state.reviews.clear();
10128        self.state.gate.clear();
10129        self.state.gate_ran = false;
10130        self.state.gate_head = None;
10131        self.state.gate_fixes.clear();
10132        self.state.gate_repair_e2e = None;
10133        self.state.pre_gate.clear();
10134        self.state.pre_gate_commit = None;
10135        self.state.contested_handoff = None;
10136        self.state.land_reruns = None;
10137        self.state.land_approval = None;
10138        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10139        self.state.status = RunStatus::Reviewing;
10140        self.state.save()
10141    }
10142
10143    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10144        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10145        self.state.merge_repairs[index].failure = Some(why.clone());
10146        self.state.status = RunStatus::Blocked;
10147        self.state.event("repair", why);
10148        self.state.save()
10149    }
10150
10151    // -------------------------------------------------------------- helpers
10152
10153    /// Fetch or create a seat, keeping its conversation across nodes.
10154    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10155        if let Some(existing) = self.state.seats.get(key)
10156            && existing.agent == agent
10157        {
10158            return existing.clone();
10159        }
10160        // A seat that changes agent mints its session id from the agent too,
10161        // like a handover: the old agent's uuid is already taken by the CLI.
10162        let fresh = if self.state.seats.contains_key(key) {
10163            handover_seat(key, agent, self.state.next_seat_seed())
10164        } else {
10165            SeatState::new(key, agent, self.state.seed)
10166        };
10167        self.state.seats.insert(key.to_owned(), fresh.clone());
10168        fresh
10169    }
10170
10171    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10172    /// seat to another roster agent, in which case that agent. Nodes that
10173    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10174    /// reconsideration) must keep talking to whoever answered it, not slip
10175    /// back to the agent that failed it.
10176    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10177        match self.state.seats.get(key) {
10178            Some(s) if s.agent != spec.id => {
10179                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10180            }
10181            _ => spec,
10182        }
10183    }
10184
10185    /// A candidate rendered for judging, with the leak policy applied.
10186    fn view(&self, c: &Candidate) -> CandidateView {
10187        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10188            .unwrap_or_default();
10189        let (patch, _) = blind::sanitize_patch(
10190            &format!("candidate {} patch", c.label),
10191            &raw,
10192            &self.state.config.blind,
10193        );
10194        CandidateView {
10195            label: c.label,
10196            branch: c.branch.clone(),
10197            summary: c.summary.clone(),
10198            stat: c.stat.clone(),
10199            patch,
10200        }
10201    }
10202
10203    /// The full candidate set as prompt text, for seats with no live session.
10204    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10205        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10206        prompt::judge(
10207            "(see above)",
10208            &views,
10209            self.roles.judges.len(),
10210            base_short,
10211            "en",
10212        )
10213    }
10214
10215    /// The final-vote prompt with everything a seat that has no session of its
10216    /// own needs: the candidates, the seat's own ranking and reasons, and the
10217    /// anonymised deliberation it took part in (only when there was one, so a
10218    /// handed-over seat never sees more than the seat it replaces did). The
10219    /// `Final vote` heading stays first.
10220    fn vote_prompt_full(
10221        &self,
10222        j: usize,
10223        labels: &[char],
10224        language: &str,
10225        candidates: &[Candidate],
10226        base_short: &str,
10227    ) -> String {
10228        let mut text = format!(
10229            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10230            prompt::final_vote(labels, language),
10231            self.state.instruction,
10232            self.candidate_block(candidates, base_short)
10233        );
10234        if let Some(own) = self
10235            .state
10236            .judgements
10237            .get(j)
10238            .filter(|r| !r.ranking.is_empty())
10239        {
10240            let reasons = own
10241                .reasons
10242                .iter()
10243                .map(|(k, v)| format!("- {k}: {v}"))
10244                .collect::<Vec<_>>()
10245                .join("\n");
10246            text.push_str(&format!(
10247                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10248                own.ranking.iter().collect::<String>(),
10249                if reasons.is_empty() {
10250                    ""
10251                } else {
10252                    ", because:\n"
10253                }
10254            ));
10255        }
10256        if !self.state.deliberation.is_empty() {
10257            text.push_str("\n# What was argued before this vote\n");
10258            for t in self.transcript(&[], j) {
10259                text.push_str(&format!(
10260                    "\n## {}{}\n\n{}\n",
10261                    t.who,
10262                    if t.is_self { " (you)" } else { "" },
10263                    t.body.trim()
10264                ));
10265            }
10266        }
10267        text
10268    }
10269
10270    /// Anonymised transcript for judge `self_idx`.
10271    ///
10272    /// The initial rankings are always the opening statements. Seeding them
10273    /// only when no turn had been taken yet meant every judge after the first
10274    /// argued against a single voice instead of against the actual split — the
10275    /// disagreement is the information, so it is always on the table.
10276    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10277        let mut turns = Vec::new();
10278        for j in &self.state.judgements {
10279            if j.ranking.is_empty() {
10280                continue;
10281            }
10282            let reasons = j
10283                .reasons
10284                .iter()
10285                .map(|(k, v)| format!("- {k}: {v}"))
10286                .collect::<Vec<_>>()
10287                .join("\n");
10288            turns.push(Turn {
10289                who: format!("Judge {} (opening ranking)", j.judge),
10290                is_self: j.judge == self_idx + 1,
10291                body: format!(
10292                    "Ranked {}{}{reasons}",
10293                    j.ranking.iter().collect::<String>(),
10294                    if reasons.is_empty() {
10295                        ""
10296                    } else {
10297                        ", because:\n"
10298                    }
10299                ),
10300            });
10301        }
10302        for t in self
10303            .state
10304            .deliberation
10305            .iter()
10306            .flat_map(|r| r.turns.iter())
10307            .chain(current)
10308        {
10309            turns.push(Turn {
10310                who: format!("Judge {}", t.judge),
10311                is_self: t.judge == self_idx + 1,
10312                body: t.body.clone(),
10313            });
10314        }
10315        turns
10316    }
10317}
10318
10319/// Does this seat still hold the context a follow-up prompt would rely on?
10320fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10321    agent::has_session(spec.kind, seat, sessions)
10322}
10323
10324/// The next entry in `roster` after `start`, never wrapping back to the
10325/// front, whose id is not in `tried` yet.
10326///
10327/// Starts one past `start` rather than at the front of `roster`: `start` is
10328/// the seat's own original position, and a seat whose candidate slot already
10329/// sits on the roster's second entry must fall through to the third next, not
10330/// restart at the first — which is very likely a different candidate's own
10331/// agent already. Never wraps back past `start`, for the same reason: an
10332/// entry earlier in the roster than the seat's own position is almost
10333/// certainly some *other* candidate slot's own agent, and once the tail of
10334/// the roster is exhausted there are no more untried agents for *this* seat
10335/// to fall through to — the caller's fallback chain ends there, exactly as
10336/// "no further untried agents remain in the list for that seat" asks for.
10337///
10338/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10339/// the same id twice (an operator's `roles.implementers` typo, or a
10340/// `[[agents]]` list reused across roles) must not let
10341/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10342/// over `roster` either finds an untried id or runs out, so this always
10343/// terminates regardless of duplicates.
10344fn next_untried_in_roster<'a>(
10345    roster: &'a [AgentSpec],
10346    start: usize,
10347    tried: &BTreeSet<String>,
10348) -> Option<&'a AgentSpec> {
10349    roster
10350        .get(start + 1..)?
10351        .iter()
10352        .find(|s| !tried.contains(&s.id))
10353}
10354
10355/// The successor for a seat, shared by all four seat kinds (implement, judge,
10356/// review, advise). `others` holds the ids that *currently* occupy the other
10357/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10358/// sees them), so roster entries beyond the seat count act as spares: a failed
10359/// seat goes to an agent no other seat holds whenever the roster permits.
10360///
10361/// `carried` is `Some` only for the review loop's carried failure history.
10362/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10363/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10364/// whole roster: untried and not another seat's occupant; (3) the plain walk
10365/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10366/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10367/// a better outcome than an empty seat, and it keeps the answer to "is there
10368/// a successor at all" exactly what it was before occupants were considered,
10369/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10370/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10371/// carried failure is a cheaper bet than doubling an agent on the panel, and
10372/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10373/// the same round are handled one at a time, so a seat later in the batch
10374/// sees an earlier one's new occupant but not yet its own freed agent.
10375fn pick_successor<'a>(
10376    roster: &'a [AgentSpec],
10377    start: usize,
10378    tried: &BTreeSet<String>,
10379    carried: Option<&BTreeSet<String>>,
10380    others: &BTreeSet<String>,
10381) -> Option<&'a AgentSpec> {
10382    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10383    roster
10384        .get(start + 1..)
10385        .and_then(|tail| {
10386            tail.iter()
10387                .filter(free)
10388                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10389        })
10390        .or_else(|| {
10391            carried
10392                .is_some()
10393                .then(|| roster.iter().find(free))
10394                .flatten()
10395        })
10396        .or_else(|| match carried {
10397            Some(c) => next_for_seat(roster, start, tried, c),
10398            None => next_untried_in_roster(roster, start, tried),
10399        })
10400}
10401
10402/// The next agent for a seat that carries its failure history across rounds
10403/// (the review loop). `round_tried` is this round's own bound and starts
10404/// empty every round; `carried_failed` only decides priority.
10405///
10406/// First: an id walking forward from `start`, never wrapping, that is neither
10407/// tried this round nor failed in an earlier one. Only when that is exhausted
10408/// does it rescue: the first roster id (in roster order, so this one *does*
10409/// look before `start`) not yet tried this round, which is by then a carried
10410/// failure. Each id is rescued at most once per round, so it cannot loop.
10411fn next_for_seat<'a>(
10412    roster: &'a [AgentSpec],
10413    start: usize,
10414    round_tried: &BTreeSet<String>,
10415    carried_failed: &BTreeSet<String>,
10416) -> Option<&'a AgentSpec> {
10417    roster
10418        .get(start + 1..)?
10419        .iter()
10420        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10421        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10422}
10423
10424/// Where a reviewer seat starts a round: the agent that last answered it when
10425/// it is still on the roster and not marked failed, else the spec's own agent
10426/// unless it failed, else the next roster agent that has not failed, else the
10427/// spec's own agent again (the whole roster failed). Ids no longer on the
10428/// roster are ignored. An empty roster has no handover, so the spec stands.
10429fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10430    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10431        return spec;
10432    };
10433    let ok = |id: &str| !h.failed.contains(id);
10434    if let Some(last) = h.last_ok.as_deref()
10435        && ok(last)
10436        && let Some(s) = roster.iter().find(|s| s.id == last)
10437    {
10438        return s.clone();
10439    }
10440    if ok(&spec.id) {
10441        return spec;
10442    }
10443    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10444    roster
10445        .get(from + 1..)
10446        .into_iter()
10447        .flatten()
10448        .chain(roster.iter())
10449        .find(|s| ok(&s.id))
10450        .cloned()
10451        .unwrap_or(spec)
10452}
10453
10454/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10455/// seed so a CLI that mints its session id up front (`--session-id`) never
10456/// reuses the uuid the previous agent already opened under the same seat key.
10457pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10458    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10459}
10460
10461/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10462/// so the classifier below and the code that builds the message cannot drift.
10463const TIMED_OUT: &str = "timed out";
10464
10465impl FailClass {
10466    /// `None` for an answer; otherwise how the turn failed.
10467    fn of(out: &AgentOutcome) -> Option<Self> {
10468        match out {
10469            AgentOutcome::Ok(_) => None,
10470            AgentOutcome::Quota(_) => Some(Self::Quota),
10471            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10472            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10473            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10474            // Not a failure of the agent. Callers check for it first; `None`
10475            // here means "no class to hand over on".
10476            AgentOutcome::ResourceBlocked(_) => None,
10477        }
10478    }
10479
10480    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10481    fn stem_word(&self) -> &'static str {
10482        match self {
10483            Self::Quota => "quota",
10484            _ => "handover",
10485        }
10486    }
10487}
10488
10489/// The message's first line with its variable parts removed — digit runs and
10490/// path-like tokens — so "exit code 2" and "exit code 7" read
10491/// as one kind of failure.
10492fn failure_signature(msg: &str) -> String {
10493    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10494    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10495    // the historical machine key while the process diagnosis uses readable prose.
10496    if let Some(code) = line
10497        .strip_prefix("process exited with no usable output (exit code ")
10498        .and_then(|s| s.strip_suffix(')'))
10499        .and_then(|s| s.parse::<i32>().ok())
10500    {
10501        line = format!("exited with some({code}) and no usable output");
10502    } else if matches!(
10503        line.as_str(),
10504        "process exited with no usable output (no exit code reported)"
10505            | "process exited with no usable output (exit code unavailable)"
10506    ) {
10507        line = "exited with none and no usable output".to_owned();
10508    }
10509    let mut out = Vec::new();
10510    for word in line.split_whitespace() {
10511        if word.contains('/') || word.contains('\\') {
10512            out.push("<path>".to_owned());
10513            continue;
10514        }
10515        let mut w = String::new();
10516        let mut in_digits = false;
10517        for c in word.chars() {
10518            if c.is_ascii_digit() {
10519                if !in_digits {
10520                    w.push('#');
10521                }
10522                in_digits = true;
10523            } else {
10524                in_digits = false;
10525                w.push(c);
10526            }
10527        }
10528        out.push(w);
10529    }
10530    out.join(" ").chars().take(120).collect()
10531}
10532
10533/// Whether a seat that just failed with `cur` may go to the next roster agent.
10534/// A quota or a timeout always may. Any other failure may not when the agent
10535/// before it failed the same way: an error the prompt causes would otherwise
10536/// walk the whole roster. `prev` is the class of the immediately preceding
10537/// agent's failure, so a quota or timeout in between breaks the run of
10538/// identical failures by itself.
10539fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10540    match cur {
10541        FailClass::Quota | FailClass::Timeout => true,
10542        FailClass::Other(_) => prev != Some(cur),
10543    }
10544}
10545
10546/// A short human reason for a failed outcome, for the handover record.
10547fn fail_reason(out: &AgentOutcome) -> String {
10548    match out {
10549        AgentOutcome::Ok(_) => String::new(),
10550        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10551        AgentOutcome::Dropped(o) => format!(
10552            "the CLI dropped the stream ({})",
10553            o.dropped
10554                .as_ref()
10555                .map(|d| d.why.as_str())
10556                .unwrap_or("it ended without delivering its answer")
10557        ),
10558        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10559            e.lines().next().unwrap_or("").chars().take(160).collect()
10560        }
10561    }
10562}
10563
10564/// Note one handover in the run: the structured record and, in the timeline,
10565/// the sentence a person reads. A quota keeps the wording it always had.
10566pub(crate) fn record_handover(
10567    state: &mut RunState,
10568    node: &str,
10569    seat: &str,
10570    from: &str,
10571    to: &str,
10572    class: &FailClass,
10573    reason: &str,
10574) {
10575    let message = if *class == FailClass::Quota {
10576        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10577    } else {
10578        format!("{seat}: handed over {from} -> {to} ({reason})")
10579    };
10580    state.event(node, message);
10581    state.handovers.push(Handover {
10582        at: Timestamp::now(),
10583        node: node.to_owned(),
10584        seat: seat.to_owned(),
10585        from: from.to_owned(),
10586        to: to.to_owned(),
10587        reason: reason.to_owned(),
10588    });
10589}
10590
10591/// Did this reply report running a command whose own CLI never confirmed an
10592/// exit status?
10593///
10594/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10595/// command *finished* (see that type's own doc), so this can only be `true`
10596/// for a command whose completion event carried no readable exit code — not
10597/// for one that simply is not mentioned at all. That is the one signal this
10598/// crate can read, from the same record `state.jobs` renders, about a reply
10599/// standing next to work its own CLI cannot vouch for finishing; it is
10600/// deliberately not a check on the exit code's *value* (a fixer legitimately
10601/// runs a command that fails mid-iteration before it succeeds) and not a
10602/// guess at a command still running in the background (which emits no event
10603/// at all, and so leaves no evidence here to find).
10604fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10605    commands.iter().any(|c| c.exit_code.is_none())
10606}
10607
10608/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10609/// trusted as a verified no-op — the adoption guard's own text-level half.
10610///
10611/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10612/// timed out): a marker only earns the benefit of the doubt from a turn the
10613/// CLI itself vouches for finishing properly, the same house style
10614/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10615/// *fix* report to for `commands`. A candidate that timed out, exited
10616/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10617/// is, whatever prose it wrote — this returns `None` before it ever looks at
10618/// `text`. The remaining guards (the tree really is empty, the evidence is
10619/// non-empty) are the caller's: this only reads what the reply *claimed*.
10620fn verified_noop_claim(
10621    usable: bool,
10622    commands: &[agent::CommandEvidence],
10623    text: &str,
10624) -> Option<String> {
10625    (usable && !has_unconfirmed_command(commands))
10626        .then(|| verdict::verified_noop(text))
10627        .flatten()
10628}
10629
10630fn short(commit: &str) -> String {
10631    commit.chars().take(7).collect()
10632}
10633
10634fn make_executable(path: &Path) -> Result<()> {
10635    #[cfg(unix)]
10636    {
10637        use std::os::unix::fs::PermissionsExt as _;
10638        let mut perms = std::fs::metadata(path)?.permissions();
10639        perms.set_mode(0o755);
10640        std::fs::set_permissions(path, perms)?;
10641    }
10642    #[cfg(not(unix))]
10643    {
10644        let _ = path;
10645    }
10646    Ok(())
10647}
10648
10649/// What every seat in one batch shares: where the answers are attributed, the
10650/// prompt overlay they inherit, and the build cache they are told to use.
10651///
10652/// A struct rather than four more parameters: `wave` also needs the run's
10653/// state (to record who is answering right now) and the attempt number, and
10654/// eight positional arguments is both unreadable and a clippy error.
10655struct WaveCtx<'a> {
10656    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10657    /// paid for it.
10658    run: &'a str,
10659    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10660    node: &'a str,
10661    prompts: &'a Prompts,
10662    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10663    cache: Option<&'a Path>,
10664    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10665    /// `JobRecord::round`. `None` for every other node.
10666    round: Option<usize>,
10667    /// Carry each seat's failed-agent history across waves (the review loop
10668    /// only): start-of-round priority and handover choice read
10669    /// [`RunState::seat_history`], and every answer or failure writes it.
10670    carry_seats: bool,
10671    /// The run's park / interrupt requests, so a wait for a shared resource
10672    /// can be abandoned the moment the operator asks to park.
10673    pauses: Option<(&'a Pause, &'a Pause)>,
10674}
10675
10676/// Shared scheduling policy for operations and spawned seat invocations.
10677fn repository_wait(
10678    state: &RunState,
10679    pauses: Option<(&Pause, &Pause)>,
10680) -> crate::repo_lock::WaitPolicy {
10681    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10682    let inherited = crate::repo_lock::WaitPolicy::current();
10683    crate::repo_lock::WaitPolicy::new(
10684        Duration::from_secs(state.config.graph.timeout_resource_wait),
10685        move || {
10686            inherited.cancelled()
10687                || pauses
10688                    .as_ref()
10689                    .is_some_and(|(a, b)| a.parked() || b.parked())
10690        },
10691    )
10692}
10693
10694fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10695    if error.is::<crate::repo_lock::ResourceBlocked>() {
10696        state.resource_block = Some(crate::run::ResourceBlock {
10697            cause: crate::run::ResourceBlockCause::Other,
10698            node: "repository".into(),
10699            kind: if error.to_string().contains("disk pressure") {
10700                "disk pressure"
10701            } else if error.to_string().contains("disk admission") {
10702                "disk admission"
10703            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10704                "incremental cache admission"
10705            } else {
10706                "repository coordination"
10707            }
10708            .into(),
10709            detail: error.to_string(),
10710            at: Timestamp::now(),
10711        });
10712        state.event("resource", error.to_string());
10713    }
10714}
10715
10716async fn coordinate_operation(
10717    state: &mut RunState,
10718    cwd: &Path,
10719    wait: &crate::repo_lock::WaitPolicy,
10720) -> Result<Option<crate::repo_lock::SeatLease>> {
10721    let result =
10722        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10723            .await;
10724    if let Err(e) = &result {
10725        record_repository_block(state, e);
10726    }
10727    result
10728}
10729
10730/// Run one job, honouring the parallelism budget.
10731async fn run_one(
10732    job: SeatJob,
10733    sem: Arc<Semaphore>,
10734    ctx: &WaveCtx<'_>,
10735    state: &mut RunState,
10736    attempt: usize,
10737) -> (SeatState, AgentOutcome) {
10738    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10739        .await
10740        .pop()
10741        .expect("one job in, one result out");
10742    (seat, out)
10743}
10744
10745/// Run every job concurrently, capped by the semaphore, preserving order.
10746///
10747/// Every seat in the batch is recorded into [`RunState::active`] before the
10748/// wave starts and cleared as each answer lands, so the run's own record says
10749/// who is still being waited on rather than only who finished.
10750async fn wave(
10751    jobs: Vec<SeatJob>,
10752    sem: Arc<Semaphore>,
10753    ctx: &WaveCtx<'_>,
10754    state: &mut RunState,
10755    attempt: usize,
10756) -> Vec<(usize, SeatState, AgentOutcome)> {
10757    let WaveCtx {
10758        run,
10759        node,
10760        prompts,
10761        cache,
10762        round,
10763        carry_seats: _,
10764        pauses,
10765    } = *ctx;
10766    for job in &jobs {
10767        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10768    }
10769    if let Err(e) = state.save() {
10770        // A failed persist of "who is answering right now" must not abort the
10771        // wave: the seats are already being asked, and the alternative is
10772        // losing the answers to save a status line nobody may even be
10773        // watching.
10774        tracing::warn!("could not persist in-progress seats: {e:#}");
10775    }
10776    // Hold the shared build cache's lease for the whole batch, not per job:
10777    // several candidates (an implement wave) or a fixer legitimately share
10778    // one cache concurrently within this run, and that stays untouched — a
10779    // single lease taken once for the whole wave and released once it is
10780    // done is what stops a *different* borrower (another run's own wave, its
10781    // e2e/gate, a human's `magi review`) from interleaving a build into the
10782    // same directory while this one is in flight. Best-effort, not
10783    // all-or-nothing: a wave that cannot get the lease within its own
10784    // longest job's budget still runs — an hour of paid implementer calls is
10785    // not thrown away over cache contention — but every write-allowed seat
10786    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10787    // below), the same fallback a read-only seat always gets, rather than
10788    // building into a directory this run was never granted. The identity
10789    // record is still invalidated below either way, so the next tracked
10790    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10791    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10792    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10793    // never borrowed from a job's agent budget (and never charged to it either,
10794    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10795    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10796    let cache_guard = if let Some(cache_dir) = cache {
10797        if jobs_had_a_writer {
10798            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10799            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10800            match acquire_cache_lease(
10801                state,
10802                cache_dir,
10803                &owner,
10804                resource_limit,
10805                node,
10806                node,
10807                &cancel,
10808            )
10809            .await
10810            {
10811                Ok(Some(g)) => Some(g),
10812                // Parking: leave without spawning anything. The run parks at
10813                // the next boundary and the seats are asked afresh on resume.
10814                Ok(None) => {
10815                    if let Some(block) = &mut state.resource_block {
10816                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10817                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10818                        };
10819                    }
10820                    return block_wave(
10821                        jobs,
10822                        state,
10823                        node,
10824                        attempt,
10825                        "parked while waiting for the shared build cache",
10826                    );
10827                }
10828                // An unreadable lease is not contention: keep the old
10829                // fallback (no shared cache for this wave's seats).
10830                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10831                Err(e) => {
10832                    // Writing seats may not build into a cache this run does
10833                    // not hold, and an agent that cannot be given one is not
10834                    // asked at all: the run stops resumably instead.
10835                    let why = format!("{e:#}");
10836                    state.event(
10837                        "resource",
10838                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10839                    );
10840                    return block_wave(jobs, state, node, attempt, &why);
10841                }
10842            }
10843        } else {
10844            None
10845        }
10846    } else {
10847        None
10848    };
10849    // Every job keeps its whole budget: the wait above was not the agent's.
10850    // Restart the seats' clocks so "started" means "being spawned now".
10851    state.seats_spawning(node, attempt);
10852    if let Err(e) = state.save() {
10853        tracing::warn!("could not persist the start of a wave: {e:#}");
10854    }
10855    // Queueing behind this wave's own seats for a parallelism slot is normal
10856    // (each of them is bounded by its own agent timeout); only a wait beyond
10857    // all of them plus the resource limit means the slots never came back.
10858    let slot_limit = resource_limit
10859        + jobs
10860            .iter()
10861            .map(|j| j.timeout)
10862            .max()
10863            .unwrap_or_default()
10864            .saturating_mul(jobs.len() as u32);
10865    let mut set = tokio::task::JoinSet::new();
10866    // Seats announce themselves here the moment they hold a parallelism
10867    // permit, i.e. just before their CLI is spawned.
10868    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10869    let overlay = prompts.overlay(node);
10870    let repository_wait = repository_wait(state, pauses);
10871    for (i, mut job) in jobs.into_iter().enumerate() {
10872        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10873        if cache.is_some() {
10874            job.prompt.push('\n');
10875            job.prompt
10876                .push_str(&prompt::build_cache_note(node, job.allow_write));
10877        }
10878        let sem = Arc::clone(&sem);
10879        let run = run.to_owned();
10880        let node = node.to_owned();
10881        // Only implementers were told about the task's attachments, so only
10882        // their seats get the directory widened for reading.
10883        let attachments = if node == "implement" {
10884            state.attachments.clone()
10885        } else {
10886            Vec::new()
10887        };
10888        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10889        // `prompt::build_cache_note`'s doc for why setting it anyway is
10890        // exactly how a sandboxed reviewer's write refusal got reported as a
10891        // defect in the patch, not a property of its own seat. And a
10892        // write-allowed one is handed it only when the lease above was
10893        // actually acquired: a wave that could not get it (`cache_guard` is
10894        // `None`, see its own comment) must not send seats to build into a
10895        // directory this run does not hold - that is the exact concurrent,
10896        // unmanaged-write race this module exists to prevent, not something
10897        // "proceeding anyway" is allowed to reintroduce.
10898        let cache = cache
10899            .filter(|_| job.allow_write && cache_guard.is_some())
10900            .map(Path::to_path_buf);
10901        let spawned_tx = spawned_tx.clone();
10902        let repository_wait = repository_wait.clone();
10903        let coordination = state.config.coordination.clone();
10904        let fetched_base = state.fetched_base.clone();
10905        set.spawn(async move {
10906            // A wait for a parallelism slot is scheduling, not the agent: it
10907            // is bounded by its own limit and ends as a resource block, never
10908            // as a timeout of the agent that was not started.
10909            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10910                Ok(p) => p,
10911                Err(_) => {
10912                    return (
10913                        i,
10914                        job.seat,
10915                        AgentOutcome::ResourceBlocked(format!(
10916                            "no parallel slot became free within {}s",
10917                            slot_limit.as_secs()
10918                        )),
10919                        false,
10920                    );
10921                }
10922            };
10923            spawned_tx.send(job.seat.key.clone()).ok();
10924            let mut seat = job.seat;
10925            let out = repository_wait
10926                .scope(agent::invoke(
10927                    &job.spec,
10928                    &mut seat,
10929                    &Invocation {
10930                        coordination: coordination.as_ref(),
10931                        fetched_base: fetched_base.as_ref(),
10932                        cwd: &job.cwd,
10933                        prompt: &job.prompt,
10934                        timeout: job.timeout,
10935                        allow_write: job.allow_write,
10936                        sessions: job.sessions,
10937                        artifacts: &job.artifacts,
10938                        stem: &job.stem,
10939                        run: &run,
10940                        node: &node,
10941                        cache_dir: cache.as_deref(),
10942                        attachments: &attachments,
10943                        writable: &[],
10944                    },
10945                ))
10946                .await;
10947            let interrupted = out
10948                .as_ref()
10949                .err()
10950                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10951            let out = match out {
10952                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10953                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10954                // Billed work the CLI failed to hand over is not an ordinary
10955                // failure, but its text is the CLI's raw error JSON, not an
10956                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10957                // read it as one by forgetting to check. `usable()` is always
10958                // false here (dropped implies an empty response), so this has
10959                // to be checked before the catch-all `Failed` below or the
10960                // one shape this exists for is lost with the rest.
10961                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10962                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10963                Ok(o) => AgentOutcome::Failed(format!(
10964                    "process exited with no usable output ({})",
10965                    crate::notices::exit_status(o.exit_code)
10966                )),
10967                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10968                    AgentOutcome::ResourceBlocked(e.to_string())
10969                }
10970                Err(e) => AgentOutcome::Failed(e.to_string()),
10971            };
10972            (i, seat, out, interrupted)
10973        });
10974    }
10975    drop(spawned_tx);
10976    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10977    loop {
10978        let joined = tokio::select! {
10979            Some(key) = spawned_rx.recv() => {
10980                // The clock of this seat starts now, not when it queued.
10981                state.seat_spawned(&key);
10982                if let Err(e) = state.save() {
10983                    tracing::warn!("could not persist a seat's start: {e:#}");
10984                }
10985                continue;
10986            }
10987            joined = set.join_next() => joined,
10988        };
10989        let Some(joined) = joined else {
10990            break;
10991        };
10992        let (i, seat, out, interrupted) = match joined {
10993            Ok(v) => v,
10994            // No seat to clear: a panicked task never reported which one it
10995            // was. The defensive sweep below this loop is what stops that
10996            // seat's `active` entry from surviving forever.
10997            Err(e) => {
10998                tracing::error!("agent task panicked: {e}");
10999                continue;
11000            }
11001        };
11002        state.seat_finished(&seat.key);
11003        if interrupted
11004            && node == "implement"
11005            && let Some(label) = state
11006                .candidates
11007                .iter()
11008                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11009                .map(|candidate| candidate.label)
11010        {
11011            state.candidate_preservation.insert(
11012                label,
11013                "interrupted implementation awaiting persistence".into(),
11014            );
11015            state.interrupted_implementations.insert(label);
11016        }
11017        if let AgentOutcome::ResourceBlocked(why) = &out
11018            && state.resource_block.is_none()
11019        {
11020            state.resource_block = Some(crate::run::ResourceBlock {
11021                cause: crate::run::ResourceBlockCause::Other,
11022                node: node.to_owned(),
11023                kind: if why.contains("disk pressure") {
11024                    "disk pressure"
11025                } else if why.contains("disk admission") {
11026                    "disk admission"
11027                } else if why.contains(crate::incremental::BLOCKED) {
11028                    "incremental cache admission"
11029                } else if why.contains("repository coordination") {
11030                    "repository coordination"
11031                } else {
11032                    "parallel slot"
11033                }
11034                .to_owned(),
11035                detail: why.clone(),
11036                at: Timestamp::now(),
11037            });
11038            let activity = if interrupted {
11039                "agent execution interrupted"
11040            } else {
11041                "no agent was started"
11042            };
11043            state.event("resource", format!("{node}: {activity} - {why}"));
11044        }
11045        record_jobs(state, node, round, &seat.key, &out);
11046        if let Err(e) = state.save() {
11047            tracing::warn!("could not persist a seat's completion: {e:#}");
11048        }
11049        if collected.len() <= i {
11050            collected.resize_with(i + 1, || None);
11051        }
11052        collected[i] = Some((i, seat, out));
11053    }
11054    // Belt-and-braces for the panic branch above: every seat this exact batch
11055    // started shares this `(node, attempt)` pair, and every seat that finished
11056    // normally already cleared itself, so anything left tagged with it here
11057    // can only be a panicked task's leftover. Cleared unconditionally rather
11058    // than left to read as still answering forever.
11059    if state
11060        .active
11061        .values()
11062        .any(|a| a.node == node && a.attempt == attempt)
11063    {
11064        state
11065            .active
11066            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11067        if let Err(e) = state.save() {
11068            tracing::warn!("could not persist the end of a wave: {e:#}");
11069        }
11070    }
11071    // Whether or not the lease above was actually held, several worktrees
11072    // may just have built into the cache with nothing here able to name one
11073    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11074    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11075    // clean it might not have strictly needed; trusting a stale match would
11076    // cost it a wrong answer.
11077    if let Some(cache_dir) = cache
11078        && jobs_had_a_writer
11079    {
11080        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11081    }
11082    if let Some(guard) = cache_guard {
11083        guard.release();
11084    }
11085    collected.into_iter().flatten().collect()
11086}
11087
11088/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11089/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11090/// this is data collection, not the fix-specific completion contract in
11091/// [`Runner::continue_fix_report`], and applies regardless of which node
11092/// asked.
11093///
11094/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11095/// evidence from; `Failed` does not, and correctly contributes nothing — a
11096/// timeout or crash is not itself evidence about a command the seat may have
11097/// started.
11098fn record_jobs(
11099    state: &mut RunState,
11100    node: &str,
11101    round: Option<usize>,
11102    seat: &str,
11103    out: &AgentOutcome,
11104) {
11105    let commands: &[agent::CommandEvidence] = match out {
11106        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11107        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11108    };
11109    let checked_at = Timestamp::now();
11110    for c in commands {
11111        state.jobs.push(JobRecord {
11112            node: node.to_owned(),
11113            round,
11114            seat: seat.to_owned(),
11115            id: c.id.clone(),
11116            description: c.description.clone(),
11117            checked_at,
11118            status: match c.exit_code {
11119                Some(0) => JobStatus::Completed,
11120                Some(_) => JobStatus::Failed,
11121                None => JobStatus::Unknown,
11122            },
11123            exit_code: c.exit_code,
11124            result_summary: c.result_summary.clone(),
11125            source: c.source.clone(),
11126        });
11127    }
11128}
11129
11130/// Is a review round clean, given how many reviewer seats answered against
11131/// how many the round expected?
11132///
11133/// A seat that never answered (timeout, crash, unparsable output) is not a
11134/// seat that read the patch and found nothing — treating it as such is
11135/// exactly the bug this function exists to close. Under the default `block`
11136/// policy a missing seat can never be clean; `warn` still requires the seats
11137/// that *did* answer to have found nothing blocking and verification to be
11138/// green.
11139///
11140/// `quota_missing` narrows that `block` default for exactly one cause of
11141/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11142/// a session limit lifts by the very next round buys nothing — the seat is
11143/// asked again with the same quota — so once every missing seat is accounted
11144/// for by a quota loss (and at least one seat *did* answer, so a decision has
11145/// something to rest on) the round is decided on the panel that could answer,
11146/// same as `warn` would. A panel that lost every seat to quota is not
11147/// decided here: `answered == 0` falls through to the existing `block`
11148/// fallback so a fully collapsed panel still waits rather than landing on no
11149/// review at all.
11150fn round_is_clean(
11151    blocking: usize,
11152    e2e_ok: bool,
11153    answered: usize,
11154    expected: usize,
11155    quota_missing: usize,
11156    policy: IncompleteReviewPolicy,
11157) -> bool {
11158    if blocking != 0 || !e2e_ok {
11159        return false;
11160    }
11161    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11162        return true;
11163    }
11164    answered > 0 && expected - answered <= quota_missing
11165}
11166
11167/// The review loop's own conclusion, derived entirely from its persisted
11168/// round records and the round budget that produced them — never from
11169/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11170/// recomputes the identical answer regardless of what an earlier node in the
11171/// same walk, or a previous walk, did to `status`.
11172///
11173/// `None` while more rounds remain to try, including when review never ran
11174/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11175/// gone clean, or the budget is spent, or the tree has stopped moving (see
11176/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11177///
11178/// - An incomplete panel that raised nothing is missing input, not a
11179///   verified tree — never a hand-off candidate, whatever verification said
11180///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11181/// - Otherwise, green e2e on the last round hands off (see
11182///   [`Runner::stop_reviewing`]); red e2e blocks.
11183///
11184/// A last round whose own verification is still `ResourceBlocked` — magi
11185/// itself never got a command to run, not evidence the patch is broken —
11186/// is neither: this returns `None` for it too, the same as "more rounds
11187/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11188/// handling of that shape) instead of this cheap recomputation guessing a
11189/// verdict a real attempt never produced.
11190pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11191    if max_rounds == 0
11192        || reviews.last().is_some_and(|r| {
11193            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11194        })
11195    {
11196        return Some(RunStatus::Gating);
11197    }
11198    let last = reviews.last()?;
11199    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11200    if reviews.len() < max_rounds && !stagnant {
11201        return None;
11202    }
11203    if last.incomplete() && last.blocking == 0 {
11204        return Some(RunStatus::Blocked);
11205    }
11206    if last.e2e_status() == E2eStatus::ResourceBlocked
11207        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11208    {
11209        return None;
11210    }
11211    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11212        RunStatus::Gating
11213    } else {
11214        RunStatus::Blocked
11215    })
11216}
11217
11218/// How long a re-ask may take, given the budget the first attempt had.
11219///
11220/// A `nudged` retry is a request to restate an answer the seat has already
11221/// worked out: it carries no new work, so it does not deserve the original
11222/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11223/// 133 seconds while a third sat for over ten minutes on a resumed session
11224/// holding 410 KB of prior output - and because the retry had inherited the
11225/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11226/// judging round whose other seats were long finished.
11227///
11228/// A quarter of the budget, with a floor so that a deliberately short timeout
11229/// does not collapse to nothing. A retry that re-sends the whole prompt
11230/// (because the seat kept no context) is the original job again, and keeps the
11231/// original budget.
11232pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11233    if nudged {
11234        (full / 4).max(Duration::from_secs(120)).min(full)
11235    } else {
11236        full
11237    }
11238}
11239
11240/// Run a wave and parse each reply, re-asking the seats whose reply was
11241/// unusable.
11242///
11243/// The re-ask is a nudge rather than the whole prompt again when the seat still
11244/// holds its conversation, which is the difference between a cheap retry and
11245/// paying for the entire candidate set twice.
11246///
11247/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11248/// successor in `roster` is handed to it instead of being re-asked: the
11249/// handover is the retry. A seat with no successor left (a single-agent
11250/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11251/// the asks to one seat in one node number at most
11252/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11253/// is asked once (a dropped stream is nudged first), and only the last agent
11254/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11255/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11256/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11257/// other failure stops the chain when the previous agent failed the same way
11258/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11259/// [`has_context`] is false and the job's own full prompt and full budget are
11260/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11261/// intermediate ones are not losses) and is returned as a failure like any
11262/// other absent seat — the caller decides whether the panel still has a
11263/// quorum. An empty `roster` disables handover: failures are nudged as they
11264/// always were, and a quota is simply lost. A reply that fails to parse or
11265/// validate is the prompt's doing and is only ever nudged, never handed over.
11266///
11267/// The returned [`SeatState`] names the agent that answered (or tried last).
11268#[allow(clippy::too_many_arguments)]
11269async fn ask_json_wave<T>(
11270    jobs: Vec<SeatJob>,
11271    sem: Arc<Semaphore>,
11272    retries: usize,
11273    roster: &[AgentSpec],
11274    ctx: &WaveCtx<'_>,
11275    losses: &mut Vec<QuotaLoss>,
11276    state: &mut RunState,
11277    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11278) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11279where
11280    T: serde::de::DeserializeOwned + Send + 'static,
11281{
11282    ask_wave_with(
11283        jobs,
11284        sem,
11285        retries,
11286        roster,
11287        ctx,
11288        losses,
11289        state,
11290        &|text: &str| {
11291            let v = verdict::extract_json::<T>(text)?;
11292            validate(&v)?;
11293            Ok(v)
11294        },
11295    )
11296    .await
11297}
11298
11299/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11300/// a node whose answer is prose (deliberation) shares the same handover,
11301/// failure classification, quota bookkeeping and bounds instead of a copy.
11302///
11303/// A seat handed to another roster agent is sent the job's `handover` prompt
11304/// (when it has one) rather than `prompt`: the new agent has no session, so a
11305/// resume-style prompt would be incomplete. That holds for the handover ask
11306/// and for every nudge to that agent whose `has_context` is false.
11307#[allow(clippy::too_many_arguments)]
11308async fn ask_wave_with<T>(
11309    jobs: Vec<SeatJob>,
11310    sem: Arc<Semaphore>,
11311    retries: usize,
11312    roster: &[AgentSpec],
11313    ctx: &WaveCtx<'_>,
11314    losses: &mut Vec<QuotaLoss>,
11315    state: &mut RunState,
11316    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11317) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11318where
11319    T: Send + 'static,
11320{
11321    let n = jobs.len();
11322    let originals: Vec<SeatJob> = jobs;
11323    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11324    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11325    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11326    // answer, N once it has gone through N nudges. Read back once this
11327    // returns, so a caller building a history record (`ReviewRecord`) can
11328    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11329    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11330    // field's own doc.
11331    let mut nudges: Vec<usize> = vec![0; n];
11332    // The agent now occupying each seat, the ids it has already been through,
11333    // where in the roster the walk began, the class of the last failure, and
11334    // the stem word of a handover not yet asked (full prompt, full budget).
11335    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11336    let mut tried: Vec<BTreeSet<String>> = specs
11337        .iter()
11338        .map(|s| BTreeSet::from([s.id.clone()]))
11339        .collect();
11340    let starts: Vec<usize> = specs
11341        .iter()
11342        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11343        .collect();
11344    let carry = ctx.carry_seats && !roster.is_empty();
11345    // Carried across rounds: ids that failed the seat earlier, and how the
11346    // last failure went (so a repeat of it is not handed over again).
11347    let carried: Vec<BTreeSet<String>> = originals
11348        .iter()
11349        .map(|j| {
11350            state
11351                .seat_history
11352                .get(&j.seat.key)
11353                .filter(|_| carry)
11354                .map(|h| h.failed.clone())
11355                .unwrap_or_default()
11356        })
11357        .collect();
11358    let mut prev: Vec<Option<FailClass>> = originals
11359        .iter()
11360        .map(|j| {
11361            state
11362                .seat_history
11363                .get(&j.seat.key)
11364                .filter(|_| carry)
11365                .and_then(|h| h.last_fail.clone())
11366        })
11367        .collect();
11368    let next_agent =
11369        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11370            let others: BTreeSet<String> = specs
11371                .iter()
11372                .enumerate()
11373                .filter(|(j, _)| *j != i)
11374                .map(|(_, s)| s.id.clone())
11375                .collect();
11376            pick_successor(
11377                roster,
11378                starts[i],
11379                tried,
11380                carry.then(|| &carried[i]),
11381                &others,
11382            )
11383            .cloned()
11384        };
11385    let mut fresh: Vec<Option<String>> = vec![None; n];
11386    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11387    let mut pending: Vec<usize> = (0..n).collect();
11388
11389    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11390    // asks: an agent with a successor is asked once and handed over, and only
11391    // a seat with no successor spends `retries` nudges on the same agent. This
11392    // only guarantees the loop's own termination whatever those say.
11393    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11394    for round in 0..max_rounds {
11395        if pending.is_empty() {
11396            break;
11397        }
11398        let mut batch = Vec::with_capacity(pending.len());
11399        let mut renudged: Vec<&str> = Vec::new();
11400        for &i in &pending {
11401            let src = &originals[i];
11402            // A seat now held by another agent than the job named has no
11403            // session of its own: it gets the full-context prompt whenever
11404            // it is asked in full (the handover ask, a nudge it cannot
11405            // resume).
11406            let full: &str = match &src.handover {
11407                Some(h) if specs[i].id != src.spec.id => h,
11408                _ => &src.prompt,
11409            };
11410            // The prompt and the budget are one decision: a nudge restates
11411            // finished work, a re-sent prompt redoes it.
11412            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11413                (
11414                    full.to_owned(),
11415                    src.timeout,
11416                    format!("{}-{word}-{}", src.stem, specs[i].id),
11417                )
11418            } else if nudges[i] == 0 {
11419                (full.to_owned(), src.timeout, src.stem.clone())
11420            } else {
11421                renudged.push(src.seat.key.as_str());
11422                let why = done[i]
11423                    .as_ref()
11424                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11425                    .unwrap_or_else(|| "no parsable answer".to_owned());
11426                let nudge = prompt::nudge(&why);
11427                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11428                let prompt = if nudged {
11429                    nudge
11430                } else {
11431                    format!("{full}\n\n---\n\n{nudge}")
11432                };
11433                (
11434                    prompt,
11435                    retry_budget(src.timeout, nudged),
11436                    format!("{}-retry{}", src.stem, nudges[i]),
11437                )
11438            };
11439            batch.push(SeatJob {
11440                spec: specs[i].clone(),
11441                seat: seats[i].clone(),
11442                cwd: src.cwd.clone(),
11443                prompt,
11444                timeout,
11445                allow_write: src.allow_write,
11446                sessions: src.sessions,
11447                artifacts: src.artifacts.clone(),
11448                stem,
11449                handover: None,
11450            });
11451        }
11452
11453        if !renudged.is_empty() {
11454            state.event(
11455                ctx.node,
11456                format!("retry {round}: re-asking {}", renudged.join(", ")),
11457            );
11458        }
11459        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11460        let mut still = Vec::new();
11461        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11462            seats[i] = seat;
11463            // No agent was asked: nothing to hand over, nudge, remember or
11464            // charge. The caller sees `resource_block` and stops the node.
11465            if let AgentOutcome::ResourceBlocked(why) = &out {
11466                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11467                continue;
11468            }
11469            let class = FailClass::of(&out);
11470            // A dropped stream is nudged first (the conversation is still
11471            // there to pick up); only a seat whose nudges are spent hands over.
11472            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11473                && nudges[i] < retries
11474                && !roster.is_empty();
11475            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11476                let next = should_hand_over(prev[i].as_ref(), &cur)
11477                    .then(|| next_agent(i, &tried[i], &specs))
11478                    .flatten();
11479                if carry {
11480                    let h = state
11481                        .seat_history
11482                        .entry(originals[i].seat.key.clone())
11483                        .or_default();
11484                    h.failed.insert(specs[i].id.clone());
11485                    h.last_fail = Some(cur.clone());
11486                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11487                        h.last_ok = None;
11488                    }
11489                    // Saved before the next agent is asked, so a restart in
11490                    // between does not forget who failed.
11491                    if let Err(e) = state.save() {
11492                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11493                    }
11494                }
11495                if let Some(next) = next {
11496                    record_handover(
11497                        state,
11498                        ctx.node,
11499                        &originals[i].seat.key,
11500                        &specs[i].id,
11501                        &next.id,
11502                        &cur,
11503                        &fail_reason(&out),
11504                    );
11505                    tried[i].insert(next.id.clone());
11506                    prev[i] = Some(cur.clone());
11507                    seats[i] =
11508                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11509                    specs[i] = next;
11510                    fresh[i] = Some(cur.stem_word().to_owned());
11511                    nudges[i] = 0;
11512                    done[i] = Some(Err(anyhow::anyhow!(
11513                        "handed over after: {}",
11514                        fail_reason(&out)
11515                    )));
11516                    still.push(i);
11517                    continue;
11518                }
11519            }
11520            if carry
11521                && !nudge_first
11522                && let Some(cur) = class.clone()
11523            {
11524                // The chain ended here (no successor, or a repeated failure).
11525                let h = state
11526                    .seat_history
11527                    .entry(originals[i].seat.key.clone())
11528                    .or_default();
11529                h.failed.insert(specs[i].id.clone());
11530                h.last_fail = Some(cur);
11531            }
11532            let parsed = match out {
11533                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11534                AgentOutcome::Quota(o) => {
11535                    last_quota[i] = o.quota.clone();
11536                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11537                }
11538                // Not a parseable answer: the nudge loop re-asks it, which is
11539                // exactly what a dropped stream needs. Just don't hand its raw
11540                // error JSON to `extract_json`.
11541                AgentOutcome::Dropped(o) => {
11542                    let why = o
11543                        .dropped
11544                        .as_ref()
11545                        .map(|d| d.why.as_str())
11546                        .unwrap_or("the CLI ended the stream without delivering its answer");
11547                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11548                }
11549                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11550                    Err(anyhow::anyhow!(e))
11551                }
11552            };
11553            let quota = class == Some(FailClass::Quota);
11554            let failed = parsed.is_err();
11555            if !failed && matches!(ctx.node, "judge" | "vote") {
11556                // A validated answer restores this seat's quorum presence,
11557                // including on the normal path after an all-seat block.
11558                state
11559                    .quota
11560                    .retain(|loss| loss.seat != originals[i].seat.key);
11561            }
11562            done[i] = Some(parsed);
11563            if carry && !failed {
11564                let h = state
11565                    .seat_history
11566                    .entry(originals[i].seat.key.clone())
11567                    .or_default();
11568                h.failed.remove(&specs[i].id);
11569                h.last_ok = Some(specs[i].id.clone());
11570                h.last_fail = None;
11571            }
11572            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11573            // fail the same way; and never re-ask a seat that already parsed.
11574            // A failed agent that still has a successor is not re-asked
11575            // either: the handover was its remedy and has just been refused
11576            // (the chain stops on a repeated failure class). A seat with no
11577            // successor left (a single-agent roster, the roster's tail, or an
11578            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11579            let agent_failure = class.is_some()
11580                && !nudge_first
11581                && !roster.is_empty()
11582                && next_agent(i, &tried[i], &specs).is_some();
11583            if failed && !quota && !agent_failure && nudges[i] < retries {
11584                nudges[i] += 1;
11585                still.push(i);
11586            }
11587        }
11588        pending = still;
11589    }
11590
11591    // One loss per seat whose chain ended on a quota: the intermediate ones
11592    // were absorbed by a handover and are not losses.
11593    for (i, q) in last_quota.into_iter().enumerate() {
11594        if let Some(quota) = q {
11595            losses.push(QuotaLoss::from_report(
11596                originals[i].seat.key.clone(),
11597                ctx.node,
11598                Some(&quota),
11599            ));
11600        }
11601    }
11602
11603    // A stalled quorum already has its own recovery checkpoint. Refresh
11604    // its missing seats before deciding whether it can resume.
11605    if state.status != RunStatus::Stalled
11606        && ctx.node != "deliberate"
11607        && n > 0
11608        && losses.len() == n
11609        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11610    {
11611        block_machine_outage(state, ctx.node, losses);
11612        state.quota.append(losses);
11613    }
11614
11615    seats
11616        .into_iter()
11617        .zip(done)
11618        .zip(nudges)
11619        .map(|((seat, res), attempts)| {
11620            (
11621                seat,
11622                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11623                attempts,
11624            )
11625        })
11626        .collect()
11627}
11628
11629/// A node with no available agent has no task verdict. Preserve the node
11630/// checkpoint and record why a subsequent explicit resume is needed.
11631pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11632    let mut auth: Vec<_> = losses
11633        .iter()
11634        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11635        .map(|loss| loss.agent.clone())
11636        .collect();
11637    auth.sort();
11638    auth.dedup();
11639    let (kind, detail) = if auth.is_empty() {
11640        (
11641            "agent usage limit",
11642            "All seats reached their agent usage limits; resume after usage is available"
11643                .to_owned(),
11644        )
11645    } else {
11646        (
11647            "agent authentication",
11648            format!(
11649                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11650                auth.join(", ")
11651            ),
11652        )
11653    };
11654    state.resource_block = Some(crate::run::ResourceBlock {
11655        node: node.to_owned(),
11656        kind: kind.to_owned(),
11657        cause: crate::run::ResourceBlockCause::Other,
11658        detail: detail.clone(),
11659        at: Timestamp::now(),
11660    });
11661    state.event(node, detail);
11662}
11663
11664/// Did the wave end without asking any agent (see [`block_wave`])?
11665fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11666    results
11667        .iter()
11668        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11669}
11670
11671/// Settle a wave that never spawned anything: every job answers
11672/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11673/// else about the seat (session, history, quota) is touched.
11674fn block_wave(
11675    jobs: Vec<SeatJob>,
11676    state: &mut RunState,
11677    node: &str,
11678    attempt: usize,
11679    why: &str,
11680) -> Vec<(usize, SeatState, AgentOutcome)> {
11681    state.resource_wait = None;
11682    state
11683        .active
11684        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11685    if let Err(e) = state.save() {
11686        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11687    }
11688    jobs.into_iter()
11689        .enumerate()
11690        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11691        .collect()
11692}
11693
11694/// Acquire the shared build cache's lease, waiting out contention within
11695/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11696/// unbounded wait is never acceptable).
11697///
11698/// A first, non-blocking check happens before ever waiting; if it finds the
11699/// lease busy, that fact is logged as a `verify` event *and* flushed with
11700/// [`RunState::save`] immediately — not only once the wait finally succeeds
11701/// or gives up — so a `magi show` run by a different process while this one
11702/// is still waiting reads a `run.json` that says so, rather than whatever it
11703/// looked like before the wait started. The same applies to the terminal
11704/// failure: logged and saved before this returns `Err`, so a caller that
11705/// could not get the lease at all still leaves a legible record of why.
11706async fn acquire_cache_lease(
11707    state: &mut RunState,
11708    cache_dir: &Path,
11709    owner: &crate::cache::Owner,
11710    budget: Duration,
11711    node: &str,
11712    context: &str,
11713    cancel: &(dyn Fn() -> bool + Sync),
11714) -> Result<Option<crate::cache::Guard>> {
11715    let home = crate::run::home();
11716    let started = Instant::now();
11717    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11718        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11719        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11720        Err(e) => {
11721            state.event(
11722                "verify",
11723                format!("{context}: could not check the shared build cache: {e:#}"),
11724            );
11725            if let Err(e2) = state.save() {
11726                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11727            }
11728            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11729        }
11730    };
11731    state.event(
11732        "verify",
11733        format!(
11734            "{context}: waiting for the shared build cache at {} ({})",
11735            cache_dir.display(),
11736            busy.describe()
11737        ),
11738    );
11739    // Seats of this node are waiting, not running: say so, so the countdown a
11740    // reader sees only ever measures an agent that has been spawned.
11741    state.resource_wait = Some(crate::run::ResourceWait {
11742        node: context.to_owned(),
11743        kind: "shared build cache".to_owned(),
11744        holder: busy.describe(),
11745        since: Timestamp::now(),
11746        limit_secs: budget.as_secs(),
11747    });
11748    if let Err(e) = state.save() {
11749        tracing::warn!("could not persist a cache wait: {e:#}");
11750    }
11751    let remaining = budget.saturating_sub(started.elapsed());
11752    let waited = crate::cache::wait_for_unless(
11753        &home,
11754        cache_dir,
11755        owner,
11756        remaining,
11757        Duration::from_secs(5),
11758        cancel,
11759    )
11760    .await;
11761    state.resource_wait = None;
11762    let cause = match &waited {
11763        Ok(Some(_)) => None,
11764        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11765        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11766            Some(crate::run::ResourceBlockCause::Timeout {
11767                waited_secs: started.elapsed().as_secs(),
11768                limit_secs: budget.as_secs(),
11769            })
11770        }
11771        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11772    };
11773    if let Some(cause) = cause {
11774        let detail = match &waited {
11775            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11776            Err(error) => format!("{error:#}"),
11777        };
11778        state.resource_block = Some(crate::run::ResourceBlock {
11779            cause,
11780            node: node.to_owned(),
11781            kind: "shared build cache".into(),
11782            detail,
11783            at: Timestamp::now(),
11784        });
11785        state.event(
11786            "resource",
11787            state.resource_block.as_ref().unwrap().description(),
11788        );
11789    }
11790    if let Err(error) = state.save() {
11791        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11792    }
11793    waited
11794}
11795
11796/// The lease state could not be read at all (unreadable lease file, a
11797/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11798/// known to be contended, so a wave degrades to running without a shared cache
11799/// instead of stopping on a hold that [`crate::triage`] could never release.
11800#[derive(Debug)]
11801struct CacheCheckError(String);
11802
11803impl std::fmt::Display for CacheCheckError {
11804    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11805        f.write_str(&self.0)
11806    }
11807}
11808
11809impl std::error::Error for CacheCheckError {}
11810
11811/// Keep a specific acquisition outcome when the caller adds command evidence.
11812fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11813    state
11814        .resource_block
11815        .get_or_insert_with(|| crate::run::ResourceBlock {
11816            cause: crate::run::ResourceBlockCause::Other,
11817            node: node.to_owned(),
11818            kind: "shared build cache".into(),
11819            detail: format!("{error:#}"),
11820            at: Timestamp::now(),
11821        });
11822    if let Err(error) = state.save() {
11823        tracing::warn!("could not persist cache resource block: {error:#}");
11824    }
11825}
11826
11827/// Run `body` — a verify command batch — while holding the shared build
11828/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11829/// never interleave with another borrower's build against the same
11830/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11831/// timeout, or a human's own `magi review`. See the `cache` module doc for
11832/// why this matters more than Cargo's own per-target locking covers — two
11833/// *different* worktrees building the same package name/version into one
11834/// cache directory is a staleness bug, not a lock contention one.
11835///
11836/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11837/// agent waves do. Repository scheduling also has its own bounded,
11838/// cancellable resource-wait policy. Contention must not consume the
11839/// verification budget: `body` receives the full `budget` after both leases
11840/// are acquired and the cache is fresh. Without a shared cache, only
11841/// repository coordination is acquired before handing over the full budget.
11842///
11843/// A lease that cannot be acquired within the resource wait is reported as a single
11844/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11845/// verification — the same shape a spawn failure already takes in
11846/// [`run_commands`], so a caller need not special-case it.
11847#[allow(clippy::too_many_arguments)]
11848async fn with_cache_lease<'s, F, Fut>(
11849    state: &'s mut RunState,
11850    cache_dir: Option<&Path>,
11851    node: &str,
11852    seat: &str,
11853    worktree: &Path,
11854    head: &str,
11855    budget: Duration,
11856    context: &str,
11857    wait: &crate::repo_lock::WaitPolicy,
11858    body: F,
11859) -> Result<(Vec<CommandOutcome>, bool)>
11860where
11861    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11862    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11863{
11864    // Match editing waves: cache first, repository second. In particular,
11865    // pre_gate must never hold the repository while waiting for a wave's cache.
11866    let started = Instant::now();
11867    let home = crate::run::home();
11868    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11869    let guard = if let Some(cache_dir) = cache_dir {
11870        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11871        match acquire_cache_lease(
11872            state,
11873            cache_dir,
11874            &owner,
11875            resource_limit,
11876            node,
11877            context,
11878            &|| wait.cancelled(),
11879        )
11880        .await
11881        .and_then(|g| {
11882            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11883        }) {
11884            Ok(g) => Some(g),
11885            Err(e) => {
11886                record_cache_error(state, node, &e);
11887                return Ok((
11888                    vec![CommandOutcome {
11889                        command: "(waiting for the shared build cache)".into(),
11890                        code: None,
11891                        output_tail: e.to_string(),
11892                        duration_ms: started.elapsed().as_millis() as u64,
11893                        resource_blocked: true,
11894                    }],
11895                    false,
11896                ));
11897            }
11898        }
11899    } else {
11900        None
11901    };
11902    let _coordination = coordinate_operation(state, worktree, wait).await?;
11903    if let Some(cache_dir) = cache_dir {
11904        let identity = crate::cache::Identity::new(worktree, head);
11905        if let Err(e) = crate::cache::ensure_fresh(
11906            &home,
11907            cache_dir,
11908            &identity,
11909            &state.config.verify.cargo_manifests(),
11910        ) {
11911            record_cache_error(state, node, &e);
11912            state.event(
11913                "verify",
11914                format!(
11915                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11916                    worktree.display(),
11917                    short(head)
11918                ),
11919            );
11920            if let Some(guard) = guard {
11921                guard.release();
11922            }
11923            return Ok((
11924                vec![CommandOutcome {
11925                    command: "(confirming the shared build cache is fresh)".into(),
11926                    code: None,
11927                    output_tail: e.to_string(),
11928                    duration_ms: started.elapsed().as_millis() as u64,
11929                    resource_blocked: true,
11930                }],
11931                false,
11932            ));
11933        }
11934    }
11935    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11936    if !timed_out_pids.is_empty() {
11937        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11938    }
11939    // Drop repository coordination before releasing the cache, reversing the
11940    // acquisition order. A cancelled/error future also drops both guards.
11941    drop(_coordination);
11942    if let Some(guard) = guard {
11943        guard.release();
11944    }
11945    Ok((outcomes, retried))
11946}
11947
11948/// Poll `pids` — commands [`run_commands`] reports as still running when its
11949/// own timeout elapsed — until every one is confirmed gone, or
11950/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11951///
11952/// Real confirmation where confirmation is possible, not a substitute for
11953/// full process-tree observation: a grandchild the timed-out process spawned
11954/// and that survives independently of it is invisible to a pid check the
11955/// same way it always was, and continuing to observe and collect *that*
11956/// stays a different piece of work with its own owner. This only narrows a
11957/// fixed blind wait into an actual check of the pids this process does know
11958/// about.
11959async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11960    wait_for_pids_with(
11961        pids,
11962        crate::proc::pid_alive,
11963        LEASE_RELEASE_POLL,
11964        LEASE_RELEASE_MAX_WAIT,
11965    )
11966    .await;
11967}
11968
11969/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11970/// interval and ceiling supplied by the caller, so the polling *logic* -
11971/// returns as soon as every pid reports dead, gives up at the ceiling
11972/// otherwise - is testable on millisecond durations without asking the real
11973/// OS about a pid at all.
11974async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11975    pids: &[u32],
11976    alive: F,
11977    poll: Duration,
11978    max_wait: Duration,
11979) {
11980    let deadline = Instant::now() + max_wait;
11981    loop {
11982        if pids.iter().all(|&pid| !alive(pid)) {
11983            return;
11984        }
11985        if Instant::now() >= deadline {
11986            return;
11987        }
11988        tokio::time::sleep(poll).await;
11989    }
11990}
11991
11992/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11993/// admission that it could not even get a verify command to run, as opposed
11994/// to evidence the command actually produced? A caller that would otherwise
11995/// read a resource-blocked outcome as a red command must check this first:
11996/// see [`Runner::gate`], which retries rather than records `Blocked` when
11997/// this is true.
11998fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11999    outcomes.iter().any(|o| o.resource_blocked)
12000}
12001
12002/// What [`Runner::gate_fix_round`] decided.
12003pub(crate) enum GateFix {
12004    /// The tree changed and `verify.e2e` is still green: run the gate again.
12005    Retry,
12006    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12007    /// last failure stands and the run ends blocked.
12008    Stop,
12009    /// `verify.e2e` could not run after the fix (magi's own contention):
12010    /// decide nothing now, a later reentry retries.
12011    Defer,
12012}
12013
12014/// Is every red command in `outcomes` an ordinary failure the code could
12015/// explain: it ran, exited non-zero, and said something?
12016///
12017/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12018/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12019/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12020/// what the command is stays the gate's business.
12021pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12022    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12023    red.peek().is_some()
12024        && red.all(|o| {
12025            !o.resource_blocked
12026                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12027                && !o.output_tail.trim().is_empty()
12028        })
12029}
12030
12031/// Describe one verify command's outcome for the event log, distinguishing a
12032/// build/link failure — the toolchain never produced a binary to run — from
12033/// an actual test failure, since only the latter is a verdict on the patch.
12034fn e2e_outcome_label(o: &CommandOutcome) -> String {
12035    if o.ok() {
12036        return "pass".to_owned();
12037    }
12038    let reason = if o.build_failed() {
12039        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12040    } else {
12041        format!("FAIL ({:?})", o.code)
12042    };
12043    format!(
12044        "{reason}\n{}",
12045        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12046    )
12047}
12048
12049/// Run `verify.e2e`, retrying once if the first attempt could not build or
12050/// link — a build/link failure is frequently a race against a shared
12051/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12052/// `verify` event per command, tagged with `context` (normally `"round N"`)
12053/// so the two call sites that need this — the ordinary per-round leg in
12054/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
12055/// it will ever call a round green — read identically in the event log.
12056async fn run_e2e_with_retry(
12057    state: &mut RunState,
12058    shell: &[String],
12059    commands: &[String],
12060    worktree: &Path,
12061    timeout: Duration,
12062    context: &str,
12063) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12064    let (mut e2e, mut timed_out_pids) = run_commands(
12065        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12066    )
12067    .await;
12068    for o in &e2e {
12069        state.event(
12070            "verify",
12071            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12072        );
12073    }
12074    // A build/link failure is not a verdict on the patch — it is frequently a
12075    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12076    // one retry before letting a red like that decide the round.
12077    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12078    if verify_retried {
12079        state.event(
12080            "verify",
12081            format!(
12082                "{context}: verify could not build/link, not a test result — retrying once \
12083                 before concluding"
12084            ),
12085        );
12086        let retried = run_commands(
12087            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12088        )
12089        .await;
12090        e2e = retried.0;
12091        // Both attempts' timeouts matter, not just the last one: the first
12092        // attempt's descendants may still be alive alongside the retry's.
12093        timed_out_pids.extend(retried.1);
12094        for o in &e2e {
12095            state.event(
12096                "verify",
12097                format!(
12098                    "{context}: retry `{}` -> {}",
12099                    o.command,
12100                    e2e_outcome_label(o)
12101                ),
12102            );
12103        }
12104    }
12105    (e2e, verify_retried, timed_out_pids)
12106}
12107
12108/// Run configured shell commands in `cwd`, in order. The second element is
12109/// the pid of every command that hit `timeout` and was still running when
12110/// this stopped waiting on it (best-effort: `None` when the platform did not
12111/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12112/// that releases a shared resource afterward needs to know.
12113///
12114/// Records `task` into [`RunState::active`] at every command boundary
12115/// (`RunState::task_command`) and clears it once the whole list has run
12116/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12117/// for minutes with no seat and no output of its own to show for it (see
12118/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12119/// apart from "not yet run" without this), and this is the only place that
12120/// knows which command is running right now and how many are left. Three
12121/// saves per command — start, not per second — matching the same "only at a
12122/// boundary" rule [`wave`] already follows for seats.
12123#[allow(clippy::too_many_arguments)]
12124async fn run_commands(
12125    state: &mut RunState,
12126    node: &str,
12127    task: &str,
12128    attempt: usize,
12129    shell: &[String],
12130    commands: &[String],
12131    cwd: &Path,
12132    timeout: Duration,
12133) -> (Vec<CommandOutcome>, Vec<u32>) {
12134    if commands.is_empty() {
12135        // Nothing to mark as running and nothing to clear — an empty list
12136        // means "not configured", and touching `active` (or the disk) over
12137        // that would be a write for every round of a repo with no
12138        // `verify.e2e` / `verify.gate` commands at all.
12139        return (Vec::new(), Vec::new());
12140    }
12141    let mut out = Vec::new();
12142    let mut timed_out_pids = Vec::new();
12143    let total = commands.len();
12144    for (idx, command) in commands.iter().enumerate() {
12145        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12146        if let Err(e) = state.save() {
12147            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12148        }
12149        let started = Instant::now();
12150        let mut cmd = tokio::process::Command::new(&shell[0]);
12151        cmd.quiet();
12152        // Verify the checkout as a standalone caller. Worker ownership and
12153        // pinned executable selection belong to this driver, not to tests or
12154        // tools it runs. Preserve home/seat context and Running's containment.
12155        for key in crate::worker::INVOCATION_ENV {
12156            cmd.env_remove(key);
12157        }
12158        cmd.args(&shell[1..])
12159            .arg(command)
12160            .current_dir(cwd)
12161            .stdin(std::process::Stdio::null())
12162            .stdout(std::process::Stdio::piped())
12163            .stderr(std::process::Stdio::piped())
12164            .kill_on_drop(true);
12165        let mut outputs = Vec::new();
12166        let protected = (|| -> Result<()> {
12167            crate::incremental::configure(&mut cmd, cwd)?;
12168            let mut paths = vec![cwd.to_owned()];
12169            if let Some(cache) = state.config.cache_dir() {
12170                paths.push(cache);
12171            }
12172            outputs.extend(crate::retention::OutputLease::admit(
12173                &crate::run::home(),
12174                &paths,
12175                state.config.disk.min_free_bytes,
12176            )?);
12177            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12178                outputs.push(guard);
12179            }
12180            if let Some(cache) = state.config.cache_dir() {
12181                outputs.push(crate::retention::OutputLease::cache(
12182                    &crate::run::home(),
12183                    &cache,
12184                    false,
12185                )?);
12186            }
12187            Ok(())
12188        })();
12189        if let Err(e) = protected {
12190            let detail = format!("execution admission unavailable: {e:#}");
12191            state.resource_block = Some(crate::run::ResourceBlock {
12192                cause: crate::run::ResourceBlockCause::Other,
12193                node: node.to_owned(),
12194                kind: if detail.contains("disk admission") {
12195                    "disk admission"
12196                } else {
12197                    "output ownership"
12198                }
12199                .to_owned(),
12200                detail: detail.clone(),
12201                at: Timestamp::now(),
12202            });
12203            out.push(CommandOutcome {
12204                command: command.clone(),
12205                code: None,
12206                output_tail: detail,
12207                duration_ms: 0,
12208                resource_blocked: true,
12209            });
12210            break;
12211        }
12212        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12213        let (code, body) = match spawned {
12214            Ok(mut running) => {
12215                let pid = running.child().id();
12216                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12217                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12218                let mut paths = vec![cwd.to_owned()];
12219                if let Some(cache) = state.config.cache_dir() {
12220                    paths.push(cache);
12221                }
12222                let mut pressure = None;
12223                let status = tokio::select! {
12224                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12225                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12226                        pressure = Some(reason); None
12227                    }
12228                };
12229                let (code, failure) = match status {
12230                    Some(Ok(Ok(status))) => (
12231                        status.code(),
12232                        status
12233                            .code()
12234                            .is_none()
12235                            .then(|| format!("command terminated: {status}")),
12236                    ),
12237                    Some(Ok(Err(error))) => {
12238                        (None, Some(format!("failed waiting for command: {error}")))
12239                    }
12240                    Some(Err(_)) => {
12241                        // Leased cleanup confirms process-tree death itself.
12242                        // Unleased commands still need the caller's PID check
12243                        // after kill-on-drop, before releasing cache ownership.
12244                        if !running.coordinated()
12245                            && let Some(pid) = pid
12246                        {
12247                            timed_out_pids.push(pid);
12248                        }
12249                        (
12250                            None,
12251                            Some(format!("timed out after {}s", timeout.as_secs())),
12252                        )
12253                    }
12254                    None => (None, None),
12255                };
12256                running.finish().await;
12257                if let Some(reason) = pressure {
12258                    state.resource_block = Some(crate::run::ResourceBlock {
12259                        cause: crate::run::ResourceBlockCause::Other,
12260                        node: node.to_owned(),
12261                        kind: "disk pressure".to_owned(),
12262                        detail: reason.clone(),
12263                        at: Timestamp::now(),
12264                    });
12265                    out.push(CommandOutcome {
12266                        command: command.clone(),
12267                        code: None,
12268                        output_tail: reason,
12269                        duration_ms: started.elapsed().as_millis() as u64,
12270                        resource_blocked: true,
12271                    });
12272                    break;
12273                }
12274                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12275                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12276                if let Some(failure) = failure {
12277                    body.push('\n');
12278                    body.push_str(&failure);
12279                }
12280                (code, body)
12281            }
12282            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12283        };
12284        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12285        if !body.is_empty() {
12286            // The full text is kept whatever the summary dropped. A fresh
12287            // name per command so a resume or retry never overwrites an
12288            // earlier run's output. A failed write changes no verdict.
12289            match crate::run::write_verification_artifact(state, task, &body) {
12290                Ok(name) if summary != body => {
12291                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12292                }
12293                Ok(_) => {}
12294                Err(e) => {
12295                    let warning = format!("could not save the full {task} output: {e:#}");
12296                    tracing::warn!("{warning}");
12297                    state.event(node, warning);
12298                    if summary != body {
12299                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12300                    }
12301                }
12302            }
12303        }
12304        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12305        out.push(CommandOutcome {
12306            command: command.clone(),
12307            code,
12308            output_tail: summary,
12309            duration_ms: started.elapsed().as_millis() as u64,
12310            resource_blocked,
12311        });
12312        if resource_blocked {
12313            state.resource_block = Some(crate::run::ResourceBlock {
12314                cause: crate::run::ResourceBlockCause::Other,
12315                node: node.to_owned(),
12316                kind: "incremental cache admission".into(),
12317                detail: body,
12318                at: Timestamp::now(),
12319            });
12320            break;
12321        }
12322    }
12323    state.task_finished(task);
12324    if let Err(e) = state.save() {
12325        tracing::warn!("could not persist the end of {task}: {e:#}");
12326    }
12327    (out, timed_out_pids)
12328}
12329
12330/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12331/// section (`report::run`) and in the `merge` event this node records — for
12332/// the operator to run by hand.
12333///
12334/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12335/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12336/// merge commits", or "require linear history") rejects the push a `--no-ff`
12337/// merge would produce, which is exactly the guidance this function replaces.
12338/// `message`'s first line becomes the squash commit's subject, matching the
12339/// note `report::run` prints alongside this command — see that function for
12340/// why an explicit subject is not optional there.
12341fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12342    let repo = repo.display();
12343    match style {
12344        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12345        MergeStyle::Squash => {
12346            // The subject sits inside double quotes, and a title an agent
12347            // wrote may carry the characters that break out of them.
12348            let subject = message
12349                .lines()
12350                .next()
12351                .unwrap_or(branch)
12352                .replace(['\\', '"', '$', '`'], "");
12353            format!(
12354                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12355            )
12356        }
12357        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12358    }
12359}
12360
12361/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12362/// under the hood, rejects a `title` over 256 characters and the whole
12363/// command fails — no PR at all, for a run whose body was otherwise fine
12364/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12365/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12366/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12367/// this bound. It is a margin, not a guarantee — a title packed
12368/// with multi-unit characters could still in principle land close to the
12369/// edge, but a real task title's occasional emoji or accented letter fits
12370/// comfortably inside it.
12371const PR_TITLE_MAX: usize = 240;
12372
12373/// A pull request title from the opening line of `text`, or `None` when that
12374/// line is not English (GitHub text is) or has no letters.
12375///
12376/// A line within `max` is kept as is. A longer one is cut at the end of its
12377/// first sentence when that falls inside `max`, else at a word boundary with a
12378/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12379/// make every merely-truncated title look non-English). The language check
12380/// runs on the kept text before any mark is added, so only what GitHub will
12381/// show is judged: an English opening followed by non-ASCII far past the cut
12382/// still passes.
12383fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12384    let line = queue::first_line(text)?;
12385    let chars: Vec<char> = line.chars().collect();
12386    let (kept, mark) = if chars.len() <= max {
12387        (line.to_owned(), "")
12388    } else if let Some(end) = sentence_end(&chars, max) {
12389        (chars[..end].iter().collect::<String>(), "")
12390    } else {
12391        let room = max.saturating_sub(3);
12392        // Cut at the last space inside the room; when the char just past the
12393        // room is a space the room already ends on a word.
12394        let cut = if chars[room].is_whitespace() {
12395            room
12396        } else {
12397            chars[..room]
12398                .iter()
12399                .rposition(|c| c.is_whitespace())
12400                .unwrap_or(room)
12401        };
12402        let head: String = chars[..cut].iter().collect();
12403        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12404        (head.to_owned(), "...")
12405    };
12406    if kept.is_empty() {
12407        return None;
12408    }
12409    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12410        return None;
12411    }
12412    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12413        return None;
12414    }
12415    Some(format!("{kept}{mark}"))
12416}
12417
12418/// The char length of the first sentence of `chars` when it ends within `max`
12419/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12420/// abbreviations so `e.g. foo` does not end a title early.
12421fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12422    const MIN: usize = 20;
12423    for i in MIN..max.min(chars.len()) {
12424        if !matches!(chars[i], '.' | '!' | '?') {
12425            continue;
12426        }
12427        let Some(&next) = chars.get(i + 1) else {
12428            continue;
12429        };
12430        if !next.is_whitespace() {
12431            continue;
12432        }
12433        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12434        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12435            continue;
12436        }
12437        let word: String = chars[..i]
12438            .iter()
12439            .rev()
12440            .take_while(|c| !c.is_whitespace())
12441            .collect::<Vec<_>>()
12442            .into_iter()
12443            .rev()
12444            .collect();
12445        let word = word.to_ascii_lowercase();
12446        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12447            continue;
12448        }
12449        let end = chars[..i]
12450            .iter()
12451            .rposition(|c| !c.is_whitespace())
12452            .map_or(i, |p| p + 1);
12453        return Some(end);
12454    }
12455    None
12456}
12457
12458/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12459/// change: a title and a body describing what was *implemented*, not the task
12460/// that asked for it. A task reads as a request; a reader of the merged
12461/// history wants the change.
12462struct PrMessage {
12463    title: String,
12464    body: String,
12465}
12466
12467impl PrMessage {
12468    /// Title, blank line, body. The first line is the squash/merge commit
12469    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12470    /// has to stay one sensible line.
12471    fn commit_message(&self) -> String {
12472        format!("{}\n\n{}", self.title, self.body)
12473    }
12474}
12475
12476/// The text after a leading `TITLE:` (any case) on `line`.
12477fn title_marker(line: &str) -> Option<&str> {
12478    let line = line.trim();
12479    let head = line.get(..6)?;
12480    head.eq_ignore_ascii_case("title:")
12481        .then(|| line[6..].trim())
12482}
12483
12484/// The implementer's own one-line title: the `TITLE:` line the implement
12485/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12486/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12487/// source, and a title that says as much is refused here too.
12488fn summary_title(summary: &str) -> Option<String> {
12489    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12490    let raw = title_marker(first)?;
12491    if raw.is_empty() {
12492        return None;
12493    }
12494    let title = queue::title_from(raw, PR_TITLE_MAX);
12495    let lower = title.to_ascii_lowercase();
12496    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12497        return None;
12498    }
12499    Some(title)
12500}
12501
12502/// How `open_review`'s instruction begins; see [`landing_title`].
12503const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12504
12505/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12506pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12507
12508/// Does this run review work that already existed, rather than implement a
12509/// task? Runs recorded before `reviewed_commits` existed carry only the
12510/// candidate marker.
12511pub(crate) fn is_review_run(state: &RunState) -> bool {
12512    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12513}
12514
12515/// The title of a review-only run: the subject of the oldest commit under
12516/// review. Later commits are usually fixups, and `instruction` is the review
12517/// prompt, which says nothing about the change. GitHub text is English, so a
12518/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12519fn review_title(state: &RunState) -> Option<String> {
12520    english_subject(
12521        state.reviewed_commits.as_ref()?.first()?,
12522        state.config.graph.github_english(),
12523    )
12524}
12525
12526/// `raw` as a pull request title, or `None` when it is blank, not English
12527/// (GitHub text is), or one of magi's own candidate commit subjects.
12528fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12529    let raw = raw.trim();
12530    if raw.is_empty() {
12531        return None;
12532    }
12533    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12534        return None;
12535    }
12536    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12537        return None;
12538    }
12539    let title = queue::title_from(raw, PR_TITLE_MAX);
12540    let lower = title.to_ascii_lowercase();
12541    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12542        return None;
12543    }
12544    Some(title)
12545}
12546
12547/// What a review-only run's branch says about itself, read at the moment the
12548/// pull request is opened.
12549#[derive(Debug, Clone, PartialEq, Eq)]
12550struct BranchFacts {
12551    /// `(subject, body)` of each commit, oldest first.
12552    commits: Vec<(String, String)>,
12553    /// Trimmed `git diff --stat`.
12554    stat: String,
12555}
12556
12557/// Longest diff stat shown: this many file lines plus the summary line.
12558const STAT_FILE_LINES: usize = 25;
12559/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12560const COMMITS_MAX_CHARS: usize = 20_000;
12561
12562/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12563/// say or finds nothing, so the caller falls back to what the run recorded.
12564async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12565    let commits = git::commit_log(repo, base, branch).await.ok()?;
12566    if commits.is_empty() {
12567        return None;
12568    }
12569    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12570    let lines: Vec<&str> = stat.lines().collect();
12571    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12572        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12573        let more = format!(" ... {omitted} more file(s)");
12574        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12575        kept.push(&more);
12576        kept.push(lines[lines.len() - 1]);
12577        kept.join("\n")
12578    } else {
12579        lines.join("\n")
12580    };
12581    Some(BranchFacts { commits, stat })
12582}
12583
12584/// Defang what would break the surrounding markdown: a closing `</details>`
12585/// and a code fence.
12586fn markdown_safe(text: &str) -> String {
12587    text.replace("</details>", "&lt;/details&gt;")
12588        .replace("\x60\x60\x60", "~~~")
12589}
12590
12591fn neutral_title(state: &RunState, winner: char) -> String {
12592    format!(
12593        "chore: land candidate {} of run {}",
12594        winner.to_ascii_uppercase(),
12595        state.id
12596    )
12597}
12598
12599/// Reuse the graph's bounded fixer without starting another graph or driver.
12600/// Copy the working state back even on errors so spent rounds remain visible.
12601pub(crate) async fn repair_landing_gate(
12602    state: &mut RunState,
12603    winner: &Candidate,
12604    outcomes: &[CommandOutcome],
12605) -> Result<GateFix> {
12606    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12607        return Ok(GateFix::Stop);
12608    }
12609    let mut runner = Runner {
12610        roles: state.config.resolve_roles()?,
12611        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12612        state: state.clone(),
12613        pause: Pause::new(),
12614        interrupt: Pause::new(),
12615    };
12616    let result = runner.gate_fix_round(winner, outcomes).await;
12617    *state = runner.state;
12618    result
12619}
12620
12621/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12622/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12623pub(crate) async fn verify_current_head(
12624    state: &mut RunState,
12625    winner: &Candidate,
12626    wait: &crate::repo_lock::WaitPolicy,
12627) -> Result<bool> {
12628    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12629    // Preserve the previous record before replacing it, including on crash recovery.
12630    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12631        state
12632            .verification_history
12633            .push(crate::run::VerificationEvidence {
12634                superseded_by: Some(head.clone()),
12635                archived_at: Some(Timestamp::now()),
12636                fixes_spent: Some(state.gate_fixes.len()),
12637                gate_head: None,
12638                gate_ran: false,
12639                gate: Vec::new(),
12640                e2e: state.head_e2e.take(),
12641            });
12642    }
12643    if let Some(e) = &state.head_e2e
12644        && e.head == head
12645    {
12646        if e.passed {
12647            return Ok(true);
12648        }
12649        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12650            state.status = RunStatus::Blocked;
12651            state.save()?;
12652            return Ok(false);
12653        }
12654    } else if state.reviews.last().is_some_and(|r| {
12655        r.verified_head.as_deref() == Some(head.as_str())
12656            && (r.e2e_status() == E2eStatus::Passed
12657                || (r.e2e_status() == E2eStatus::NotConfigured
12658                    && state.config.verify.e2e.is_empty()))
12659    }) || state
12660        .gate_repair_e2e
12661        .as_ref()
12662        .is_some_and(|e| e.head == head && e.passed)
12663    {
12664        return Ok(true);
12665    }
12666    if let Some(e) = &state.gate_repair_e2e
12667        && e.head == head
12668        && !e.passed
12669        && !e.outcomes.is_empty()
12670        && !verify_inconclusive(&e.outcomes)
12671    {
12672        state.status = RunStatus::Blocked;
12673        state.save()?;
12674        return Ok(false);
12675    }
12676    state.head_e2e = Some(crate::run::GateRepairE2e {
12677        head: head.clone(),
12678        passed: false,
12679        outcomes: Vec::new(),
12680    });
12681    state.save()?;
12682    let commands = state.config.verify.e2e.clone();
12683    let shell = state.config.shell();
12684    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12685    let cache_dir = state.config.cache_dir();
12686    let outcomes = if commands.is_empty() {
12687        Vec::new()
12688    } else {
12689        let (outcomes, _) = with_cache_lease(
12690            state,
12691            cache_dir.as_deref(),
12692            "e2e",
12693            "e2e",
12694            &winner.worktree,
12695            &head,
12696            timeout,
12697            "current head e2e",
12698            wait,
12699            |state, budget| async move {
12700                run_e2e_with_retry(
12701                    state,
12702                    &shell,
12703                    &commands,
12704                    &winner.worktree,
12705                    budget,
12706                    "current head e2e",
12707                )
12708                .await
12709            },
12710        )
12711        .await?;
12712        outcomes
12713    };
12714    let unresolved = verify_inconclusive(&outcomes);
12715    let passed = !unresolved
12716        && outcomes.iter().all(CommandOutcome::ok)
12717        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12718    if !outcomes.is_empty() && !unresolved {
12719        state.record_base_progress(&head);
12720    }
12721    state.head_e2e = Some(crate::run::GateRepairE2e {
12722        head: head.clone(),
12723        passed,
12724        outcomes,
12725    });
12726    if !passed && !unresolved {
12727        state.status = RunStatus::Blocked;
12728        state.event(
12729            "e2e",
12730            format!("verify.e2e failed on head {head}; nothing published"),
12731        );
12732    }
12733    state.save()?;
12734    Ok(passed)
12735}
12736
12737/// Recheck a repair's e2e requirement across failures, resource deferrals and
12738/// process interruptions. A successful result only applies to its exact SHA.
12739pub(crate) async fn check_gate_repair_e2e(
12740    state: &mut RunState,
12741    winner: &Candidate,
12742    wait: &crate::repo_lock::WaitPolicy,
12743) -> Result<GateFix> {
12744    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12745        return Ok(GateFix::Retry);
12746    }
12747    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12748    if state
12749        .gate_repair_e2e
12750        .as_ref()
12751        .is_some_and(|r| r.head == head && r.passed)
12752    {
12753        return Ok(GateFix::Retry);
12754    }
12755    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12756        head: head.clone(),
12757        passed: false,
12758        outcomes: Vec::new(),
12759    });
12760    state.save()?;
12761    let commands = state.config.verify.e2e.clone();
12762    let outcomes = if commands.is_empty() {
12763        Vec::new()
12764    } else {
12765        let shell = state.config.shell();
12766        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12767        let cache_dir = state.config.cache_dir();
12768        let (outcomes, _) = with_cache_lease(
12769            state,
12770            cache_dir.as_deref(),
12771            "e2e",
12772            "e2e",
12773            &winner.worktree,
12774            &head,
12775            timeout,
12776            "gate repair e2e",
12777            wait,
12778            |state, budget| async move {
12779                run_e2e_with_retry(
12780                    state,
12781                    &shell,
12782                    &commands,
12783                    &winner.worktree,
12784                    budget,
12785                    "gate repair e2e",
12786                )
12787                .await
12788            },
12789        )
12790        .await?;
12791        outcomes
12792    };
12793    let deferred = verify_inconclusive(&outcomes);
12794    let passed = outcomes.iter().all(CommandOutcome::ok)
12795        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12796    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12797        head,
12798        passed,
12799        outcomes,
12800    });
12801    state.save()?;
12802    if deferred {
12803        Ok(GateFix::Defer)
12804    } else if !passed {
12805        state.event(
12806            "gate",
12807            "verify.e2e failed after gate repair; nothing published",
12808        );
12809        Ok(GateFix::Stop)
12810    } else {
12811        Ok(GateFix::Retry)
12812    }
12813}
12814
12815/// Run the same configured gate for a landing head and a fresh graph.
12816pub(crate) async fn run_landing_gate(
12817    state: &mut RunState,
12818    winner: &Candidate,
12819    wait: &crate::repo_lock::WaitPolicy,
12820) -> Result<Vec<CommandOutcome>> {
12821    let shell = state.config.shell();
12822    let gate_commands = state.config.verify.gate.clone();
12823    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12824    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12825        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12826        Vec::new()
12827    } else {
12828        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12829        let cache_dir = state.config.cache_dir();
12830        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12831        let (outcomes, _) = with_cache_lease(
12832            state,
12833            cache_dir.as_deref(),
12834            "gate",
12835            "gate",
12836            &winner.worktree,
12837            &head,
12838            timeout,
12839            "final gate",
12840            wait,
12841            |state, budget| {
12842                let shell = shell.clone();
12843                let gate_commands = gate_commands.clone();
12844                let winner = winner.clone();
12845                async move {
12846                    Runner::run_pre_gate(state, &winner).await;
12847                    // Bind command evidence after mechanical fixes, before commands start.
12848                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12849                    let (outcomes, timed_out_pids) = run_commands(
12850                        state,
12851                        "gate",
12852                        "gate",
12853                        0,
12854                        &shell,
12855                        &gate_commands,
12856                        &winner.worktree,
12857                        budget,
12858                    )
12859                    .await;
12860                    (outcomes, false, timed_out_pids)
12861                }
12862            },
12863        )
12864        .await?;
12865        outcomes
12866    };
12867    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12868    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12869        let mut outcomes = outcomes;
12870        outcomes.push(CommandOutcome {
12871            command: "gate head stability".into(), code: None,
12872            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12873            duration_ms: 0, resource_blocked: false,
12874        });
12875        return Ok(outcomes);
12876    }
12877    if outcomes.is_empty() {
12878        // Nothing configured to check — distinct from every other
12879        // silence in this run's event log, since an empty `gate` alone
12880        // no longer says whether the gate ran at all (see
12881        // `RunState::gate_ran`'s own doc).
12882        state.event("gate", "no gate commands configured; not tested");
12883    }
12884    for o in &outcomes {
12885        state.event(
12886            "gate",
12887            format!(
12888                "`{}` -> {}",
12889                o.command,
12890                if o.ok() {
12891                    "pass".to_owned()
12892                } else {
12893                    format!(
12894                        "FAIL ({:?})\n{}",
12895                        o.code,
12896                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12897                    )
12898                }
12899            ),
12900        );
12901    }
12902    Ok(outcomes)
12903}
12904
12905/// The pull request title to hand to `land::merge_subject`. A review-only run
12906/// opened by an earlier build titled its pull request with the review prompt;
12907/// that title is dropped (empty, so the fallback applies) rather than landed.
12908/// Any other title, including an operator's rename, passes through untouched,
12909/// and so does every title of a run that implements a task.
12910pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12911    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12912        ""
12913    } else {
12914        pr_title
12915    }
12916}
12917
12918/// What the squash subject falls back to when the pull request title is empty
12919/// or candidate-shaped: for a review-only run the derived title, never the
12920/// review prompt held in `instruction`.
12921pub fn landing_subject_source(state: &RunState) -> String {
12922    if is_review_run(state) {
12923        let winner = state.candidates.first().map_or('A', |c| c.label);
12924        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12925    }
12926    state.instruction.clone()
12927}
12928
12929/// `summary` without its `TITLE:` line, which the pull request title already
12930/// carries.
12931fn summary_without_title(summary: &str) -> String {
12932    let mut lines = summary.trim().lines().peekable();
12933    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12934        lines.next();
12935    }
12936    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12937}
12938
12939/// The pull request title and body for the winning candidate.
12940///
12941/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12942/// the task's own opening line via [`queue::title_from`] when there is none.
12943/// `state.instruction` can open with blank lines (`task_text` only rejects a
12944/// body that is blank *entirely*), which `title_from` skips.
12945///
12946/// Publishing uses an independent description generated from the final diff
12947/// and structured evidence. Internal summaries and reports remain local.
12948/// Non-publishing previews may display the original task and candidate summary.
12949/// The footer retains the run and candidate provenance.
12950#[cfg(test)]
12951fn pr_message(state: &RunState, winner: char) -> PrMessage {
12952    pr_message_with(state, winner, None)
12953}
12954
12955/// [`pr_message`] with what the branch of a review-only run says about itself.
12956/// `facts` is ignored for a run that implements a task.
12957#[cfg(test)]
12958fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12959    let raw = pr_message_raw(
12960        state,
12961        winner,
12962        facts,
12963        crate::pr_outcome::FinalHead::Unreported,
12964        ChangeNote::NotAsked,
12965    );
12966    let id = crate::scrub::Identity::current();
12967    PrMessage {
12968        title: crate::scrub::scrub(&raw.title, &id),
12969        body: crate::scrub::scrub(&raw.body, &id),
12970    }
12971}
12972
12973/// What the public description may say about the change itself.
12974#[derive(Debug, Clone, Copy)]
12975enum ChangeNote<'a> {
12976    /// A local preview or non-publishing merge mode: no public generation needed.
12977    NotAsked,
12978    /// A summary written from the final diff.
12979    Summary(&'a crate::run::ChangeSummary),
12980    /// A summary was asked for and could not be produced; the text says why.
12981    Unavailable(&'a str),
12982}
12983
12984fn pr_message_raw(
12985    state: &RunState,
12986    winner: char,
12987    facts: Option<&BranchFacts>,
12988    head: crate::pr_outcome::FinalHead<'_>,
12989    change: ChangeNote<'_>,
12990) -> PrMessage {
12991    if !matches!(change, ChangeNote::NotAsked) {
12992        let (title, mut body) = match change {
12993            ChangeNote::Summary(c) => (
12994                c.title.clone(),
12995                format!("## Summary\n\n{}\n\n", c.body.trim()),
12996            ),
12997            ChangeNote::Unavailable(why) => (
12998                neutral_title(state, winner),
12999                format!(
13000                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13001                ),
13002            ),
13003            ChangeNote::NotAsked => unreachable!(),
13004        };
13005        body.push_str(&public_pr_evidence(state, head));
13006        body.push_str(&format!(
13007            "\n---\nmagi:run/{} magi:candidate-{}\n",
13008            state.id,
13009            winner.to_ascii_lowercase()
13010        ));
13011        return PrMessage { title, body };
13012    }
13013    pr_preview(state, winner, facts, head)
13014}
13015
13016/// Append public facts from the run records after every prose generation,
13017/// including posting-gate rewrites. The agent cannot drop this provenance.
13018fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13019    let mut body = crate::pr_outcome::review_rounds(state);
13020    body.push_str(&crate::pr_outcome::final_verification(state, head));
13021    let open = state.last_round_findings();
13022    if !open.is_empty() {
13023        body.push_str("## Open review findings\n\n");
13024        for f in open {
13025            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13026        }
13027        body.push('\n');
13028    }
13029    if let Some(c) = state.change_summary.as_ref()
13030        && head == crate::pr_outcome::FinalHead::At(&c.head)
13031    {
13032        body.push_str(&format!(
13033            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13034            c.base, c.head
13035        ));
13036    }
13037    body
13038}
13039
13040/// Local-only preview of the operator's task and candidate summary.
13041fn pr_preview(
13042    state: &RunState,
13043    winner: char,
13044    facts: Option<&BranchFacts>,
13045    head: crate::pr_outcome::FinalHead<'_>,
13046) -> PrMessage {
13047    let summary = state
13048        .candidates
13049        .iter()
13050        .find(|c| c.label == winner)
13051        .map(|c| c.summary.as_str())
13052        .unwrap_or_default();
13053    // The fallback is the operator's own words and may not be English; GitHub
13054    // text always is, so a non-English task gets a neutral title instead.
13055    let review = is_review_run(state);
13056    let title = if review {
13057        facts
13058            .and_then(|f| {
13059                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13060            })
13061            .or_else(|| review_title(state))
13062            .or_else(|| {
13063                state
13064                    .candidates
13065                    .iter()
13066                    .find(|c| c.label == winner)
13067                    .filter(|c| !c.branch.starts_with("magi/"))
13068                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13069            })
13070            .unwrap_or_else(|| neutral_title(state, winner))
13071    } else {
13072        summary_title(summary).unwrap_or_else(|| {
13073            english_title(
13074                &state.instruction,
13075                PR_TITLE_MAX,
13076                state.config.graph.github_english(),
13077            )
13078            .unwrap_or_else(|| neutral_title(state, winner))
13079        })
13080    };
13081
13082    let mut body = String::new();
13083    let what = summary_without_title(summary);
13084    if !what.is_empty() {
13085        body.push_str("## Summary\n\n");
13086        body.push_str(&what);
13087        body.push_str("\n\n");
13088    }
13089
13090    // Review history uses structured facts across all rounds, never reports.
13091    body.push_str(&crate::pr_outcome::review_rounds(state));
13092    body.push_str(&crate::pr_outcome::final_verification(state, head));
13093    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13094
13095    let open = state.open_findings();
13096    if !open.is_empty() {
13097        body.push_str("## Open review findings\n\n");
13098        for f in &open {
13099            body.push_str(&format!(
13100                "- `{}` [{:?}] {}\n",
13101                f.id, f.severity, "Context retained in local review records."
13102            ));
13103        }
13104        body.push('\n');
13105    }
13106
13107    if let Some(fix) = fix
13108        && !fix.rejected.is_empty()
13109    {
13110        body.push_str("## Declined by the fixer\n\n");
13111        for r in &fix.rejected {
13112            body.push_str(&format!(
13113                "- `{}`: {}\n",
13114                r.id, "Reason retained in local review records."
13115            ));
13116        }
13117        body.push('\n');
13118    }
13119
13120    if review {
13121        // The review prompt is not the task; list what the branch carries.
13122        body.push_str("## Commits under review\n\n");
13123        if let Some(facts) = facts {
13124            let mut left = COMMITS_MAX_CHARS;
13125            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13126                let mut entry = format!("- {}\n", markdown_safe(subject));
13127                for l in markdown_safe(text).lines() {
13128                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13129                }
13130                if left == 0 {
13131                    body.push_str(&format!(
13132                        "- ... {} more commit(s)\n",
13133                        facts.commits.len() - i
13134                    ));
13135                    break;
13136                }
13137                if entry.len() > left {
13138                    // Even the first commit is cut: one huge body must not
13139                    // push the whole description past GitHub's limit.
13140                    let mut end = left;
13141                    while !entry.is_char_boundary(end) {
13142                        end -= 1;
13143                    }
13144                    entry.truncate(end);
13145                    entry.push_str("\n  ... (truncated)\n");
13146                    left = 0;
13147                } else {
13148                    left -= entry.len();
13149                }
13150                body.push_str(&entry);
13151            }
13152            if !facts.stat.trim().is_empty() {
13153                body.push_str(&format!(
13154                    "\n## Diff stat\n\n```\n{}\n```\n",
13155                    markdown_safe(facts.stat.trim())
13156                ));
13157            }
13158        } else {
13159            match &state.reviewed_commits {
13160                Some(subjects) => {
13161                    for s in subjects {
13162                        body.push_str(&format!("- {}\n", s.trim()));
13163                    }
13164                }
13165                None => {
13166                    // An older run kept only the prompt, with the commit list
13167                    // after its first paragraph.
13168                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13169                    body.push_str(rest.trim());
13170                    body.push('\n');
13171                }
13172            }
13173        }
13174    } else {
13175        let task = state.instruction.trim();
13176        let task = if task.is_empty() {
13177            "(empty task)"
13178        } else {
13179            task
13180        };
13181        body.push_str(&format!(
13182            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13183            task.replace("</details>", "&lt;/details&gt;")
13184        ));
13185    }
13186
13187    body.push_str(&format!(
13188        "\n---\nmagi:run/{} magi:candidate-{}\n",
13189        state.id,
13190        winner.to_ascii_lowercase()
13191    ));
13192
13193    PrMessage { title, body }
13194}
13195
13196/// Source imports have already been prepared from structured inputs. Legacy
13197/// inferred references must never instruct a seat to reconstruct those imports.
13198fn seeded_instruction(state: &RunState) -> String {
13199    let mut inputs = Vec::new();
13200    if let Some(start) = &state.sources.start_from {
13201        inputs.push(format!(
13202            "Start from `{}` at `{}` (including ancestry).",
13203            start.token, start.sha
13204        ));
13205    }
13206    for pick in &state.sources.pick_commits {
13207        inputs.push(format!(
13208            "Pick `{}` at `{}` (single commit).",
13209            pick.token, pick.sha
13210        ));
13211    }
13212    if inputs.is_empty() {
13213        state.instruction.clone()
13214    } else {
13215        format!(
13216            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13217            state.instruction,
13218            inputs.join("\n")
13219        )
13220    }
13221}
13222
13223/// Does the winner have no commits ahead of the base it would land on?
13224/// Any failure to find out reads as "not empty": the merge then behaves as it
13225/// always did rather than refusing on a guess.
13226async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13227    let base = &state.base_branch;
13228    let mut against = base.clone();
13229    if mode == MergeMode::Pr {
13230        // A pull request lands on the remote's base, never the local branch
13231        // of the same name: if that cannot be read, "not empty" is the safe
13232        // answer.
13233        let remote = &state.config.merge.remote;
13234        let tracking = format!("{remote}/{base}");
13235        let fetched = git::fetch(repo, remote, base).await;
13236        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13237            against = tracking;
13238        } else {
13239            return false;
13240        }
13241    }
13242    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13243}
13244
13245/// Why nothing was opened for an empty winner, with what the task's own
13246/// references resolved to.
13247fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13248    let mut detail = format!(
13249        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13250         nothing to open a pull request for"
13251    );
13252    detail.push_str(
13253        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13254    );
13255    detail
13256}
13257
13258/// What the `Pr` merge does once it knows whether the branch already has an
13259/// open pull request.
13260#[derive(Debug, PartialEq, Eq)]
13261enum PrPlan {
13262    Create,
13263    Adopt { url: String, title: String },
13264    Stop(String),
13265}
13266
13267/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13268/// a failed lookup -> stop with the real reason. Never guesses.
13269fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13270    match found {
13271        Ok(land::OpenPr::None) => PrPlan::Create,
13272        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13273        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13274            "several open pull requests exist for this branch, not picking one: {}",
13275            urls.join(" ")
13276        )),
13277        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13278    }
13279}
13280
13281/// `gh pr create`, returning the PR url.
13282pub(crate) async fn gh_pr_create(
13283    cwd: &Path,
13284    base: &str,
13285    head: &str,
13286    title: &str,
13287    body: &str,
13288) -> Result<String> {
13289    let out = tokio::process::Command::new("gh")
13290        .args([
13291            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13292        ])
13293        .current_dir(cwd)
13294        .quiet()
13295        .stdin(std::process::Stdio::null())
13296        .output()
13297        .await
13298        .context("spawn gh")?;
13299    if out.status.success() {
13300        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13301    } else {
13302        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13303    }
13304}
13305
13306/// Never force deletion of uncommitted work, including unknown legacy trees.
13307async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13308    match std::fs::symlink_metadata(path) {
13309        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13310            // A previous cleanup may have removed the tree before a branch
13311            // deletion or state save failed. Unregister it if still present.
13312            git::worktree_remove_clean(repo, path).await?;
13313            return Ok(());
13314        }
13315        Err(e) => return Err(e.into()),
13316        Ok(_) => {}
13317    }
13318    if !git::is_clean(path).await? {
13319        bail!(
13320            "worktree has uncommitted work; refusing removal: {}",
13321            path.display()
13322        );
13323    }
13324    if !git::worktree_remove_clean(repo, path).await? {
13325        bail!("worktree removal refused: {}", path.display());
13326    }
13327    Ok(())
13328}
13329
13330/// Tear a run's worktrees and branches down.
13331///
13332/// `home` is where the updated `run.json` is saved (via
13333/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13334/// a housekeeping pass already has its own honest `home` handed to it, and
13335/// falling through to the global here would write back through whichever
13336/// directory some other process or test pinned into that `OnceLock` first,
13337/// not the one the caller actually resolved its `runs` and `state` from.
13338pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13339    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13340        bail!("refusing fold: {reason}");
13341    }
13342    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13343        bail!("candidate preservation incomplete; refusing cleanup");
13344    }
13345    // Serialize ordinary folding with the same starts/resumes as target-only
13346    // eviction. Otherwise a second janitor could remove the whole worktree
13347    // around a live output borrower even though target eviction was fenced.
13348    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13349    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13350    let _driver = crate::driver::maintenance(home, &state.id)?;
13351    let mut output_guards = Vec::new();
13352    for entry in std::fs::read_dir(state.worktree_root())
13353        .into_iter()
13354        .flatten()
13355        .flatten()
13356    {
13357        if entry.path().join(".git").is_file() {
13358            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13359        }
13360    }
13361    let repo = state.repo.clone();
13362    // CLI, Web and automatic cleanup all enter here, independently of
13363    // Runner::fold_losers. Protect deletion and shared config release too.
13364    let wait = repository_wait(state, None);
13365    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13366        Ok(lease) => lease,
13367        Err(e) => {
13368            state.save_under(home)?;
13369            return Err(e);
13370        }
13371    };
13372    // Confirmed merges dispose of their owned sources after durable evidence
13373    // capture. Preflight every source before removing anything.
13374    let _source_outputs = if state.status == RunStatus::Merged
13375        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13376    {
13377        verify_merged_fold_sources(state, drop_winner, home).await?
13378    } else {
13379        Vec::new()
13380    };
13381    let root = state.worktree_root();
13382    let winner = state.tally.as_ref().map(|t| t.winner);
13383    let mut removed = Vec::new();
13384
13385    for i in 0..state.candidates.len() {
13386        let c = state.candidates[i].clone();
13387        let is_winner = Some(c.label) == winner;
13388        if is_winner && !drop_winner {
13389            continue;
13390        }
13391        let existed = c.worktree.exists();
13392        // A vanished directory can still have a Git registration that keeps
13393        // its branch checked out. Remove that registration before the branch.
13394        remove_preserved_worktree(&repo, &c.worktree).await?;
13395        if existed {
13396            removed.push(c.worktree.to_string_lossy().into_owned());
13397        }
13398        // A branch handed to a later run (and its pull request) is not this
13399        // run's to delete.
13400        let handed_over = state.released_branches.contains(&c.branch);
13401        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13402            if !git::branch_delete(&repo, &c.branch).await? {
13403                bail!("candidate branch removal refused");
13404            }
13405            removed.push(c.branch.clone());
13406        }
13407        state.candidates[i].folded = true;
13408    }
13409
13410    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13411        let path = name.path();
13412        let keep = !drop_winner
13413            && winner.is_some_and(|w| {
13414                path.file_name()
13415                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13416            });
13417        if keep {
13418            continue;
13419        }
13420        remove_preserved_worktree(&repo, &path).await?;
13421        removed.push(path.to_string_lossy().into_owned());
13422    }
13423
13424    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13425    // judge worktrees, so once the loop above has cleared all of them out,
13426    // the parent is a bare directory nobody else was ever going to remove -
13427    // git only ever managed what was inside it. Left alone, one of these
13428    // accumulates per fully-folded run; the operator's own machine had 74.
13429    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13430    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13431    // so does anything a run never claimed that happens to share the bay.
13432    remove_if_empty(&root);
13433
13434    if state.enabled_worktree_config && drop_winner {
13435        // A release, not a raw disable: some sibling run in this repository
13436        // may still hold its own reference (see `git::acquire_worktree_config`),
13437        // and only the last release actually turns the setting back off.
13438        git::release_worktree_config(&repo).await.ok();
13439        state.enabled_worktree_config = false;
13440    }
13441    state.save_under(home)?;
13442    Ok(removed)
13443}
13444
13445async fn verify_merged_fold_sources(
13446    state: &RunState,
13447    drop_winner: bool,
13448    home: &Path,
13449) -> Result<Vec<std::fs::File>> {
13450    let winner = state.tally.as_ref().map(|t| t.winner);
13451    let mut heads = BTreeSet::new();
13452    for candidate in &state.candidates {
13453        if Some(candidate.label) == winner && !drop_winner {
13454            continue;
13455        }
13456        anyhow::ensure!(
13457            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13458            "candidate recovery source is outside the inventoried run root: {}",
13459            candidate.worktree.display()
13460        );
13461        match std::fs::symlink_metadata(&candidate.worktree) {
13462            Ok(_) => {}
13463            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13464            Err(error) => return Err(error.into()),
13465        }
13466        if !state.released_branches.contains(&candidate.branch)
13467            && git::branch_exists(&state.repo, &candidate.branch).await?
13468        {
13469            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13470        }
13471    }
13472    let mut cargo_guards = Vec::new();
13473    let entries = match std::fs::read_dir(state.worktree_root()) {
13474        Ok(entries) => Some(entries),
13475        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13476        Err(error) => return Err(error.into()),
13477    };
13478    for entry in entries.into_iter().flatten() {
13479        let path = entry?.path();
13480        let metadata = std::fs::symlink_metadata(&path)?;
13481        anyhow::ensure!(
13482            metadata.is_dir() && !metadata.file_type().is_symlink(),
13483            "non-directory or symlink recovery source retained: {}",
13484            path.display()
13485        );
13486        if !drop_winner
13487            && winner.is_some_and(|w| {
13488                path.file_name()
13489                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13490            })
13491        {
13492            continue;
13493        }
13494        anyhow::ensure!(
13495            path.join(".git").is_file(),
13496            "unregistered recovery source retained: {}",
13497            path.display()
13498        );
13499        anyhow::ensure!(
13500            git::git(
13501                &path,
13502                &[
13503                    "status",
13504                    "--porcelain",
13505                    "--untracked-files=all",
13506                    "--ignored",
13507                    "--",
13508                    ".",
13509                    ":(exclude)target"
13510                ]
13511            )
13512            .await?
13513            .is_empty(),
13514            "source changes or ignored recovery files retained: {}",
13515            path.display()
13516        );
13517        match std::fs::symlink_metadata(path.join("target")) {
13518            Ok(_) => {
13519                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13520            }
13521            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13522            Err(error) => return Err(error.into()),
13523        }
13524        for checkpoint in [
13525            "rebase-merge",
13526            "rebase-apply",
13527            "sequencer",
13528            "CHERRY_PICK_HEAD",
13529            "MERGE_HEAD",
13530            "REBASE_HEAD",
13531            "REVERT_HEAD",
13532            "BISECT_LOG",
13533            "AUTO_MERGE",
13534            "MERGE_MSG",
13535        ] {
13536            let location = git::git(
13537                &path,
13538                &[
13539                    "rev-parse",
13540                    "--path-format=absolute",
13541                    "--git-path",
13542                    checkpoint,
13543                ],
13544            )
13545            .await?;
13546            anyhow::ensure!(
13547                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13548                if e.kind() == std::io::ErrorKind::NotFound),
13549                "Git recovery checkpoint retained: {checkpoint}"
13550            );
13551        }
13552        heads.insert(git::rev_parse(&path, "HEAD").await?);
13553    }
13554    crate::fold_evidence::preserve(state, home, &heads).await?;
13555
13556    Ok(cargo_guards)
13557}
13558
13559/// Remove `dir` if it exists and has nothing in it.
13560///
13561/// Best-effort and silent by design: a directory that is not empty (a run
13562/// whose winner is still parked there, a stray file some other process left)
13563/// is exactly the case this must refuse, and a directory that is already gone
13564/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13565/// already refuses a non-empty directory, so the emptiness check below is
13566/// belt, not suspenders - it is what keeps this from ever attempting the
13567/// removal in the case that matters, rather than trusting `remove_dir`'s
13568/// error path to have no side effects if it ever changed.
13569fn remove_if_empty(dir: &Path) {
13570    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13571        std::fs::remove_dir(dir).ok();
13572    }
13573}
13574
13575/// Severity of the worst open finding in the last review round, for reporting.
13576pub fn worst_open(state: &RunState) -> Option<Severity> {
13577    state
13578        .reviews
13579        .last()?
13580        .reviews
13581        .iter()
13582        .flat_map(|r| r.findings.iter())
13583        .map(|f| f.severity)
13584        .max()
13585}
13586
13587/// `[worktree] setup` hides its products per worktree, which needs
13588/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13589/// worktrees, like the hook's; every place that runs setup calls this first.
13590async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13591    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13592        git::acquire_worktree_config(repo).await?;
13593        state.enabled_worktree_config = true;
13594    }
13595    Ok(())
13596}
13597
13598#[cfg(test)]
13599mod tests {
13600    #[tokio::test]
13601    async fn launcher_base_fetch_requires_remote_success() {
13602        let tmp = tempfile::tempdir().unwrap();
13603        let repo = tmp.path().join("repo");
13604        let remote = tmp.path().join("remote");
13605        std::fs::create_dir(&repo).unwrap();
13606        std::fs::create_dir(&remote).unwrap();
13607        git::git(&remote, &["init", "--bare", "-b", "main"])
13608            .await
13609            .unwrap();
13610        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13611        git::git(
13612            &repo,
13613            &[
13614                "-c",
13615                "user.name=Fixture",
13616                "-c",
13617                "user.email=fixture@example.invalid",
13618                "commit",
13619                "--allow-empty",
13620                "-m",
13621                "fixture",
13622            ],
13623        )
13624        .await
13625        .unwrap();
13626        git::git(
13627            &repo,
13628            &["remote", "add", "origin", remote.to_str().unwrap()],
13629        )
13630        .await
13631        .unwrap();
13632        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13633        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13634        assert_eq!(
13635            resolve_base(&repo, "main", "origin", None)
13636                .await
13637                .unwrap()
13638                .commit,
13639            head
13640        );
13641        #[cfg(unix)]
13642        {
13643            let policy: crate::config::Coordination = toml::from_str(
13644                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13645            ).unwrap();
13646            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13647            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13648            let held = crate::repo_lock::SeatLease::operation_with(
13649                &repo,
13650                Some(&policy),
13651                &crate::repo_lock::WaitPolicy::default(),
13652            )
13653            .await
13654            .unwrap();
13655            assert!(held.is_some());
13656            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13657            tokio::pin!(fetch);
13658            assert!(
13659                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13660                    .await
13661                    .is_err()
13662            );
13663            assert!(
13664                !repo.join(".git/FETCH_HEAD").exists(),
13665                "no fetch before admission"
13666            );
13667            drop(held);
13668            let fetched = fetch.await.unwrap();
13669            assert_eq!(fetched.commit, head);
13670            assert_eq!(fetched.remote, "origin");
13671            assert_eq!(fetched.branch, "main");
13672        }
13673        // A surviving tracking ref is not successful fetch evidence.
13674        git::git(
13675            &repo,
13676            &[
13677                "remote",
13678                "set-url",
13679                "origin",
13680                tmp.path().join("missing").to_str().unwrap(),
13681            ],
13682        )
13683        .await
13684        .unwrap();
13685        let error = resolve_base(&repo, "main", "origin", None)
13686            .await
13687            .unwrap_err()
13688            .to_string();
13689        assert!(error.contains("cannot read origin/main"));
13690        assert!(error.contains("refusing to branch off"));
13691    }
13692
13693    #[test]
13694    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13695        let leaked = vec!["chore(deps): update a crate".to_owned()];
13696        let own = "fix(daemon): apply a chosen action";
13697        assert!(should_retitle("", own, &leaked));
13698        assert!(should_retitle(
13699            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13700            own,
13701            &leaked
13702        ));
13703        assert!(should_retitle(
13704            "chore: land candidate A of run 1",
13705            own,
13706            &leaked
13707        ));
13708        assert!(should_retitle(
13709            "magi: candidate A (uncommitted work)",
13710            own,
13711            &leaked
13712        ));
13713        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13714        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13715        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13716    }
13717
13718    #[test]
13719    fn pr_merge_plan_creates_adopts_or_stops() {
13720        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13721        assert_eq!(
13722            pr_merge_plan(Ok(land::OpenPr::One {
13723                url: "u".into(),
13724                title: "t".into()
13725            })),
13726            PrPlan::Adopt {
13727                url: "u".into(),
13728                title: "t".into()
13729            }
13730        );
13731        let PrPlan::Stop(many) =
13732            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13733        else {
13734            panic!("many must stop");
13735        };
13736        assert!(many.contains('a') && many.contains('b'));
13737        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13738            panic!("a failed lookup must stop");
13739        };
13740        assert!(err.contains("bad token"));
13741    }
13742
13743    use super::*;
13744    use crate::run::GateStatus;
13745    use std::collections::BTreeMap;
13746    use std::time::Duration;
13747
13748    fn conductor() -> AgentSpec {
13749        AgentSpec {
13750            id: "conductor".to_owned(),
13751            kind: crate::config::AgentKind::Command,
13752            model: None,
13753            command: vec!["true".to_owned()],
13754            extra_args: Vec::new(),
13755            env: BTreeMap::new(),
13756            prompt_delivery: None,
13757            max_turn_secs: None,
13758        }
13759    }
13760
13761    fn spec(id: &str) -> AgentSpec {
13762        AgentSpec {
13763            id: id.to_owned(),
13764            kind: crate::config::AgentKind::Command,
13765            model: None,
13766            command: vec!["true".to_owned()],
13767            extra_args: Vec::new(),
13768            env: BTreeMap::new(),
13769            prompt_delivery: None,
13770            max_turn_secs: None,
13771        }
13772    }
13773
13774    fn ids(xs: &[&str]) -> BTreeSet<String> {
13775        xs.iter().map(|s| (*s).to_owned()).collect()
13776    }
13777
13778    #[test]
13779    fn pick_successor_skips_an_agent_another_seat_holds() {
13780        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13781        let roster = [spec("a"), spec("b"), spec("c")];
13782        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13783        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13784    }
13785
13786    #[test]
13787    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13788        // The other seat started on c but was handed to d; c is free again.
13789        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13790        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13791        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13792        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13793        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13794    }
13795
13796    #[test]
13797    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13798        let roster = [spec("a"), spec("b")];
13799        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13800        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13801        let carried = ids(&["b"]);
13802        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13803        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13804    }
13805
13806    #[test]
13807    fn pick_successor_returns_none_without_any_untried_successor() {
13808        let roster = [spec("a"), spec("b")];
13809        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13810        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13811        let carried = ids(&["a"]);
13812        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13813    }
13814
13815    #[test]
13816    fn pick_successor_rescue_avoids_another_seats_occupant() {
13817        // b is a carried failure and free; a is held by the other seat.
13818        let roster = [spec("a"), spec("b"), spec("c")];
13819        let carried = ids(&["b", "c"]);
13820        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13821        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13822    }
13823
13824    #[test]
13825    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13826        let roster = [spec("a"), spec("b"), spec("c")];
13827        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13828        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13829    }
13830
13831    #[test]
13832    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13833        let roster = [spec("a"), spec("b"), spec("c")];
13834        let failed = ids(&["a", "b", "c"]);
13835        // Rescue looks at the whole roster, once per id, then runs out.
13836        let mut tried = ids(&["b"]);
13837        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13838        assert_eq!(first.id, "a");
13839        tried.insert(first.id.clone());
13840        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13841        assert_eq!(second.id, "c");
13842        tried.insert(second.id.clone());
13843        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13844    }
13845
13846    #[test]
13847    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13848        let roster = [spec("a"), spec("b")];
13849        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13850        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13851    }
13852
13853    #[test]
13854    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13855        let roster = [spec("a"), spec("b"), spec("c")];
13856        let h = SeatHistory {
13857            failed: ids(&["a"]),
13858            last_ok: Some("b".to_owned()),
13859            last_fail: None,
13860        };
13861        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13862        // A last answerer that left the roster, or later failed, is ignored.
13863        let gone = SeatHistory {
13864            last_ok: Some("zzz".to_owned()),
13865            ..h.clone()
13866        };
13867        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13868        let failed = SeatHistory {
13869            failed: ids(&["a", "b"]),
13870            last_ok: Some("b".to_owned()),
13871            last_fail: None,
13872        };
13873        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13874    }
13875
13876    #[test]
13877    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13878        let first = SeatState::new("review-1", "alpha", 7);
13879        let next = handover_seat("review-1", "gamma", 7);
13880        assert_ne!(first.claude_session, next.claude_session);
13881    }
13882
13883    #[test]
13884    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13885        let mut state = state_with_summary("x", "y");
13886        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13887        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13888        assert_ne!(a.claude_session, b.claude_session);
13889    }
13890
13891    #[test]
13892    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13893        let roster = [spec("a"), spec("b")];
13894        let h = SeatHistory {
13895            failed: ids(&["a", "b"]),
13896            ..SeatHistory::default()
13897        };
13898        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13899        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13900        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13901    }
13902
13903    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13904    // fallback loop depends on to terminate: it must walk forward from the
13905    // seat's own position, never restart at the front of the roster, and it
13906    // must never hand back an id already tried, however many times that id
13907    // happens to appear.
13908
13909    #[test]
13910    fn failure_signature_ignores_numbers_and_paths() {
13911        assert_eq!(
13912            failure_signature("exited with Some(2) and no usable output"),
13913            failure_signature("exited with Some(137) and no usable output")
13914        );
13915        assert_eq!(
13916            failure_signature("process exited with no usable output (exit code 2)"),
13917            failure_signature("process exited with no usable output (exit code 137)")
13918        );
13919        for (old, new) in [
13920            (
13921                "exited with Some(2) and no usable output",
13922                "process exited with no usable output (exit code 2)",
13923            ),
13924            (
13925                "exited with None and no usable output",
13926                "process exited with no usable output (exit code unavailable)",
13927            ),
13928        ] {
13929            let previous = FailClass::Other(failure_signature(old));
13930            let current = FailClass::Other(failure_signature(new));
13931            assert_eq!(previous, current, "resumption retains the failure class");
13932            assert!(!should_hand_over(Some(&previous), &current));
13933        }
13934        assert_eq!(
13935            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13936            failure_signature("cannot open /var/x.txt: denied")
13937        );
13938        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13939    }
13940
13941    #[test]
13942    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13943        let other = FailClass::Other("x".into());
13944        assert!(should_hand_over(None, &FailClass::Quota));
13945        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13946        assert!(should_hand_over(
13947            Some(&FailClass::Timeout),
13948            &FailClass::Timeout
13949        ));
13950        assert!(should_hand_over(None, &other));
13951        assert!(!should_hand_over(Some(&other), &other));
13952        assert!(should_hand_over(
13953            Some(&other),
13954            &FailClass::Other("y".into())
13955        ));
13956        // A quota or timeout in between ends the run of identical failures.
13957        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13958        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13959    }
13960
13961    #[test]
13962    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13963        let a = SeatState::new("judge-1", "alpha", 7);
13964        let b = handover_seat("judge-1", "beta", 7);
13965        assert_ne!(a.claude_session, b.claude_session);
13966        assert_eq!(b.turns, 0);
13967    }
13968
13969    #[test]
13970    fn a_timeout_is_classified_apart_from_other_failures() {
13971        assert_eq!(
13972            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13973            Some(FailClass::Timeout)
13974        );
13975        assert!(matches!(
13976            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13977            Some(FailClass::Other(_))
13978        ));
13979    }
13980
13981    #[test]
13982    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13983        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13984        let tried = BTreeSet::from(["beta".to_owned()]);
13985        // beta sits at index 1; the next candidate is gamma, never alpha —
13986        // which is very likely a different candidate slot's own agent.
13987        let next = next_untried_in_roster(&roster, 1, &tried);
13988        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13989    }
13990
13991    #[test]
13992    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13993        let roster = vec![spec("alpha"), spec("beta")];
13994        let tried = BTreeSet::from(["beta".to_owned()]);
13995        // beta is the roster's last entry: nothing follows it, and alpha —
13996        // earlier in the roster, almost certainly a different candidate
13997        // slot's own agent — must not be reached by wrapping back to it.
13998        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13999    }
14000
14001    #[test]
14002    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14003        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14004        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14005        // beta (index 1) and gamma (index 2, the only entry after it) have
14006        // both been tried; alpha (index 0) never has, but it comes before
14007        // beta's own position, so there is nothing further for this seat.
14008        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14009    }
14010
14011    #[test]
14012    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14013        let roster = vec![spec("a"), spec("a"), spec("b")];
14014        let tried = BTreeSet::from(["a".to_owned()]);
14015        let next = next_untried_in_roster(&roster, 0, &tried);
14016        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14017    }
14018
14019    #[test]
14020    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14021        let roster = vec![spec("a"), spec("b")];
14022        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14023        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14024    }
14025
14026    #[test]
14027    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14028        let dir = tempfile::tempdir().unwrap();
14029        let bay = dir.path().join("ffff");
14030
14031        // Not there yet: nothing to do, nothing to panic on.
14032        remove_if_empty(&bay);
14033        assert!(!bay.exists());
14034
14035        // Something still inside - the winner's worktree, or a stray file -
14036        // keeps the directory standing.
14037        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14038        remove_if_empty(&bay);
14039        assert!(bay.exists(), "non-empty directory must survive");
14040
14041        // Once the last entry is gone, so is the directory itself.
14042        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14043        remove_if_empty(&bay);
14044        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14045    }
14046
14047    // `round_is_clean` is the exact decision this task fixed: a round with a
14048    // seat that never answered must not read the same as a round every seat
14049    // actually reviewed. These are deterministic and process-free by design —
14050    // the equivalent end-to-end check (a real reviewer timing out under a
14051    // live graph run) is a genuine race against wall-clock contention, and a
14052    // spawn slow enough to blow even a generous budget under a loaded test
14053    // run must not turn this specific regression check flaky.
14054
14055    #[test]
14056    fn a_full_panel_that_found_nothing_is_clean() {
14057        assert!(round_is_clean(
14058            0,
14059            true,
14060            2,
14061            2,
14062            0,
14063            IncompleteReviewPolicy::Block
14064        ));
14065    }
14066
14067    #[test]
14068    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14069        assert!(!round_is_clean(
14070            0,
14071            true,
14072            1,
14073            2,
14074            0,
14075            IncompleteReviewPolicy::Block
14076        ));
14077    }
14078
14079    #[test]
14080    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14081        assert!(!round_is_clean(
14082            1,
14083            true,
14084            1,
14085            2,
14086            0,
14087            IncompleteReviewPolicy::Warn
14088        ));
14089    }
14090
14091    #[test]
14092    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14093        assert!(round_is_clean(
14094            0,
14095            true,
14096            1,
14097            2,
14098            0,
14099            IncompleteReviewPolicy::Warn
14100        ));
14101    }
14102
14103    #[test]
14104    fn a_full_panel_with_an_open_finding_is_not_clean() {
14105        assert!(!round_is_clean(
14106            1,
14107            true,
14108            2,
14109            2,
14110            0,
14111            IncompleteReviewPolicy::Block
14112        ));
14113    }
14114
14115    #[test]
14116    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14117        assert!(!round_is_clean(
14118            0,
14119            false,
14120            2,
14121            2,
14122            0,
14123            IncompleteReviewPolicy::Block
14124        ));
14125    }
14126
14127    // The stall this task closes: under the default `block` policy, a seat
14128    // missing only because it was rate limited must not force a wait for a
14129    // session limit that will not lift by the next round. `round_is_clean`
14130    // is where that quorum carve-out lives; the review loop around it never
14131    // changes what a reviewer's vote or a finding's severity means.
14132
14133    #[test]
14134    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14135        // 1 of 2 answered, and the one missing was quota'd — the exact
14136        // "review-2 rate limited (quota)" shape from the field report.
14137        assert!(round_is_clean(
14138            0,
14139            true,
14140            1,
14141            2,
14142            1,
14143            IncompleteReviewPolicy::Block
14144        ));
14145    }
14146
14147    #[test]
14148    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14149        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14150        // not a quota loss (`quota_missing` stays 0) — worth another try.
14151        assert!(!round_is_clean(
14152            0,
14153            true,
14154            1,
14155            2,
14156            0,
14157            IncompleteReviewPolicy::Block
14158        ));
14159    }
14160
14161    #[test]
14162    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14163        assert!(!round_is_clean(
14164            1,
14165            true,
14166            1,
14167            2,
14168            1,
14169            IncompleteReviewPolicy::Block
14170        ));
14171        assert!(!round_is_clean(
14172            0,
14173            false,
14174            1,
14175            2,
14176            1,
14177            IncompleteReviewPolicy::Block
14178        ));
14179    }
14180
14181    #[test]
14182    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14183        // Every seat quota'd, nobody answered: there is no panel to decide
14184        // on, so this must fall through to the existing block-and-retry
14185        // fallback rather than call an unreviewed patch clean.
14186        assert!(!round_is_clean(
14187            0,
14188            true,
14189            0,
14190            2,
14191            2,
14192            IncompleteReviewPolicy::Block
14193        ));
14194    }
14195
14196    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14197        CommandOutcome {
14198            command: "test".to_owned(),
14199            code,
14200            output_tail: String::new(),
14201            duration_ms: 0,
14202            resource_blocked,
14203        }
14204    }
14205
14206    #[cfg(unix)]
14207    #[tokio::test]
14208    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14209        ask_test_home();
14210        let temp = tempfile::tempdir().unwrap();
14211        let mut runner = runner_at(RunStatus::Gating);
14212        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14213        let commands = vec![
14214            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14215            "touch must-not-run".to_owned(),
14216        ];
14217        let (outcomes, retried, pids) = run_e2e_with_retry(
14218            &mut runner.state,
14219            &shell,
14220            &commands,
14221            temp.path(),
14222            Duration::from_secs(5),
14223            "fixture",
14224        )
14225        .await;
14226        assert_eq!(outcomes.len(), 1);
14227        assert!(outcomes[0].resource_blocked);
14228        assert!(!outcomes[0].build_failed());
14229        assert!(!retried);
14230        assert!(pids.is_empty());
14231        assert!(!temp.path().join("must-not-run").exists());
14232        assert_eq!(
14233            runner.state.resource_block.as_ref().unwrap().kind,
14234            "incremental cache admission"
14235        );
14236    }
14237
14238    #[test]
14239    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14240        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14241        assert!(
14242            !verify_inconclusive(&[outcome(Some(1), false)]),
14243            "an ordinary failure is still evidence about the patch"
14244        );
14245        assert!(verify_inconclusive(&[outcome(None, true)]));
14246        assert!(
14247            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14248            "one inconclusive outcome taints the whole batch"
14249        );
14250        assert!(!verify_inconclusive(&[]));
14251    }
14252
14253    #[tokio::test]
14254    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14255        // Alive for the first two checks, then dead - confirms the loop
14256        // actually re-polls rather than deciding once and sleeping out the
14257        // ceiling regardless.
14258        let calls = std::sync::atomic::AtomicUsize::new(0);
14259        let started = Instant::now();
14260        wait_for_pids_with(
14261            &[123],
14262            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14263            Duration::from_millis(5),
14264            Duration::from_secs(5),
14265        )
14266        .await;
14267        assert!(
14268            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14269            "must keep checking rather than deciding on the first answer"
14270        );
14271        assert!(
14272            started.elapsed() < Duration::from_secs(1),
14273            "must return the moment it is confirmed dead, not wait out the ceiling"
14274        );
14275    }
14276
14277    #[tokio::test]
14278    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14279        let started = Instant::now();
14280        wait_for_pids_with(
14281            &[123],
14282            |_| true, // never reports dead
14283            Duration::from_millis(5),
14284            Duration::from_millis(30),
14285        )
14286        .await;
14287        let elapsed = started.elapsed();
14288        assert!(
14289            elapsed >= Duration::from_millis(30),
14290            "must not give up before its own ceiling: {elapsed:?}"
14291        );
14292        assert!(
14293            elapsed < Duration::from_secs(1),
14294            "must not wait past its own ceiling either: {elapsed:?}"
14295        );
14296    }
14297
14298    #[tokio::test]
14299    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14300        let started = Instant::now();
14301        wait_for_pids_with(
14302            &[],
14303            |_| true,
14304            Duration::from_secs(5),
14305            Duration::from_secs(5),
14306        )
14307        .await;
14308        assert!(
14309            started.elapsed() < Duration::from_millis(200),
14310            "an empty pid list has nothing to confirm"
14311        );
14312    }
14313
14314    // `review_conclusion` is the exact decision the review hand-off task
14315    // fixed: a round budget spent (or a tree that stopped moving) must not
14316    // collapse into `Blocked` regardless of what verification actually
14317    // said. Deterministic and process-free for the same reason the
14318    // `round_is_clean` family above is.
14319    fn review_round(
14320        clean: bool,
14321        blocking: usize,
14322        answered: usize,
14323        expected: usize,
14324        progressed: bool,
14325        e2e_ok: bool,
14326    ) -> ReviewRound {
14327        ReviewRound {
14328            round: 1,
14329            head: "h".to_owned(),
14330            verified_head: None,
14331            verified_at: None,
14332            reviews: Vec::new(),
14333            e2e: vec![CommandOutcome {
14334                command: "test".to_owned(),
14335                code: Some(if e2e_ok { 0 } else { 1 }),
14336                output_tail: String::new(),
14337                duration_ms: 0,
14338                resource_blocked: false,
14339            }],
14340            verify_retried: false,
14341            e2e_deferred: false,
14342            e2e_defer_reason: None,
14343            fix: None,
14344            blocking,
14345            answered,
14346            expected,
14347            clean,
14348            progressed,
14349            vote_split: false,
14350            reconsideration: Vec::new(),
14351            verdict: None,
14352        }
14353    }
14354
14355    #[test]
14356    fn review_conclusion_is_none_when_nothing_has_run() {
14357        assert_eq!(review_conclusion(&[], 3), None);
14358    }
14359
14360    #[test]
14361    fn review_conclusion_is_none_while_rounds_remain() {
14362        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14363        assert_eq!(review_conclusion(&rounds, 3), None);
14364    }
14365
14366    #[test]
14367    fn review_conclusion_is_gating_once_a_round_is_clean() {
14368        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14369        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14370    }
14371
14372    #[test]
14373    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14374        let rounds = vec![
14375            review_round(false, 1, 2, 2, true, true),
14376            review_round(false, 1, 2, 2, true, true),
14377        ];
14378        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14379    }
14380
14381    #[test]
14382    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14383        let rounds = vec![
14384            review_round(false, 1, 2, 2, true, true),
14385            review_round(false, 1, 2, 2, true, false),
14386        ];
14387        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14388    }
14389
14390    #[test]
14391    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14392        // Magi never got a command to run against this round's own head — a
14393        // resource-blocked attempt, not a red one — so this must never
14394        // settle on `Blocked` the way a genuine e2e failure would. `None`
14395        // here is what tells `Runner::review_loop` to retry the check
14396        // itself rather than trust this cheap recomputation with a verdict
14397        // it cannot actually produce.
14398        let mut blocked = review_round(false, 1, 2, 2, true, false);
14399        blocked.e2e[0].resource_blocked = true;
14400        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14401        assert_eq!(review_conclusion(&rounds, 2), None);
14402    }
14403
14404    #[test]
14405    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14406        // Missing input, not a verified tree — never a hand-off candidate.
14407        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14408        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14409    }
14410
14411    #[test]
14412    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14413        let rounds = vec![
14414            review_round(false, 1, 2, 2, false, true),
14415            review_round(false, 1, 2, 2, false, true),
14416        ];
14417        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14418    }
14419
14420    fn secs(n: u64) -> Duration {
14421        Duration::from_secs(n)
14422    }
14423
14424    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14425    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14426    #[tokio::test]
14427    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14428        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14429        if std::env::var_os(CHILD).is_none() {
14430            // Other tests reconcile every run in the process-global home,
14431            // briefly taking owner locks. This ancestry test expects an idle
14432            // predecessor, so give it its own process and pinned test home.
14433            let output = std::process::Command::new(std::env::current_exe().unwrap())
14434                .args([
14435                    "--exact",
14436                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14437                    "--nocapture",
14438                ])
14439                .env(CHILD, "1")
14440                .env_remove("MAGI_WORKER_HOME")
14441                .env_remove("MAGI_WORKER_ID")
14442                .output()
14443                .unwrap();
14444            assert!(
14445                output.status.success()
14446                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14447                "{}\n{}",
14448                String::from_utf8_lossy(&output.stdout),
14449                String::from_utf8_lossy(&output.stderr)
14450            );
14451            return;
14452        }
14453        ask_test_home();
14454        let tmp = tempfile::tempdir().unwrap();
14455        let repo = tmp.path().join("repo");
14456        std::fs::create_dir_all(&repo).unwrap();
14457        init_repo(&repo);
14458        let g = |args: &[&str]| {
14459            let out = std::process::Command::new("git")
14460                .args(args)
14461                .current_dir(&repo)
14462                .quiet()
14463                .output()
14464                .unwrap();
14465            assert!(
14466                out.status.success(),
14467                "{}",
14468                String::from_utf8_lossy(&out.stderr)
14469            );
14470        };
14471        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14472        g(&["checkout", "-b", "historical"]);
14473        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14474        g(&["add", "."]);
14475        g(&["commit", "-m", "old alternative"]);
14476        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14477        g(&["checkout", "-b", "magi/task-selected", &base]);
14478        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14479        g(&["add", "."]);
14480        g(&["commit", "-m", "measurement infrastructure"]);
14481        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14482        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14483        g(&["add", "."]);
14484        g(&["commit", "-m", "dependent implementation"]);
14485        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14486        g(&["checkout", "main"]);
14487        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14488        g(&["add", "."]);
14489        g(&["commit", "-m", "unrelated base change"]);
14490        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14491        let origin = tmp.path().join("origin.git");
14492        g(&[
14493            "clone",
14494            "--bare",
14495            repo.to_str().unwrap(),
14496            origin.to_str().unwrap(),
14497        ]);
14498        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14499        g(&["fetch", "origin"]);
14500        let selection = crate::pr_recovery::Selection {
14501            url: "https://example.test/project/pull/1".into(),
14502            branch: "magi/task-selected".into(),
14503            head: head.clone(),
14504            source_head: None,
14505            source_run: None,
14506            source_proof: None,
14507            disposition: Some(crate::pr_recovery::Disposition::Review),
14508            instruction: "preserve measured behavior".into(),
14509            approved_at: jiff::Timestamp::now(),
14510        };
14511        let mut template = runner_at(RunStatus::Reviewing);
14512        template.state.repo = repo.clone();
14513        template.state.base_commit = current.clone();
14514        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14515        template.state.origin = Some(crate::run::Origin::queue("selected"));
14516        template.state.pr_head = Some(selection.branch.clone());
14517        template.state.pr_recovery = Some(selection.clone());
14518        let mut runner =
14519            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14520                .await
14521                .unwrap();
14522        let wt = runner.state.winner().unwrap().worktree.clone();
14523        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14524        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14525        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14526        assert!(runner.state.seeds.is_empty());
14527        assert_eq!(runner.state.instruction, "preserve measured behavior");
14528        runner.sync_to_base().await.unwrap();
14529        assert_eq!(
14530            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14531            "prerequisite"
14532        );
14533        assert_eq!(
14534            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14535            "measured implementation"
14536        );
14537        assert_eq!(
14538            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14539            "current main change"
14540        );
14541        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14542        assert_eq!(
14543            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14544            head
14545        );
14546
14547        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14548        let mut partial = runner_at(RunStatus::Reviewing);
14549        partial.state.repo = repo.clone();
14550        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14551        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14552            run: Some("exit 1".into()),
14553            ..Default::default()
14554        }];
14555        let selection = runner.state.pr_recovery.clone().unwrap();
14556        partial.state.pr_recovery = Some(selection.clone());
14557        partial.state.pr_head = Some(selection.branch.clone());
14558        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14559        let id = partial.state.id.clone();
14560        let error = match Runner::open_selected(
14561            &repo,
14562            partial.state,
14563            partial.roles,
14564            1,
14565            runner.state.base_commit.clone(),
14566            selection,
14567        )
14568        .await
14569        {
14570            Ok(_) => panic!("setup must fail"),
14571            Err(e) => e,
14572        };
14573        assert_eq!(
14574            error
14575                .downcast_ref::<crate::run::StartupFailure>()
14576                .unwrap()
14577                .run,
14578            id
14579        );
14580        let failed = RunState::load(&id).unwrap();
14581        assert_eq!(failed.status, RunStatus::Failed);
14582        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14583        assert!(Runner::resume(&id).is_err());
14584        runner.state.status = RunStatus::Ready;
14585        runner.state.driver_exited = true;
14586        runner.state.save().unwrap();
14587        let mut retry = runner_at(RunStatus::Reviewing);
14588        retry.state.repo = repo.clone();
14589        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14590        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14591        reselected.approved_at = jiff::Timestamp::now();
14592        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14593        retry.state.pr_head = Some(reselected.branch.clone());
14594        retry.state.pr_recovery = Some(reselected.clone());
14595        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14596            .await
14597            .unwrap();
14598        let restarted = Runner::open_selected(
14599            &repo,
14600            retry.state,
14601            retry.roles,
14602            1,
14603            runner.state.base_commit.clone(),
14604            reselected,
14605        )
14606        .await
14607        .unwrap();
14608        assert_eq!(
14609            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14610                .await
14611                .unwrap(),
14612            head
14613        );
14614        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14615        assert_eq!(
14616            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14617            head
14618        );
14619    }
14620
14621    #[tokio::test]
14622    #[cfg(unix)]
14623    async fn fallback_and_interrupted_restoration_require_coordination() {
14624        use std::os::fd::AsRawFd as _;
14625        ask_test_home();
14626        let tmp = tempfile::tempdir().unwrap();
14627        let mut runner = registered_runner(tmp.path()).await;
14628        runner.state.config.graph.timeout_resource_wait = 1;
14629        let winner = runner.state.candidates[0].clone();
14630        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14631        let leftover = winner.worktree.join("leftover");
14632        std::fs::write(&leftover, "untrusted edits").unwrap();
14633        let next = tmp.path().join("next-fixer");
14634        let mut failed = spec("first");
14635        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14636        let mut successor = spec("second");
14637        successor.command = vec![
14638            "sh".into(),
14639            "-c".into(),
14640            format!("touch '{}'; echo done", next.display()),
14641        ];
14642        let attempts = vec![
14643            (failed.clone(), "first".into()),
14644            (successor.clone(), "second".into()),
14645        ];
14646        let lock_path = runner.state.repo.join(".git/updater.lock");
14647        let external = std::fs::File::create(&lock_path).unwrap();
14648        assert_eq!(
14649            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14650            0
14651        );
14652        let artifacts = tmp.path().join("artifacts");
14653        let build = |spec, seat| SeatJob {
14654            spec,
14655            seat,
14656            cwd: winner.worktree.clone(),
14657            prompt: "fixture".into(),
14658            timeout: Duration::from_secs(10),
14659            allow_write: true,
14660            sessions: false,
14661            artifacts: artifacts.clone(),
14662            stem: "fix".into(),
14663            handover: None,
14664        };
14665        let (_, _, out) = runner
14666            .ask_fixer_among(
14667                attempts.clone(),
14668                "graft",
14669                None,
14670                Some((&winner.worktree, &head)),
14671                build,
14672            )
14673            .await;
14674        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14675        assert!(leftover.exists(), "contention must prevent clean");
14676        assert!(!next.exists());
14677        assert!(runner.state.handovers.is_empty());
14678        // Early interrupted-repair rollback uses the same protected route.
14679        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14680            from_run: "earlier".into(),
14681            generation: 1,
14682            mode: crate::recovery::Mode::VerifyRepair,
14683            reasons: Vec::new(),
14684            reviewed_head: None,
14685            rounds: Vec::new(),
14686            open_findings: Vec::new(),
14687            evidence: None,
14688            criteria_inherited: true,
14689            repair: crate::recovery::Repair::Started {
14690                before: head.clone(),
14691            },
14692        });
14693        assert!(
14694            runner
14695                .recovery_repair(&winner)
14696                .await
14697                .unwrap_err()
14698                .is::<crate::repo_lock::ResourceBlocked>()
14699        );
14700        assert!(matches!(
14701            runner.state.recovery.as_ref().unwrap().repair,
14702            crate::recovery::Repair::Started { .. }
14703        ));
14704        assert!(leftover.exists());
14705        drop(external);
14706        runner.recovery_repair(&winner).await.unwrap();
14707        assert!(!leftover.exists());
14708        assert!(matches!(
14709            runner.state.recovery.as_ref().unwrap().repair,
14710            crate::recovery::Repair::Done { .. }
14711        ));
14712        // A hard lock-open failure also leaves HEAD and untracked work intact.
14713        std::fs::remove_file(&lock_path).unwrap();
14714        std::fs::create_dir(&lock_path).unwrap();
14715        std::fs::write(&leftover, "untrusted edits").unwrap();
14716        let (_, _, out) = runner
14717            .ask_fixer_among(
14718                attempts,
14719                "graft",
14720                None,
14721                Some((&winner.worktree, &head)),
14722                build,
14723            )
14724            .await;
14725        assert!(matches!(out, AgentOutcome::Failed(_)));
14726        assert!(leftover.exists());
14727        assert!(!next.exists());
14728        assert_eq!(
14729            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14730            head
14731        );
14732    }
14733
14734    #[cfg(unix)]
14735    async fn registered_runner(root: &Path) -> Runner {
14736        repository_runner(root, crate::config::test_coordination()).await
14737    }
14738
14739    async fn repository_runner(
14740        root: &Path,
14741        coordination: Option<crate::config::Coordination>,
14742    ) -> Runner {
14743        ask_test_home();
14744        let repo = root.join("repo");
14745        std::fs::create_dir_all(&repo).unwrap();
14746        init_repo(&repo);
14747        if coordination.is_some() {
14748            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14749        }
14750        let mut runner = runner_at(RunStatus::Prep);
14751        runner.state.repo = repo.clone();
14752        runner.state.config.coordination = coordination;
14753        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14754        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14755        runner.state.config.graph.advise = false;
14756        runner.roles.implementers = vec![spec("implement")];
14757        runner.roles.judges = vec![spec("judge")];
14758        runner.prep().await.unwrap();
14759        runner
14760    }
14761
14762    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14763
14764    #[tokio::test]
14765    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14766        let tmp = tempfile::tempdir().unwrap();
14767        let mut runner = repository_runner(tmp.path(), None).await;
14768        let prompts = runner.state.config.prompts.clone();
14769        let run = runner.state.id.clone();
14770        let ctx = WaveCtx {
14771            carry_seats: false,
14772            pauses: None,
14773            run: &run,
14774            node: "advise",
14775            prompts: &prompts,
14776            cache: None,
14777            round: None,
14778        };
14779        let jobs: Vec<_> = ["alpha", "beta"]
14780            .into_iter()
14781            .map(|id| SeatJob {
14782                spec: scripted(id, AUTH_OUTAGE),
14783                seat: SeatState::new(id, id, 7),
14784                prompt: "propose".into(),
14785                cwd: runner.state.candidates[0].worktree.clone(),
14786                timeout: Duration::from_secs(10),
14787                allow_write: false,
14788                sessions: false,
14789                artifacts: agent::artifacts_dir(&runner.state.dir()),
14790                stem: id.into(),
14791                handover: None,
14792            })
14793            .collect();
14794        let mut losses = Vec::new();
14795        let results = ask_wave_with(
14796            jobs.clone(),
14797            runner.sem.clone(),
14798            2,
14799            &[],
14800            &ctx,
14801            &mut losses,
14802            &mut runner.state,
14803            &|_: &str| Ok(()),
14804        )
14805        .await;
14806        assert!(
14807            results
14808                .iter()
14809                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14810        );
14811        assert_eq!(runner.state.quota.len(), 2);
14812        assert!(
14813            runner
14814                .state
14815                .quota
14816                .iter()
14817                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14818        );
14819        assert!(!runner.state.advise_attempted);
14820        runner.stop_for_resource().unwrap();
14821        assert_eq!(runner.state.status, RunStatus::Blocked);
14822        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14823        assert_eq!(
14824            runner.state.resource_block.as_ref().unwrap().kind,
14825            "agent authentication"
14826        );
14827        // A machine loss followed by an ordinary failure is not evidence
14828        // that the whole node was unavailable, and must not earn a refund.
14829        runner.state.resource_block = None;
14830        runner.state.quota.clear();
14831        let mut mixed = jobs;
14832        mixed[1].spec.command = vec!["false".into()];
14833        let results = ask_wave_with(
14834            mixed,
14835            runner.sem.clone(),
14836            0,
14837            &[],
14838            &ctx,
14839            &mut losses,
14840            &mut runner.state,
14841            &|_: &str| Ok(()),
14842        )
14843        .await;
14844        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14845        assert!(runner.state.resource_block.is_none());
14846        assert_eq!(losses.len(), 1);
14847    }
14848
14849    #[tokio::test]
14850    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14851        let tmp = tempfile::tempdir().unwrap();
14852        let mut runner = repository_runner(tmp.path(), None).await;
14853        let implementer = scripted("implement", AUTH_OUTAGE);
14854        runner.roles.implementers = vec![implementer.clone()];
14855        runner.state.config.agents = vec![implementer];
14856        runner.implement().await.unwrap();
14857        assert_eq!(runner.state.status, RunStatus::Blocked);
14858        assert_eq!(runner.state.quota.len(), 1);
14859        assert_eq!(
14860            runner.state.quota[0].cause,
14861            agent::QuotaCause::Authentication
14862        );
14863        let candidate = &runner.state.candidates[0];
14864        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14865        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14866    }
14867
14868    #[tokio::test]
14869    #[cfg(unix)]
14870    async fn parked_implementer_persists_completed_work() {
14871        for registered in [false, true] {
14872            let tmp = tempfile::tempdir().unwrap();
14873            let mut runner = registered_runner(tmp.path()).await;
14874            if !registered {
14875                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14876            }
14877            let marker = tmp.path().join("started");
14878            let release = tmp.path().join("release");
14879            let mut implementer = spec("implement");
14880            implementer.command = vec![
14881                "sh".into(),
14882                "-c".into(),
14883                format!(
14884                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14885                    marker.display(),
14886                    release.display()
14887                ),
14888            ];
14889            runner.roles.implementers = vec![implementer.clone()];
14890            runner.state.config.agents = vec![implementer];
14891            let pause = runner.pause.clone();
14892            let (result, ()) = tokio::join!(runner.implement(), async {
14893                tokio::time::timeout(Duration::from_secs(10), async {
14894                    while !marker.exists() {
14895                        tokio::time::sleep(Duration::from_millis(10)).await;
14896                    }
14897                })
14898                .await
14899                .unwrap();
14900                pause.park();
14901                std::fs::write(&release, "go").unwrap();
14902            });
14903            result.unwrap();
14904            let candidate = &runner.state.candidates[0];
14905            assert!(candidate.commits > 0, "{candidate:?}");
14906            assert!(candidate.summary.contains("saved-result"));
14907            assert_eq!(candidate.files, 1);
14908            let before = candidate.commits;
14909            runner.implement().await.unwrap();
14910            assert_eq!(runner.state.candidates[0].commits, before);
14911        }
14912    }
14913
14914    #[tokio::test]
14915    #[cfg(unix)]
14916    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14917        let tmp = tempfile::tempdir().unwrap();
14918        let mut runner = registered_runner(tmp.path()).await;
14919        let winner = runner.state.candidates[0].clone();
14920        let cache = tmp.path().join("cache");
14921        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14922        runner.state.config.graph.timeout_resource_wait = 0;
14923        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14924        let crate::cache::AcquireOutcome::Acquired(held) =
14925            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14926        else {
14927            panic!("cache must be free")
14928        };
14929        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14930            from_run: "earlier".into(),
14931            generation: 1,
14932            mode: crate::recovery::Mode::VerifyRepair,
14933            reasons: Vec::new(),
14934            reviewed_head: None,
14935            rounds: Vec::new(),
14936            open_findings: Vec::new(),
14937            evidence: None,
14938            criteria_inherited: true,
14939            repair: crate::recovery::Repair::NotStarted,
14940        });
14941        runner.recovery_repair(&winner).await.unwrap();
14942        assert!(matches!(
14943            runner.state.recovery.as_ref().unwrap().repair,
14944            crate::recovery::Repair::WaitingForRestore { .. }
14945        ));
14946        drop(held);
14947        runner.state.resource_block = None;
14948        runner.state.config.graph.timeout_resource_wait = 1;
14949        runner.recovery_repair(&winner).await.unwrap();
14950        assert!(matches!(
14951            runner.state.recovery.as_ref().unwrap().repair,
14952            crate::recovery::Repair::Done { .. }
14953        ));
14954        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14955    }
14956
14957    #[tokio::test]
14958    #[cfg(unix)]
14959    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14960        use std::os::fd::AsRawFd as _;
14961        for graft in [false, true] {
14962            let tmp = tempfile::tempdir().unwrap();
14963            let mut runner = registered_runner(tmp.path()).await;
14964            runner.state.config.graph.timeout_resource_wait = 1;
14965            let winner = runner.state.candidates[0].clone();
14966            let marker = tmp.path().join("fixer-started");
14967            let mut fixer = spec("fixer");
14968            fixer.command = vec![
14969                "sh".into(),
14970                "-c".into(),
14971                format!("echo started >> '{}'; echo done", marker.display()),
14972            ];
14973            runner.roles.fixer = Some(vec![fixer]);
14974            if graft {
14975                let mut loser = winner.clone();
14976                loser.label = 'B';
14977                loser.index = 1;
14978                runner.state.candidates.push(loser);
14979                runner.state.config.graph.graft_runner_up = true;
14980                runner.state.tally = Some(Tally {
14981                    first_choice: BTreeMap::new(),
14982                    borda: BTreeMap::new(),
14983                    winner: winner.label,
14984                    rankings: 1,
14985                    unanimous_initial: true,
14986                    deliberated: false,
14987                    changed_votes: 0,
14988                    unanimous_final: true,
14989                    tie_break: None,
14990                    judges: 1,
14991                    present: 1,
14992                    quorum: 1,
14993                    met_quorum: true,
14994                    uncontested: None,
14995                });
14996                runner.state.judgements.push(crate::run::Judgement {
14997                    judge: 1,
14998                    seat: "judge-1".into(),
14999                    agent: "judge".into(),
15000                    ranking: vec![winner.label, 'B'],
15001                    reasons: BTreeMap::new(),
15002                    confidence: None,
15003                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15004                    order: vec![0, 1],
15005                    failed: None,
15006                    duration_ms: 0,
15007                });
15008            } else {
15009                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15010                    from_run: "earlier".into(),
15011                    generation: 1,
15012                    mode: crate::recovery::Mode::VerifyRepair,
15013                    reasons: Vec::new(),
15014                    reviewed_head: None,
15015                    rounds: Vec::new(),
15016                    open_findings: Vec::new(),
15017                    evidence: None,
15018                    criteria_inherited: true,
15019                    repair: crate::recovery::Repair::NotStarted,
15020                });
15021            }
15022            let reader = crate::repo_lock::SeatLease::for_checkout(
15023                &winner.worktree,
15024                false,
15025                crate::config::test_coordination().as_ref(),
15026            )
15027            .await
15028            .unwrap();
15029            let pause = runner.pause.clone();
15030            let (result, ()) = tokio::join!(
15031                async {
15032                    if graft {
15033                        runner.graft().await
15034                    } else {
15035                        runner.recovery_repair(&winner).await
15036                    }
15037                },
15038                async {
15039                    tokio::time::sleep(Duration::from_millis(100)).await;
15040                    pause.park();
15041                }
15042            );
15043            result.unwrap();
15044            assert!(!marker.exists(), "a blocked seat must not spawn");
15045            // Exercise the persisted state, as a binary replacement/resume does.
15046            runner.state = RunState::load(&runner.state.id).unwrap();
15047            if graft {
15048                assert_eq!(
15049                    runner.state.graft.as_ref().unwrap().outcome,
15050                    GraftOutcome::WaitingForRestore
15051                );
15052            } else {
15053                assert!(matches!(
15054                    runner.state.recovery.as_ref().unwrap().repair,
15055                    crate::recovery::Repair::WaitingForRestore { .. }
15056                ));
15057            }
15058            drop(reader);
15059            runner.pause = Pause::new();
15060            runner.state.resource_block = None;
15061            // Deferred restoration must still respect an external lock owner.
15062            let external =
15063                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15064            assert_eq!(
15065                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15066                0
15067            );
15068            let leftover = winner.worktree.join("leftover");
15069            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15070            let result = if graft {
15071                runner.graft().await
15072            } else {
15073                runner.recovery_repair(&winner).await
15074            };
15075            assert!(
15076                result
15077                    .unwrap_err()
15078                    .is::<crate::repo_lock::ResourceBlocked>()
15079            );
15080            assert!(leftover.exists());
15081            assert!(!marker.exists());
15082            drop(external);
15083            runner.state.resource_block = None;
15084            let result = if graft {
15085                runner.graft().await
15086            } else {
15087                runner.recovery_repair(&winner).await
15088            };
15089            result.unwrap();
15090            assert!(!leftover.exists(), "restore must precede retry");
15091            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15092            if graft {
15093                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15094            } else {
15095                assert!(matches!(
15096                    runner.state.recovery.as_ref().unwrap().repair,
15097                    crate::recovery::Repair::Done { .. }
15098                ));
15099            }
15100        }
15101    }
15102
15103    #[tokio::test]
15104    #[cfg(unix)]
15105    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15106        use std::os::fd::AsRawFd as _;
15107        let tmp = tempfile::tempdir().unwrap();
15108        let mut runner = registered_runner(tmp.path()).await;
15109        runner.state.config.graph.timeout_resource_wait = 1;
15110        let winner = runner.state.candidates[0].clone();
15111        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15112        runner.state.tally = Some(Tally {
15113            first_choice: BTreeMap::new(),
15114            borda: BTreeMap::new(),
15115            winner: winner.label,
15116            rankings: 0,
15117            unanimous_initial: false,
15118            deliberated: false,
15119            changed_votes: 0,
15120            unanimous_final: false,
15121            tie_break: None,
15122            judges: 0,
15123            present: 0,
15124            quorum: 0,
15125            met_quorum: true,
15126            uncontested: Some("fixture".into()),
15127        });
15128        runner.state.graft = Some(GraftRecord {
15129            pre: pre.clone(),
15130            post: None,
15131            lines: None,
15132            limit: None,
15133            agent: None,
15134            outcome: GraftOutcome::Running,
15135            requested: Vec::new(),
15136            adoption: None,
15137            adoption_error: None,
15138        });
15139        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15140        git::commit_all(&winner.worktree, "unfinished graft")
15141            .await
15142            .unwrap();
15143        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15144        let leftover = winner.worktree.join("leftover");
15145        std::fs::write(&leftover, "untrusted edits").unwrap();
15146        let path = runner.state.repo.join(".git/updater.lock");
15147        let external = std::fs::File::create(&path).unwrap();
15148        assert_eq!(
15149            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15150            0
15151        );
15152        assert!(
15153            runner
15154                .graft()
15155                .await
15156                .unwrap_err()
15157                .is::<crate::repo_lock::ResourceBlocked>()
15158        );
15159        assert!(matches!(
15160            runner.state.graft.as_ref().unwrap().outcome,
15161            GraftOutcome::Running
15162        ));
15163        assert_eq!(
15164            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15165            unfinished
15166        );
15167        assert!(leftover.exists());
15168        drop(external);
15169        std::fs::remove_file(&path).unwrap();
15170        std::fs::create_dir(&path).unwrap();
15171        assert!(runner.graft().await.is_err());
15172        assert!(matches!(
15173            runner.state.graft.as_ref().unwrap().outcome,
15174            GraftOutcome::Running
15175        ));
15176        assert_eq!(
15177            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15178            unfinished
15179        );
15180        assert!(leftover.exists());
15181        std::fs::remove_dir(&path).unwrap();
15182        runner.graft().await.unwrap();
15183        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15184        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15185        assert!(!leftover.exists());
15186    }
15187
15188    #[tokio::test]
15189    #[cfg(unix)]
15190    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15191        use std::os::fd::AsRawFd as _;
15192        let tmp = tempfile::tempdir().unwrap();
15193        let mut runner = registered_runner(tmp.path()).await;
15194        runner.state.prep_complete = Some(false);
15195        runner.state.config.graph.timeout_resource_wait = 1;
15196        runner.state.save().unwrap();
15197        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15198        assert_eq!(
15199            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15200            0
15201        );
15202        runner.execute().await.unwrap();
15203        let disk = RunState::load(&runner.state.id).unwrap();
15204        assert!(disk.driver_exited);
15205        assert_eq!(disk.driver_pid, None);
15206        assert_eq!(runner.state.status, RunStatus::Blocked);
15207        assert_eq!(
15208            runner.state.resource_block.as_ref().unwrap().kind,
15209            "repository coordination"
15210        );
15211        assert!(runner.state.prep_failure.is_none());
15212        assert!(runner.state.seats.is_empty());
15213        runner.pause.park();
15214        // Include headroom for Git subprocesses under parallel test load;
15215        // the repository contention deadline itself remains one second.
15216        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15217            .await
15218            .unwrap()
15219            .unwrap();
15220        assert!(runner.state.parked);
15221        assert!(runner.state.prep_failure.is_none());
15222        let disk = RunState::load(&runner.state.id).unwrap();
15223        assert!(disk.driver_exited);
15224        assert_eq!(disk.driver_pid, None);
15225    }
15226
15227    #[tokio::test]
15228    #[cfg(unix)]
15229    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15230        use std::os::fd::AsRawFd as _;
15231        let tmp = tempfile::tempdir().unwrap();
15232        let mut runner = registered_runner(tmp.path()).await;
15233        runner.state.prep_complete = Some(false);
15234        runner.state.config.graph.timeout_resource_wait = 30;
15235        runner.state.save().unwrap();
15236        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15237        assert_eq!(
15238            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15239            0
15240        );
15241        assert!(
15242            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15243                .await
15244                .is_err()
15245        );
15246        let disk = RunState::load(&runner.state.id).unwrap();
15247        assert!(disk.driver_exited);
15248        assert_eq!(disk.driver_pid, None);
15249        assert!(disk.status.resumable());
15250        assert!(runner.state.driver_exited);
15251        assert_eq!(runner.state.driver_pid, None);
15252        runner.state.save().unwrap();
15253        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15254    }
15255
15256    #[tokio::test]
15257    #[cfg(unix)]
15258    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15259        let tmp = tempfile::tempdir().unwrap();
15260        let mut runner = registered_runner(tmp.path()).await;
15261        let winner = runner.state.candidates[0].clone();
15262        let cache = tmp.path().join("cache");
15263        let started = tmp.path().join("pre-gate-started");
15264        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15265        runner.state.config.verify.gate =
15266            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15267        let home = crate::run::home();
15268        let owner =
15269            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15270        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15271            crate::cache::AcquireOutcome::Acquired(g) => g,
15272            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15273        };
15274        let mut gate = Box::pin(runner.run_gate(&winner));
15275        assert!(
15276            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15277                .await
15278                .is_err()
15279        );
15280        assert!(!started.exists(), "pre-gate must wait for the cache too");
15281        // Simulate the wave that already owns the cache entering invoke.
15282        // It must obtain the editor reservation and finish without waiting
15283        // for the gate that is queued behind its cache.
15284        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15285        let editor = crate::repo_lock::SeatLease::checkout_with(
15286            &winner.worktree,
15287            true,
15288            crate::config::test_coordination().as_ref(),
15289            &wait,
15290        )
15291        .await
15292        .unwrap();
15293        drop(editor);
15294        guard.release();
15295        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15296            .await
15297            .unwrap()
15298            .unwrap();
15299        drop(gate);
15300        assert!(outcomes.iter().all(CommandOutcome::ok));
15301        assert!(started.exists());
15302    }
15303
15304    #[tokio::test]
15305    #[cfg(unix)]
15306    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15307        use std::os::fd::AsRawFd as _;
15308
15309        ask_test_home();
15310        let tmp = tempfile::tempdir().unwrap();
15311        let repo = tmp.path().join("repo");
15312        std::fs::create_dir_all(&repo).unwrap();
15313        init_repo(&repo);
15314        let common = repo.join(".git");
15315        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15316        let mut runner = runner_at(RunStatus::Prep);
15317        runner.state.repo = repo.clone();
15318        runner.state.config.coordination = crate::config::test_coordination();
15319        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15320        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15321        runner.state.config.graph.advise = false;
15322        runner.roles.implementers = vec![spec("implement")];
15323        runner.roles.judges = vec![spec("judge")];
15324        runner.prep().await.unwrap();
15325        let winner = runner.state.candidates[0].clone();
15326        // Emulate edits already left by a completed seat. The fixture CLI
15327        // only reports completion; it performs no repository operations.
15328        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15329        let completed = tmp.path().join("seat-completed");
15330        runner.roles.implementers[0].command = vec![
15331            "sh".into(),
15332            "-c".into(),
15333            format!("touch '{}'; echo completed", completed.display()),
15334        ];
15335        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15336        assert_eq!(
15337            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15338            0
15339        );
15340        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15341        let mut implement = Box::pin(runner.implement());
15342        tokio::select! {
15343            result = &mut implement => panic!("rescue must wait: {result:?}"),
15344            result = tokio::time::timeout(Duration::from_secs(10), async {
15345                while !completed.exists() {
15346                    tokio::time::sleep(Duration::from_millis(20)).await;
15347                }
15348            }) => result.unwrap(),
15349        }
15350        assert!(
15351            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15352                .await
15353                .is_err()
15354        );
15355        assert_eq!(
15356            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15357            before
15358        );
15359        assert_eq!(
15360            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15361            0
15362        );
15363        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15364            .await
15365            .unwrap()
15366            .unwrap();
15367        drop(implement);
15368        assert_ne!(
15369            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15370            before
15371        );
15372        assert!(git::is_clean(&winner.worktree).await.unwrap());
15373
15374        let started = tmp.path().join("pre-gate-started");
15375        let proceed = tmp.path().join("pre-gate-proceed");
15376        let checked = tmp.path().join("gate-ran");
15377        runner.state.config.verify.pre_gate = vec![format!(
15378            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15379            started.display(),
15380            proceed.display()
15381        )];
15382        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15383        assert_eq!(
15384            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15385            0
15386        );
15387        let mut gate = Box::pin(runner.run_gate(&winner));
15388        assert!(
15389            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15390                .await
15391                .is_err()
15392        );
15393        assert!(!started.exists());
15394        assert!(!checked.exists());
15395        assert!(git::is_clean(&winner.worktree).await.unwrap());
15396        assert_eq!(
15397            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15398            0
15399        );
15400        tokio::select! {
15401            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15402            result = tokio::time::timeout(Duration::from_secs(10), async {
15403                while !started.exists() {
15404                    tokio::time::sleep(Duration::from_millis(20)).await;
15405                }
15406            }) => result.unwrap(),
15407        }
15408        assert_eq!(
15409            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15410            -1,
15411            "mechanical edits must hold the real lock"
15412        );
15413        std::fs::write(proceed, "ready").unwrap();
15414        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15415            .await
15416            .unwrap()
15417            .unwrap();
15418        drop(gate);
15419        assert!(outcomes.iter().all(CommandOutcome::ok));
15420        assert!(checked.exists());
15421        assert!(runner.state.pre_gate_commit.is_some());
15422        assert!(git::is_clean(&winner.worktree).await.unwrap());
15423        assert_eq!(
15424            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15425            0,
15426            "gate completion must release coordination"
15427        );
15428        drop(outsider);
15429        std::fs::remove_file(common.join("updater.lock")).unwrap();
15430        std::fs::create_dir(common.join("updater.lock")).unwrap();
15431        std::fs::remove_file(&started).unwrap();
15432        std::fs::remove_file(&checked).unwrap();
15433        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15434        let error = runner.run_gate(&winner).await.unwrap_err();
15435        assert!(
15436            error
15437                .to_string()
15438                .contains("acquire mandatory repository lock")
15439        );
15440        assert!(
15441            !started.exists(),
15442            "failed acquisition must prevent pre-gate"
15443        );
15444        assert!(!checked.exists(), "failed acquisition must prevent gate");
15445        assert!(git::is_clean(&winner.worktree).await.unwrap());
15446        assert_eq!(
15447            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15448            head
15449        );
15450    }
15451
15452    #[tokio::test]
15453    #[cfg(unix)]
15454    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15455        use std::os::fd::AsRawFd as _;
15456        use std::time::Duration;
15457
15458        ask_test_home();
15459        let tmp = tempfile::tempdir().unwrap();
15460        let repo = tmp.path().join("repo");
15461        std::fs::create_dir_all(&repo).unwrap();
15462        init_repo(&repo);
15463        let common = repo.join(".git");
15464        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15465        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15466        assert_eq!(
15467            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15468            0
15469        );
15470        let mut runner = runner_at(RunStatus::Prep);
15471        runner.state.repo = repo.clone();
15472        runner.state.config.coordination = crate::config::test_coordination();
15473        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15474        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15475        runner.state.config.graph.advise = true;
15476        runner.state.config.graph.advisors = 1;
15477        runner.roles.implementers = vec![spec("implement")];
15478        runner.roles.judges = vec![spec("judge")];
15479        let root = runner.state.worktree_root();
15480        let started = tmp.path().join("setup-started");
15481        let proceed = tmp.path().join("setup-proceed");
15482        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15483            run: Some(format!(
15484                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15485                started.display(),
15486                proceed.display()
15487            )),
15488            ..Default::default()
15489        }];
15490        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15491        let config = git::git(&repo, &["config", "--local", "--list"])
15492            .await
15493            .unwrap();
15494        let mut prep = Box::pin(runner.prep());
15495        assert!(
15496            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15497                .await
15498                .is_err()
15499        );
15500        assert!(
15501            !root.exists(),
15502            "contention must prevent even checkout creation"
15503        );
15504        assert!(!started.exists(), "contention must prevent setup");
15505        assert_eq!(
15506            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15507            branches
15508        );
15509        assert_eq!(
15510            git::git(&repo, &["config", "--local", "--list"])
15511                .await
15512                .unwrap(),
15513            config
15514        );
15515        drop(lock);
15516        tokio::select! {
15517            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15518            result = tokio::time::timeout(Duration::from_secs(10), async {
15519                while !started.exists() {
15520                    tokio::time::sleep(Duration::from_millis(20)).await;
15521                }
15522            }) => result.unwrap(),
15523        }
15524        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15525        assert_eq!(
15526            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15527            -1
15528        );
15529        assert!(
15530            tokio::time::timeout(
15531                Duration::from_millis(100),
15532                crate::repo_lock::SeatLease::for_checkout(
15533                    &repo,
15534                    true,
15535                    crate::config::test_coordination().as_ref()
15536                )
15537            )
15538            .await
15539            .is_err(),
15540            "preparation must also exclude local editors"
15541        );
15542        std::fs::write(proceed, "ready").unwrap();
15543        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15544            .await
15545            .unwrap()
15546            .unwrap();
15547        drop(prep);
15548        assert_eq!(runner.state.prep_complete, Some(true));
15549        assert!(root.join("judge-1").exists());
15550        assert!(root.join("advisor-1").exists());
15551        assert_eq!(
15552            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15553            0
15554        );
15555    }
15556
15557    #[tokio::test]
15558    #[cfg(unix)]
15559    async fn preparation_lock_failure_prevents_repository_changes() {
15560        ask_test_home();
15561        let tmp = tempfile::tempdir().unwrap();
15562        let repo = tmp.path().join("repo");
15563        std::fs::create_dir_all(&repo).unwrap();
15564        init_repo(&repo);
15565        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15566        // A directory at the required lock path makes opening the real lock
15567        // fail, rather than merely exercising an invalid registration.
15568        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15569        let mut runner = runner_at(RunStatus::Prep);
15570        runner.state.repo = repo.clone();
15571        runner.state.config.coordination = crate::config::test_coordination();
15572        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15573        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15574        runner.roles.implementers = vec![spec("implement")];
15575        let error = runner.prep().await.unwrap_err();
15576        assert!(
15577            error
15578                .to_string()
15579                .contains("acquire mandatory repository lock")
15580        );
15581        assert!(!runner.state.worktree_root().exists());
15582        assert!(runner.state.candidates.is_empty());
15583        assert!(!repo.join(".git/config.worktree").exists());
15584        assert_eq!(
15585            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15586            "* main"
15587        );
15588    }
15589
15590    #[tokio::test]
15591    async fn preparation_records_creation_intent_before_git_can_fail() {
15592        ask_test_home();
15593        let tmp = tempfile::tempdir().unwrap();
15594        let repo = tmp.path().join("repo");
15595        std::fs::create_dir_all(&repo).unwrap();
15596        init_repo(&repo);
15597        let mut runner = runner_at(RunStatus::Prep);
15598        runner.state.repo = repo.clone();
15599        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15600        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15601        runner.state.config.blind.commit_msg_hook = false;
15602        runner.roles.implementers = vec![spec("one")];
15603        let label = blind::assign_labels(1, runner.state.seed)[0];
15604        let branch = runner.state.branch_for(label);
15605        // A conflicting branch makes Git refuse creation before a candidate
15606        // can be registered. The durable intent must still identify it.
15607        git::git(&repo, &["branch", &branch]).await.unwrap();
15608        assert!(runner.prep().await.is_err());
15609        let saved = RunState::load(&runner.state.id).unwrap();
15610        assert!(saved.candidates.is_empty());
15611        let event = saved
15612            .events
15613            .iter()
15614            .find(|e| e.node == "prep-intent")
15615            .unwrap();
15616        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15617        assert_eq!(intent["branch"], branch);
15618        assert_eq!(intent["start"], runner.state.base_commit);
15619        assert_eq!(
15620            intent["worktree"],
15621            runner
15622                .state
15623                .worktree_root()
15624                .join(format!("cand-{label}"))
15625                .to_string_lossy()
15626                .as_ref()
15627        );
15628        assert_eq!(intent["picks"], serde_json::json!([]));
15629    }
15630
15631    #[tokio::test]
15632    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15633        ask_test_home();
15634        let tmp = tempfile::tempdir().unwrap();
15635        let repo = tmp.path().join("repo");
15636        std::fs::create_dir_all(&repo).unwrap();
15637        init_repo(&repo);
15638        let mut runner = runner_at(RunStatus::Prep);
15639        runner.state.repo = repo.clone();
15640        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15641        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15642        runner.state.sources.pick_commits.push(refs::Input {
15643            token: "missing".into(),
15644            sha: "0".repeat(40),
15645            provenance: refs::Provenance::ExplicitCli,
15646        });
15647        assert!(runner.prep().await.is_err());
15648        assert!(runner.state.candidates.is_empty());
15649        assert!(!runner.state.worktree_root().exists());
15650    }
15651
15652    #[tokio::test]
15653    async fn prose_and_legacy_inference_never_import_into_candidates() {
15654        ask_test_home();
15655        let tmp = tempfile::tempdir().unwrap();
15656        let repo = tmp.path().join("repo");
15657        std::fs::create_dir_all(&repo).unwrap();
15658        init_repo(&repo);
15659        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15660        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15661            .await
15662            .unwrap();
15663        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15664        git::git(&repo, &["add", "."]).await.unwrap();
15665        git::git(&repo, &["commit", "-m", "historical implementation"])
15666            .await
15667            .unwrap();
15668        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15669        git::git(&repo, &["checkout", "main"]).await.unwrap();
15670        for instruction in [
15671            format!("Example: magi/history/A at {source}"),
15672            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15673            format!("Never import magi/history/A or {source}"),
15674            format!("Conversation answer: I mentioned {source} only as background"),
15675        ] {
15676            let mut runner = runner_at(RunStatus::Prep);
15677            runner.state.repo = repo.clone();
15678            runner.state.base_commit = base.clone();
15679            runner.state.instruction = instruction.clone();
15680            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15681            runner.state.config.blind.commit_msg_hook = false;
15682            runner.state.seeds = vec![refs::Seed {
15683                token: "magi/history/A".into(),
15684                sha: source.clone(),
15685                kind: refs::SeedKind::Unmerged,
15686                branch: true,
15687                detail: String::new(),
15688            }];
15689            runner.roles.implementers = vec![spec("one")];
15690            assert_eq!(seeded_instruction(&runner.state), instruction);
15691            // Model an old record with inferred seeds but no structured input field.
15692            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15693            legacy.as_object_mut().unwrap().remove("sources");
15694            runner.state = serde_json::from_value(legacy).unwrap();
15695            assert_eq!(runner.state.sources, refs::Sources::default());
15696            runner.state.save().unwrap();
15697            runner.state = RunState::load(&runner.state.id).unwrap();
15698            runner.prep().await.unwrap();
15699            for candidate in &runner.state.candidates {
15700                assert_eq!(
15701                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15702                    base
15703                );
15704                assert!(!candidate.worktree.join("unwanted").exists());
15705            }
15706        }
15707    }
15708
15709    #[tokio::test]
15710    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15711        ask_test_home();
15712        let tmp = tempfile::tempdir().unwrap();
15713        let repo = tmp.path().join("repo");
15714        std::fs::create_dir_all(&repo).unwrap();
15715        init_repo(&repo);
15716        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15717        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15718        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15719        git::git(&repo, &["add", "."]).await.unwrap();
15720        git::git(&repo, &["commit", "-m", "required seed"])
15721            .await
15722            .unwrap();
15723        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15724        git::git(&repo, &["checkout", "main"]).await.unwrap();
15725        let mut runner = runner_at(RunStatus::Prep);
15726        runner.state.repo = repo.clone();
15727        runner.state.base_commit = base.clone();
15728        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15729        runner.state.config.graph.advise = false;
15730        runner.state.config.blind.commit_msg_hook = false;
15731        runner.state.sources =
15732            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15733                .await
15734                .unwrap();
15735        runner.roles.implementers = vec![spec("one"), spec("two")];
15736        runner.roles.judges = vec![spec("judge")];
15737        let label = blind::assign_labels(2, runner.state.seed)[0];
15738        let branch = runner.state.branch_for(label);
15739        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15740        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15741            .await
15742            .unwrap();
15743        runner.state.candidates.push(Candidate {
15744            index: 0,
15745            label,
15746            agent: "one".into(),
15747            branch,
15748            worktree,
15749            summary: String::new(),
15750            stat: String::new(),
15751            files: 0,
15752            commits: 0,
15753            empty: false,
15754            failed: None,
15755            verified_noop: None,
15756            duration_ms: 0,
15757            folded: false,
15758        });
15759        runner.state.save().unwrap();
15760        // Reload the checkpoint immediately before its first cherry-pick.
15761        runner.state = RunState::load(&runner.state.id).unwrap();
15762        assert_eq!(runner.state.prep_complete, Some(false));
15763        runner.prep().await.unwrap();
15764        assert_eq!(runner.state.prep_complete, Some(true));
15765        assert_eq!(runner.state.candidates.len(), 2);
15766        for candidate in &runner.state.candidates {
15767            assert_eq!(
15768                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15769                git::tree_of(&repo, &seed).await.unwrap()
15770            );
15771            assert_eq!(
15772                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15773                "required commit"
15774            );
15775        }
15776        assert!(runner.state.worktree_root().join("judge-1").exists());
15777        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15778        runner.prep().await.unwrap();
15779        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15780
15781        runner.state.prep_complete = Some(false);
15782        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15783            run: Some(format!(
15784                "test -f '{}'",
15785                tmp.path().join("dependency-ready").display()
15786            )),
15787            ..Default::default()
15788        }];
15789        assert!(runner.prep().await.is_err());
15790        assert!(runner.state.prep_failure.is_some());
15791        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15792        runner.state = RunState::load(&runner.state.id).unwrap();
15793        assert!(
15794            runner.prep().await.is_err(),
15795            "correcting the cause alone must not auto-retry"
15796        );
15797        let failed_at = runner.state.events.last().unwrap().at;
15798        assert!(
15799            runner.retry_preparation(failed_at).is_err(),
15800            "old approvals cannot unlock the hold"
15801        );
15802        let mut task = crate::queue::Task::new(
15803            "retry".into(),
15804            "retry setup".into(),
15805            repo.clone(),
15806            crate::queue::Source::Human,
15807        );
15808        task.start(runner.state.id.clone());
15809        task.hold_machine(Some("setup failed".into()));
15810        task.release_operator("cli", None);
15811        runner
15812            .retry_preparation(task.resumption.as_ref().unwrap().at)
15813            .unwrap();
15814        runner.prep().await.unwrap();
15815        assert!(runner.state.prep_failure.is_none());
15816        assert_eq!(runner.state.prep_complete, Some(true));
15817        assert_eq!(runner.state.candidates.len(), 2);
15818        assert!(runner.state.worktree_root().join("judge-1").exists());
15819        for candidate in &runner.state.candidates {
15820            assert_eq!(
15821                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15822                git::tree_of(&repo, &seed).await.unwrap()
15823            );
15824        }
15825    }
15826
15827    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15828        let mut heads = Vec::new();
15829        for candidate in candidates {
15830            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15831        }
15832        heads
15833    }
15834
15835    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15836        ask_test_home();
15837        let tmp = tempfile::tempdir().unwrap();
15838        let repo = tmp.path().join("repo");
15839        std::fs::create_dir(&repo).unwrap();
15840        init_repo(&repo);
15841        let mut runner = runner_at(RunStatus::Reviewing);
15842        runner.state.repo = repo.clone();
15843        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15844        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15845        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15846            let branch = format!("candidate-{label}");
15847            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15848            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15849                .await
15850                .unwrap();
15851            if label == 'B' {
15852                std::fs::write(worktree.join("change"), "winner").unwrap();
15853                git::rescue_commit(&worktree, "candidate change")
15854                    .await
15855                    .unwrap();
15856            }
15857            runner.state.candidates.push(Candidate {
15858                index,
15859                label,
15860                agent: "alpha".into(),
15861                branch,
15862                worktree,
15863                summary: "completed reply".into(),
15864                stat: String::new(),
15865                files: usize::from(label == 'B'),
15866                commits: usize::from(label == 'B'),
15867                empty: label == 'A',
15868                failed: None,
15869                verified_noop: None,
15870                duration_ms: 1,
15871                folded: false,
15872            });
15873        }
15874        runner.state.judge_skipped = true;
15875        runner.state.tally = Some(Tally {
15876            first_choice: BTreeMap::new(),
15877            borda: BTreeMap::new(),
15878            winner: 'B',
15879            rankings: 0,
15880            unanimous_initial: false,
15881            deliberated: false,
15882            changed_votes: 0,
15883            unanimous_final: false,
15884            tie_break: None,
15885            judges: 0,
15886            present: 0,
15887            quorum: 0,
15888            met_quorum: true,
15889            uncontested: Some("only B produced a change".into()),
15890        });
15891        runner.state.save().unwrap();
15892        (tmp, runner)
15893    }
15894
15895    #[tokio::test]
15896    #[cfg(unix)]
15897    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15898        for (solo, failed_checkpoint) in
15899            [(true, false), (false, false), (true, true), (false, true)]
15900        {
15901            let (_tmp, mut runner) = preservation_resume_fixture().await;
15902            if solo {
15903                runner.state.candidates.retain(|c| c.label == 'A');
15904            }
15905            runner.state.candidates[0].empty = false;
15906            runner.state.candidates[0].summary.clear();
15907            runner.state.tally = None;
15908            runner.state.judge_skipped = false;
15909            runner.state.status = RunStatus::Implementing;
15910            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15911            let mut implementer = spec("alpha");
15912            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15913            runner.roles.implementers = vec![implementer];
15914            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15915            if failed_checkpoint {
15916                std::fs::create_dir_all(&patch).unwrap();
15917            }
15918            runner.state.save().unwrap();
15919            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15920            drop(pressure);
15921            runner.state = RunState::load(&runner.state.id).unwrap();
15922            assert_eq!(runner.state.status, RunStatus::Blocked);
15923            assert!(runner.state.interrupted_implementations.contains(&'A'));
15924            assert_eq!(runner.state.candidates[0].commits, 0);
15925            assert!(!runner.state.candidates[0].empty);
15926            assert!(runner.state.candidates[0].failed.is_none());
15927            if failed_checkpoint {
15928                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15929                std::fs::remove_dir(patch).unwrap();
15930            }
15931            runner.state.resource_block = None;
15932            runner.state.config.disk.min_free_bytes = 0;
15933            runner.roles.implementers[0].command = vec![
15934                "sh".into(),
15935                "-c".into(),
15936                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15937            ];
15938            runner.implement().await.unwrap();
15939            assert!(runner.state.interrupted_implementations.is_empty());
15940            assert!(runner.state.candidate_preservation.is_empty());
15941            assert_eq!(runner.state.candidates[0].commits, 1);
15942            assert!(!runner.state.candidates[0].empty);
15943            assert!(
15944                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15945                    .unwrap()
15946                    .contains("retried empty interrupted seat")
15947            );
15948            if !solo {
15949                assert_eq!(runner.state.candidates[1].commits, 1);
15950            }
15951        }
15952    }
15953
15954    #[tokio::test]
15955    #[cfg(unix)]
15956    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15957        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15958            let (_tmp, mut runner) = preservation_resume_fixture().await;
15959            runner.state.candidates.retain(|c| c.label == 'A');
15960            runner.state.candidates[0].empty = false;
15961            runner.state.candidates[0].summary.clear();
15962            runner.state.tally = None;
15963            runner.state.judge_skipped = false;
15964            runner.state.status = RunStatus::Implementing;
15965            // Force only the real post-spawn watchdog, without failing the
15966            // separate admission check. Existing edits predate its measurement.
15967            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15968            let wt = runner.state.candidates[0].worktree.clone();
15969            std::fs::write(
15970                wt.join("interrupted-source"),
15971                "retain implementation work\n",
15972            )
15973            .unwrap();
15974            if committed {
15975                git::rescue_commit(&wt, "implementation checkpoint")
15976                    .await
15977                    .unwrap();
15978            }
15979            let lock = git::git(
15980                &wt,
15981                &[
15982                    "rev-parse",
15983                    "--path-format=absolute",
15984                    "--git-path",
15985                    "index.lock",
15986                ],
15987            )
15988            .await
15989            .unwrap();
15990            if block_rescue {
15991                std::fs::write(&lock, "fixture-owned lock").unwrap();
15992            }
15993            let mut implementer = spec("alpha");
15994            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15995            runner.roles.implementers = vec![implementer];
15996            runner.state.save().unwrap();
15997            let result = runner.implement().await;
15998            drop(pressure);
15999            assert_eq!(result.is_err(), block_rescue);
16000            assert_eq!(
16001                runner.state.resource_block.as_ref().unwrap().kind,
16002                "disk pressure"
16003            );
16004            assert!(
16005                runner
16006                    .state
16007                    .events
16008                    .iter()
16009                    .any(|e| e.message.contains("agent execution interrupted"))
16010            );
16011            assert!(
16012                runner
16013                    .state
16014                    .dir()
16015                    .join("artifacts/candidate-A-recovery/staged.patch")
16016                    .exists()
16017            );
16018            if block_rescue {
16019                let saved = RunState::load(&runner.state.id).unwrap();
16020                assert!(saved.candidate_preservation.contains_key(&'A'));
16021                assert!(!saved.candidates[0].empty);
16022                assert_eq!(
16023                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16024                    "retain implementation work\n"
16025                );
16026                std::fs::remove_file(lock).unwrap();
16027                runner.state = saved;
16028            } else {
16029                assert_eq!(runner.state.status, RunStatus::Blocked);
16030                assert_eq!(runner.state.candidates[0].commits, 1);
16031            }
16032            // Recovery must save the candidate before deciding whether another
16033            // implementation is needed. No implementer is available on retry.
16034            runner.roles.implementers.clear();
16035            runner.state.config.disk.min_free_bytes = 0;
16036            runner.state.resource_block = None;
16037            runner.implement().await.unwrap();
16038            assert!(runner.state.candidate_preservation.is_empty());
16039            assert_eq!(runner.state.candidates[0].commits, 1);
16040            assert!(!runner.state.candidates[0].empty);
16041            assert!(git::is_clean(&wt).await.unwrap());
16042            assert!(
16043                crate::run::read_artifact(&runner.state, "cand-A.patch")
16044                    .unwrap()
16045                    .contains("retain implementation work")
16046            );
16047        }
16048    }
16049
16050    #[tokio::test]
16051    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16052        let (_tmp, mut runner) = preservation_resume_fixture().await;
16053        // Schema 15 already confirmed the unfolded no-change result.
16054        runner.implement().await.unwrap();
16055        assert!(runner.state.candidate_preservation.is_empty());
16056        assert!(
16057            !runner
16058                .state
16059                .dir()
16060                .join("artifacts/candidate-A-recovery")
16061                .exists()
16062        );
16063        runner.fold_losers().await.unwrap();
16064        assert!(!runner.state.candidates[0].worktree.exists());
16065        for schema in [14, crate::run::SCHEMA] {
16066            runner.state.schema = schema;
16067            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16068            runner.state.save().unwrap();
16069            runner.state = RunState::load(&runner.state.id).unwrap();
16070            runner.implement().await.unwrap();
16071            assert!(runner.state.candidate_preservation.is_empty());
16072            assert!(runner.state.candidates[0].folded);
16073            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16074        }
16075    }
16076
16077    #[tokio::test]
16078    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16079        let (_tmp, mut runner) = preservation_resume_fixture().await;
16080        runner.state.candidates.retain(|c| c.label == 'A');
16081        runner.state.tally = None;
16082        runner.state.judge_skipped = false;
16083        let candidate = &mut runner.state.candidates[0];
16084        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16085        candidate.failed = Some("deployment validation unavailable".into());
16086        std::fs::write(
16087            candidate.worktree.join("repair"),
16088            "future launcher policy\n",
16089        )
16090        .unwrap();
16091        runner
16092            .state
16093            .candidate_preservation
16094            .insert('A', "awaiting persistence".into());
16095        runner.persist_candidates().await.unwrap();
16096        let candidate = &runner.state.candidates[0];
16097        assert!(!candidate.empty);
16098        assert_eq!(candidate.commits, 1);
16099        assert_eq!(candidate.files, 1);
16100        assert!(candidate.failed.is_none());
16101        assert!(candidate.summary.contains("EPERM"));
16102        assert!(
16103            crate::run::read_artifact(&runner.state, "cand-A.patch")
16104                .unwrap()
16105                .contains("future launcher policy")
16106        );
16107        assert!(runner.state.candidate_preservation.is_empty());
16108    }
16109
16110    #[tokio::test]
16111    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16112        let (_tmp, mut runner) = preservation_resume_fixture().await;
16113        let wt = runner.state.candidates[0].worktree.clone();
16114        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16115        runner.state.schema = 14;
16116        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16117        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16118        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16119        for _ in 0..2 {
16120            assert!(runner.implement().await.is_err());
16121            assert_eq!(runner.state.status, RunStatus::Blocked);
16122            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16123            assert!(
16124                runner.state.candidates[0].empty,
16125                "old verdict must remain unusable"
16126            );
16127            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16128            assert!(runner.state.judge_skipped);
16129            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16130            assert!(
16131                crate::run::read_artifact(&runner.state, "cand-A.patch")
16132                    .unwrap()
16133                    .contains("owned edits")
16134            );
16135            assert!(wt.exists());
16136            assert_eq!(
16137                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16138                    .await
16139                    .unwrap(),
16140                1
16141            );
16142            runner.state = RunState::load(&runner.state.id).unwrap();
16143        }
16144    }
16145
16146    #[tokio::test]
16147    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16148        let (_tmp, mut runner) = preservation_resume_fixture().await;
16149        let wt = runner.state.candidates[0].worktree.clone();
16150        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16151        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16152        let index_lock = PathBuf::from(
16153            git::git(
16154                &wt,
16155                &[
16156                    "rev-parse",
16157                    "--path-format=absolute",
16158                    "--git-path",
16159                    "index.lock",
16160                ],
16161            )
16162            .await
16163            .unwrap(),
16164        );
16165        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16166        runner.state.schema = 14;
16167        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16168        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16169        assert!(runner.implement().await.is_err());
16170        assert!(runner.after_implement().is_err());
16171        assert!(runner.fold_losers().await.is_err());
16172        assert_eq!(
16173            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16174            "failed seat edits"
16175        );
16176        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16177        runner.state = RunState::load(&runner.state.id).unwrap();
16178        std::fs::remove_file(index_lock).unwrap();
16179        // Recovery finds changes that the old uncontested result never judged.
16180        assert!(runner.implement().await.is_err());
16181        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16182        assert!(runner.state.candidates[0].empty);
16183        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16184        assert!(
16185            crate::run::read_artifact(&runner.state, "cand-A.patch")
16186                .unwrap()
16187                .contains("failed seat edits")
16188        );
16189        assert!(wt.exists());
16190    }
16191
16192    #[tokio::test]
16193    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16194        let (_tmp, mut runner) = preservation_resume_fixture().await;
16195        let repo = runner.state.repo.clone();
16196        let common = git::git(
16197            &repo,
16198            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16199        )
16200        .await
16201        .unwrap();
16202        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16203        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16204        assert!(runner.fold_losers().await.is_err());
16205        assert!(!runner.state.candidates[0].worktree.exists());
16206        assert!(!runner.state.candidates[0].folded);
16207        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16208        runner.state = RunState::load(&runner.state.id).unwrap();
16209        std::fs::remove_file(branch_lock).unwrap();
16210        let judge = runner.state.worktree_root().join("judge-1");
16211        git::worktree_add_detached(&repo, &judge, "main")
16212            .await
16213            .unwrap();
16214        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16215        runner.roles.judges = vec![spec("judge")];
16216        assert!(runner.fold_losers().await.is_err());
16217        runner.state = RunState::load(&runner.state.id).unwrap();
16218        assert!(
16219            runner.state.candidates[0].folded,
16220            "candidate progress saved before judge failure"
16221        );
16222        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16223        assert!(judge.join("uncommitted").exists());
16224        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16225        runner.fold_losers().await.unwrap();
16226        assert!(!judge.exists());
16227        runner.fold_losers().await.unwrap();
16228    }
16229
16230    #[tokio::test]
16231    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16232        let (_tmp, mut runner) = preservation_resume_fixture().await;
16233        runner.state.candidates.retain(|c| c.label == 'B');
16234        runner.state.tally = None;
16235        runner.state.judge_skipped = false;
16236        runner.state.prep_complete = Some(true);
16237        runner.state.config.graph.advise = false;
16238        runner.state.config.graph.selected_implementer = Some("alpha".into());
16239        let wt = runner.state.candidates[0].worktree.clone();
16240        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16241        runner
16242            .state
16243            .candidate_preservation
16244            .insert('B', "awaiting persistence".into());
16245        let index_lock = git::git(
16246            &wt,
16247            &[
16248                "rev-parse",
16249                "--path-format=absolute",
16250                "--git-path",
16251                "index.lock",
16252            ],
16253        )
16254        .await
16255        .unwrap();
16256        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16257        runner.state.save().unwrap();
16258        assert!(runner.execute_graph().await.is_err());
16259        assert_eq!(runner.state.status, RunStatus::Blocked);
16260        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16261        std::fs::remove_file(index_lock).unwrap();
16262        // Park at the next graph boundary: this tests the real selected-seat
16263        // hold guard without dispatching reviewers or publishing anything.
16264        for _ in 0..2 {
16265            runner.state = RunState::load(&runner.state.id).unwrap();
16266            runner.pause.park();
16267            runner.execute_graph().await.unwrap();
16268            assert_eq!(runner.state.status, RunStatus::Implementing);
16269            assert!(runner.state.candidate_preservation.is_empty());
16270            assert_eq!(runner.state.candidates[0].commits, 2);
16271            assert!(runner.state.tally.is_none());
16272        }
16273        runner.implement().await.unwrap();
16274        assert_eq!(runner.state.status, RunStatus::Judging);
16275        // A genuine selected-implementer hold must still require release.
16276        runner.state.status = RunStatus::Blocked;
16277        runner.state.candidates[0].failed = Some("implementation failed".into());
16278        runner.state.save().unwrap();
16279        let error = runner.execute_graph().await.unwrap_err();
16280        assert!(
16281            error
16282                .to_string()
16283                .contains("selected implementation is held")
16284        );
16285    }
16286
16287    #[tokio::test]
16288    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16289        let (_tmp, mut runner) = preservation_resume_fixture().await;
16290        let repo = runner.state.repo.clone();
16291        let candidate = runner.state.candidates[0].clone();
16292        // Remove only the disposable fixture directory, leaving Git's linked
16293        // worktree registration and branch intact, as in a vanished checkout.
16294        let registered = git::worktree_holding(&repo, &candidate.branch)
16295            .await
16296            .unwrap()
16297            .unwrap();
16298        assert_eq!(
16299            registered.canonicalize().unwrap(),
16300            candidate.worktree.canonicalize().unwrap()
16301        );
16302        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16303        assert_eq!(
16304            git::worktree_holding(&repo, &candidate.branch)
16305                .await
16306                .unwrap(),
16307            Some(registered)
16308        );
16309        assert!(
16310            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16311            "the stale registration initially keeps the branch checked out"
16312        );
16313        runner.fold_losers().await.unwrap();
16314        assert!(runner.state.candidates[0].folded);
16315        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16316        assert_eq!(
16317            git::worktree_holding(&repo, &candidate.branch)
16318                .await
16319                .unwrap(),
16320            None
16321        );
16322        runner.state = RunState::load(&runner.state.id).unwrap();
16323        runner.fold_losers().await.unwrap();
16324    }
16325
16326    #[tokio::test]
16327    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16328        let (_tmp, mut runner) = preservation_resume_fixture().await;
16329        let repo = runner.state.repo.clone();
16330        let candidate = runner.state.candidates[0].clone();
16331        let winner = runner.state.candidates[1].clone();
16332        let registered = git::worktree_holding(&repo, &candidate.branch)
16333            .await
16334            .unwrap()
16335            .unwrap();
16336        assert_eq!(
16337            registered.canonicalize().unwrap(),
16338            candidate.worktree.canonicalize().unwrap()
16339        );
16340        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16341        assert_eq!(
16342            git::worktree_holding(&repo, &candidate.branch)
16343                .await
16344                .unwrap(),
16345            Some(registered)
16346        );
16347        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16348
16349        for _ in 0..2 {
16350            fold_run(&mut runner.state, false, &crate::run::home())
16351                .await
16352                .unwrap();
16353            runner.state = RunState::load(&runner.state.id).unwrap();
16354            assert!(runner.state.candidates[0].folded);
16355            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16356            assert_eq!(
16357                git::worktree_holding(&repo, &candidate.branch)
16358                    .await
16359                    .unwrap(),
16360                None
16361            );
16362            assert!(!runner.state.candidates[1].folded);
16363            assert!(winner.worktree.exists());
16364            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16365        }
16366    }
16367
16368    #[tokio::test]
16369    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16370        ask_test_home();
16371        let tmp = tempfile::tempdir().unwrap();
16372        let repo = tmp.path().join("repo");
16373        std::fs::create_dir(&repo).unwrap();
16374        init_repo(&repo);
16375        let mut runner = runner_at(RunStatus::Implementing);
16376        runner.state.repo = repo.clone();
16377        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16378        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16379        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16380            let branch = format!("candidate-{label}");
16381            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16382            git::worktree_add_branch(&repo, &wt, &branch, "main")
16383                .await
16384                .unwrap();
16385            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16386            git::git(&wt, &["add", "README.md"]).await.unwrap();
16387            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16388            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16389            runner.state.candidates.push(Candidate {
16390                index,
16391                label,
16392                agent: "alpha".to_owned(),
16393                branch,
16394                worktree: wt,
16395                summary: "completed implementation".to_owned(),
16396                stat: String::new(),
16397                files: 0,
16398                commits: 0,
16399                empty: false,
16400                failed: None,
16401                verified_noop: None,
16402                duration_ms: 42,
16403                folded: false,
16404            });
16405            runner
16406                .state
16407                .candidate_preservation
16408                .insert(label, "awaiting persistence".to_owned());
16409        }
16410        let wt = runner.state.candidates[0].worktree.clone();
16411        let index_lock = git::git(
16412            &wt,
16413            &[
16414                "rev-parse",
16415                "--path-format=absolute",
16416                "--git-path",
16417                "index.lock",
16418            ],
16419        )
16420        .await
16421        .unwrap();
16422        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16423        runner.state.save().unwrap();
16424        assert!(runner.persist_candidates().await.is_err());
16425        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16426        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16427        assert_eq!(
16428            runner.state.candidates[1].commits, 1,
16429            "other results are still collected"
16430        );
16431        assert!(
16432            !runner.state.candidates[0].empty,
16433            "failed rescue is not no change"
16434        );
16435        assert_eq!(
16436            runner.state.candidates[0].summary,
16437            "completed implementation"
16438        );
16439        assert!(runner.after_implement().is_err());
16440        let home = tmp.path().join("fold-home");
16441        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16442        assert!(wt.join("new-file").exists());
16443        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16444        assert!(recovery.join("staged.patch").exists());
16445        assert!(recovery.join("unstaged.patch").exists());
16446        assert_eq!(
16447            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16448            "untracked edits\n"
16449        );
16450        runner.state.begin_pass();
16451        runner.state.end_pass_parked(ParkCause::Upgrade {
16452            from: "1.0".into(),
16453            to: Some("1.1".into()),
16454        });
16455        runner.state.save().unwrap();
16456        runner.state = RunState::load(&runner.state.id).unwrap();
16457        assert!(matches!(
16458            runner.state.passes.last().unwrap().end,
16459            Some(crate::run::PassEnd::Parked {
16460                cause: ParkCause::Upgrade { .. }
16461            })
16462        ));
16463        std::fs::remove_file(index_lock).unwrap();
16464        // No implementer is configured: a persistence retry cannot dispatch a seat.
16465        runner.implement().await.unwrap();
16466        assert!(runner.state.candidate_preservation.is_empty());
16467        assert!(
16468            runner
16469                .state
16470                .candidates
16471                .iter()
16472                .all(|c| c.commits == 1 && !c.empty)
16473        );
16474        assert!(git::is_clean(&wt).await.unwrap());
16475        assert!(
16476            crate::run::read_artifact(&runner.state, "cand-A.patch")
16477                .unwrap()
16478                .contains("later tracked edits")
16479        );
16480    }
16481
16482    #[tokio::test]
16483    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16484        ask_test_home();
16485        let tmp = tempfile::tempdir().unwrap();
16486        let repo = tmp.path().join("repo");
16487        std::fs::create_dir(&repo).unwrap();
16488        init_repo(&repo);
16489        let wt = tmp.path().join("candidate");
16490        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16491            .await
16492            .unwrap();
16493        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16494        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16495        assert!(wt.join("uncommitted").exists());
16496        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16497        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16498        let mut runner = runner_at(RunStatus::Implementing);
16499        runner.state.repo = repo.clone();
16500        runner.state.base_commit = "missing-base".to_owned();
16501        runner.state.candidates.push(Candidate {
16502            index: 0,
16503            label: 'A',
16504            agent: "alpha".into(),
16505            branch: "candidate".into(),
16506            worktree: wt.clone(),
16507            summary: "completed".into(),
16508            stat: String::new(),
16509            files: 0,
16510            commits: 0,
16511            empty: false,
16512            failed: None,
16513            verified_noop: None,
16514            duration_ms: 1,
16515            folded: false,
16516        });
16517        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16518        assert!(
16519            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16520                .await
16521                .is_err()
16522        );
16523        assert!(!runner.state.candidates[0].folded);
16524        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16525        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16526        runner
16527            .state
16528            .candidate_preservation
16529            .insert('A', "pending".into());
16530        assert!(runner.persist_candidates().await.is_err());
16531        assert!(!runner.state.candidates[0].empty);
16532        assert!(!runner.state.candidates[0].folded);
16533        assert!(wt.exists());
16534        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16535        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16536        std::fs::create_dir_all(&patch_path).unwrap();
16537        assert!(
16538            runner.persist_candidates().await.is_err(),
16539            "artifact failure stays pending"
16540        );
16541        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16542        assert!(!runner.state.candidates[0].empty);
16543        std::fs::remove_dir(patch_path).unwrap();
16544        runner.persist_candidates().await.unwrap();
16545        assert!(
16546            runner.state.candidates[0].empty,
16547            "only successful inspection confirms no change"
16548        );
16549    }
16550
16551    fn init_repo(dir: &Path) {
16552        let run = |args: &[&str]| {
16553            let out = std::process::Command::new("git")
16554                .args(args)
16555                .current_dir(dir)
16556                .quiet()
16557                .output()
16558                .expect("spawn git");
16559            assert!(
16560                out.status.success(),
16561                "git {args:?} failed: {}",
16562                String::from_utf8_lossy(&out.stderr)
16563            );
16564        };
16565        run(&["init", "-b", "main"]);
16566        run(&["config", "user.name", "magi test"]);
16567        run(&["config", "user.email", "magi@example.com"]);
16568        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16569        run(&["add", "-A"]);
16570        run(&["commit", "-m", "init"]);
16571    }
16572
16573    // `settle_questions` is what closes the ghost the phone showed: a run's
16574    // seat asked something, the run then ended, and nothing was left to
16575    // abandon the question it left `open`. `HOME` is a process-wide
16576    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16577    // first time it runs in the binary — every test below still reaches the
16578    // same directory whichever call won, and each gets its own run id from
16579    // `RunState::new`, so they never collide there.
16580    fn ask_test_home() {
16581        crate::run::pin_test_home();
16582    }
16583
16584    #[test]
16585    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16586        ask_test_home();
16587        let mut runner = runner_at(RunStatus::Prep);
16588        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16589        // A blocked temporary output path makes the terminal save fail while
16590        // preserving the already published reservation.
16591        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16592        let error = guard
16593            .finish(
16594                &mut runner.state,
16595                Err(
16596                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16597                        .into(),
16598                ),
16599            )
16600            .unwrap_err();
16601        assert_eq!(
16602            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16603            std::io::ErrorKind::PermissionDenied
16604        );
16605        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16606        let message = format!("{error:#}");
16607        assert!(message.contains("original refusal"));
16608        assert!(!message.contains("could not save terminal"));
16609        assert_eq!(
16610            error.chain().count(),
16611            2,
16612            "only startup attribution wraps the original error"
16613        );
16614        let saved = RunState::load(&runner.state.id).unwrap();
16615        assert_eq!(saved.status, RunStatus::Blocked);
16616        assert!(!saved.startup.unwrap().complete);
16617    }
16618
16619    #[test]
16620    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16621        ask_test_home();
16622        let mut runner = runner_at(RunStatus::Prep);
16623        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16624        let id = runner.state.id.clone();
16625        drop(guard);
16626        let saved = RunState::load(&id).unwrap();
16627        assert_eq!(saved.status, RunStatus::Blocked);
16628        assert!(!saved.startup.as_ref().unwrap().complete);
16629        assert!(
16630            saved
16631                .startup
16632                .unwrap()
16633                .failure
16634                .unwrap()
16635                .contains("interrupted")
16636        );
16637        assert!(Runner::resume(&id).is_err());
16638    }
16639
16640    #[test]
16641    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16642        ask_test_home();
16643        let mut runner = runner_at(RunStatus::Prep);
16644        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16645        let error = guard
16646            .finish(
16647                &mut runner.state,
16648                Err(
16649                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16650                        .into(),
16651                ),
16652            )
16653            .unwrap_err();
16654        assert_eq!(
16655            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16656            std::io::ErrorKind::PermissionDenied
16657        );
16658        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16659        let mut task = crate::queue::Task::new(
16660            "review".into(),
16661            "criteria".into(),
16662            runner.state.repo.clone(),
16663            crate::queue::Source::Human,
16664        );
16665        task.record_startup_failure(failure.clone());
16666        task.record_startup_failure(failure.clone());
16667        assert_eq!(task.startup_failures.len(), 1);
16668        assert!(task.runs.is_empty());
16669        assert_eq!(task.attempts, 0);
16670    }
16671
16672    #[cfg(unix)]
16673    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16674        ask_test_home();
16675        let tmp = tempfile::tempdir().unwrap();
16676        let repo = tmp.path().join("repo");
16677        std::fs::create_dir(&repo).unwrap();
16678        init_repo(&repo);
16679        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16680        let origin = tmp.path().join("origin.git");
16681        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16682            .await
16683            .unwrap();
16684        git::git(
16685            &repo,
16686            &["remote", "add", "origin", origin.to_str().unwrap()],
16687        )
16688        .await
16689        .unwrap();
16690        git::git(&repo, &["checkout", "-b", "fix/retained"])
16691            .await
16692            .unwrap();
16693        std::fs::write(repo.join("retained"), "retained change").unwrap();
16694        git::git(&repo, &["add", "."]).await.unwrap();
16695        git::git(&repo, &["commit", "-m", "retained implementation"])
16696            .await
16697            .unwrap();
16698        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16699        git::git(&repo, &["checkout", "main"]).await.unwrap();
16700        std::fs::write(repo.join("base-change"), "new base").unwrap();
16701        git::git(&repo, &["add", "."]).await.unwrap();
16702        git::git(&repo, &["commit", "-m", "base moved"])
16703            .await
16704            .unwrap();
16705        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16706        git::git(&repo, &["checkout", "fix/retained"])
16707            .await
16708            .unwrap();
16709        let mut runner = runner_at(RunStatus::Voting);
16710        runner.state.repo = repo.clone();
16711        runner.state.base_commit = base;
16712        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16713        runner.state.config.disk.min_free_bytes = 0;
16714        runner.state.config.graph.review_rounds = 1;
16715        runner.state.config.verify.e2e = vec!["true".into()];
16716        runner.state.config.verify.gate = vec!["true".into()];
16717        runner.state.candidates = vec![Candidate {
16718            index: 0,
16719            label: 'A',
16720            agent: "fixture".into(),
16721            branch: "fix/retained".into(),
16722            worktree: repo.clone(),
16723            summary: String::new(),
16724            stat: String::new(),
16725            files: 1,
16726            commits: 1,
16727            empty: false,
16728            failed: None,
16729            verified_noop: None,
16730            duration_ms: 0,
16731            folded: false,
16732        }];
16733        runner.state.tally = Some(Tally {
16734            first_choice: BTreeMap::new(),
16735            borda: BTreeMap::new(),
16736            winner: 'A',
16737            rankings: 0,
16738            unanimous_initial: true,
16739            deliberated: false,
16740            changed_votes: 0,
16741            unanimous_final: true,
16742            tie_break: None,
16743            judges: 0,
16744            present: 0,
16745            quorum: 0,
16746            met_quorum: true,
16747            uncontested: Some("review-only".into()),
16748        });
16749        let mut task = crate::queue::Task::new(
16750            "fixture".into(),
16751            "fixture".into(),
16752            repo,
16753            crate::queue::Source::Human,
16754        );
16755        task.link_run(&runner.state.id);
16756        crate::queue::Queue::open().put(&mut task).unwrap();
16757        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16758        runner.state.base_sync = Some(BaseSync {
16759            tip: String::new(),
16760            behind: 1,
16761            attempts: BASE_SYNC_ROUNDS,
16762            conflict: None,
16763            already_in: None,
16764        });
16765        runner.sync_to_base().await.unwrap();
16766        assert_eq!(runner.state.status, RunStatus::Blocked);
16767        (tmp, runner, head, task.id)
16768    }
16769
16770    #[cfg(unix)]
16771    #[tokio::test]
16772    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
16773        for routed in [false, true] {
16774            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
16775            let home = crate::run::home();
16776            let queue = crate::queue::Queue::open();
16777            let task = queue.get(&owner).unwrap();
16778            runner.state.task_instruction = Some(task.instruction.clone());
16779            runner.state.driver_exited = true;
16780            runner.state.save().unwrap();
16781            let questions = crate::ask::Questions::at(home.join("questions"));
16782            let mut q = crate::ask::Question::new(
16783                runner.state.id.clone(),
16784                "base-recovery".into(),
16785                "owner".into(),
16786                "Recover bounded base movement?".into(),
16787                String::new(),
16788                vec!["recover".into()],
16789            );
16790            q.actions.insert(
16791                "recover".into(),
16792                crate::ask::ChoiceAction::RecoverBase {
16793                    run: runner.state.id.clone(),
16794                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
16795                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
16796                },
16797            );
16798            q.routed_task = Some(owner.clone());
16799            q.answer(crate::ask::Answer::Choice("recover".into()))
16800                .unwrap();
16801            questions.put(&mut q).unwrap();
16802            crate::base_recovery_control::recover(
16803                &queue,
16804                &home,
16805                crate::base_recovery_control::Request {
16806                    run: runner.state.id.clone(),
16807                    head: head.clone(),
16808                    actor: "decisions".into(),
16809                    request: q.id.clone(),
16810                    decision: Some((owner.clone(), q.answered_at.unwrap())),
16811                    expected_stop: Some((
16812                        runner.state.base_sync_recovery.stop.clone().unwrap(),
16813                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
16814                    )),
16815                },
16816            )
16817            .await
16818            .unwrap();
16819            if routed {
16820                questions
16821                    .update(&q.id, |q| {
16822                        q.routed_task = Some("another-owner".into());
16823                        Ok(())
16824                    })
16825                    .unwrap();
16826            } else {
16827                let mut task = queue.get(&owner).unwrap();
16828                task.hold_manual(Some("new owner decision".into()));
16829                // Keep the generation to exercise the separate owner-decision binding.
16830                task.status = crate::queue::TaskStatus::Queued;
16831                queue.put(&mut task).unwrap();
16832            }
16833            runner.state = RunState::load(&runner.state.id).unwrap();
16834            runner.sync_to_base().await.unwrap();
16835            let approval = runner
16836                .state
16837                .base_sync_recovery
16838                .authorizations
16839                .last()
16840                .unwrap();
16841            assert!(approval.consumed);
16842            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
16843                "routing"
16844            } else {
16845                "owner decision"
16846            }));
16847            assert_eq!(
16848                runner.state.base_sync.as_ref().unwrap().attempts,
16849                BASE_SYNC_ROUNDS
16850            );
16851            assert_eq!(
16852                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16853                head
16854            );
16855        }
16856    }
16857
16858    #[cfg(unix)]
16859    #[tokio::test]
16860    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16861        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16862        if std::env::var_os(CHILD).is_none() {
16863            let home = tempfile::tempdir().unwrap();
16864            let output = std::process::Command::new(std::env::current_exe().unwrap())
16865                .args([
16866                    "--exact",
16867                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16868                    "--nocapture",
16869                ])
16870                .env(CHILD, "1")
16871                .env("MAGI_HOME", home.path())
16872                .env("MAGI_WORKER_HOME", home.path())
16873                .env("MAGI_WORKER_ID", "abcdef")
16874                .output()
16875                .unwrap();
16876            assert!(
16877                output.status.success()
16878                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16879                "{}\n{}",
16880                String::from_utf8_lossy(&output.stdout),
16881                String::from_utf8_lossy(&output.stderr)
16882            );
16883            return;
16884        }
16885        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16886        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16887        let home = crate::run::home();
16888        let queue = crate::queue::Queue::open();
16889        let mut owner = queue.get(&task).unwrap();
16890        owner.release_operator("test-release", None);
16891        owner.start(runner.state.id.clone());
16892        queue.put(&mut owner).unwrap();
16893        runner.state.driver_exited = true;
16894        runner.state.task_instruction = Some(owner.instruction.clone());
16895        runner.state.save().unwrap();
16896        let _claim = queue.claim(&task).unwrap();
16897        let reserve = |id: &str| {
16898            let directory = home.join("workers/jobs").join(id);
16899            std::fs::create_dir_all(&directory).unwrap();
16900            let job = crate::worker::Job {
16901                id: id.into(),
16902                sequence: 1,
16903                kind: "task".into(),
16904                key: format!("task:{task}"),
16905                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16906                generation: "fixture-generation".into(),
16907                version: None,
16908                status: crate::worker::Status::Running,
16909                result: None,
16910                pid: Some(std::process::id()),
16911                created_at: 0,
16912                finished_at: None,
16913            };
16914            std::fs::write(
16915                directory.join("job.json"),
16916                serde_json::to_vec(&job).unwrap(),
16917            )
16918            .unwrap();
16919            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16920        };
16921        let _own = reserve("abcdef");
16922        let other = reserve("fedcba");
16923        assert!(
16924            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16925                .await
16926                .is_err()
16927        );
16928        assert!(
16929            RunState::load(&runner.state.id)
16930                .unwrap()
16931                .base_sync_recovery
16932                .authorizations
16933                .is_empty()
16934        );
16935        drop(other);
16936        assert!(crate::daemon::is_working_on(
16937            &home,
16938            &runner.state.id,
16939            Timestamp::now()
16940        ));
16941        assert!(
16942            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16943        );
16944        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16945            .await
16946            .unwrap();
16947        assert!(
16948            RunState::load(&runner.state.id)
16949                .unwrap()
16950                .base_recovery_pending()
16951        );
16952    }
16953
16954    #[cfg(unix)]
16955    #[tokio::test]
16956    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16957        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16958        let home = crate::run::home();
16959        runner.state.driver_pid = Some(std::process::id());
16960        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16961        runner.state.driver_exited = false;
16962        runner.state.begin_pass();
16963        runner.state.save().unwrap();
16964        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16965        driver.record(&runner.state).unwrap();
16966        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16967        // Preserve the durable claim as if the process died before Drop saved.
16968        drop(driver);
16969        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16970        assert_eq!(
16971            runner.state.liveness_under(&home, false),
16972            crate::run::Liveness::Dead
16973        );
16974        let queue = crate::queue::Queue::open();
16975        let _claim = queue.claim(&task).unwrap();
16976        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16977            .await
16978            .unwrap();
16979        let disk = RunState::load(&runner.state.id).unwrap();
16980        assert!(!disk.driver_exited);
16981        assert!(disk.base_recovery_pending());
16982    }
16983
16984    #[cfg(unix)]
16985    #[tokio::test]
16986    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16987        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16988        if std::env::var_os(CHILD).is_none() {
16989            use std::os::unix::fs::PermissionsExt;
16990            let tmp = tempfile::tempdir().unwrap();
16991            let gh = tmp.path().join("gh");
16992            std::fs::write(&gh, r#"#!/bin/sh
16993case "$1 $2" in
16994  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16995  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16996  *) exit 2 ;;
16997esac
16998"#).unwrap();
16999            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17000            let mut paths = vec![tmp.path().to_path_buf()];
17001            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17002            let output = std::process::Command::new(std::env::current_exe().unwrap())
17003                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17004                .env(CHILD, "1")
17005                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17006                .env("PATH", std::env::join_paths(paths).unwrap())
17007                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17008                .output().unwrap();
17009            assert!(
17010                output.status.success()
17011                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17012                "{}\n{}",
17013                String::from_utf8_lossy(&output.stdout),
17014                String::from_utf8_lossy(&output.stderr)
17015            );
17016            return;
17017        }
17018        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17019        let repo = runner.state.repo.clone();
17020        git::git(&repo, &["checkout", "main"]).await.unwrap();
17021        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17022        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17023        git::git(&repo, &["checkout", "fix/retained"])
17024            .await
17025            .unwrap();
17026        runner.state.pr = Some(crate::run::PrRecord {
17027            url: "https://example.test/project/pull/1".into(),
17028            number: 1,
17029            state: "open".into(),
17030            checks: "unknown".into(),
17031            round: 0,
17032            rounds: 1,
17033            red_at_merge: Vec::new(),
17034        });
17035        runner.sync_to_base().await.unwrap();
17036        assert_eq!(runner.state.status, RunStatus::Blocked);
17037        assert_eq!(
17038            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17039            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17040        );
17041        assert!(runner.state.base_sync_closure_pending());
17042        assert!(crate::handover::can_resume(&runner.state));
17043        let queue = crate::queue::Queue::open();
17044        assert!(!crate::base_sync_recovery::held_task(
17045            &queue,
17046            &queue.get(&task).unwrap()
17047        ));
17048        assert!(runner.state.authorize_base_sync(&task).is_err());
17049        // Legacy closure stops have no typed stop but must retain the checkpoint.
17050        runner.state.base_sync_recovery.stop = None;
17051        runner.state.status = RunStatus::Voting;
17052        runner.state.gate_ran = true;
17053        runner.state.config.graph.review_rounds = 0;
17054        runner.state.save().unwrap();
17055        assert!(crate::handover::can_resume(&runner.state));
17056        runner.state = RunState::load(&runner.state.id).unwrap();
17057        runner.execute_graph().await.unwrap();
17058        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17059        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17060        assert_eq!(
17061            runner.state.base_sync.as_ref().unwrap().attempts,
17062            BASE_SYNC_ROUNDS
17063        );
17064        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17065        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17066        assert!(runner.state.reviews.is_empty());
17067        assert!(runner.state.gate.is_empty());
17068    }
17069
17070    #[cfg(unix)]
17071    #[tokio::test]
17072    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17073        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17074        runner.state.driver_exited = true;
17075        runner.state.save().unwrap();
17076        let home = crate::run::home();
17077        let queue = crate::queue::Queue::open();
17078        let _claim = queue.claim(&task).unwrap();
17079        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17080        assert!(
17081            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17082                .await
17083                .is_err()
17084        );
17085        drop(lease);
17086        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17087        assert!(
17088            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17089                .await
17090                .is_err()
17091        );
17092        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17093        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17094            .await
17095            .unwrap();
17096        let disk = RunState::load(&runner.state.id).unwrap();
17097        assert!(disk.base_recovery_pending());
17098        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17099        assert!(
17100            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17101                .await
17102                .is_err()
17103        );
17104        assert_eq!(
17105            RunState::load(&runner.state.id)
17106                .unwrap()
17107                .base_sync_recovery
17108                .authorizations
17109                .len(),
17110            1
17111        );
17112        let mut owner = queue.get(&task).unwrap();
17113        owner.runs.push("20260101-000000-ffff".into());
17114        queue.put(&mut owner).unwrap();
17115        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17116            .await
17117            .unwrap_err();
17118        assert!(error.to_string().contains("run task ownership changed"));
17119    }
17120
17121    #[cfg(unix)]
17122    #[tokio::test]
17123    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17124        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17125        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17126        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17127        legacy["status"] = serde_json::json!("voting");
17128        runner.state = serde_json::from_value(legacy).unwrap();
17129        runner.state.save().unwrap();
17130        for _ in 0..3 {
17131            runner.execute_graph().await.unwrap();
17132            assert_eq!(runner.state.status, RunStatus::Blocked);
17133            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17134            assert!(!crate::handover::can_resume(&runner.state));
17135            assert_eq!(
17136                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17137                head
17138            );
17139            assert!(runner.state.reviews.is_empty());
17140            assert!(!runner.state.gate_ran);
17141        }
17142        let notice = crate::notices::run_ended(&runner.state).unwrap();
17143        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17144        runner.state.begin_pass();
17145        runner.state.status = RunStatus::Voting;
17146        runner.deliberate().await.unwrap();
17147        runner.vote().await.unwrap();
17148        runner.finish_after_tally().await.unwrap();
17149        assert_eq!(runner.state.status, RunStatus::Blocked);
17150        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17151        assert_eq!(
17152            crate::notices::run_ended(&runner.state).unwrap().since,
17153            notice.since
17154        );
17155    }
17156
17157    #[cfg(unix)]
17158    #[tokio::test]
17159    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17160        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17161        for detail in [
17162            "fix/retained conflicts with origin/main and did not rebase: conflict",
17163            "rebased fix/retained locally but origin refused the push",
17164            "unknown failure",
17165        ] {
17166            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17167            runner.state.record_base_sync_stop(&head);
17168            assert!(runner.state.authorize_base_sync(&task).is_err());
17169            runner.state.status = RunStatus::Voting;
17170            runner.execute_graph().await.unwrap();
17171            assert_eq!(runner.state.status, RunStatus::Blocked);
17172            assert!(!crate::handover::can_resume(&runner.state));
17173            assert_eq!(
17174                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17175                head
17176            );
17177        }
17178    }
17179
17180    #[cfg(unix)]
17181    #[tokio::test]
17182    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17183        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17184        runner.state.authorize_base_sync(&task).unwrap();
17185        assert!(runner.state.consume_base_recovery(&head));
17186        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17187        runner.state.reserve_base_operation(&head, &tip);
17188        runner.state.save().unwrap();
17189        runner.state = RunState::load(&runner.state.id).unwrap();
17190        assert!(!runner.state.consume_base_recovery(&head));
17191        runner.sync_to_base().await.unwrap();
17192        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17193        assert_eq!(runner.state.base_episode_used(), 1);
17194        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17195        assert!(
17196            runner
17197                .state
17198                .base_sync_recovery
17199                .operations
17200                .last()
17201                .unwrap()
17202                .finished
17203        );
17204        assert!(!runner.state.base_sync_blocked());
17205        runner.sync_to_base().await.unwrap();
17206        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17207    }
17208
17209    #[cfg(unix)]
17210    #[tokio::test]
17211    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17212        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17213        // Both historical diagnostic spellings retain an exhaustion identity.
17214        let original = runner
17215            .state
17216            .base_sync
17217            .as_ref()
17218            .unwrap()
17219            .conflict
17220            .clone()
17221            .unwrap();
17222        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17223            original
17224                .replace("synchronization(s)", "rebase(s)")
17225                .replace("synchronizing", "rebasing"),
17226        );
17227        runner.state.base_sync_recovery = Default::default();
17228        runner.state.record_base_sync_stop(&head);
17229        assert_eq!(
17230            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17231            crate::base_sync_recovery::StopKind::Exhausted
17232        );
17233        runner.state.authorize_base_sync(&task).unwrap();
17234        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17235        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17236            .as_object_mut()
17237            .unwrap();
17238        auth.remove("interval");
17239        auth.remove("scope");
17240        runner.state = serde_json::from_value(legacy).unwrap();
17241        assert!(runner.state.consume_base_recovery(&head));
17242        assert_eq!(runner.state.base_episode_used(), 3);
17243        runner.sync_to_base().await.unwrap();
17244        assert_eq!(runner.state.base_episode_used(), 4);
17245        runner.state.resource_block = Some(crate::run::ResourceBlock {
17246            node: "gate".into(),
17247            kind: "cache".into(),
17248            detail: "legacy wait".into(),
17249            at: Timestamp::now(),
17250            cause: crate::run::ResourceBlockCause::Unrecorded,
17251        });
17252        runner.state.save().unwrap();
17253        assert!(
17254            runner
17255                .state
17256                .base_sync_recovery
17257                .episodes
17258                .last()
17259                .unwrap()
17260                .boundary
17261                .is_none()
17262        );
17263        runner.state.resource_block = None;
17264        advance_base_fixture(&runner).await;
17265        runner.sync_to_base().await.unwrap();
17266        assert!(runner.state.base_sync_blocked());
17267        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17268    }
17269
17270    #[cfg(unix)]
17271    async fn advance_base_fixture(runner: &Runner) {
17272        let repo = &runner.state.repo;
17273        for args in [
17274            vec!["checkout", "main"],
17275            vec!["commit", "--allow-empty", "-m", "base advanced"],
17276            vec!["push", "origin", "main"],
17277            vec!["checkout", "fix/retained"],
17278        ] {
17279            git::git(repo, &args).await.unwrap();
17280        }
17281    }
17282
17283    #[cfg(unix)]
17284    fn fresh_base_interval_fixture(runner: &mut Runner) {
17285        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17286        runner.state.base_sync.as_mut().unwrap().conflict = None;
17287        runner.state.base_sync_recovery = Default::default();
17288        runner.state.status = RunStatus::Voting;
17289    }
17290
17291    #[cfg(unix)]
17292    #[tokio::test]
17293    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17294        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17295        fresh_base_interval_fixture(&mut runner);
17296        runner.state.begin_pass();
17297        runner.sync_to_base().await.unwrap();
17298        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17299        runner.state.resource_block = Some(crate::run::ResourceBlock {
17300            node: "gate".into(),
17301            kind: "build cache".into(),
17302            detail: "busy".into(),
17303            at: Timestamp::now(),
17304            cause: crate::run::ResourceBlockCause::Timeout {
17305                waited_secs: 1,
17306                limit_secs: 1,
17307            },
17308        });
17309        runner.state.save().unwrap();
17310        let queue = crate::queue::Queue::open();
17311        let mut owner = queue.get(&task).unwrap();
17312        runner.state.task_instruction = Some(owner.instruction.clone());
17313        owner.release_operator("fixture-owner", None);
17314        queue.put(&mut owner).unwrap();
17315        let approval = owner.resumption.clone().unwrap();
17316        runner.state.resource_block = None;
17317        runner.state.save().unwrap();
17318        runner.state = RunState::load(&runner.state.id).unwrap();
17319        advance_base_fixture(&runner).await;
17320        runner.sync_to_base().await.unwrap();
17321        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17322        assert_eq!(runner.state.base_episode_used(), 1);
17323        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17324        assert!(
17325            runner.state.base_sync_recovery.episodes[0]
17326                .boundary
17327                .as_ref()
17328                .unwrap()
17329                .consumed
17330        );
17331        assert!(
17332            !runner
17333                .state
17334                .renew_base_episode(&task, &approval, &head, "base")
17335        );
17336        for _ in 0..3 {
17337            owner.release_operator("fixture-owner", None);
17338            queue.put(&mut owner).unwrap();
17339            runner.state = RunState::load(&runner.state.id).unwrap();
17340            runner.state.begin_pass();
17341            runner.sync_to_base().await.unwrap();
17342            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17343            assert_eq!(runner.state.base_episode_used(), 1);
17344        }
17345        runner.state.resource_block = Some(crate::run::ResourceBlock {
17346            node: "gate".into(),
17347            kind: "disk pressure".into(),
17348            detail: "free space required".into(),
17349            at: Timestamp::now(),
17350            cause: crate::run::ResourceBlockCause::Other,
17351        });
17352        runner.state.end_pass_parked(ParkCause::Interrupt {
17353            reason: "disk pressure".into(),
17354        });
17355        runner.state.save().unwrap();
17356        owner.release_operator("fixture-owner-after-disk", None);
17357        queue.put(&mut owner).unwrap();
17358        runner.state.resource_block = None;
17359        advance_base_fixture(&runner).await;
17360        runner.sync_to_base().await.unwrap();
17361        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17362        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17363        assert_eq!(runner.state.base_episode_used(), 1);
17364        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17365    }
17366
17367    #[cfg(unix)]
17368    #[tokio::test]
17369    async fn base_recovery_continuous_advancement_stops_at_four() {
17370        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17371        fresh_base_interval_fixture(&mut runner);
17372        for spent in 1..=BASE_SYNC_ROUNDS {
17373            runner.sync_to_base().await.unwrap();
17374            assert!(!runner.state.base_sync_blocked());
17375            assert_eq!(runner.state.base_episode_used(), spent);
17376            advance_base_fixture(&runner).await;
17377        }
17378        runner.sync_to_base().await.unwrap();
17379        assert_eq!(
17380            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17381            crate::base_sync_recovery::StopKind::Exhausted
17382        );
17383        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17384        assert!(stop.unverified_sync);
17385        let action = runner.state.base_sync_next_action();
17386        assert!(action.contains("never reached its gate"));
17387        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17388        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17389        runner.state = RunState::load(&runner.state.id).unwrap();
17390        runner.sync_to_base().await.unwrap();
17391        assert_eq!(runner.state.base_episode_used(), 4);
17392        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17393    }
17394
17395    #[cfg(unix)]
17396    #[tokio::test]
17397    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17398        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17399        runner.state.ensure_base_episode();
17400        // Verification progress alone is not evidence of a successful sync.
17401        runner.state.record_base_progress(&head);
17402        runner.state.base_sync_recovery.stop = None;
17403        runner.state.record_base_sync_stop(&head);
17404        assert!(
17405            !runner
17406                .state
17407                .base_sync_recovery
17408                .stop
17409                .as_ref()
17410                .unwrap()
17411                .unverified_sync
17412        );
17413
17414        runner.state.finish_base_operation(Some(&head));
17415        // A gate for another head cannot verify this synchronization.
17416        runner.state.gate_ran = true;
17417        runner.state.gate_head = Some("older-head".into());
17418        runner.state.base_sync_recovery.stop = None;
17419        runner.state.record_base_sync_stop(&head);
17420        runner.state.save().unwrap();
17421        runner.state = RunState::load(&runner.state.id).unwrap();
17422        assert!(
17423            runner
17424                .state
17425                .base_sync_recovery
17426                .stop
17427                .as_ref()
17428                .unwrap()
17429                .unverified_sync
17430        );
17431
17432        runner.state.gate_head = Some(head.clone());
17433        runner.state.base_sync_recovery.stop = None;
17434        runner.state.record_base_sync_stop(&head);
17435        assert!(
17436            !runner
17437                .state
17438                .base_sync_recovery
17439                .stop
17440                .as_ref()
17441                .unwrap()
17442                .unverified_sync
17443        );
17444        assert!(
17445            !runner
17446                .state
17447                .base_sync_next_action()
17448                .contains("never reached its gate")
17449        );
17450
17451        // Historical records cannot establish the missing synchronization evidence.
17452        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17453        legacy["base_sync_recovery"]["stop"]
17454            .as_object_mut()
17455            .unwrap()
17456            .remove("unverified_sync");
17457        legacy["base_sync_recovery"]["episodes"][0]
17458            .as_object_mut()
17459            .unwrap()
17460            .remove("last_sync_head");
17461        runner.state = serde_json::from_value(legacy).unwrap();
17462        assert!(
17463            !runner
17464                .state
17465                .base_sync_recovery
17466                .stop
17467                .as_ref()
17468                .unwrap()
17469                .unverified_sync
17470        );
17471        assert!(
17472            runner.state.base_sync_recovery.episodes[0]
17473                .last_sync_head
17474                .is_none()
17475        );
17476    }
17477
17478    #[cfg(unix)]
17479    #[tokio::test]
17480    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17481        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17482        runner.state.authorize_base_sync(&task).unwrap();
17483        assert!(
17484            runner
17485                .state
17486                .base_sync_next_action()
17487                .contains("Approval recorded")
17488        );
17489        assert!(
17490            runner
17491                .state
17492                .base_sync_next_action()
17493                .contains("4/4 in episode 0, 4 lifetime")
17494        );
17495        assert!(runner.state.consume_base_recovery(&head));
17496        assert!(
17497            runner
17498                .state
17499                .base_sync_next_action()
17500                .contains("0/4 in episode 1, 4 lifetime")
17501        );
17502    }
17503
17504    #[cfg(unix)]
17505    #[tokio::test]
17506    async fn base_recovery_changed_head_invalidates_without_spending() {
17507        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17508        runner.state.authorize_base_sync(&task).unwrap();
17509        git::git(
17510            &runner.state.repo,
17511            &["commit", "--allow-empty", "-m", "external head change"],
17512        )
17513        .await
17514        .unwrap();
17515        runner.sync_to_base().await.unwrap();
17516        let a = runner
17517            .state
17518            .base_sync_recovery
17519            .authorizations
17520            .last()
17521            .unwrap();
17522        assert!(a.consumed);
17523        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17524        runner.state = RunState::load(&runner.state.id).unwrap();
17525        assert!(
17526            runner
17527                .state
17528                .base_sync_next_action()
17529                .contains("Approval invalidated")
17530        );
17531        assert!(!runner.state.base_recovery_pending());
17532        assert!(runner.state.base_sync_blocked());
17533        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17534    }
17535
17536    #[cfg(unix)]
17537    #[tokio::test]
17538    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17539        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17540        runner.state.driver_exited = true;
17541        runner.state.save().unwrap();
17542        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17543            .await
17544            .unwrap();
17545        runner.state = RunState::load(&runner.state.id).unwrap();
17546        advance_base_fixture(&runner).await;
17547        runner.sync_to_base().await.unwrap();
17548        let a = runner
17549            .state
17550            .base_sync_recovery
17551            .authorizations
17552            .last()
17553            .unwrap();
17554        assert!(
17555            a.consumed
17556                && a.invalidated
17557                    .as_deref()
17558                    .unwrap()
17559                    .contains("fetched base changed")
17560        );
17561        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17562        assert_eq!(
17563            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17564            head
17565        );
17566        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17567            .await
17568            .unwrap();
17569        runner.state = RunState::load(&runner.state.id).unwrap();
17570        runner.sync_to_base().await.unwrap();
17571        assert!(!runner.state.base_sync_blocked());
17572        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17573    }
17574
17575    #[cfg(unix)]
17576    #[tokio::test]
17577    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17578        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17579        runner.state.authorize_base_sync(&task).unwrap();
17580        assert!(runner.state.consume_base_recovery(&head));
17581        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17582        runner.state.reserve_base_operation(&head, &tip);
17583        runner.state.save().unwrap();
17584        let scratch = runner.state.dir().join("base-sync");
17585        assert_eq!(
17586            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17587                .await
17588                .unwrap(),
17589            crate::rebase::Rebased::Applied
17590        );
17591        runner.state = RunState::load(&runner.state.id).unwrap();
17592        runner.sync_to_base().await.unwrap();
17593        assert!(
17594            runner
17595                .state
17596                .base_sync_recovery
17597                .operations
17598                .last()
17599                .unwrap()
17600                .finished
17601        );
17602        assert_eq!(runner.state.base_episode_used(), 1);
17603        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17604        assert!(!runner.state.base_sync_blocked());
17605    }
17606
17607    #[cfg(unix)]
17608    #[tokio::test]
17609    async fn base_recovery_crash_preserves_untracked_files() {
17610        for ignored in [false, true] {
17611            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17612            runner.state.authorize_base_sync(&task).unwrap();
17613            assert!(runner.state.consume_base_recovery(&head));
17614            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17615            runner.state.reserve_base_operation(&head, &tip);
17616            runner.state.save().unwrap();
17617            let scratch = runner.state.dir().join("base-sync");
17618            assert_eq!(
17619                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17620                    .await
17621                    .unwrap(),
17622                crate::rebase::Rebased::Applied
17623            );
17624            let repo = &runner.state.repo;
17625            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17626            assert_ne!(result, head);
17627            if ignored {
17628                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17629                    .await
17630                    .unwrap();
17631                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17632            }
17633            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17634            assert!(
17635                git::git_raw(repo, &["diff", "--quiet", &head])
17636                    .await
17637                    .unwrap()
17638                    .ok()
17639            );
17640            // Resume after the branch moved, before its checkout was reset.
17641            runner.state = RunState::load(&runner.state.id).unwrap();
17642            runner.sync_to_base().await.unwrap();
17643            assert!(runner.state.base_sync_blocked());
17644            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17645            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17646            assert_eq!(
17647                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17648                result
17649            );
17650            assert_eq!(
17651                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17652                "retained work\n"
17653            );
17654            assert!(
17655                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17656                    .await
17657                    .unwrap()
17658                    .ok()
17659            );
17660        }
17661    }
17662
17663    #[cfg(unix)]
17664    #[tokio::test]
17665    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17666        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17667        runner.state.authorize_base_sync(&task).unwrap();
17668        assert!(runner.state.consume_base_recovery(&head));
17669        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17670        runner.state.reserve_base_operation(&head, &tip);
17671        runner.state.save().unwrap();
17672        let scratch = runner.state.dir().join("base-sync");
17673        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17674            .await
17675            .unwrap();
17676        git::sync_to_head(&runner.state.repo).await.unwrap();
17677        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17678        git::git(&runner.state.repo, &["add", "foreign"])
17679            .await
17680            .unwrap();
17681        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17682            .await
17683            .unwrap();
17684        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17685        runner.state = RunState::load(&runner.state.id).unwrap();
17686        runner.sync_to_base().await.unwrap();
17687        assert!(runner.state.base_sync_blocked());
17688        assert!(
17689            !runner
17690                .state
17691                .base_sync_recovery
17692                .operations
17693                .last()
17694                .unwrap()
17695                .finished
17696        );
17697        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17698        assert_eq!(
17699            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17700            foreign_head
17701        );
17702    }
17703
17704    #[cfg(unix)]
17705    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17706    {
17707        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17708        fresh_base_interval_fixture(&mut runner);
17709        let repo = &runner.state.repo;
17710        git::git(repo, &["checkout", "main"]).await.unwrap();
17711        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17712        git::git(repo, &["add", "retained"]).await.unwrap();
17713        git::git(repo, &["commit", "-m", "upstream retained change"])
17714            .await
17715            .unwrap();
17716        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17717        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17718        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17719        let scratch = runner.state.dir().join("base-sync");
17720        assert!(matches!(
17721            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17722                .await
17723                .unwrap(),
17724            git::RebaseStart::Conflicted(_)
17725        ));
17726        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17727            agent: "fixture".into(),
17728            paths: vec!["retained".into()],
17729            from: Some(head.clone()),
17730            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17731            index_outside: None,
17732            finished: false,
17733            error: None,
17734            launcher: None,
17735            infra: None,
17736            outage: None,
17737        });
17738        (tmp, runner, head, tip, scratch)
17739    }
17740
17741    #[cfg(unix)]
17742    #[tokio::test]
17743    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17744        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17745        // A resource stop from before reservations existed, with resolved
17746        // contents left for the launcher's ordinary staging continuation.
17747        std::fs::write(
17748            scratch.join("retained"),
17749            "upstream and retained resolution\n",
17750        )
17751        .unwrap();
17752        runner.state.resource_block = Some(crate::run::ResourceBlock {
17753            node: "rebase".into(),
17754            kind: "repository coordination".into(),
17755            detail: "busy".into(),
17756            at: Timestamp::now(),
17757            cause: crate::run::ResourceBlockCause::Other,
17758        });
17759        runner.state.save().unwrap();
17760        // Advance the remote without moving either checkout participating
17761        // in the standing rebase.
17762        let newer = git::git(
17763            &runner.state.repo,
17764            &[
17765                "commit-tree",
17766                "main^{tree}",
17767                "-p",
17768                "main",
17769                "-m",
17770                "base advanced during wait",
17771            ],
17772        )
17773        .await
17774        .unwrap();
17775        git::git(
17776            &runner.state.repo,
17777            &[
17778                "push",
17779                "origin",
17780                &format!("{}:refs/heads/main", newer.trim()),
17781            ],
17782        )
17783        .await
17784        .unwrap();
17785        runner.state = RunState::load(&runner.state.id).unwrap();
17786        runner.state.resource_block = None;
17787        runner.sync_to_base().await.unwrap();
17788        assert!(!runner.state.base_sync_blocked());
17789        let op = &runner.state.base_sync_recovery.operations[0];
17790        assert_eq!(op.head, head);
17791        assert_eq!(op.tip, tip);
17792        assert!(op.finished);
17793        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17794        assert_eq!(runner.state.rebase_fixes.len(), 1);
17795        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17796            .await
17797            .unwrap();
17798        assert_ne!(latest, tip);
17799        runner.state = RunState::load(&runner.state.id).unwrap();
17800        runner.sync_to_base().await.unwrap();
17801        assert!(!runner.state.base_sync_blocked());
17802        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17803        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17804        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17805    }
17806
17807    #[cfg(unix)]
17808    #[tokio::test]
17809    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17810        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17811        runner.state.reserve_base_operation(&head, &tip);
17812        runner.state.save().unwrap();
17813        std::fs::write(
17814            scratch.join("retained"),
17815            "upstream and retained resolution\n",
17816        )
17817        .unwrap();
17818        git::git(&scratch, &["add", "retained"]).await.unwrap();
17819        git::git(
17820            &scratch,
17821            &["-c", "core.editor=true", "rebase", "--continue"],
17822        )
17823        .await
17824        .unwrap();
17825        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17826        assert!(
17827            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17828                .await
17829                .unwrap()
17830                .is_empty()
17831        );
17832        assert_eq!(
17833            crate::rebase::finish(
17834                &mut runner.state,
17835                &scratch,
17836                "fix/retained",
17837                &head,
17838                &tip,
17839                &["retained".into()],
17840                ""
17841            )
17842            .await
17843            .unwrap(),
17844            crate::rebase::Rebased::Applied
17845        );
17846        assert!(!scratch.exists());
17847        // Simulate losing the caller before finish_base_operation/save.
17848        runner.state = RunState::load(&runner.state.id).unwrap();
17849        let op = &runner.state.base_sync_recovery.operations[0];
17850        assert!(!op.finished);
17851        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17852        runner.sync_to_base().await.unwrap();
17853        assert!(!runner.state.base_sync_blocked());
17854        assert!(runner.state.base_sync_recovery.operations[0].finished);
17855        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17856        assert_eq!(runner.state.rebase_fixes.len(), 1);
17857        let winner = runner.state.winner().unwrap().clone();
17858        let wait = runner.repository_wait();
17859        assert!(
17860            verify_current_head(&mut runner.state, &winner, &wait)
17861                .await
17862                .unwrap()
17863        );
17864        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17865    }
17866
17867    #[cfg(unix)]
17868    #[tokio::test]
17869    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17870        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17871            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17872            runner.state.reserve_base_operation(&head, &tip);
17873            runner.state.save().unwrap();
17874            std::fs::write(
17875                scratch.join("retained"),
17876                "upstream and retained resolution\n",
17877            )
17878            .unwrap();
17879            git::git(&scratch, &["add", "retained"]).await.unwrap();
17880            git::git(
17881                &scratch,
17882                &["-c", "core.editor=true", "rebase", "--continue"],
17883            )
17884            .await
17885            .unwrap();
17886            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17887            assert!(!git::rebase_in_progress(&scratch).await);
17888            assert!(
17889                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17890                    .await
17891                    .unwrap()
17892                    .is_empty()
17893            );
17894            match damage {
17895                Some("record") => {
17896                    let path =
17897                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17898                            .await
17899                            .unwrap();
17900                    std::fs::write(scratch.join(path), "").unwrap();
17901                }
17902                Some("foreign") => {
17903                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17904                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17905                    git::git(&scratch, &["commit", "-m", "foreign work"])
17906                        .await
17907                        .unwrap();
17908                }
17909                Some("dirty") => {
17910                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17911                }
17912                _ => {}
17913            }
17914            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17915            // Crash immediately after Git continuation, before finish runs.
17916            runner.state = RunState::load(&runner.state.id).unwrap();
17917            assert!(
17918                runner.state.base_sync_recovery.operations[0]
17919                    .validated_head
17920                    .is_none()
17921            );
17922            runner.sync_to_base().await.unwrap();
17923            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17924            assert_eq!(runner.state.base_episode_used(), 1);
17925            assert_eq!(runner.state.rebase_fixes.len(), 1);
17926            assert_eq!(
17927                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17928                retained_head
17929            );
17930            if damage.is_some() {
17931                assert!(runner.state.base_sync_blocked());
17932                assert!(scratch.exists());
17933                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17934                assert!(
17935                    runner.state.base_sync_recovery.operations[0]
17936                        .validated_head
17937                        .is_none()
17938                );
17939            } else {
17940                assert!(!runner.state.base_sync_blocked());
17941                assert!(!scratch.exists());
17942                assert!(runner.state.base_sync_recovery.operations[0].finished);
17943                assert_eq!(
17944                    runner.state.base_sync_recovery.operations[0]
17945                        .validated_head
17946                        .as_deref(),
17947                    Some(result.as_str())
17948                );
17949                let winner = runner.state.winner().unwrap().clone();
17950                let wait = runner.repository_wait();
17951                assert!(
17952                    verify_current_head(&mut runner.state, &winner, &wait)
17953                        .await
17954                        .unwrap()
17955                );
17956                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17957                runner.state = RunState::load(&runner.state.id).unwrap();
17958                runner.sync_to_base().await.unwrap();
17959                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17960            }
17961        }
17962    }
17963
17964    #[cfg(unix)]
17965    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17966        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17967        // This is an ordinary gate stop, not an authorized base-budget recovery.
17968        runner.state.prep_complete = Some(true);
17969        runner.state.base_sync = None;
17970        runner.state.base_sync_recovery = Default::default();
17971        runner.state.config.graph.gate_fix_rounds = 1;
17972        let mut review = review_round(true, 0, 1, 1, false, true);
17973        review.head = head.clone();
17974        review.verified_head = Some(head.clone());
17975        runner.state.reviews = vec![review];
17976        runner.state.gate = vec![CommandOutcome {
17977            command: "old gate".into(),
17978            code: Some(1),
17979            output_tail: "old failure".into(),
17980            duration_ms: 0,
17981            resource_blocked: false,
17982        }];
17983        runner.state.gate_head = Some(head.clone());
17984        runner.state.gate_ran = true;
17985        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17986            agent: "fixture".into(),
17987            failed: runner.state.gate.clone(),
17988            notes: "spent".into(),
17989            committed: false,
17990            error: None,
17991        }];
17992        runner.state.land_approval = Some(crate::run::LandApproval {
17993            question: "old approval".into(),
17994            head: head.clone(),
17995        });
17996        runner.state.status = RunStatus::Blocked;
17997        (tmp, runner, head)
17998    }
17999
18000    #[cfg(unix)]
18001    #[tokio::test]
18002    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18003        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18004            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18005            let phases = tmp.path().join("phases");
18006            runner.state.config.verify.e2e = vec![format!(
18007                "echo e2e >> '{}'; {}",
18008                phases.display(),
18009                if e2e_ok {
18010                    "true"
18011                } else {
18012                    "echo new-e2e-failure; false"
18013                }
18014            )];
18015            runner.state.config.verify.gate = vec![format!(
18016                "echo gate >> '{}'; {}",
18017                phases.display(),
18018                if gate_ok {
18019                    "true"
18020                } else {
18021                    "echo new-gate-failure; false"
18022                }
18023            )];
18024            runner.state.save().unwrap();
18025            runner.state = RunState::load(&runner.state.id).unwrap();
18026            runner.sync_to_base().await.unwrap();
18027            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18028            assert_ne!(head, old);
18029            // Reentry after Git updated but before verification was invalidated.
18030            runner.state = RunState::load(&runner.state.id).unwrap();
18031            runner.execute_graph().await.unwrap();
18032            assert_eq!(runner.state.gate_fixes.len(), 1);
18033            assert_eq!(runner.state.reviews.len(), 1);
18034            assert_eq!(runner.state.reviews[0].head, old);
18035            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18036            assert_eq!(runner.state.verification_history.len(), 1);
18037            let history = &runner.state.verification_history[0];
18038            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18039            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18040            assert!(history.archived_at.is_some());
18041            assert_eq!(history.fixes_spent, Some(1));
18042            assert_eq!(history.gate[0].output_tail, "old failure");
18043            let archive_events: Vec<_> = runner
18044                .state
18045                .events
18046                .iter()
18047                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18048                .collect();
18049            assert_eq!(archive_events.len(), 1);
18050            assert!(archive_events[0].message.contains(&short(&old)));
18051            assert!(archive_events[0].message.contains(&short(&head)));
18052            assert!(
18053                archive_events[0]
18054                    .message
18055                    .contains("1 gate-fix round(s) already spent stay spent")
18056            );
18057            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18058            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18059            if e2e_ok {
18060                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18061                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18062            } else {
18063                assert!(!runner.state.gate_ran);
18064            }
18065            assert_eq!(
18066                runner.state.status,
18067                if e2e_ok && gate_ok {
18068                    RunStatus::Ready
18069                } else {
18070                    RunStatus::Blocked
18071                }
18072            );
18073            if !e2e_ok || !gate_ok {
18074                let cause = runner.state.verification_blocking_cause().unwrap();
18075                assert!(cause.contains(&head));
18076                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18077                // A later successful progress event must not become the cause.
18078                runner.state.event("land", "rebased successfully");
18079                let notice = crate::notices::run_ended(&runner.state).unwrap();
18080                assert!(!format!("{notice:?}").contains("rebased successfully"));
18081                assert!(runner.state.merge.is_none());
18082            }
18083            let before = std::fs::read_to_string(&phases).unwrap();
18084            runner.state.save().unwrap();
18085            runner.state = RunState::load(&runner.state.id).unwrap();
18086            runner.review_loop().await.unwrap();
18087            runner.gate().await.unwrap();
18088            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18089            assert_eq!(runner.state.verification_history.len(), 1);
18090            let history = &runner.state.verification_history[0];
18091            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18092            assert!(history.archived_at.is_some());
18093            assert_eq!(history.fixes_spent, Some(1));
18094        }
18095    }
18096
18097    #[cfg(unix)]
18098    #[tokio::test]
18099    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18100        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18101        let marker = tmp.path().join("gate-ran");
18102        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18103        runner.review_loop().await.unwrap();
18104        runner.gate().await.unwrap();
18105        assert_eq!(runner.state.status, RunStatus::Blocked);
18106        assert!(!marker.exists());
18107        assert!(runner.state.verification_history.is_empty());
18108        runner.state.gate_head = None;
18109        runner.gate().await.unwrap();
18110        assert!(marker.exists());
18111        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18112        assert_eq!(runner.state.verification_history.len(), 1);
18113        assert_eq!(runner.state.gate_fixes.len(), 1);
18114    }
18115
18116    #[cfg(unix)]
18117    #[tokio::test]
18118    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18119        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18120        runner.sync_to_base().await.unwrap();
18121        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18122        runner.state.reconcile_verification_head(&head);
18123        // Both crash-before-command and resource wait are unresolved evidence.
18124        for outcomes in [
18125            Vec::new(),
18126            vec![CommandOutcome {
18127                command: "true".into(),
18128                code: None,
18129                output_tail: "cache unavailable".into(),
18130                duration_ms: 0,
18131                resource_blocked: true,
18132            }],
18133        ] {
18134            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18135                head: head.clone(),
18136                passed: false,
18137                outcomes,
18138            });
18139            runner.state.gate_ran = false;
18140            runner.state.gate.clear();
18141            runner.state.save().unwrap();
18142            runner.state = RunState::load(&runner.state.id).unwrap();
18143            runner.merge().await.unwrap();
18144            assert!(runner.state.merge.is_none());
18145            runner.review_loop().await.unwrap();
18146            runner.gate().await.unwrap();
18147            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18148            assert!(runner.state.gate_allows_progress());
18149            assert_eq!(runner.state.gate_fixes.len(), 1);
18150            assert!(runner.state.merge.is_none());
18151        }
18152        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18153        git::git(
18154            &runner.state.repo,
18155            &["commit", "--allow-empty", "-m", "external update"],
18156        )
18157        .await
18158        .unwrap();
18159        runner.merge().await.unwrap();
18160        assert!(runner.state.merge.is_none());
18161        assert_eq!(runner.state.status, RunStatus::Blocked);
18162    }
18163
18164    #[cfg(unix)]
18165    #[tokio::test]
18166    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18167        for waiting_on_gate in [false, true] {
18168            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18169            runner.sync_to_base().await.unwrap();
18170            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18171            runner.state.reconcile_verification_head(&head);
18172            if waiting_on_gate {
18173                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18174                    head: head.clone(),
18175                    passed: true,
18176                    outcomes: Vec::new(),
18177                });
18178            }
18179            let cache = tmp.path().join("cache");
18180            runner.state.config.verify.e2e =
18181                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18182            runner.state.config.verify.gate = vec!["true".into()];
18183            runner.state.config.graph.timeout_resource_wait = 1;
18184            let owner =
18185                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18186            let guard =
18187                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18188                    crate::cache::AcquireOutcome::Acquired(g) => g,
18189                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18190                };
18191            runner.review_loop().await.unwrap();
18192            runner.gate().await.unwrap();
18193            assert!(!runner.state.gate_ran);
18194            assert_eq!(runner.state.gate_fixes.len(), 1);
18195            assert!(runner.state.resource_block.is_some());
18196            runner.merge().await.unwrap();
18197            assert!(runner.state.merge.is_none());
18198            drop(guard);
18199            runner.state = RunState::load(&runner.state.id).unwrap();
18200            // The driver clears the previous pass's resource stop on resume.
18201            runner.state.resource_block = None;
18202            runner.review_loop().await.unwrap();
18203            runner.gate().await.unwrap();
18204            assert!(runner.state.gate_allows_progress());
18205            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18206            assert_eq!(runner.state.gate_fixes.len(), 1);
18207        }
18208    }
18209
18210    #[cfg(unix)]
18211    #[tokio::test]
18212    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18213        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18214        runner.sync_to_base().await.unwrap();
18215        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18216        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18217        // Verification passes before pre_gate and fails on the mechanically changed head.
18218        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18219        runner.review_loop().await.unwrap();
18220        runner.gate().await.unwrap();
18221        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18222        assert_ne!(before, after);
18223        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18224        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18225        assert!(!runner.state.gate_allows_progress());
18226        runner.merge().await.unwrap();
18227        assert!(runner.state.merge.is_none());
18228        assert!(
18229            runner
18230                .state
18231                .verification_history
18232                .iter()
18233                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18234        );
18235
18236        // An incomplete review is still an independent hold on a changed head.
18237        runner.state.reviews[0].clean = false;
18238        runner.state.reviews[0].answered = 0;
18239        runner.state.config.verify.e2e = vec!["true".into()];
18240        runner.review_loop().await.unwrap();
18241        runner.gate().await.unwrap();
18242        runner.merge().await.unwrap();
18243        assert_eq!(runner.state.status, RunStatus::Blocked);
18244        assert!(runner.state.merge.is_none());
18245        assert!(
18246            runner
18247                .state
18248                .verification_blocking_cause()
18249                .unwrap()
18250                .contains("review blocked")
18251        );
18252    }
18253
18254    #[cfg(unix)]
18255    #[tokio::test]
18256    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18257        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18258        runner.state.gate_head = None;
18259        runner.state.config.verify.gate =
18260            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18261        runner.review_loop().await.unwrap();
18262        runner.gate().await.unwrap();
18263        assert_ne!(
18264            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18265            old
18266        );
18267        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18268        assert!(!runner.state.gate_allows_progress());
18269        assert_eq!(runner.state.status, RunStatus::Blocked);
18270        runner.merge().await.unwrap();
18271        assert!(runner.state.merge.is_none());
18272    }
18273
18274    #[cfg(unix)]
18275    #[tokio::test]
18276    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18277        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18278        let mut old_review = review_round(true, 0, 1, 1, false, true);
18279        old_review.head = head.clone();
18280        old_review.verified_head = Some(head.clone());
18281        runner.state.reviews = vec![old_review];
18282        runner.state.gate = vec![CommandOutcome {
18283            command: "old gate".into(),
18284            code: None,
18285            output_tail: "timed out after 2400s".into(),
18286            duration_ms: 2_400_000,
18287            resource_blocked: false,
18288        }];
18289        runner.state.gate_ran = true;
18290        runner.state.gate_head = Some(head.clone());
18291        runner.state.land_approval = Some(crate::run::LandApproval {
18292            question: "old-head-approval".into(),
18293            head: head.clone(),
18294        });
18295        runner.state.authorize_base_sync(&task).unwrap();
18296        runner.sync_to_base().await.unwrap();
18297        assert!(!runner.state.base_sync_blocked());
18298        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18299        assert_ne!(head, fresh);
18300        assert_eq!(runner.state.reviews.len(), 1);
18301        assert!(!runner.state.gate_ran);
18302        let saved = &runner.state.base_sync_recovery.authorizations[0];
18303        assert_eq!(saved.reviews[0].head, head);
18304        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18305        assert!(saved.gate_ran);
18306        assert!(!saved.gate[0].ok());
18307        runner.review_loop().await.unwrap();
18308        runner.gate().await.unwrap();
18309        assert_eq!(runner.state.reviews[0].head, head);
18310        assert_eq!(
18311            runner.state.reviews[0].verified_head.as_deref(),
18312            Some(head.as_str())
18313        );
18314        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18315        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18316        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18317        assert!(runner.state.gate_ran);
18318        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18319        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18320        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18321        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18322        assert_eq!(
18323            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18324            "retained change"
18325        );
18326    }
18327
18328    #[cfg(unix)]
18329    #[tokio::test]
18330    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18331        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18332        let queue = crate::queue::Queue::open();
18333        let mut owner = queue.get(&task).unwrap();
18334        owner.release_operator("test-resume", None);
18335        queue.put(&mut owner).unwrap();
18336        let generation = owner.resumption.as_ref().unwrap().generation;
18337        runner.state.task_instruction = Some(owner.instruction.clone());
18338        runner.state.driver_exited = true;
18339        runner.state.save().unwrap();
18340        let home = crate::run::home();
18341        let _claim = queue.claim(&task).unwrap();
18342        owner.resumption.as_mut().unwrap().at =
18343            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18344        queue.put(&mut owner).unwrap();
18345        assert!(
18346            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18347                .await
18348                .is_err(),
18349            "an approval from before the stop grants nothing"
18350        );
18351        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18352        queue.put(&mut owner).unwrap();
18353        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18354            .await
18355            .unwrap();
18356        assert!(
18357            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18358                .await
18359                .is_err()
18360        );
18361        runner.state = RunState::load(&runner.state.id).unwrap();
18362        assert_eq!(
18363            runner.state.base_sync_recovery.authorizations[0].generation,
18364            Some(generation)
18365        );
18366        runner.sync_to_base().await.unwrap();
18367        let repo = runner.state.repo.clone();
18368        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18369        git::git(&repo, &["checkout", "main"]).await.unwrap();
18370        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18371        git::git(&repo, &["add", "."]).await.unwrap();
18372        git::git(&repo, &["commit", "-m", "move base again"])
18373            .await
18374            .unwrap();
18375        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18376        git::git(&repo, &["checkout", "fix/retained"])
18377            .await
18378            .unwrap();
18379        runner.sync_to_base().await.unwrap();
18380        assert!(!runner.state.base_sync_blocked());
18381        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18382        assert_eq!(runner.state.base_episode_used(), 2);
18383        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18384        assert_ne!(fresh, head);
18385        assert!(!runner.state.base_recovery_pending());
18386        // Even replaying the generation with a later timestamp grants nothing.
18387        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18388        queue.put(&mut owner).unwrap();
18389        assert!(
18390            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18391                .await
18392                .is_err()
18393        );
18394        runner.state = RunState::load(&runner.state.id).unwrap();
18395        runner.sync_to_base().await.unwrap();
18396        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18397        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18398        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18399    }
18400
18401    #[cfg(unix)]
18402    #[tokio::test]
18403    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18404        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18405        runner.state.base_sync = None;
18406        runner.state.status = RunStatus::Blocked;
18407        let mut second = runner.state.candidates[0].clone();
18408        second.label = 'B';
18409        runner.state.candidates.push(second);
18410        assert!(runner.state.votes.is_empty());
18411        assert!(runner.state.deliberation.is_empty());
18412        runner.deliberate().await.unwrap();
18413        runner.vote().await.unwrap();
18414        assert_eq!(runner.state.status, RunStatus::Blocked);
18415        assert!(runner.state.votes.is_empty());
18416        assert!(runner.state.deliberation.is_empty());
18417    }
18418
18419    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18420    /// reads nothing else off it.
18421    fn runner_at(status: RunStatus) -> Runner {
18422        let mut state = RunState::new(
18423            PathBuf::from("/nonexistent/repo"),
18424            "main".to_owned(),
18425            "deadbeef".to_owned(),
18426            "task".to_owned(),
18427            Config::default(),
18428        );
18429        // These fixtures test execution flow, independent of host disk capacity.
18430        state.config.disk.min_free_bytes = 0;
18431        // These fixtures share one home and save concurrently. Short clock
18432        // IDs can collide, making an ownership test read another test's run.
18433        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18434        state.id = format!(
18435            "20000101-000000-{:04x}",
18436            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18437        );
18438        state.status = status;
18439        Runner {
18440            state,
18441            roles: ResolvedRoles {
18442                implementers: Vec::new(),
18443                judges: Vec::new(),
18444                reviewers: Vec::new(),
18445                fixer: None,
18446                conductor: conductor(),
18447                implementer_roster: Vec::new(),
18448                judge_roster: Vec::new(),
18449                reviewer_roster: Vec::new(),
18450            },
18451            sem: Arc::new(Semaphore::new(1)),
18452            pause: Pause::new(),
18453            interrupt: Pause::new(),
18454        }
18455    }
18456
18457    #[tokio::test]
18458    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18459        crate::run::pin_test_home();
18460        let temp = tempfile::tempdir().unwrap();
18461        let root = temp.path().canonicalize().unwrap();
18462        let repo = root.join("repo");
18463        std::fs::create_dir(&repo).unwrap();
18464        init_repo(&repo);
18465        git::git(&repo, &["config", "core.autocrlf", "false"])
18466            .await
18467            .unwrap();
18468        let mut runner = runner_at(RunStatus::Blocked);
18469        runner.state.repo = repo.clone();
18470        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18471        runner.state.config.agents = vec![conductor()];
18472        runner.state.config.graph.selected_implementer = Some("conductor".into());
18473        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18474        let (shell, command, setup_output) = if cfg!(windows) {
18475            (
18476                vec!["cmd".into(), "/C".into()],
18477                "echo ready>seat-ready",
18478                "ready\r\n",
18479            )
18480        } else {
18481            (
18482                vec!["sh".into(), "-c".into()],
18483                "printf ready > seat-ready",
18484                "ready",
18485            )
18486        };
18487        runner.state.config.verify.shell = Some(shell);
18488        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18489            run: Some(command.into()),
18490            ..Default::default()
18491        }];
18492        runner.state.save().unwrap();
18493        let path = runner.state.worktree_root().join("judge-1");
18494        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18495            .await
18496            .unwrap();
18497        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18498        git::git(&path, &["add", "."]).await.unwrap();
18499        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18500        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18501            .await
18502            .unwrap();
18503        git::git(&path, &["checkout", "--detach", &head])
18504            .await
18505            .unwrap();
18506        let evidence = crate::fold_evidence::snapshot(
18507            &runner.state,
18508            &crate::run::home().canonicalize().unwrap(),
18509            &head,
18510            64 * 1024 * 1024,
18511        )
18512        .await
18513        .unwrap();
18514        let mut contents = BTreeMap::new();
18515        contents.insert(
18516            repo.join("logs/HEAD"),
18517            format!("{head} {head} fixture\n").into_bytes(),
18518        );
18519        let record = serde_json::json!({
18520            "version":1, "run":runner.state.id, "source":path, "head":head,
18521            "evidence":evidence, "admin":{}, "admin_contents":contents,
18522            "history":{head.clone():evidence}, "phase":"evicted"
18523        });
18524        std::fs::write(
18525            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18526            serde_json::to_vec(&record).unwrap(),
18527        )
18528        .unwrap();
18529        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18530        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18531            .await
18532            .unwrap();
18533        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18534        assert!(
18535            !git::git_raw(&repo, &["cat-file", "-e", &head])
18536                .await
18537                .unwrap()
18538                .ok()
18539        );
18540        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18541        let error = resumed.execute().await.unwrap_err();
18542        assert!(
18543            error
18544                .to_string()
18545                .contains("selected implementation is held"),
18546            "{error:#}"
18547        );
18548        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18549        assert_eq!(
18550            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18551            setup_output
18552        );
18553        assert_eq!(
18554            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18555            "unrelated seat"
18556        );
18557        let error = resumed.execute().await.unwrap_err();
18558        assert!(
18559            error
18560                .to_string()
18561                .contains("selected implementation is held"),
18562            "{error:#}"
18563        );
18564        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18565        git::git(&path, &["add", "."]).await.unwrap();
18566        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18567        let next = git::git(
18568            &path,
18569            &[
18570                "commit-tree",
18571                &next_tree,
18572                "-p",
18573                &head,
18574                "-m",
18575                "next review checkpoint",
18576            ],
18577        )
18578        .await
18579        .unwrap();
18580        git::git(&path, &["checkout", "--detach", &next])
18581            .await
18582            .unwrap();
18583        resumed.state.reviews.push(
18584            serde_json::from_value(serde_json::json!({
18585                "round":1, "head":next, "reviews":[]
18586            }))
18587            .unwrap(),
18588        );
18589        resumed.state.save().unwrap();
18590        let error = resumed.execute().await.unwrap_err();
18591        assert!(
18592            error
18593                .to_string()
18594                .contains("selected implementation is held"),
18595            "{error:#}"
18596        );
18597        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18598    }
18599
18600    #[tokio::test]
18601    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18602        crate::run::pin_test_home();
18603        let mut runner = runner_at(RunStatus::Gating);
18604        runner.state.save().unwrap();
18605        let id = runner.state.id.clone();
18606        let passes = runner.state.passes.len();
18607        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18608        let mut claiming = Box::pin(runner.claim_driver());
18609        std::future::poll_fn(|cx| {
18610            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18611            std::task::Poll::Ready(())
18612        })
18613        .await;
18614        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18615        drop(owner);
18616        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18617            .await
18618            .unwrap()
18619            .unwrap();
18620        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18621        assert!(execution.runner.state.resource_block.is_none());
18622        assert_eq!(
18623            execution.runner.state.liveness(false),
18624            crate::run::Liveness::Live
18625        );
18626        drop(execution);
18627        assert!(RunState::load(&id).unwrap().driver_exited);
18628    }
18629
18630    #[tokio::test]
18631    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18632        crate::run::pin_test_home();
18633        let mut runner = runner_at(RunStatus::Gating);
18634        runner.state.save().unwrap();
18635        // run.json can commit, but the subsequent lease metadata write fails.
18636        // Cleanup must run after releasing the short ownership lock.
18637        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18638        assert!(runner.claim_driver().await.is_err());
18639        assert!(runner.state.driver_exited);
18640        assert_eq!(runner.state.driver_pid, None);
18641        let disk = RunState::load(&runner.state.id).unwrap();
18642        assert!(disk.driver_exited);
18643        assert_eq!(disk.driver_pid, None);
18644        assert_eq!(disk.status, RunStatus::Gating);
18645    }
18646
18647    #[tokio::test]
18648    async fn an_early_graph_error_releases_its_driver() {
18649        crate::run::pin_test_home();
18650        let mut runner = runner_at(RunStatus::Blocked);
18651        runner.state.config.graph.selected_implementer = Some("selected".into());
18652        runner.state.save().unwrap();
18653        assert!(runner.execute().await.is_err());
18654        let disk = RunState::load(&runner.state.id).unwrap();
18655        assert!(disk.driver_exited);
18656        assert_eq!(disk.driver_pid, None);
18657        assert_eq!(disk.status, RunStatus::Blocked);
18658    }
18659
18660    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18661        crate::run::MergeRepair {
18662            question: "repair-question".into(),
18663            head: "old-head".into(),
18664            pr: "https://github.com/o/r/pull/42".into(),
18665            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18666            gate: vec![outcome(Some(0), false)],
18667            gate_head: Some("old-head".into()),
18668            gate_fixes: Vec::new(),
18669            pre_gate: Vec::new(),
18670            phase,
18671            failure: None,
18672        }
18673    }
18674
18675    #[tokio::test]
18676    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18677        crate::run::pin_test_home();
18678        let mut runner = runner_at(RunStatus::Landing);
18679        runner.state.merge_repairs.push(merge_repair_request(
18680            crate::run::MergeRepairPhase::Requested,
18681        ));
18682        runner.state.save().unwrap();
18683        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18684        assert!(
18685            runner
18686                .execute()
18687                .await
18688                .unwrap_err()
18689                .to_string()
18690                .contains("already running")
18691        );
18692        assert_eq!(
18693            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18694            crate::run::MergeRepairPhase::Requested
18695        );
18696        drop(claim);
18697        let mut disk = runner.state.clone();
18698        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18699        disk.save().unwrap();
18700        assert!(
18701            runner
18702                .execute()
18703                .await
18704                .unwrap_err()
18705                .to_string()
18706                .contains("decision changed")
18707        );
18708        assert_eq!(
18709            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18710            crate::run::MergeRepairPhase::Complete
18711        );
18712    }
18713
18714    #[test]
18715    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18716        crate::run::pin_test_home();
18717        let mut runner = runner_at(RunStatus::Landing);
18718        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18719        runner.state.reviews = request.reviews.clone();
18720        runner.state.gate = request.gate.clone();
18721        runner.state.gate_ran = true;
18722        runner.state.land_approval = Some(crate::run::LandApproval {
18723            question: "old".into(),
18724            head: "old-head".into(),
18725        });
18726        runner.state.merge_repairs.push(request);
18727        runner.begin_merge_repair_review(0).unwrap();
18728        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18729        assert!(!runner.state.gate_status().ok());
18730        assert!(runner.state.land_approval.is_none());
18731        let disk = RunState::load(&runner.state.id).unwrap();
18732        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18733        assert!(disk.merge_repairs[0].gate[0].ok());
18734        assert_eq!(
18735            disk.merge_repairs[0].phase,
18736            crate::run::MergeRepairPhase::Reviewing
18737        );
18738        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18739        assert!(disk.reviews.is_empty());
18740    }
18741
18742    #[tokio::test]
18743    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18744        crate::run::pin_test_home();
18745        for phase in [
18746            crate::run::MergeRepairPhase::Fixing,
18747            crate::run::MergeRepairPhase::Requested,
18748        ] {
18749            let mut runner = runner_at(RunStatus::Landing);
18750            runner.state.config.graph.land_rounds = 0;
18751            runner.state.merge_repairs.push(merge_repair_request(phase));
18752            runner.state.save().unwrap();
18753            assert!(!runner.repair_merge_review().await.unwrap());
18754            assert_eq!(runner.state.status, RunStatus::Blocked);
18755            assert_eq!(
18756                runner.state.merge_repairs[0].phase,
18757                crate::run::MergeRepairPhase::Failed
18758            );
18759            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18760            assert!(
18761                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18762                    "interrupted"
18763                } else {
18764                    "budget exhausted"
18765                })
18766            );
18767            assert!(runner.state.merge.is_none());
18768        }
18769    }
18770
18771    /// `park_here` folding in the reason `Pause::park_because` recorded -
18772    /// this is what lets an operator reading a run's events tell an
18773    /// interrupt-driven park from an ordinary shutdown park.
18774    #[test]
18775    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18776        crate::run::pin_test_home();
18777        let mut runner = runner_at(RunStatus::Implementing);
18778        let interrupt = Pause::new();
18779        runner.watch_interrupt(interrupt.clone());
18780
18781        interrupt.park_because("task a1b2 asked to run first");
18782
18783        assert!(runner.park_here().expect("park_here"));
18784        assert!(runner.state.parked);
18785        let last = runner.state.events.last().expect("a park event");
18786        assert_eq!(last.node, "park");
18787        assert!(
18788            last.message.contains("task a1b2 asked to run first"),
18789            "expected the interrupt reason in {:?}",
18790            last.message
18791        );
18792    }
18793
18794    /// The cause is stored on the pass when it parks, and the first cause a
18795    /// shared pause was given is the one that stays.
18796    #[test]
18797    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18798        crate::run::pin_test_home();
18799        let mut runner = runner_at(RunStatus::Implementing);
18800        runner.state.begin_pass();
18801        let shutdown = Pause::new();
18802        runner.on_pause(shutdown.clone());
18803        let upgrade = ParkCause::Upgrade {
18804            from: "0.121.0".to_owned(),
18805            to: Some("0.121.1".to_owned()),
18806        };
18807        shutdown.park_for(upgrade.clone());
18808        shutdown.park_for(ParkCause::Operator);
18809
18810        assert!(runner.park_here().expect("park_here"));
18811        assert_eq!(
18812            runner.state.passes[0].end,
18813            Some(crate::run::PassEnd::Parked { cause: upgrade })
18814        );
18815        let last = runner.state.events.last().expect("a park event");
18816        assert!(last.message.contains("0.121.0"), "{}", last.message);
18817    }
18818
18819    #[tokio::test]
18820    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18821        crate::run::pin_test_home();
18822        let mut runner = runner_at(RunStatus::Reviewing);
18823        runner.state.save().unwrap();
18824        let mut execution = runner.claim_driver().await.expect("first execution");
18825        assert_eq!(execution.runner.state.passes.len(), 1);
18826        execution
18827            .driver
18828            .finish(&mut execution.runner.state)
18829            .await
18830            .unwrap();
18831    }
18832
18833    #[tokio::test]
18834    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18835        crate::run::pin_test_home();
18836        let mut runner = runner_at(RunStatus::Reviewing);
18837        runner.state.config.agents = vec![conductor()];
18838        runner.state.config.graph.timeout_resource_wait = 0;
18839        runner.state.save().unwrap();
18840        let id = runner.state.id.clone();
18841        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18842        assert!(runner.claim_driver().await.is_err());
18843        drop(owner);
18844        drop(runner);
18845        let mut resumed = Runner::resume(&id).unwrap();
18846        assert_eq!(resumed.state.driver_claimed, Some(false));
18847        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18848        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18849        let mut execution = resumed
18850            .claim_driver()
18851            .await
18852            .expect("retry initial claim after reload");
18853        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18854        assert_eq!(execution.runner.state.passes.len(), 1);
18855        execution
18856            .driver
18857            .finish(&mut execution.runner.state)
18858            .await
18859            .unwrap();
18860    }
18861
18862    #[tokio::test]
18863    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18864        crate::run::pin_test_home();
18865        let mut runner = runner_at(RunStatus::Reviewing);
18866        runner.state.config.agents = vec![conductor()];
18867        runner.state.save().unwrap();
18868        let id = runner.state.id.clone();
18869        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18870        let mut resumed = Runner::resume(&id).unwrap();
18871        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18872        assert!(resumed.claim_driver().await.is_err());
18873        assert!(RunState::load(&id).unwrap().passes.is_empty());
18874        drop(lease);
18875        let mut execution = resumed.claim_driver().await.unwrap();
18876        execution
18877            .driver
18878            .finish(&mut execution.runner.state)
18879            .await
18880            .unwrap();
18881    }
18882
18883    #[tokio::test]
18884    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18885        crate::run::pin_test_home();
18886        let mut runner = runner_at(RunStatus::Reviewing);
18887        let state = &mut runner.state;
18888        state.config.agents = vec![conductor()];
18889        state.driver_claimed = None;
18890        state.driver_pid = Some(std::process::id());
18891        state.driver_started_at = None;
18892        state.begin_pass();
18893        state.save().unwrap();
18894        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18895        std::fs::write(
18896            state.dir().join("driver.json"),
18897            serde_json::to_vec(&serde_json::json!({
18898                "pid": state.driver_pid, "started_at": null,
18899                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18900            }))
18901            .unwrap(),
18902        )
18903        .unwrap();
18904        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18905        drop(lease);
18906        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18907        let mut resumed = Runner::resume(&state.id).unwrap();
18908        assert!(!resumed.state.driver_exited);
18909        let mut execution = resumed
18910            .claim_driver()
18911            .await
18912            .expect("lease proves previous execution ended");
18913        assert_eq!(execution.runner.state.passes.len(), 2);
18914        execution
18915            .driver
18916            .finish(&mut execution.runner.state)
18917            .await
18918            .unwrap();
18919    }
18920
18921    #[tokio::test]
18922    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18923        crate::run::pin_test_home();
18924        let mut runner = runner_at(RunStatus::Reviewing);
18925        let mut json = serde_json::to_value(&runner.state).unwrap();
18926        json.as_object_mut().unwrap().remove("driver_claimed");
18927        runner.state = serde_json::from_value(json).unwrap();
18928        assert_eq!(runner.state.driver_claimed, None);
18929        runner.state.save().unwrap();
18930        let mut execution = runner
18931            .claim_driver()
18932            .await
18933            .expect("the execution lease, not a missing marker, excludes a second driver");
18934        assert_eq!(execution.runner.state.passes.len(), 1);
18935        execution
18936            .driver
18937            .finish(&mut execution.runner.state)
18938            .await
18939            .unwrap();
18940    }
18941
18942    #[tokio::test]
18943    async fn driver_admission_refuses_a_proven_live_driver() {
18944        crate::run::pin_test_home();
18945        let mut runner = runner_at(RunStatus::Reviewing);
18946        let pid = std::process::id();
18947        runner.state.driver_pid = Some(pid);
18948        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18949        assert!(runner.state.driver_started_at.is_some());
18950        runner.state.save().unwrap();
18951        let error = runner
18952            .claim_driver()
18953            .await
18954            .err()
18955            .expect("a live driver must refuse");
18956        assert!(error.to_string().contains("live driver"), "{error:#}");
18957        let saved = RunState::load(&runner.state.id).unwrap();
18958        assert_eq!(saved.driver_pid, Some(pid));
18959        assert_eq!(saved.status, RunStatus::Reviewing);
18960        assert!(saved.passes.is_empty());
18961    }
18962
18963    #[test]
18964    fn parking_resource_halt_persists_the_pass_once() {
18965        crate::run::pin_test_home();
18966        let mut runner = runner_at(RunStatus::Reviewing);
18967        runner.state.begin_pass();
18968        runner.state.resource_block = Some(crate::run::ResourceBlock {
18969            node: "fix".into(),
18970            kind: "shared build cache".into(),
18971            detail: "cancelled".into(),
18972            at: Timestamp::now(),
18973            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18974        });
18975        let cause = ParkCause::Upgrade {
18976            from: "old".into(),
18977            to: Some("new".into()),
18978        };
18979        runner.pause.park_for(cause.clone());
18980        // Both direct fixer exits and graph resource boundaries must complete
18981        // parking, rather than returning a stale Reviewing record.
18982        runner.stop_for_resource().unwrap();
18983        assert!(runner.resource_halt().unwrap());
18984        assert!(runner.park_here().unwrap());
18985        let saved = RunState::load(&runner.state.id).unwrap();
18986        assert!(saved.parked);
18987        assert_eq!(saved.status, RunStatus::Reviewing);
18988        assert!(saved.passes[0].ended_at.is_some());
18989        assert_eq!(
18990            saved.passes[0].end,
18991            Some(crate::run::PassEnd::Parked {
18992                cause: cause.clone()
18993            })
18994        );
18995        assert_eq!(
18996            saved.resource_block.unwrap().cause,
18997            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18998        );
18999        assert_eq!(
19000            saved
19001                .events
19002                .iter()
19003                .filter(|event| event.node == "park")
19004                .count(),
19005            1
19006        );
19007    }
19008
19009    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19010    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19011    /// cleared) must not make a *different* run - one only watching its own,
19012    /// unshared interrupt `Pause` - see itself as parked. If a future change
19013    /// ever collapsed these back into one handle, the interrupt scheduler
19014    /// would park every run for the rest of the daemon's life, not just the
19015    /// one it meant to interrupt.
19016    #[test]
19017    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19018        crate::run::pin_test_home();
19019        let mut runner = runner_at(RunStatus::Implementing);
19020        let shutdown = Pause::new();
19021        runner.on_pause(shutdown.clone());
19022        let interrupt = Pause::new();
19023        runner.watch_interrupt(interrupt.clone());
19024
19025        // Nobody has asked for anything yet.
19026        assert!(!runner.park_here().expect("park_here"));
19027        assert!(!runner.state.parked);
19028
19029        // Only the interrupt handle fires; the shutdown handle stays clear.
19030        interrupt.park_because("test");
19031        assert!(!shutdown.parked());
19032        assert!(runner.park_here().expect("park_here"));
19033    }
19034
19035    /// The property every prior attempt at this feature failed to pin down:
19036    /// asking a run to park while one of its nodes has a real, in-flight
19037    /// async operation running (an agent call, in production) must not cut
19038    /// that operation short. `park_here` is only ever consulted *between*
19039    /// `execute`'s node calls - see its own doc - so nothing inside a node
19040    /// can observe a park request until the node itself returns. This proves
19041    /// that structurally, with real `tokio` concurrency and a channel
19042    /// handshake (never a sleep, which would only prove "usually", not
19043    /// "cannot"): the "node" below reports that it has genuinely started,
19044    /// and only then is the park requested; the node still has to be told to
19045    /// finish before `park_here` is ever called, exactly mirroring every
19046    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19047    /// in `execute`.
19048    #[tokio::test]
19049    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19050        crate::run::pin_test_home();
19051        let mut runner = runner_at(RunStatus::Implementing);
19052        let interrupt = Pause::new();
19053        runner.watch_interrupt(interrupt.clone());
19054
19055        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19056        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19057
19058        // Stands in for one node's in-flight agent call: it proves it has
19059        // genuinely started, then blocks - exactly as a spawned CLI process
19060        // does - until told to finish.
19061        let node = async move {
19062            started_tx.send(()).expect("send started");
19063            finish_rx.await.expect("recv finish");
19064            "node finished"
19065        };
19066
19067        let interrupter = async move {
19068            started_rx.await.expect("recv started");
19069            // The call is now genuinely in flight. Ask it to park.
19070            interrupt.park_because("higher-priority task waiting");
19071            // Nothing the node does can observe this yet - there is no
19072            // check inside it, by construction - so let the executor run
19073            // anything pending and then let the node finish on its own.
19074            tokio::task::yield_now().await;
19075            finish_tx.send(()).expect("send finish");
19076        };
19077
19078        let (node_result, ()) = tokio::join!(node, interrupter);
19079        assert_eq!(
19080            node_result, "node finished",
19081            "the in-flight call ran to completion"
19082        );
19083
19084        // Only now, at the boundary the real `execute` would check right
19085        // after this node, does the park take effect.
19086        assert!(runner.park_here().expect("park_here"));
19087        assert!(runner.state.parked);
19088    }
19089
19090    /// A run parked mid-competition carries every field it had accumulated
19091    /// through the exact same disk round-trip an ordinary resume uses -
19092    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19093    /// Nothing about parking for an interrupt is a special case of that path;
19094    /// this is what proves it rather than assuming it.
19095    #[test]
19096    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19097        crate::run::pin_test_home();
19098        let mut runner = runner_at(RunStatus::Judging);
19099        // `Runner::resume` re-resolves roles from the saved config, which
19100        // refuses an empty roster - give it the same minimal one `conductor`
19101        // itself uses.
19102        runner.state.config.agents = vec![conductor()];
19103        runner.state.candidates = vec![Candidate {
19104            index: 0,
19105            label: 'A',
19106            agent: "alpha".to_owned(),
19107            branch: "magi/x/A".to_owned(),
19108            worktree: PathBuf::from("/nonexistent/worktree"),
19109            summary: "did the thing".to_owned(),
19110            stat: "1 file changed".to_owned(),
19111            files: 1,
19112            commits: 1,
19113            empty: false,
19114            failed: None,
19115            verified_noop: None,
19116            duration_ms: 1234,
19117            folded: false,
19118        }];
19119        let run_id = runner.state.id.clone();
19120
19121        let interrupt = Pause::new();
19122        runner.watch_interrupt(interrupt.clone());
19123        interrupt.park_because("task c3d4 asked to run first");
19124        assert!(runner.park_here().expect("park_here"));
19125
19126        let resumed = Runner::resume(&run_id).expect("resume");
19127        assert_eq!(resumed.state.candidates.len(), 1);
19128        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19129        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19130        assert_eq!(resumed.state.status, runner.state.status);
19131        assert!(
19132            resumed.state.parked,
19133            "still parked until `execute` actually walks the graph again"
19134        );
19135        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19136    }
19137
19138    /// A fresh open question on `run`, stored and handed back for assertions.
19139    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19140        let mut q = ask::Question::new(
19141            run.to_owned(),
19142            "implement".to_owned(),
19143            "impl-A".to_owned(),
19144            "Which storage backend should the cache use?".to_owned(),
19145            String::new(),
19146            vec!["SQLite".to_owned(), "Redis".to_owned()],
19147        );
19148        store.put(&mut q).unwrap();
19149        q
19150    }
19151
19152    #[test]
19153    fn a_failed_runs_open_question_is_abandoned() {
19154        ask_test_home();
19155        let store = ask::Questions::open();
19156        let mut runner = runner_at(RunStatus::Failed);
19157        let run = runner.state.id.clone();
19158        let q = ask_open_question(&store, &run);
19159
19160        runner.settle_questions();
19161
19162        let back = store.get(&q.id).unwrap();
19163        assert!(
19164            !back.status.open(),
19165            "the seat that asked died with the run; nobody is left to read an answer"
19166        );
19167        assert!(
19168            back.detail.contains(&run) && back.detail.contains("failed"),
19169            "the reason names what the run became, not just that it is gone: {}",
19170            back.detail
19171        );
19172    }
19173
19174    #[test]
19175    fn a_merged_runs_open_question_is_abandoned_too() {
19176        ask_test_home();
19177        let store = ask::Questions::open();
19178        // A run that finishes cleanly still leaves nobody to read an answer -
19179        // this is not only a failure-path cleanup.
19180        for status in [RunStatus::Merged, RunStatus::Ready] {
19181            let mut runner = runner_at(status);
19182            let run = runner.state.id.clone();
19183            let q = ask_open_question(&store, &run);
19184
19185            runner.settle_questions();
19186
19187            let back = store.get(&q.id).unwrap();
19188            assert!(
19189                !back.status.open(),
19190                "{status:?} run's question must not outlive the run"
19191            );
19192        }
19193    }
19194
19195    #[test]
19196    fn a_still_resumable_runs_open_question_is_left_alone() {
19197        ask_test_home();
19198        let store = ask::Questions::open();
19199        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19200        // review round and the seat sessions are all still on disk — so a
19201        // question asked mid-round may yet get a real answer from a real
19202        // resume. Sweeping it here would be exactly the failure mode this
19203        // whole feature exists to avoid on the other side.
19204        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19205            let mut runner = runner_at(status);
19206            let run = runner.state.id.clone();
19207            let q = ask_open_question(&store, &run);
19208
19209            runner.settle_questions();
19210
19211            let back = store.get(&q.id).unwrap();
19212            assert!(
19213                back.status.open(),
19214                "{status:?} is still alive; the question must still be waiting"
19215            );
19216        }
19217    }
19218
19219    #[test]
19220    fn settle_questions_never_touches_an_already_answered_question() {
19221        ask_test_home();
19222        let store = ask::Questions::open();
19223        let mut runner = runner_at(RunStatus::Failed);
19224        let run = runner.state.id.clone();
19225        let mut q = ask_open_question(&store, &run);
19226        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19227            .unwrap();
19228        store.put(&mut q).unwrap();
19229
19230        // Called twice, the way a crash-recovered daemon reclaim and the
19231        // graph's own cleanup both can for the same run — `abandon_for_run`
19232        // only ever touches what is still open, so this must be inert both
19233        // times, not merely the second.
19234        runner.settle_questions();
19235        runner.settle_questions();
19236
19237        let back = store.get(&q.id).unwrap();
19238        assert_eq!(
19239            back.status,
19240            ask::QuestionStatus::Answered,
19241            "a real answer is a decision on record, never overwritten by a sweep"
19242        );
19243    }
19244
19245    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19246    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19247    /// without merging, whose winner is still the operator's answer to read.
19248    /// Nothing previously called `fold_run` itself with a real `tally`, so
19249    /// this is the first test to pin down the one distinction the whole
19250    /// automatic-fold feature depends on: the winner's worktree and branch
19251    /// must survive, everything else sharing the run's worktree bay - a
19252    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19253    /// second sweep treats every non-winner directory under the bay alike -
19254    /// must not.
19255    #[tokio::test]
19256    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19257        crate::run::pin_test_home();
19258        let tmp = tempfile::tempdir().expect("tempdir");
19259        let repo = tmp.path().join("repo");
19260        std::fs::create_dir_all(&repo).unwrap();
19261        init_repo(&repo);
19262
19263        let mut config = Config::default();
19264        config.graph.worktree_root = Some(tmp.path().join("wt"));
19265
19266        let mut state = RunState::new(
19267            repo.clone(),
19268            "main".to_owned(),
19269            "deadbeef".to_owned(),
19270            "task".to_owned(),
19271            config,
19272        );
19273        let root = state.worktree_root();
19274        let wt_a = root.join("cand-A");
19275        let wt_b = root.join("cand-B");
19276        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19277            .await
19278            .expect("worktree A");
19279        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19280            .await
19281            .expect("worktree B");
19282
19283        state.candidates = vec![
19284            Candidate {
19285                index: 0,
19286                label: 'A',
19287                agent: "alpha".to_owned(),
19288                branch: "magi/x/A".to_owned(),
19289                worktree: wt_a.clone(),
19290                summary: String::new(),
19291                stat: String::new(),
19292                files: 0,
19293                commits: 0,
19294                empty: false,
19295                failed: None,
19296                verified_noop: None,
19297                duration_ms: 0,
19298                folded: false,
19299            },
19300            Candidate {
19301                index: 1,
19302                label: 'B',
19303                agent: "beta".to_owned(),
19304                branch: "magi/x/B".to_owned(),
19305                worktree: wt_b.clone(),
19306                summary: String::new(),
19307                stat: String::new(),
19308                files: 0,
19309                commits: 0,
19310                empty: false,
19311                failed: None,
19312                verified_noop: None,
19313                duration_ms: 0,
19314                folded: false,
19315            },
19316        ];
19317        state.tally = Some(Tally {
19318            first_choice: BTreeMap::from([('A', 1)]),
19319            borda: BTreeMap::new(),
19320            winner: 'A',
19321            rankings: 1,
19322            unanimous_initial: true,
19323            deliberated: false,
19324            changed_votes: 0,
19325            unanimous_final: true,
19326            tie_break: None,
19327            judges: 1,
19328            present: 1,
19329            quorum: 1,
19330            met_quorum: true,
19331            uncontested: None,
19332        });
19333        state.status = RunStatus::Ready;
19334
19335        fold_run(&mut state, false, &crate::run::home())
19336            .await
19337            .expect("fold_run");
19338
19339        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19340        assert!(
19341            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19342            "the unmerged winner's branch survives"
19343        );
19344        assert!(
19345            !state.candidates[0].folded,
19346            "the winner is not marked folded"
19347        );
19348
19349        assert!(!wt_b.exists(), "the loser's worktree is removed");
19350        assert!(
19351            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19352            "the loser's branch is removed"
19353        );
19354        assert!(state.candidates[1].folded, "the loser is marked folded");
19355    }
19356
19357    #[tokio::test]
19358    #[cfg(unix)]
19359    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19360        use std::os::fd::AsRawFd as _;
19361        use std::time::Duration;
19362
19363        let tmp = tempfile::tempdir().unwrap();
19364        let repo = tmp.path().join("repo");
19365        std::fs::create_dir_all(&repo).unwrap();
19366        init_repo(&repo);
19367        let common = repo.join(".git");
19368        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19369        let home = tmp.path().join("home");
19370        let mut config = Config::default();
19371        config.coordination = crate::config::test_coordination();
19372        config.graph.worktree_root = Some(tmp.path().join("wt"));
19373        config.graph.timeout_resource_wait = 1;
19374        let mut state = RunState::new(
19375            repo.clone(),
19376            "main".into(),
19377            "deadbeef".into(),
19378            "task".into(),
19379            config,
19380        );
19381        let root = state.worktree_root();
19382        let candidate = root.join("cand-A");
19383        let reviewer = root.join("review-1");
19384        let branch = state.branch_for('A');
19385        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19386            .await
19387            .unwrap();
19388        git::worktree_add_detached(&repo, &reviewer, "main")
19389            .await
19390            .unwrap();
19391        git::acquire_worktree_config(&repo).await.unwrap();
19392        state.enabled_worktree_config = true;
19393        state.status = RunStatus::Ready;
19394        state.candidates.push(Candidate {
19395            index: 0,
19396            label: 'A',
19397            agent: "implement".into(),
19398            branch: branch.clone(),
19399            worktree: candidate.clone(),
19400            summary: String::new(),
19401            stat: String::new(),
19402            files: 0,
19403            commits: 0,
19404            empty: false,
19405            failed: None,
19406            verified_noop: None,
19407            duration_ms: 0,
19408            folded: false,
19409        });
19410        state.save_under(&home).unwrap();
19411        let config_before = std::fs::read(common.join("config")).unwrap();
19412        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19413            .await
19414            .unwrap();
19415        // A separate OS handle bypasses the launcher's local gate, exactly as
19416        // an external updater does, and must still prevent all folding work.
19417        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19418        assert_eq!(
19419            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19420            0
19421        );
19422        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19423        assert!(
19424            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19425                .await
19426                .is_err()
19427        );
19428        assert!(candidate.exists());
19429        assert!(reviewer.exists());
19430        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19431        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19432        assert_eq!(
19433            git::git(&repo, &["worktree", "list", "--porcelain"])
19434                .await
19435                .unwrap(),
19436            worktrees_before
19437        );
19438        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19439            .await
19440            .unwrap()
19441            .unwrap_err();
19442        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19443        drop(fold);
19444        assert!(candidate.exists() && reviewer.exists());
19445        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19446        assert_eq!(
19447            state.resource_block.as_ref().unwrap().kind,
19448            "repository coordination"
19449        );
19450        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19451        drop(external);
19452        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19453        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19454            .await
19455            .unwrap()
19456            .unwrap();
19457        drop(fold);
19458        assert!(removed.contains(&branch));
19459        assert!(!root.exists());
19460        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19461        assert!(state.candidates[0].folded);
19462        assert!(!state.enabled_worktree_config);
19463        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19464        let enabled = git::git_raw(
19465            &repo,
19466            &["config", "--local", "--get", "extensions.worktreeConfig"],
19467        )
19468        .await
19469        .unwrap();
19470        assert!(!enabled.ok(), "the last config reference was released");
19471    }
19472
19473    /// A branch handed to a later run is that run's (and its pull request's):
19474    /// folding the run that released it must not delete it.
19475    #[tokio::test]
19476    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19477        let tmp = tempfile::tempdir().expect("tempdir");
19478        let repo = tmp.path().join("repo");
19479        std::fs::create_dir_all(&repo).unwrap();
19480        init_repo(&repo);
19481        let home = tmp.path().join("home");
19482
19483        let mut config = Config::default();
19484        config.graph.worktree_root = Some(tmp.path().join("wt"));
19485        let mut state = RunState::new(
19486            repo.clone(),
19487            "main".to_owned(),
19488            "deadbeef".to_owned(),
19489            "task".to_owned(),
19490            config,
19491        );
19492        // The worktree is already gone (released); the branch survives.
19493        git::git(&repo, &["branch", "magi/x/A", "main"])
19494            .await
19495            .expect("branch");
19496        state.candidates = vec![Candidate {
19497            index: 0,
19498            label: 'A',
19499            agent: "alpha".to_owned(),
19500            branch: "magi/x/A".to_owned(),
19501            worktree: state.worktree_root().join("cand-A"),
19502            summary: String::new(),
19503            stat: String::new(),
19504            files: 0,
19505            commits: 0,
19506            empty: false,
19507            failed: None,
19508            verified_noop: None,
19509            duration_ms: 0,
19510            folded: true,
19511        }];
19512        state.released_to = Some("20260901-000000-new1".to_owned());
19513        state.released_branches = vec!["magi/x/A".to_owned()];
19514
19515        fold_run(&mut state, true, &home).await.expect("fold_run");
19516        state.status = RunStatus::Merged;
19517        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19518        state.merge = Some(crate::run::MergeOutcome {
19519            mode: MergeMode::Pr,
19520            ok: true,
19521            detail: "confirmed merged".into(),
19522            empty: false,
19523        });
19524        fold_run(&mut state, true, &home)
19525            .await
19526            .expect("merged fold");
19527
19528        assert!(
19529            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19530            "the handed-over branch survives a fold"
19531        );
19532    }
19533
19534    /// A winner with nothing ahead of the base is caught before `gh` is ever
19535    /// asked for a pull request, and the message carries what the task's
19536    /// references resolved to.
19537    #[tokio::test]
19538    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19539        let tmp = tempfile::tempdir().expect("tempdir");
19540        let repo = tmp.path().join("repo");
19541        std::fs::create_dir_all(&repo).unwrap();
19542        init_repo(&repo);
19543        let run = |args: &[&str]| {
19544            let out = std::process::Command::new("git")
19545                .quiet()
19546                .args(args)
19547                .current_dir(&repo)
19548                .output()
19549                .expect("spawn git");
19550            assert!(out.status.success(), "git {args:?}");
19551        };
19552        run(&["branch", "magi/x/A"]);
19553        run(&["checkout", "-q", "-b", "magi/x/B"]);
19554        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19555        run(&["add", "-A"]);
19556        run(&["commit", "-q", "-m", "work"]);
19557        run(&["checkout", "-q", "main"]);
19558
19559        // A pull request is compared against the remote's base, so the
19560        // fixture needs one. Before it exists nothing can be read, and the
19561        // answer must be "not empty".
19562        let probe = RunState::new(
19563            repo.clone(),
19564            "main".to_owned(),
19565            "deadbeef".to_owned(),
19566            "task".to_owned(),
19567            Config::default(),
19568        );
19569        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19570        let bare = tmp.path().join("origin.git");
19571        let out = std::process::Command::new("git")
19572            .quiet()
19573            .args(["init", "-q", "--bare"])
19574            .arg(&bare)
19575            .output()
19576            .expect("spawn git");
19577        assert!(out.status.success(), "git init --bare");
19578        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19579        run(&["push", "-q", "origin", "main"]);
19580
19581        let mut state = RunState::new(
19582            repo.clone(),
19583            "main".to_owned(),
19584            "deadbeef".to_owned(),
19585            "task".to_owned(),
19586            Config::default(),
19587        );
19588        state.seeds = vec![refs::Seed {
19589            token: "magi/27b2/A".to_owned(),
19590            kind: refs::SeedKind::Unresolved,
19591            sha: String::new(),
19592            branch: true,
19593            detail: "no branch or commit named magi/27b2/A".to_owned(),
19594        }];
19595
19596        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19597        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19598        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19599        let detail = empty_candidate_detail(&state, "main");
19600        assert!(detail.starts_with("empty candidate"), "{detail}");
19601        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19602        assert!(detail.contains("--start-from"), "{detail}");
19603    }
19604
19605    /// `status == Ready` used to be read as "this is the harmless
19606    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19607    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19608    /// run's PR was closed without merging — and reentering `merge` with
19609    /// `mode` still `Pr` does not know the difference, so it pushed and
19610    /// opened a second pull request. `mode == Local` reproduces the same
19611    /// blind spot without a network call: reentry must not attempt another
19612    /// git merge once this node has already recorded an outcome.
19613    #[tokio::test]
19614    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19615        let tmp = tempfile::tempdir().expect("tempdir");
19616        let repo = tmp.path().join("repo");
19617        std::fs::create_dir_all(&repo).unwrap();
19618        init_repo(&repo);
19619
19620        let mut config = Config::default();
19621        config.merge.mode = MergeMode::Local;
19622
19623        let mut state = RunState::new(
19624            repo.clone(),
19625            "main".to_owned(),
19626            "deadbeef".to_owned(),
19627            "task".to_owned(),
19628            config,
19629        );
19630        state.candidates = vec![Candidate {
19631            index: 0,
19632            label: 'A',
19633            agent: "alpha".to_owned(),
19634            branch: "does-not-exist".to_owned(),
19635            worktree: repo.clone(),
19636            summary: String::new(),
19637            stat: String::new(),
19638            files: 0,
19639            commits: 0,
19640            empty: false,
19641            failed: None,
19642            verified_noop: None,
19643            duration_ms: 0,
19644            folded: false,
19645        }];
19646        state.tally = Some(Tally {
19647            first_choice: BTreeMap::from([('A', 1)]),
19648            borda: BTreeMap::new(),
19649            winner: 'A',
19650            rankings: 1,
19651            unanimous_initial: true,
19652            deliberated: false,
19653            changed_votes: 0,
19654            unanimous_final: true,
19655            tie_break: None,
19656            judges: 0,
19657            present: 0,
19658            quorum: 0,
19659            met_quorum: true,
19660            uncontested: Some("only candidate A produced a change".to_owned()),
19661        });
19662        state.reviews = vec![ReviewRound {
19663            round: 1,
19664            head: "deadbeef".to_owned(),
19665            verified_head: None,
19666            verified_at: None,
19667            reviews: Vec::new(),
19668            e2e: Vec::new(),
19669            fix: None,
19670            blocking: 0,
19671            answered: 0,
19672            expected: 0,
19673            clean: true,
19674            verify_retried: false,
19675            e2e_deferred: false,
19676            e2e_defer_reason: None,
19677            progressed: false,
19678            vote_split: false,
19679            reconsideration: Vec::new(),
19680            verdict: None,
19681        }];
19682        state.gate = vec![CommandOutcome {
19683            command: "test".to_owned(),
19684            code: Some(0),
19685            output_tail: String::new(),
19686            duration_ms: 0,
19687            resource_blocked: false,
19688        }];
19689        state.gate_ran = true;
19690        // Reached its conclusion already — e.g. `land` closing the PR without
19691        // merging it, which (like the honest `MergeMode::None` path) leaves
19692        // `status` at `Ready`. The recorded outcome is what actually marks
19693        // this node done.
19694        state.status = RunStatus::Ready;
19695        state.merge = Some(MergeOutcome {
19696            mode: MergeMode::Local,
19697            ok: false,
19698            detail: "already concluded".to_owned(),
19699            empty: false,
19700        });
19701
19702        let mut runner = Runner {
19703            state,
19704            roles: ResolvedRoles {
19705                implementers: Vec::new(),
19706                judges: Vec::new(),
19707                reviewers: Vec::new(),
19708                fixer: None,
19709                conductor: conductor(),
19710                implementer_roster: Vec::new(),
19711                judge_roster: Vec::new(),
19712                reviewer_roster: Vec::new(),
19713            },
19714            sem: Arc::new(Semaphore::new(1)),
19715            pause: Pause::new(),
19716            interrupt: Pause::new(),
19717        };
19718
19719        runner.merge().await.expect("merge");
19720
19721        assert_eq!(
19722            runner.state.status,
19723            RunStatus::Ready,
19724            "a concluded run's status must not change on reentry"
19725        );
19726        assert_eq!(
19727            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19728            Some("already concluded"),
19729            "merge must not run again once the node already recorded an outcome"
19730        );
19731    }
19732
19733    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19734    /// when its last attempt was resource-blocked (the shared build cache
19735    /// could not be acquired or confirmed fresh in time - see
19736    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19737    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19738    /// a run merge a tree the gate never actually checked - exactly the case
19739    /// a contended cache produces on every retry until it clears. `merge`
19740    /// must refuse until `gate` has actually recorded an attempt.
19741    #[tokio::test]
19742    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19743        let tmp = tempfile::tempdir().expect("tempdir");
19744        let repo = tmp.path().join("repo");
19745        std::fs::create_dir_all(&repo).unwrap();
19746        init_repo(&repo);
19747
19748        let mut config = Config::default();
19749        config.merge.mode = MergeMode::Local;
19750
19751        let mut state = RunState::new(
19752            repo.clone(),
19753            "main".to_owned(),
19754            "deadbeef".to_owned(),
19755            "task".to_owned(),
19756            config,
19757        );
19758        state.candidates = vec![Candidate {
19759            index: 0,
19760            label: 'A',
19761            agent: "alpha".to_owned(),
19762            branch: "does-not-exist".to_owned(),
19763            worktree: repo.clone(),
19764            summary: String::new(),
19765            stat: String::new(),
19766            files: 0,
19767            commits: 0,
19768            empty: false,
19769            failed: None,
19770            verified_noop: None,
19771            duration_ms: 0,
19772            folded: false,
19773        }];
19774        state.tally = Some(Tally {
19775            first_choice: BTreeMap::from([('A', 1)]),
19776            borda: BTreeMap::new(),
19777            winner: 'A',
19778            rankings: 1,
19779            unanimous_initial: true,
19780            deliberated: false,
19781            changed_votes: 0,
19782            unanimous_final: true,
19783            tie_break: None,
19784            judges: 0,
19785            present: 0,
19786            quorum: 0,
19787            met_quorum: true,
19788            uncontested: Some("only candidate A produced a change".to_owned()),
19789        });
19790        state.reviews = vec![ReviewRound {
19791            round: 1,
19792            head: "deadbeef".to_owned(),
19793            verified_head: None,
19794            verified_at: None,
19795            reviews: Vec::new(),
19796            e2e: Vec::new(),
19797            fix: None,
19798            blocking: 0,
19799            answered: 0,
19800            expected: 0,
19801            clean: true,
19802            verify_retried: false,
19803            e2e_deferred: false,
19804            e2e_defer_reason: None,
19805            progressed: false,
19806            vote_split: false,
19807            reconsideration: Vec::new(),
19808            verdict: None,
19809        }];
19810        // The point: `gate` has not recorded anything yet.
19811        state.gate = Vec::new();
19812        state.gate_ran = false;
19813        state.status = RunStatus::Gating;
19814
19815        let mut runner = Runner {
19816            state,
19817            roles: ResolvedRoles {
19818                implementers: Vec::new(),
19819                judges: Vec::new(),
19820                reviewers: Vec::new(),
19821                fixer: None,
19822                conductor: conductor(),
19823                implementer_roster: Vec::new(),
19824                judge_roster: Vec::new(),
19825                reviewer_roster: Vec::new(),
19826            },
19827            sem: Arc::new(Semaphore::new(1)),
19828            pause: Pause::new(),
19829            interrupt: Pause::new(),
19830        };
19831
19832        runner.merge().await.expect("merge");
19833
19834        assert!(
19835            runner.state.merge.is_none(),
19836            "an empty gate must never be read as a passing one: {:?}",
19837            runner.state.merge
19838        );
19839    }
19840
19841    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19842    /// commands configured and `merge.mode` is `none` (a review-only run).
19843    /// `gate` must still record a real attempt — zero commands, vacuously
19844    /// passed — rather than leaving `state.gate` empty in a way `merge`
19845    /// cannot tell apart from "never ran"; otherwise the run reaches
19846    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19847    #[tokio::test]
19848    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19849        let tmp = tempfile::tempdir().expect("tempdir");
19850        let repo = tmp.path().join("repo");
19851        std::fs::create_dir_all(&repo).unwrap();
19852        init_repo(&repo);
19853
19854        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19855        let config = Config::default();
19856
19857        let mut state = RunState::new(
19858            repo.clone(),
19859            "main".to_owned(),
19860            "deadbeef".to_owned(),
19861            "task".to_owned(),
19862            config,
19863        );
19864        state.candidates = vec![Candidate {
19865            index: 0,
19866            label: 'A',
19867            agent: "alpha".to_owned(),
19868            branch: "does-not-exist".to_owned(),
19869            worktree: repo.clone(),
19870            summary: String::new(),
19871            stat: String::new(),
19872            files: 0,
19873            commits: 0,
19874            empty: false,
19875            failed: None,
19876            verified_noop: None,
19877            duration_ms: 0,
19878            folded: false,
19879        }];
19880        state.tally = Some(Tally {
19881            first_choice: BTreeMap::from([('A', 1)]),
19882            borda: BTreeMap::new(),
19883            winner: 'A',
19884            rankings: 1,
19885            unanimous_initial: true,
19886            deliberated: false,
19887            changed_votes: 0,
19888            unanimous_final: true,
19889            tie_break: None,
19890            judges: 0,
19891            present: 0,
19892            quorum: 0,
19893            met_quorum: true,
19894            uncontested: Some("only candidate A produced a change".to_owned()),
19895        });
19896        state.reviews = vec![ReviewRound {
19897            round: 1,
19898            head: "deadbeef".to_owned(),
19899            verified_head: None,
19900            verified_at: None,
19901            reviews: Vec::new(),
19902            e2e: Vec::new(),
19903            fix: None,
19904            blocking: 0,
19905            answered: 0,
19906            expected: 0,
19907            clean: true,
19908            verify_retried: false,
19909            e2e_deferred: false,
19910            e2e_defer_reason: None,
19911            progressed: false,
19912            vote_split: false,
19913            reconsideration: Vec::new(),
19914            verdict: None,
19915        }];
19916
19917        let mut runner = Runner {
19918            state,
19919            roles: ResolvedRoles {
19920                implementers: Vec::new(),
19921                judges: Vec::new(),
19922                reviewers: Vec::new(),
19923                fixer: None,
19924                conductor: conductor(),
19925                implementer_roster: Vec::new(),
19926                judge_roster: Vec::new(),
19927                reviewer_roster: Vec::new(),
19928            },
19929            sem: Arc::new(Semaphore::new(1)),
19930            pause: Pause::new(),
19931            interrupt: Pause::new(),
19932        };
19933
19934        runner.gate().await.expect("gate");
19935        assert!(
19936            runner.state.gate_ran,
19937            "zero configured commands is still a real attempt, not an unrun gate"
19938        );
19939        assert!(runner.state.gate.is_empty());
19940        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19941        assert_ne!(
19942            runner.state.status,
19943            RunStatus::Blocked,
19944            "a gate with nothing to check must not read as failed"
19945        );
19946
19947        runner.merge().await.expect("merge");
19948        assert_eq!(
19949            runner.state.status,
19950            RunStatus::Ready,
19951            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19952        );
19953    }
19954
19955    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19956    /// `verify.gate` (so the e2e leg and the final gate never build against
19957    /// different directories). With zero `verify.gate` commands but a
19958    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19959    /// that lease before discovering it had nothing to run - so a repo with
19960    /// no gate commands could come back `resource_blocked` (and therefore
19961    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19962    /// cache, exactly the contention this run's own zero commands could
19963    /// never have touched. `gate` must recognise there is nothing to check
19964    /// before it ever asks for the lease.
19965    #[tokio::test]
19966    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19967        crate::run::pin_test_home();
19968        let home = crate::run::home();
19969
19970        let tmp = tempfile::tempdir().expect("tempdir");
19971        let repo = tmp.path().join("repo");
19972        std::fs::create_dir_all(&repo).unwrap();
19973        init_repo(&repo);
19974        // Unique to this test, so holding its lease cannot collide with
19975        // another test sharing the same process-wide `home`.
19976        let cache_dir = tmp.path().join("target");
19977
19978        let mut config = Config::default();
19979        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19980        // `verify.gate` stays empty (the default). Any accidental cache
19981        // acquisition must report contention immediately, regardless of how
19982        // long unrelated Git subprocesses take under parallel test load.
19983        config.graph.timeout_verify = Some(2);
19984        config.graph.timeout_resource_wait = 0;
19985
19986        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19987        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19988            .expect("no io error acquiring directly")
19989        {
19990            crate::cache::AcquireOutcome::Acquired(g) => g,
19991            crate::cache::AcquireOutcome::Busy(b) => {
19992                panic!("expected the direct acquire to win the lease first: {b:?}")
19993            }
19994        };
19995
19996        let mut state = RunState::new(
19997            repo.clone(),
19998            "main".to_owned(),
19999            "deadbeef".to_owned(),
20000            "task".to_owned(),
20001            config,
20002        );
20003        state.candidates = vec![Candidate {
20004            index: 0,
20005            label: 'A',
20006            agent: "alpha".to_owned(),
20007            branch: "does-not-exist".to_owned(),
20008            worktree: repo.clone(),
20009            summary: String::new(),
20010            stat: String::new(),
20011            files: 0,
20012            commits: 0,
20013            empty: false,
20014            failed: None,
20015            verified_noop: None,
20016            duration_ms: 0,
20017            folded: false,
20018        }];
20019        state.tally = Some(Tally {
20020            first_choice: BTreeMap::from([('A', 1)]),
20021            borda: BTreeMap::new(),
20022            winner: 'A',
20023            rankings: 1,
20024            unanimous_initial: true,
20025            deliberated: false,
20026            changed_votes: 0,
20027            unanimous_final: true,
20028            tie_break: None,
20029            judges: 0,
20030            present: 0,
20031            quorum: 0,
20032            met_quorum: true,
20033            uncontested: Some("only candidate A produced a change".to_owned()),
20034        });
20035        state.reviews = vec![ReviewRound {
20036            round: 1,
20037            head: "deadbeef".to_owned(),
20038            verified_head: None,
20039            verified_at: None,
20040            reviews: Vec::new(),
20041            e2e: Vec::new(),
20042            fix: None,
20043            blocking: 0,
20044            answered: 0,
20045            expected: 0,
20046            clean: true,
20047            verify_retried: false,
20048            e2e_deferred: false,
20049            e2e_defer_reason: None,
20050            progressed: false,
20051            vote_split: false,
20052            reconsideration: Vec::new(),
20053            verdict: None,
20054        }];
20055
20056        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20057        state.reviews[0].e2e = vec![CommandOutcome {
20058            command: state.config.verify.e2e[0].clone(),
20059            code: Some(0),
20060            output_tail: String::new(),
20061            duration_ms: 0,
20062            resource_blocked: false,
20063        }];
20064        let mut runner = Runner {
20065            state,
20066            roles: ResolvedRoles {
20067                implementers: Vec::new(),
20068                judges: Vec::new(),
20069                reviewers: Vec::new(),
20070                fixer: None,
20071                conductor: conductor(),
20072                implementer_roster: Vec::new(),
20073                judge_roster: Vec::new(),
20074                reviewer_roster: Vec::new(),
20075            },
20076            sem: Arc::new(Semaphore::new(1)),
20077            pause: Pause::new(),
20078            interrupt: Pause::new(),
20079        };
20080
20081        runner.gate().await.expect("gate");
20082        assert!(runner.state.resource_block.is_none());
20083        assert!(
20084            !runner
20085                .state
20086                .events
20087                .iter()
20088                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20089            "a gate with nothing to run must never request the held cache lease"
20090        );
20091        assert!(
20092            runner.state.gate_ran,
20093            "zero commands is still a real, immediate attempt"
20094        );
20095        assert!(runner.state.gate.is_empty());
20096        assert_ne!(
20097            runner.state.status,
20098            RunStatus::Blocked,
20099            "must not read as resource-blocked on a lease it never asked for"
20100        );
20101    }
20102
20103    /// A separate process carries a live worker's environment throughout all
20104    /// verification entry points; the parallel harness never changes its env.
20105    #[cfg(unix)]
20106    #[tokio::test]
20107    async fn verification_is_standalone_inside_a_live_worker() {
20108        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20109        if std::env::var_os(CHILD).is_none() {
20110            let home = tempfile::tempdir().unwrap();
20111            let generation = crate::worker::pin_current(home.path()).unwrap();
20112            let output = std::process::Command::new(std::env::current_exe().unwrap())
20113                .args([
20114                    "--exact",
20115                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20116                    "--nocapture",
20117                ])
20118                .env(CHILD, "1")
20119                .env("MAGI_HOME", home.path())
20120                .env("MAGI_WORKER_HOME", home.path())
20121                .env("MAGI_WORKER_ID", "abcdef")
20122                .env("MAGI_WORKER_GENERATION", &generation.id)
20123                .env("MAGI_WORKER_EXE", &generation.executable)
20124                .env("MAGI_RUN", "verification-parent")
20125                .env("MAGI_NODE", "gate")
20126                .env("MAGI_SEAT", "worker-fixture")
20127                .env("MAGI_PROCESS_OWNER", "fixture-owner")
20128                .env("MAGI_NO_AUTOUPDATE", "0")
20129                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
20130                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
20131                .env(
20132                    "VERIFY_EXPECT_TOOLCHAIN",
20133                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
20134                )
20135                .env(
20136                    "VERIFY_EXPECT_CACHE",
20137                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
20138                )
20139                .output()
20140                .unwrap();
20141            assert!(
20142                output.status.success()
20143                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
20144                "{}\n{}",
20145                String::from_utf8_lossy(&output.stdout),
20146                String::from_utf8_lossy(&output.stderr)
20147            );
20148            return;
20149        }
20150        let home = crate::run::home();
20151        let directory = home.join("workers/jobs/abcdef");
20152        std::fs::create_dir_all(&directory).unwrap();
20153        let job = crate::worker::Job {
20154            id: "abcdef".into(),
20155            sequence: 1,
20156            kind: "task".into(),
20157            key: "task:verification".into(),
20158            payload: serde_json::json!({}),
20159            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
20160            version: Some(env!("CARGO_PKG_VERSION").into()),
20161            status: crate::worker::Status::Running,
20162            result: None,
20163            pid: Some(std::process::id()),
20164            created_at: 0,
20165            finished_at: None,
20166        };
20167        std::fs::write(
20168            directory.join("job.json"),
20169            serde_json::to_vec(&job).unwrap(),
20170        )
20171        .unwrap();
20172        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
20173        let assert_worker = || {
20174            assert_eq!(
20175                crate::worker::current_job(&home).unwrap().unwrap().id,
20176                job.id
20177            );
20178            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
20179            assert_eq!(
20180                crate::worker::helper_executable(&home).unwrap(),
20181                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20182            );
20183            assert!(
20184                crate::updater::Checker::new(&crate::config::Update {
20185                    mode: crate::config::UpdateMode::Notify,
20186                    interval: None,
20187                    auto_restart: false,
20188                })
20189                .is_none(),
20190                "the driver remains pinned, even while tests are standalone"
20191            );
20192        };
20193        assert_worker();
20194        let repo = home.join("repo");
20195        std::fs::create_dir_all(&repo).unwrap();
20196        init_repo(&repo);
20197        // Keep storage, seat/control context, containment and tool lookup.
20198        // No inherited values (or credentials) are printed on success.
20199        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20200            test -z "${MAGI_WORKER_GENERATION+x}" &&
20201            test -z "${MAGI_WORKER_HOME+x}" &&
20202            test -z "${MAGI_WORKER_EXE+x}" &&
20203            test "$MAGI_HOME/repo" -ef . &&
20204            test "$MAGI_RUN" = verification-parent &&
20205            test "$MAGI_NODE" = gate &&
20206            test "$MAGI_SEAT" = worker-fixture &&
20207            test -n "$MAGI_PROCESS_OWNER" &&
20208            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20209            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20210            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20211            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20212        // Negative control: an ordinary child inherits the contamination.
20213        assert!(
20214            !std::process::Command::new("sh")
20215                .args(["-c", probe])
20216                .current_dir(&repo)
20217                .status()
20218                .unwrap()
20219                .success()
20220        );
20221        let unrelated_home = tempfile::tempdir().unwrap();
20222        assert!(
20223            crate::worker::current_job(unrelated_home.path()).is_err(),
20224            "an inherited worker id cannot be resolved in a unit fixture's home"
20225        );
20226        let mut runner = runner_at(RunStatus::Gating);
20227        runner.state.repo = repo.clone();
20228        runner.state.config.disk.min_free_bytes = 0;
20229        runner.state.config.coordination = crate::config::test_coordination();
20230        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20231        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20232        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20233        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20234        let winner = Candidate {
20235            index: 0,
20236            label: 'A',
20237            agent: "fixture".into(),
20238            branch: "main".into(),
20239            worktree: repo.clone(),
20240            summary: String::new(),
20241            stat: String::new(),
20242            files: 0,
20243            commits: 0,
20244            empty: false,
20245            failed: None,
20246            verified_noop: None,
20247            duration_ms: 0,
20248            folded: false,
20249        };
20250        let (e2e, retried, pids) = run_e2e_with_retry(
20251            &mut runner.state,
20252            &["sh".into(), "-c".into()],
20253            &[format!("{probe} && printf initial")],
20254            &repo,
20255            Duration::from_secs(10),
20256            "initial",
20257        )
20258        .await;
20259        assert!(!retried && pids.is_empty());
20260        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20261        assert_eq!(e2e[0].output_tail, "initial");
20262        let gate = runner.run_gate(&winner).await.unwrap();
20263        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20264        // An interrupted repair must recheck e2e from saved state. Landing
20265        // reuses that state and must run both pre-gate and gate again.
20266        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20267            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20268            passed: false,
20269            outcomes: Vec::new(),
20270        });
20271        runner.state.save().unwrap();
20272        let mut resumed = RunState::load(&runner.state.id).unwrap();
20273        let wait = crate::repo_lock::WaitPolicy::default();
20274        assert!(matches!(
20275            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20276                .await
20277                .unwrap(),
20278            GateFix::Retry
20279        ));
20280        let saved = RunState::load(&resumed.id).unwrap();
20281        let repair = saved.gate_repair_e2e.unwrap();
20282        assert!(repair.passed);
20283        assert_eq!(repair.outcomes.len(), 1);
20284        assert!(repair.outcomes[0].ok());
20285        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20286            .await
20287            .unwrap();
20288        assert_eq!(landing.len(), 1);
20289        assert!(landing[0].ok(), "{landing:?}");
20290        assert_eq!(
20291            std::fs::read_to_string(repo.join("phases")).unwrap(),
20292            "pre\ngate\ne2e\npre\ngate\n"
20293        );
20294        assert_worker();
20295    }
20296
20297    #[cfg(unix)]
20298    #[tokio::test]
20299    async fn verification_keeps_the_panic_and_the_full_output() {
20300        crate::run::pin_test_home();
20301        let tmp = tempfile::tempdir().unwrap();
20302        let mut config = Config::default();
20303        config.disk.min_free_bytes = 0;
20304        let mut state = RunState::new(
20305            tmp.path().to_owned(),
20306            "main".into(),
20307            "base".into(),
20308            "verification evidence".into(),
20309            config,
20310        );
20311        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20312                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20313                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20314        let mut names = Vec::new();
20315        for _ in 0..2 {
20316            let (outcomes, _) = run_commands(
20317                &mut state,
20318                "gate",
20319                "gate",
20320                0,
20321                &["sh".into(), "-c".into()],
20322                &[script.into()],
20323                tmp.path(),
20324                Duration::from_secs(60),
20325            )
20326            .await;
20327            let tail = &outcomes[0].output_tail;
20328            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20329            assert!(tail.contains("boom assertion"));
20330            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20331            let line = tail.lines().next().unwrap();
20332            names.push(line.to_owned());
20333        }
20334        assert_ne!(names[0], names[1], "the second run must not overwrite");
20335        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20336        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20337        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20338
20339        // Storage failures are evidence too; preserve the command's verdict.
20340        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20341        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20342        let (outcomes, _) = run_commands(
20343            &mut state,
20344            "gate",
20345            "gate",
20346            0,
20347            &["sh".into(), "-c".into()],
20348            &["echo failure-output; exit 7".into()],
20349            tmp.path(),
20350            Duration::from_secs(10),
20351        )
20352        .await;
20353        assert_eq!(outcomes[0].code, Some(7));
20354        assert!(state.events.iter().any(|e| {
20355            e.node == "gate" && e.message.contains("could not save the full gate output:")
20356        }));
20357    }
20358
20359    #[cfg(unix)]
20360    #[tokio::test]
20361    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20362        crate::run::pin_test_home();
20363        let tmp = tempfile::tempdir().unwrap();
20364        let mut config = Config::default();
20365        config.disk.min_free_bytes = 0;
20366        let mut state = RunState::new(
20367            tmp.path().to_owned(),
20368            "main".into(),
20369            "base".into(),
20370            "verification evidence".into(),
20371            config,
20372        );
20373        let (outcomes, pids) = run_commands(
20374            &mut state,
20375            "gate",
20376            "gate",
20377            1,
20378            &["sh".into(), "-c".into()],
20379            &[
20380                "printf before-timeout; exec sleep 30".into(),
20381                "kill -TERM $$".into(),
20382                "exit 7".into(),
20383                "printf success".into(),
20384            ],
20385            tmp.path(),
20386            Duration::from_secs(1),
20387        )
20388        .await;
20389        wait_for_timed_out_children_to_die(&pids).await;
20390        assert_eq!(
20391            pids.len(),
20392            1,
20393            "unleased timeout retains its death-check PID"
20394        );
20395        assert_eq!(outcomes.len(), 4);
20396        assert_eq!(outcomes[0].code, None);
20397        assert!(outcomes[0].output_tail.contains("before-timeout"));
20398        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20399        assert_eq!(outcomes[1].code, None);
20400        assert!(outcomes[1].output_tail.contains("command terminated:"));
20401        assert!(!outcomes[1].output_tail.contains("timed out"));
20402        assert_eq!(outcomes[2].code, Some(7));
20403        assert!(!outcomes[2].output_tail.contains("timed out"));
20404        assert_eq!(outcomes[3].code, Some(0));
20405        assert_eq!(outcomes[3].output_tail, "success");
20406        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20407    }
20408
20409    /// The addendum's second gap: a `verify.gate` command running for real
20410    /// wall-clock time had nothing at all to show for it in `active` before
20411    /// `run_commands` learned to record it — a run could sit in `Gating` for
20412    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20413    /// what was actually happening. Proven with a genuinely still-running
20414    /// command, not just a before/after check on the final state: a poller
20415    /// task reads the same `run.json` `gate()` is writing, the same way the
20416    /// phone or `magi show` would, while the shell command is still blocked
20417    /// on its own release marker.
20418    #[tokio::test]
20419    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20420        crate::run::pin_test_home();
20421
20422        let tmp = tempfile::tempdir().expect("tempdir");
20423        let repo = tmp.path().join("repo");
20424        std::fs::create_dir_all(&repo).unwrap();
20425        init_repo(&repo);
20426
20427        let mut config = Config::default();
20428        config.disk.min_free_bytes = 0;
20429        config.verify.gate = vec![
20430            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20431             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20432                .to_owned(),
20433        ];
20434
20435        let mut state = RunState::new(
20436            repo.clone(),
20437            "main".to_owned(),
20438            "deadbeef".to_owned(),
20439            "task".to_owned(),
20440            config,
20441        );
20442        let run_id = state.id.clone();
20443        state.candidates = vec![Candidate {
20444            index: 0,
20445            label: 'A',
20446            agent: "alpha".to_owned(),
20447            branch: "does-not-exist".to_owned(),
20448            worktree: repo.clone(),
20449            summary: String::new(),
20450            stat: String::new(),
20451            files: 0,
20452            commits: 0,
20453            empty: false,
20454            failed: None,
20455            verified_noop: None,
20456            duration_ms: 0,
20457            folded: false,
20458        }];
20459        state.tally = Some(Tally {
20460            first_choice: BTreeMap::from([('A', 1)]),
20461            borda: BTreeMap::new(),
20462            winner: 'A',
20463            rankings: 1,
20464            unanimous_initial: true,
20465            deliberated: false,
20466            changed_votes: 0,
20467            unanimous_final: true,
20468            tie_break: None,
20469            judges: 0,
20470            present: 0,
20471            quorum: 0,
20472            met_quorum: true,
20473            uncontested: Some("only candidate A produced a change".to_owned()),
20474        });
20475        state.reviews = vec![ReviewRound {
20476            round: 1,
20477            head: "deadbeef".to_owned(),
20478            verified_head: None,
20479            verified_at: None,
20480            reviews: Vec::new(),
20481            e2e: Vec::new(),
20482            fix: None,
20483            blocking: 0,
20484            answered: 0,
20485            expected: 0,
20486            clean: true,
20487            verify_retried: false,
20488            e2e_deferred: false,
20489            e2e_defer_reason: None,
20490            progressed: false,
20491            vote_split: false,
20492            reconsideration: Vec::new(),
20493            verdict: None,
20494        }];
20495
20496        let mut runner = Runner {
20497            state,
20498            roles: ResolvedRoles {
20499                implementers: Vec::new(),
20500                judges: Vec::new(),
20501                reviewers: Vec::new(),
20502                fixer: None,
20503                conductor: conductor(),
20504                implementer_roster: Vec::new(),
20505                judge_roster: Vec::new(),
20506                reviewer_roster: Vec::new(),
20507            },
20508            sem: Arc::new(Semaphore::new(1)),
20509            pause: Pause::new(),
20510            interrupt: Pause::new(),
20511        };
20512
20513        let started_marker = repo.join("started.marker");
20514        let release_marker = repo.join("release.marker");
20515        let poller = tokio::spawn(async move {
20516            // Bounded so a regression that never records the task entry
20517            // fails this test in seconds instead of hanging the suite —
20518            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20519            // does_not_cut_it_short` uses for the same reason.
20520            for _ in 0..100 {
20521                if started_marker.exists()
20522                    && let Ok(s) = crate::run::RunState::load(&run_id)
20523                    && let Some(a) = s.active.get("gate")
20524                {
20525                    std::fs::write(&release_marker, b"go").expect("release marker");
20526                    return Some(a.clone());
20527                }
20528                tokio::time::sleep(Duration::from_millis(50)).await;
20529            }
20530            None
20531        });
20532
20533        runner.gate().await.expect("gate");
20534        let captured = poller.await.expect("poller task");
20535        let captured = captured.expect(
20536            "the poller never saw a `gate` task entry in run.json while the command was \
20537             still blocked on its own release marker",
20538        );
20539
20540        assert_eq!(captured.task.as_deref(), Some("gate"));
20541        assert_eq!(captured.node, "gate");
20542        assert_eq!(captured.index, Some(1));
20543        assert_eq!(captured.total, Some(1));
20544        assert!(
20545            captured
20546                .command
20547                .as_deref()
20548                .is_some_and(|c| c.contains("started.marker")),
20549            "{captured:?}"
20550        );
20551
20552        assert!(
20553            runner.state.active.is_empty(),
20554            "the entry must be cleared once the command actually finished: {:?}",
20555            runner.state.active
20556        );
20557        assert!(runner.state.gate_ran);
20558        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20559    }
20560
20561    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20562    /// record for `land`; one with only a Minor, or no reject, leaves none.
20563    #[tokio::test]
20564    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20565        use crate::verdict::{Finding, ReviewVote, Severity};
20566        crate::run::pin_test_home();
20567        let tmp = tempfile::tempdir().expect("tempdir");
20568        let repo = tmp.path().join("repo");
20569        std::fs::create_dir_all(&repo).unwrap();
20570        init_repo(&repo);
20571
20572        for (severity, vote, expect) in [
20573            (Severity::Major, ReviewVote::Reject, true),
20574            (Severity::Minor, ReviewVote::Reject, false),
20575            (Severity::Major, ReviewVote::Approve, false),
20576        ] {
20577            let mut round = review_round(false, 1, 1, 1, false, true);
20578            round.reviews = vec![ReviewRecord {
20579                reviewer: 1,
20580                agent: "alpha".to_owned(),
20581                summary: String::new(),
20582                findings: vec![Finding {
20583                    id: "R1-1-1".to_owned(),
20584                    severity,
20585                    file: None,
20586                    line: None,
20587                    title: "t".to_owned(),
20588                    detail: String::new(),
20589                }],
20590                vote: Some(vote),
20591                failed: None,
20592                duration_ms: 0,
20593                attempts: 0,
20594            }];
20595            let mut state = RunState::new(
20596                repo.clone(),
20597                "main".to_owned(),
20598                "deadbeef".to_owned(),
20599                "task".to_owned(),
20600                Config::default(),
20601            );
20602            state.reviews = vec![round];
20603            let mut runner = Runner {
20604                state,
20605                roles: ResolvedRoles {
20606                    implementers: Vec::new(),
20607                    judges: Vec::new(),
20608                    reviewers: Vec::new(),
20609                    fixer: None,
20610                    conductor: conductor(),
20611                    implementer_roster: Vec::new(),
20612                    judge_roster: Vec::new(),
20613                    reviewer_roster: Vec::new(),
20614                },
20615                sem: Arc::new(Semaphore::new(1)),
20616                pause: Pause::new(),
20617                interrupt: Pause::new(),
20618            };
20619            let shell = runner.state.config.shell();
20620            runner
20621                .stop_reviewing("round budget spent", &shell, &repo)
20622                .await
20623                .expect("stop_reviewing");
20624            assert_eq!(runner.state.status, RunStatus::Gating);
20625            assert_eq!(
20626                runner.state.contested_handoff.is_some(),
20627                expect,
20628                "{severity:?} + {vote:?}"
20629            );
20630        }
20631    }
20632
20633    /// The shape the incident this whole fix responds to actually had: the
20634    /// round budget spent, the last round's own e2e blocked on the shared
20635    /// build cache (held here by a live pid — this test process — exactly
20636    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20637    /// without forking a process). `stop_reviewing` must retry it — not
20638    /// silently leave the round looking untouched (the catch-up-only half of
20639    /// the bug), and not read the contention as a red `e2e` and block the
20640    /// run on it (the other half). Called directly, the same way
20641    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20642    /// above exercises `gate`, so this never needs a real cargo build to
20643    /// reach: the lease is never released, so `with_cache_lease` never gets
20644    /// past acquiring it into anything that would need a real workspace.
20645    #[tokio::test]
20646    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20647        crate::run::pin_test_home();
20648        let home = crate::run::home();
20649
20650        let tmp = tempfile::tempdir().expect("tempdir");
20651        let repo = tmp.path().join("repo");
20652        std::fs::create_dir_all(&repo).unwrap();
20653        init_repo(&repo);
20654        let head = crate::git::rev_parse(&repo, "HEAD")
20655            .await
20656            .expect("rev-parse");
20657        // Unique to this test, so holding its lease cannot collide with
20658        // another test sharing the same process-wide `home`.
20659        let cache_dir = tmp.path().join("target");
20660
20661        let mut config = Config::default();
20662        config.verify.e2e = vec![format!(
20663            "CARGO_TARGET_DIR='{}' test -f README.md",
20664            cache_dir.display()
20665        )];
20666        config.graph.review_rounds = 1;
20667        // Bounded so a regression that does start waiting fails the test in
20668        // seconds, not hangs it.
20669        config.graph.timeout_verify = Some(2);
20670        config.graph.timeout_resource_wait = 2;
20671
20672        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20673        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20674            .expect("no io error acquiring directly")
20675        {
20676            crate::cache::AcquireOutcome::Acquired(g) => g,
20677            crate::cache::AcquireOutcome::Busy(b) => {
20678                panic!("expected the direct acquire to win the lease first: {b:?}")
20679            }
20680        };
20681
20682        let mut state = RunState::new(
20683            repo.clone(),
20684            "main".to_owned(),
20685            head.clone(),
20686            "task".to_owned(),
20687            config,
20688        );
20689        state.candidates = vec![Candidate {
20690            index: 0,
20691            label: 'A',
20692            agent: "alpha".to_owned(),
20693            branch: "does-not-exist".to_owned(),
20694            worktree: repo.clone(),
20695            summary: String::new(),
20696            stat: String::new(),
20697            files: 0,
20698            commits: 0,
20699            empty: false,
20700            failed: None,
20701            verified_noop: None,
20702            duration_ms: 0,
20703            folded: false,
20704        }];
20705        state.tally = Some(Tally {
20706            first_choice: BTreeMap::from([('A', 1)]),
20707            borda: BTreeMap::new(),
20708            winner: 'A',
20709            rankings: 1,
20710            unanimous_initial: true,
20711            deliberated: false,
20712            changed_votes: 0,
20713            unanimous_final: true,
20714            tie_break: None,
20715            judges: 0,
20716            present: 0,
20717            quorum: 0,
20718            met_quorum: true,
20719            uncontested: Some("only candidate A produced a change".to_owned()),
20720        });
20721        // The round budget's last round, deferred: `needs_catchup_run`'s
20722        // other trigger. `stop_reviewing`'s retry machinery must treat this
20723        // exactly like a resource-blocked attempt once it actually runs.
20724        state.reviews = vec![ReviewRound {
20725            round: 1,
20726            head: head.clone(),
20727            verified_head: None,
20728            verified_at: None,
20729            reviews: Vec::new(),
20730            e2e: Vec::new(),
20731            fix: None,
20732            blocking: 1,
20733            answered: 1,
20734            expected: 1,
20735            clean: false,
20736            verify_retried: false,
20737            e2e_deferred: true,
20738            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20739            progressed: false,
20740            vote_split: false,
20741            reconsideration: Vec::new(),
20742            verdict: None,
20743        }];
20744
20745        let mut runner = Runner {
20746            state,
20747            roles: ResolvedRoles {
20748                implementers: Vec::new(),
20749                judges: Vec::new(),
20750                reviewers: Vec::new(),
20751                fixer: None,
20752                conductor: conductor(),
20753                implementer_roster: Vec::new(),
20754                judge_roster: Vec::new(),
20755                reviewer_roster: Vec::new(),
20756            },
20757            sem: Arc::new(Semaphore::new(1)),
20758            pause: Pause::new(),
20759            interrupt: Pause::new(),
20760        };
20761
20762        let shell = runner.state.config.shell();
20763        runner
20764            .stop_reviewing("round budget spent", &shell, &repo)
20765            .await
20766            .expect("stop_reviewing");
20767
20768        let last = runner.state.reviews.last().expect("round record");
20769        assert_eq!(
20770            last.e2e_status(),
20771            E2eStatus::ResourceBlocked,
20772            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20773             failed: {last:?}"
20774        );
20775        assert_eq!(
20776            last.verified_head.as_deref(),
20777            Some(head.as_str()),
20778            "which commit this attempt targeted is known even though nothing finished checking \
20779             it"
20780        );
20781        let first_attempt_at = last
20782            .verified_at
20783            .expect("when this attempt ran is known too");
20784        assert_ne!(
20785            runner.state.status,
20786            RunStatus::Blocked,
20787            "contention is evidence about the machine, not the patch — it must not settle the \
20788             run as blocked: {:?}",
20789            runner.state.status
20790        );
20791        assert!(
20792            !runner
20793                .state
20794                .events
20795                .iter()
20796                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20797            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20798            runner.state.events
20799        );
20800
20801        // The cache is still held: a later reentry must retry the same
20802        // round's verification again — not leave it looking exactly as
20803        // untouched as the first blocked attempt, which is indistinguishable
20804        // from never having tried again at all.
20805        runner
20806            .stop_reviewing("round budget spent", &shell, &repo)
20807            .await
20808            .expect("stop_reviewing retry");
20809        assert_eq!(
20810            runner.state.reviews.len(),
20811            1,
20812            "no new round was started: {:?}",
20813            runner.state.reviews
20814        );
20815        let last = runner.state.reviews.last().expect("round record");
20816        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20817        assert!(
20818            last.verified_at.expect("still known") > first_attempt_at,
20819            "a second reentry must be a fresh attempt, not a stale copy of the first"
20820        );
20821        assert_ne!(runner.state.status, RunStatus::Blocked);
20822
20823        held.release();
20824    }
20825
20826    #[cfg(unix)]
20827    #[tokio::test]
20828    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20829        ask_test_home();
20830        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20831            let tmp = tempfile::tempdir().unwrap();
20832            let repo = tmp.path().join("repo");
20833            std::fs::create_dir(&repo).unwrap();
20834            init_repo(&repo);
20835            let run_git = |args: &[&str]| {
20836                let out = std::process::Command::new("git")
20837                    .args(args)
20838                    .current_dir(&repo)
20839                    .output()
20840                    .unwrap();
20841                assert!(
20842                    out.status.success(),
20843                    "{args:?}: {}",
20844                    String::from_utf8_lossy(&out.stderr)
20845                );
20846            };
20847            let origin = tmp.path().join("origin.git");
20848            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20849            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20850            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20851            run_git(&["add", "spec.rs"]);
20852            run_git(&["commit", "-m", "initial spec"]);
20853            run_git(&["push", "origin", "main"]);
20854            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20855            run_git(&["checkout", "-b", "fix/review"]);
20856            std::fs::write(
20857                repo.join("check.rs"),
20858                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20859            )
20860            .unwrap();
20861            run_git(&["add", "check.rs"]);
20862            run_git(&["commit", "-m", "use spec"]);
20863            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20864            run_git(&["checkout", "main"]);
20865            std::fs::write(
20866                repo.join("spec.rs"),
20867                "struct AgentSpec { max_turn_secs: u64 }\n",
20868            )
20869            .unwrap();
20870            run_git(&["add", "spec.rs"]);
20871            run_git(&["commit", "-m", "extend spec"]);
20872            run_git(&["push", "origin", "main"]);
20873            run_git(&["checkout", "fix/review"]);
20874
20875            let mut runner = runner_at(RunStatus::Reviewing);
20876            runner.state.repo = repo.clone();
20877            runner.state.base_commit = base;
20878            runner.state.pr_head = Some("fix/review".into());
20879            runner.state.config.graph.review_rounds = max_rounds;
20880            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20881            runner.state.config.graph.sessions = false;
20882            runner.state.config.disk.min_free_bytes = 0;
20883            runner.state.config.verify.e2e =
20884                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20885            runner.state.candidates = vec![Candidate {
20886                index: 0,
20887                label: 'A',
20888                agent: "fixture".into(),
20889                branch: "fix/review".into(),
20890                worktree: repo.clone(),
20891                summary: String::new(),
20892                stat: String::new(),
20893                files: 1,
20894                commits: 1,
20895                empty: false,
20896                failed: None,
20897                verified_noop: None,
20898                duration_ms: 0,
20899                folded: false,
20900            }];
20901            runner.state.tally = Some(Tally {
20902                first_choice: BTreeMap::new(),
20903                borda: BTreeMap::new(),
20904                winner: 'A',
20905                rankings: 0,
20906                unanimous_initial: true,
20907                deliberated: false,
20908                changed_votes: 0,
20909                unanimous_final: true,
20910                tie_break: None,
20911                judges: 0,
20912                present: 0,
20913                quorum: 0,
20914                met_quorum: true,
20915                uncontested: Some("review-only".into()),
20916            });
20917            let mut last = review_round(false, 0, 1, 1, false, false);
20918            last.head = reviewed.clone();
20919            last.e2e_deferred = true;
20920            last.e2e[0].resource_blocked = true;
20921            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20922            if max_rounds == 1 {
20923                last.e2e.clear();
20924            }
20925            runner.state.reviews = vec![last];
20926            let calls = tmp.path().join("fix-calls");
20927            let mut fixer = spec("repair");
20928            fixer
20929                .env
20930                .insert("FIX_CALLS".into(), calls.display().to_string());
20931            fixer.command = vec!["sh".into(), "-c".into(), r#"
20932                echo fix >> "$FIX_CALLS"
20933                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20934                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20935                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20936                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20937            "#.into()];
20938            let mut reviewer = spec("reviewer");
20939            reviewer.command = vec![
20940                "sh".into(),
20941                "-c".into(),
20942                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20943                    .into(),
20944            ];
20945            runner.roles.fixer = Some(vec![fixer]);
20946            runner.roles.reviewers = vec![reviewer];
20947
20948            runner.sync_to_base().await.unwrap();
20949            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20950            assert_ne!(rebased, reviewed);
20951            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20952            if reentry {
20953                // Simulate a crash after catch-up evidence was saved but before
20954                // dispatching the fixer. No dedicated pending-repair field exists.
20955                let shell = runner.state.config.shell();
20956                runner
20957                    .stop_unresolved_review("deferred verification", &shell, &repo)
20958                    .await
20959                    .unwrap();
20960                assert!(runner.catchup_repairable());
20961                assert!(!calls.exists());
20962                runner.state = RunState::load(&runner.state.id).unwrap();
20963                assert!(runner.catchup_repairable());
20964            }
20965            runner.review_loop().await.unwrap();
20966            let first = &runner.state.reviews[0];
20967            assert_eq!(first.head, reviewed);
20968            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20969            assert!(
20970                first.verify_retried,
20971                "a reproduced build failure gets exactly one retry: {:?}",
20972                first.e2e
20973            );
20974            assert!(first.e2e[0].output_tail.contains("E0063"));
20975            assert!(!runner.catchup_repairable());
20976            if max_rounds == 2 {
20977                assert_eq!(
20978                    runner.state.status,
20979                    RunStatus::Gating,
20980                    "{}",
20981                    crate::report::run(&runner.state)
20982                );
20983                assert_eq!(runner.state.reviews.len(), 2);
20984                assert!(first.fix.as_ref().unwrap().committed);
20985                assert!(runner.state.reviews[1].clean);
20986                assert_eq!(
20987                    runner
20988                        .state
20989                        .reviews
20990                        .iter()
20991                        .filter(|r| r.fix.is_some())
20992                        .count(),
20993                    1
20994                );
20995                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20996                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20997            } else {
20998                assert_eq!(runner.state.status, RunStatus::Blocked);
20999                assert!(first.fix.is_none());
21000                assert!(!calls.exists());
21001                let recovery = crate::recovery::assess(
21002                    "prior",
21003                    Some(&runner.state),
21004                    &runner.landing_base(),
21005                    crate::recovery::Relation::Diverged,
21006                );
21007                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21008                let reasons = recovery.reasons.join("\n");
21009                assert!(
21010                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21011                    "{reasons}"
21012                );
21013                assert!(
21014                    reasons.contains("base sync to")
21015                        && reasons.contains("additional review is required"),
21016                    "{reasons}"
21017                );
21018            }
21019        }
21020    }
21021
21022    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21023    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21024    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21025    /// own top-of-function fast path (`review_conclusion`) correctly reads
21026    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21027    /// own `for` range is empty once the round budget is spent, so
21028    /// `review_loop` must retry the check itself rather than silently doing
21029    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21030    /// above exercises directly, but through `review_loop`'s own entry point
21031    /// this time, proving the wiring between the two rather than just the
21032    /// retry logic in isolation.
21033    #[tokio::test]
21034    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21035        crate::run::pin_test_home();
21036        let home = crate::run::home();
21037
21038        let tmp = tempfile::tempdir().expect("tempdir");
21039        let repo = tmp.path().join("repo");
21040        std::fs::create_dir_all(&repo).unwrap();
21041        init_repo(&repo);
21042        let head = crate::git::rev_parse(&repo, "HEAD")
21043            .await
21044            .expect("rev-parse");
21045        let cache_dir = tmp.path().join("target");
21046
21047        let mut config = Config::default();
21048        config.verify.e2e = vec![format!(
21049            "CARGO_TARGET_DIR='{}' test -f README.md",
21050            cache_dir.display()
21051        )];
21052        config.graph.review_rounds = 1;
21053        config.graph.timeout_verify = Some(2);
21054        config.graph.timeout_resource_wait = 2;
21055
21056        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21057        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21058            .expect("no io error acquiring directly")
21059        {
21060            crate::cache::AcquireOutcome::Acquired(g) => g,
21061            crate::cache::AcquireOutcome::Busy(b) => {
21062                panic!("expected the direct acquire to win the lease first: {b:?}")
21063            }
21064        };
21065
21066        let mut state = RunState::new(
21067            repo.clone(),
21068            "main".to_owned(),
21069            head.clone(),
21070            "task".to_owned(),
21071            config,
21072        );
21073        state.candidates = vec![Candidate {
21074            index: 0,
21075            label: 'A',
21076            agent: "alpha".to_owned(),
21077            branch: "does-not-exist".to_owned(),
21078            worktree: repo.clone(),
21079            summary: String::new(),
21080            stat: String::new(),
21081            files: 0,
21082            commits: 0,
21083            empty: false,
21084            failed: None,
21085            verified_noop: None,
21086            duration_ms: 0,
21087            folded: false,
21088        }];
21089        state.tally = Some(Tally {
21090            first_choice: BTreeMap::from([('A', 1)]),
21091            borda: BTreeMap::new(),
21092            winner: 'A',
21093            rankings: 1,
21094            unanimous_initial: true,
21095            deliberated: false,
21096            changed_votes: 0,
21097            unanimous_final: true,
21098            tie_break: None,
21099            judges: 0,
21100            present: 0,
21101            quorum: 0,
21102            met_quorum: true,
21103            uncontested: Some("only candidate A produced a change".to_owned()),
21104        });
21105        // The exact shape a prior process's `stop_reviewing` would have left
21106        // on disk: the round budget's last round, a real attempt already
21107        // made and already resource-blocked.
21108        state.reviews = vec![ReviewRound {
21109            round: 1,
21110            head: head.clone(),
21111            verified_head: Some(head.clone()),
21112            verified_at: Some(jiff::Timestamp::now()),
21113            reviews: Vec::new(),
21114            e2e: vec![CommandOutcome {
21115                command: format!(
21116                    "CARGO_TARGET_DIR='{}' test -f README.md",
21117                    cache_dir.display()
21118                ),
21119                code: None,
21120                output_tail: "waiting for the shared build cache".to_owned(),
21121                duration_ms: 0,
21122                resource_blocked: true,
21123            }],
21124            fix: None,
21125            blocking: 1,
21126            answered: 1,
21127            expected: 1,
21128            clean: false,
21129            verify_retried: false,
21130            e2e_deferred: false,
21131            e2e_defer_reason: None,
21132            progressed: false,
21133            vote_split: false,
21134            reconsideration: Vec::new(),
21135            verdict: None,
21136        }];
21137
21138        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
21139        let mut runner = Runner {
21140            state,
21141            roles: ResolvedRoles {
21142                implementers: Vec::new(),
21143                judges: Vec::new(),
21144                reviewers: Vec::new(),
21145                fixer: None,
21146                conductor: conductor(),
21147                implementer_roster: Vec::new(),
21148                judge_roster: Vec::new(),
21149                reviewer_roster: Vec::new(),
21150            },
21151            sem: Arc::new(Semaphore::new(1)),
21152            pause: Pause::new(),
21153            interrupt: Pause::new(),
21154        };
21155
21156        // The lease is still held throughout, so this reentry's own retry is
21157        // also contended — proving `review_loop` actually tried again (not
21158        // that it happened to succeed) is what the timestamp comparison
21159        // below is for.
21160        runner.review_loop().await.expect("review_loop");
21161
21162        assert_eq!(
21163            runner.state.reviews.len(),
21164            1,
21165            "no new round was started on top of the unresolved one: {:?}",
21166            runner.state.reviews
21167        );
21168        let last = &runner.state.reviews[0];
21169        assert_eq!(
21170            last.e2e_status(),
21171            E2eStatus::ResourceBlocked,
21172            "still contended: {last:?}"
21173        );
21174        assert!(
21175            last.verified_at.expect("still known") > first_attempt_at,
21176            "review_loop must have actually retried the check, not left it exactly as found"
21177        );
21178        assert_ne!(
21179            runner.state.status,
21180            RunStatus::Blocked,
21181            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21182            runner.state.status
21183        );
21184
21185        held.release();
21186    }
21187
21188    #[tokio::test]
21189    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21190        crate::run::pin_test_home();
21191        for status in [RunStatus::Blocked, RunStatus::Gating] {
21192            let mut runner = runner_at(status);
21193            runner.state.config.graph.land = true;
21194            runner.state.gate_ran = true;
21195            runner.state.merge = Some(MergeOutcome {
21196                mode: MergeMode::Pr,
21197                ok: false,
21198                detail: "the fixer produced no commit".into(),
21199                empty: false,
21200            });
21201            let url = "https://example.invalid/o/r/pull/1";
21202            runner.state.pr = Some(crate::run::PrRecord {
21203                url: url.into(),
21204                number: 1,
21205                state: "open".into(),
21206                checks: "red".into(),
21207                round: 1,
21208                rounds: 1,
21209                red_at_merge: Vec::new(),
21210            });
21211            runner.state.save().unwrap();
21212            // The nonexistent repository makes observation fail before a forge
21213            // command can run. We still must reach land, rather than silently
21214            // return from the graph with its old gate and merge records.
21215            runner.execute_graph().await.unwrap();
21216            assert_eq!(runner.state.status, RunStatus::Blocked);
21217            assert!(runner.state.land_sync_pending);
21218            assert!(runner.state.events.iter().any(|event|
21219                event.node == "land" && event.message == format!("watching {url}")));
21220            assert_eq!(
21221                runner.state.merge.as_ref().unwrap().detail,
21222                "the fixer produced no commit"
21223            );
21224        }
21225    }
21226
21227    #[tokio::test]
21228    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21229        crate::run::pin_test_home();
21230        let tmp = tempfile::tempdir().expect("tempdir");
21231        let repo = tmp.path().join("repo");
21232        std::fs::create_dir_all(&repo).unwrap();
21233        init_repo(&repo);
21234
21235        let mut config = Config::default();
21236        config.merge.mode = MergeMode::Pr;
21237        config.graph.land = true;
21238        config.graph.land_approval = false;
21239
21240        let mut state = RunState::new(
21241            repo.clone(),
21242            "main".to_owned(),
21243            "deadbeef".to_owned(),
21244            "task".to_owned(),
21245            config,
21246        );
21247        state.candidates = vec![Candidate {
21248            index: 0,
21249            label: 'A',
21250            agent: "alpha".to_owned(),
21251            branch: "does-not-exist".to_owned(),
21252            worktree: repo.clone(),
21253            summary: String::new(),
21254            stat: String::new(),
21255            files: 0,
21256            commits: 0,
21257            empty: false,
21258            failed: None,
21259            verified_noop: None,
21260            duration_ms: 0,
21261            folded: false,
21262        }];
21263        state.tally = Some(Tally {
21264            first_choice: BTreeMap::from([('A', 1)]),
21265            borda: BTreeMap::new(),
21266            winner: 'A',
21267            rankings: 1,
21268            unanimous_initial: true,
21269            deliberated: false,
21270            changed_votes: 0,
21271            unanimous_final: true,
21272            tie_break: None,
21273            judges: 0,
21274            present: 0,
21275            quorum: 0,
21276            met_quorum: true,
21277            uncontested: Some("only candidate A produced a change".to_owned()),
21278        });
21279        state.reviews = vec![ReviewRound {
21280            round: 1,
21281            head: "deadbeef".to_owned(),
21282            verified_head: None,
21283            verified_at: None,
21284            reviews: Vec::new(),
21285            e2e: Vec::new(),
21286            fix: None,
21287            blocking: 0,
21288            answered: 0,
21289            expected: 0,
21290            clean: true,
21291            verify_retried: false,
21292            e2e_deferred: false,
21293            e2e_defer_reason: None,
21294            progressed: false,
21295            vote_split: false,
21296            reconsideration: Vec::new(),
21297            verdict: None,
21298        }];
21299        state.gate = vec![CommandOutcome {
21300            command: "test".to_owned(),
21301            code: Some(0),
21302            output_tail: String::new(),
21303            duration_ms: 0,
21304            resource_blocked: false,
21305        }];
21306        state.gate_ran = true;
21307        // A first pass through `merge` already pushed and opened this pull
21308        // request; `status` is `Landing` because a previous call into `land`
21309        // parked or was interrupted before it reached a terminal outcome.
21310        state.status = RunStatus::Landing;
21311        state.merge = Some(MergeOutcome {
21312            mode: MergeMode::Pr,
21313            ok: true,
21314            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21315            empty: false,
21316        });
21317
21318        // The Landing-resume shortcut calls `run_land` directly rather than
21319        // through `merge`, which is exactly the call site that used to skip
21320        // `settle_questions` - see the fixture below.
21321        ask_test_home();
21322        // A real resume loads a persisted run; driver ownership is checked
21323        // against that record before the Landing shortcut is entered.
21324        state.save().expect("save landing run");
21325        let store = ask::Questions::open();
21326        let q = ask_open_question(&store, &state.id);
21327
21328        let mut runner = Runner {
21329            state,
21330            roles: ResolvedRoles {
21331                implementers: Vec::new(),
21332                judges: Vec::new(),
21333                reviewers: Vec::new(),
21334                fixer: None,
21335                conductor: conductor(),
21336                implementer_roster: Vec::new(),
21337                judge_roster: Vec::new(),
21338                reviewer_roster: Vec::new(),
21339            },
21340            sem: Arc::new(Semaphore::new(1)),
21341            pause: Pause::new(),
21342            interrupt: Pause::new(),
21343        };
21344
21345        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21346        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21347        // exactly because `review_loop` would otherwise clobber the marker
21348        // first.
21349        runner.execute().await.expect("execute");
21350
21351        assert_eq!(
21352            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21353            Some("https://example.invalid/x/y/pull/1"),
21354            "reentry must not push again or open a second pull request over the \
21355             one `land` is already watching"
21356        );
21357        assert_ne!(
21358            runner.state.status,
21359            RunStatus::Landing,
21360            "land could not actually reach the fake pull request, so it must \
21361             have given up rather than left the run silently parked forever"
21362        );
21363        // `land` could not reach the fake pull request, so it gave up into
21364        // `Blocked` - still resumable, so the question must not have been
21365        // swept just because this branch now also calls `settle_questions`.
21366        assert_eq!(runner.state.status, RunStatus::Blocked);
21367        assert!(
21368            store.get(&q.id).unwrap().status.open(),
21369            "Blocked is still alive; settle_questions must have been a no-op here"
21370        );
21371    }
21372
21373    fn state_with_round(round: ReviewRound) -> RunState {
21374        let mut s = RunState::new(
21375            PathBuf::from("/repo"),
21376            "main".to_owned(),
21377            "abc1234".to_owned(),
21378            "add retries".to_owned(),
21379            Config::default(),
21380        );
21381        s.reviews = vec![round];
21382        s
21383    }
21384
21385    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21386        crate::verdict::Finding {
21387            id: id.to_owned(),
21388            severity,
21389            file: None,
21390            line: None,
21391            title: title.to_owned(),
21392            detail: String::new(),
21393        }
21394    }
21395
21396    #[test]
21397    fn pr_body_names_open_findings_and_declined_ones() {
21398        let round = ReviewRound {
21399            round: 2,
21400            head: "deadbee".to_owned(),
21401            verified_head: None,
21402            verified_at: None,
21403            reviews: vec![ReviewRecord {
21404                attempts: 0,
21405                reviewer: 1,
21406                agent: "alpha".to_owned(),
21407                summary: String::new(),
21408                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21409                vote: None,
21410                failed: None,
21411                duration_ms: 0,
21412            }],
21413            e2e: vec![CommandOutcome {
21414                command: "cargo test".to_owned(),
21415                code: Some(0),
21416                output_tail: String::new(),
21417                duration_ms: 0,
21418                resource_blocked: false,
21419            }],
21420            verify_retried: false,
21421            e2e_deferred: false,
21422            e2e_defer_reason: None,
21423            fix: Some(FixRecord {
21424                agent: "alpha".to_owned(),
21425                addressed: Vec::new(),
21426                rejected: vec![crate::verdict::Rejection {
21427                    id: "R1-1-1".to_owned(),
21428                    why: "not reachable from any caller".to_owned(),
21429                }],
21430                notes: String::new(),
21431                committed: true,
21432                failed: None,
21433                duration_ms: 0,
21434                continuation: None,
21435            }),
21436            blocking: 0,
21437            answered: 1,
21438            expected: 1,
21439            clean: false,
21440            progressed: true,
21441            vote_split: false,
21442            reconsideration: Vec::new(),
21443            verdict: None,
21444        };
21445        let state = state_with_round(round);
21446        let body = pr_message(&state, 'A').body;
21447
21448        assert!(body.contains("add retries"), "the task must still be there");
21449        assert!(body.contains("R2-1-1"), "{body}");
21450        assert!(!body.contains("unused import"), "{body}");
21451        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21452        assert!(
21453            !body.contains("not reachable from any caller"),
21454            "the reason it was declined: {body}"
21455        );
21456    }
21457
21458    #[test]
21459    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21460        let round = ReviewRound {
21461            round: 1,
21462            head: "deadbee".to_owned(),
21463            verified_head: None,
21464            verified_at: None,
21465            reviews: vec![ReviewRecord {
21466                attempts: 0,
21467                reviewer: 1,
21468                agent: "alpha".to_owned(),
21469                summary: String::new(),
21470                findings: Vec::new(),
21471                vote: None,
21472                failed: None,
21473                duration_ms: 0,
21474            }],
21475            e2e: Vec::new(),
21476            verify_retried: false,
21477            e2e_deferred: false,
21478            e2e_defer_reason: None,
21479            fix: None,
21480            blocking: 0,
21481            answered: 1,
21482            expected: 1,
21483            clean: true,
21484            progressed: false,
21485            vote_split: false,
21486            reconsideration: Vec::new(),
21487            verdict: None,
21488        };
21489        let state = state_with_round(round);
21490        let body = pr_message(&state, 'A').body;
21491        assert!(!body.contains("Open review findings"), "{body}");
21492        assert!(!body.contains("Declined"), "{body}");
21493    }
21494
21495    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21496        let mut state = RunState::new(
21497            PathBuf::from("/repo"),
21498            "main".to_owned(),
21499            "abc1234".to_owned(),
21500            instruction.to_owned(),
21501            Config::default(),
21502        );
21503        // These fixtures test execution flow, independent of host disk capacity.
21504        state.config.disk.min_free_bytes = 0;
21505        state.candidates.push(Candidate {
21506            index: 0,
21507            label: 'A',
21508            agent: "alpha".to_owned(),
21509            branch: "magi/x/A".to_owned(),
21510            worktree: PathBuf::from("/wt"),
21511            summary: summary.to_owned(),
21512            stat: String::new(),
21513            files: 1,
21514            commits: 1,
21515            empty: false,
21516            failed: None,
21517            verified_noop: None,
21518            folded: false,
21519            duration_ms: 0,
21520        });
21521        state
21522    }
21523
21524    fn review_state(subjects: &[&str]) -> RunState {
21525        let mut state = state_with_summary(
21526            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21527            "",
21528        );
21529        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21530        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21531        state
21532    }
21533
21534    /// A branch rebased onto a main that moved past the recorded
21535    /// `base_commit` is titled from its own first commit, never main's.
21536    #[tokio::test]
21537    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21538        ask_test_home();
21539        let tmp = tempfile::tempdir().unwrap();
21540        let repo = tmp.path().join("repo");
21541        std::fs::create_dir_all(&repo).unwrap();
21542        init_repo(&repo);
21543        let origin = tmp.path().join("origin.git");
21544        let g = |dir: &Path, args: &[&str]| {
21545            let out = std::process::Command::new("git")
21546                .args(args)
21547                .current_dir(dir)
21548                .quiet()
21549                .output()
21550                .expect("spawn git");
21551            assert!(
21552                out.status.success(),
21553                "git {args:?}: {}",
21554                String::from_utf8_lossy(&out.stderr)
21555            );
21556        };
21557        g(
21558            tmp.path(),
21559            &[
21560                "clone",
21561                "--bare",
21562                "-q",
21563                repo.to_str().unwrap(),
21564                origin.to_str().unwrap(),
21565            ],
21566        );
21567        g(
21568            &repo,
21569            &["remote", "add", "origin", origin.to_str().unwrap()],
21570        );
21571        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21572
21573        // Main moves on; the branch is built on top of the new main.
21574        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21575        g(&repo, &["add", "-A"]);
21576        g(
21577            &repo,
21578            &["commit", "-q", "-m", "chore(deps): update a crate"],
21579        );
21580        g(&repo, &["push", "-q", "origin", "main"]);
21581        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21582        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21583        g(&repo, &["add", "-A"]);
21584        g(
21585            &repo,
21586            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21587        );
21588        g(&repo, &["checkout", "-q", "main"]);
21589
21590        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21591        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21592        // Without a readable tracking ref the recorded base's merge base is used.
21593        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21594        assert_eq!(fallback, c1);
21595
21596        let mut state = review_state(&[
21597            "chore(deps): update a crate",
21598            "fix(daemon): apply a chosen action",
21599        ]);
21600        state.repo = repo.clone();
21601        state.base_branch = "main".to_owned();
21602        state.base_commit = c1;
21603        refresh_reviewed_commits(&mut state, "feat/own").await;
21604        assert_eq!(
21605            state.reviewed_commits,
21606            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21607        );
21608        assert_eq!(
21609            review_title(&state).as_deref(),
21610            Some("fix(daemon): apply a chosen action")
21611        );
21612        assert_eq!(
21613            leaked_subjects(&state, "feat/own").await,
21614            Some(vec!["chore(deps): update a crate".to_owned()])
21615        );
21616        // A stale tracking ref is still used when the fetch fails, but the
21617        // leak list is withheld.
21618        state.config.merge.remote = "nowhere".to_owned();
21619        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21620    }
21621
21622    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21623    /// with `origin/main` one commit behind it, and a summarizing agent that
21624    /// prints whatever `reply` holds.
21625    struct SummaryFixture {
21626        _tmp: tempfile::TempDir,
21627        repo: PathBuf,
21628        reply: PathBuf,
21629        runner: Runner,
21630    }
21631
21632    fn summary_fixture() -> SummaryFixture {
21633        ask_test_home();
21634        let tmp = tempfile::tempdir().unwrap();
21635        let repo = tmp.path().join("repo");
21636        std::fs::create_dir_all(&repo).unwrap();
21637        init_repo(&repo);
21638        let origin = tmp.path().join("origin.git");
21639        let g = |args: &[&str]| {
21640            let out = std::process::Command::new("git")
21641                .args(args)
21642                .current_dir(&repo)
21643                .quiet()
21644                .output()
21645                .expect("spawn git");
21646            assert!(
21647                out.status.success(),
21648                "git {args:?}: {}",
21649                String::from_utf8_lossy(&out.stderr)
21650            );
21651        };
21652        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21653        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21654        g(&["push", "-q", "origin", "main"]);
21655        g(&["checkout", "-q", "-b", "feat/own"]);
21656        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21657        g(&["add", "-A"]);
21658        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21659        let reply = tmp.path().join("reply.txt");
21660        std::fs::write(
21661            &reply,
21662            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21663        )
21664        .unwrap();
21665        let mut runner = runner_at(RunStatus::Gating);
21666        let mut state = review_state(&["feat(own): add the own file"]);
21667        state.repo = repo.clone();
21668        state.base_branch = "main".to_owned();
21669        state.candidates[0].branch = "feat/own".to_owned();
21670        state.candidates[0].worktree = repo.clone();
21671        let mut author = spec("alpha");
21672        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21673        author
21674            .env
21675            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21676        state.config.agents = vec![author];
21677        runner.state = state;
21678        SummaryFixture {
21679            _tmp: tmp,
21680            repo,
21681            reply,
21682            runner,
21683        }
21684    }
21685
21686    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21687        let winner = f.runner.state.candidates[0].clone();
21688        f.runner
21689            .guarded_pr_message(&winner, None, true)
21690            .await
21691            .unwrap()
21692            .expect("an English description never parks")
21693    }
21694
21695    fn clean_round_with_minor_findings() -> ReviewRound {
21696        let mut round = passing_round(2, None);
21697        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21698        round.reviews = vec![ReviewRecord {
21699            attempts: 0,
21700            reviewer: 1,
21701            agent: "alpha".into(),
21702            summary: "内部レビュー".into(),
21703            findings: vec![
21704                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21705                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21706            ],
21707            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21708            failed: None,
21709            duration_ms: 0,
21710        }];
21711        round
21712    }
21713
21714    #[tokio::test]
21715    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21716        for review_only in [false, true] {
21717            let mut f = summary_fixture();
21718            if !review_only {
21719                f.runner.state.candidates[0].agent = "alpha".into();
21720            }
21721            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21722            assert!(f.runner.state.reviews[0].clean);
21723            assert!(f.runner.state.reviews[0].fix.is_none());
21724            assert!(f.runner.state.open_findings().is_empty());
21725            assert_eq!(
21726                crate::pr_outcome::required_finding_ids(&f.runner.state),
21727                ["R2-1-1", "R2-1-2"]
21728            );
21729            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21730            let evidence = crate::pr_outcome::summary_evidence(
21731                &f.runner.state,
21732                crate::pr_outcome::FinalHead::At(&head),
21733            );
21734            assert!(
21735                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21736            );
21737            // The ordinary reply omits both IDs: no cached description is adopted.
21738            let omitted = posted(&mut f).await;
21739            assert!(omitted.body.contains("No description"));
21740            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21741            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21742            assert!(f.runner.state.change_summary.is_none());
21743            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21744            let described = posted(&mut f).await;
21745            assert!(described.body.contains("improve documentation for callers"));
21746            assert!(described.body.contains("remaining edge case"));
21747            assert!(described.body.contains("## Open review findings"));
21748            assert!(!described.body.contains("未修正"));
21749        }
21750    }
21751
21752    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21753        let mut f = summary_fixture();
21754        if !review_only {
21755            f.runner.state.candidates[0].agent = "alpha".into();
21756        }
21757        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21758        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21759        fixed
21760            .fix
21761            .as_mut()
21762            .unwrap()
21763            .rejected
21764            .push(crate::verdict::Rejection {
21765                id: "R1-1-1".into(),
21766                why: "呼び出されないコードのため辞退".into(),
21767            });
21768        let mut clean = clean_round_with_minor_findings();
21769        clean.verified_head = Some(head.clone());
21770        f.runner.state.reviews = vec![fixed, clean];
21771        ok_gate(&mut f.runner.state, Some(&head));
21772        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21773        f.runner.state.config.agents.push(scripted("beta", reply));
21774        f.runner.state.config.roles.language_judge =
21775            Some(crate::config::AgentChoice::One("beta".into()));
21776        f
21777    }
21778
21779    #[tokio::test]
21780    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21781        for review_only in [false, true] {
21782            let mut f = rewrite_fixture_with_findings(review_only,
21783                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21784            let m = posted(&mut f).await;
21785            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21786            let expected =
21787                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21788            assert!(m.body.contains(&expected));
21789            assert!(m.body.contains("## Review rounds"));
21790            assert!(
21791                m.body
21792                    .contains("Gate (`verify.gate`): passed on the final commit")
21793            );
21794            assert!(
21795                m.body
21796                    .contains("Tests (`verify.e2e`): passed on the final commit")
21797            );
21798            let c = f.runner.state.change_summary.as_ref().unwrap();
21799            assert!(
21800                m.body
21801                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21802            );
21803            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21804            assert!(crate::pr_outcome::covers_required_findings(
21805                &f.runner.state,
21806                &m.body
21807            ));
21808            assert!(m.body.contains("documentation remains incomplete"));
21809            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21810            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21811            assert!(f.runner.state.github_text.is_none());
21812        }
21813    }
21814
21815    #[tokio::test]
21816    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21817        for review_only in [false, true] {
21818            for reply in [
21819                "TITLE: fix: public description\nA short English description.",
21820                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21821            ] {
21822                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21823                let winner = f.runner.state.candidates[0].clone();
21824                let posted = f
21825                    .runner
21826                    .guarded_pr_message(&winner, None, true)
21827                    .await
21828                    .unwrap();
21829                assert!(posted.is_none());
21830                assert!(f.runner.state.parked);
21831                let withheld = f.runner.state.github_text.as_ref().unwrap();
21832                assert!(
21833                    withheld.rewrite.is_none(),
21834                    "an incomplete rewrite must not be selectable"
21835                );
21836                let q = ask::Questions::open()
21837                    .get(withheld.question.as_ref().unwrap())
21838                    .unwrap();
21839                assert!(
21840                    !q.choices
21841                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21842                );
21843                assert!(
21844                    q.detail.contains("R1-1-1")
21845                        && q.detail.contains("R2-1-1")
21846                        && q.detail.contains("R2-1-2")
21847                );
21848            }
21849        }
21850    }
21851
21852    #[tokio::test]
21853    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21854        for review_only in [false, true] {
21855            let mut f = summary_fixture();
21856            if !review_only {
21857                f.runner.state.candidates[0].agent = "alpha".into();
21858            }
21859            f.runner.state.instruction = "内部タスクの説明".into();
21860            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21861            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21862            f.runner.state.reviews = vec![
21863                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21864                round_with_notes(2, Some("二回目の内部修正報告です。")),
21865                passing_round(3, Some(&head)),
21866            ];
21867            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21868            ok_gate(&mut f.runner.state, Some(&head));
21869            let captured = f.repo.join("public-prompt.txt");
21870            let author = &mut f.runner.state.config.agents[0];
21871            author.command = vec![
21872                "sh".into(),
21873                "-c".into(),
21874                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21875            ];
21876            author.env.insert(
21877                "PROMPT_FILE".into(),
21878                captured.to_string_lossy().into_owned(),
21879            );
21880            let m = posted(&mut f).await;
21881            assert!(m.body.contains("Adds `own.txt`"));
21882            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21883            assert!(m.body.contains("R1-1-1"));
21884            assert!(
21885                m.body
21886                    .contains("Gate (`verify.gate`): passed on the final commit")
21887            );
21888            assert!(
21889                m.body
21890                    .contains("Tests (`verify.e2e`): passed on the final commit")
21891            );
21892            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21893            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21894            let prompt = std::fs::read_to_string(captured).unwrap();
21895            assert!(prompt.contains(&format!(
21896                "git diff {} {head}",
21897                f.runner.state.change_summary.as_ref().unwrap().base
21898            )));
21899            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21900            assert_eq!(
21901                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21902                "二回目の内部修正報告です。"
21903            );
21904        }
21905    }
21906
21907    #[tokio::test]
21908    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21909        for review_only in [false, true] {
21910            let mut f = summary_fixture();
21911            if !review_only {
21912                f.runner.state.candidates[0].agent = "alpha".into();
21913            }
21914            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21915            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21916            std::fs::write(&f.reply, "no usable description").unwrap();
21917            let failed = posted(&mut f).await;
21918            assert!(
21919                failed
21920                    .body
21921                    .contains("No description of what this change does could be generated")
21922            );
21923            assert!(failed.body.is_ascii());
21924            assert!(f.runner.state.github_text.is_none());
21925            assert!(f.runner.state.change_summary.is_none());
21926            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21927            let evidence = crate::pr_outcome::summary_evidence(
21928                &f.runner.state,
21929                crate::pr_outcome::FinalHead::At(&head),
21930            );
21931            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21932            assert!(
21933                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21934                    .exists()
21935            );
21936            f.runner.state =
21937                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21938            std::fs::write(
21939                &f.reply,
21940                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21941            )
21942            .unwrap();
21943            let retried = posted(&mut f).await;
21944            assert!(retried.body.contains("Adds the committed own file."));
21945            assert_eq!(
21946                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21947                "日本語の修正報告"
21948            );
21949        }
21950    }
21951
21952    #[tokio::test]
21953    async fn same_head_verification_and_findings_invalidate_public_prose() {
21954        let mut f = summary_fixture();
21955        posted(&mut f).await;
21956        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21957        let original = f
21958            .runner
21959            .state
21960            .change_summary
21961            .as_ref()
21962            .unwrap()
21963            .evidence
21964            .clone();
21965        ok_gate(&mut f.runner.state, Some(&head));
21966        std::fs::write(
21967            &f.reply,
21968            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21969        )
21970        .unwrap();
21971        let verified = posted(&mut f).await;
21972        assert!(verified.body.contains("now verified"));
21973        assert_ne!(
21974            f.runner.state.change_summary.as_ref().unwrap().evidence,
21975            original
21976        );
21977        let mut round = round_with_notes(1, Some("内部報告"));
21978        round.reviews.push(ReviewRecord {
21979            attempts: 0,
21980            reviewer: 1,
21981            agent: "alpha".into(),
21982            summary: "内部レビュー".into(),
21983            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21984            vote: None,
21985            failed: None,
21986            duration_ms: 0,
21987        });
21988        round
21989            .fix
21990            .as_mut()
21991            .unwrap()
21992            .rejected
21993            .push(crate::verdict::Rejection {
21994                id: "R1-1-1".into(),
21995                why: "呼び出し元から到達できない".into(),
21996            });
21997        f.runner.state.reviews = vec![round];
21998        // A description omitting the required ID is unusable, even with a title.
21999        let omitted = posted(&mut f).await;
22000        assert!(omitted.body.contains("No description"));
22001        assert!(omitted.body.contains("R1-1-1"));
22002        assert!(!omitted.body.contains("到達"));
22003        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22004        let described = posted(&mut f).await;
22005        assert!(described.body.contains("unreachable from callers"));
22006        assert!(described.body.contains("R1-1-1"));
22007        assert!(!described.body.contains("到達"));
22008        let evidence = f
22009            .runner
22010            .state
22011            .change_summary
22012            .as_ref()
22013            .unwrap()
22014            .evidence
22015            .clone();
22016        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22017        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22018        assert!(posted(&mut f).await.body.contains("rationale updated"));
22019        assert_ne!(
22020            f.runner.state.change_summary.as_ref().unwrap().evidence,
22021            evidence
22022        );
22023        // Old serialized summaries have no evidence fingerprint and must regenerate.
22024        f.runner
22025            .state
22026            .change_summary
22027            .as_mut()
22028            .unwrap()
22029            .evidence
22030            .clear();
22031        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22032        assert!(posted(&mut f).await.body.contains("current specification"));
22033    }
22034
22035    #[tokio::test]
22036    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22037        let mut f = summary_fixture();
22038        let m = posted(&mut f).await;
22039        assert_eq!(m.title, "feat(own): add the own file");
22040        assert!(
22041            m.body
22042                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22043            "{}",
22044            m.body
22045        );
22046        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22047        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22048        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22049        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22050        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22051        let kept = f.runner.state.change_summary.as_ref().unwrap();
22052        assert_eq!(
22053            (kept.head.as_str(), kept.base.as_str()),
22054            (head.as_str(), base.as_str())
22055        );
22056        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22057    }
22058
22059    #[tokio::test]
22060    async fn review_only_summary_receives_the_configured_github_language() {
22061        let mut f = summary_fixture();
22062        f.runner.state.config.graph.github_language = "ja".into();
22063        let captured = f.repo.join("summary-prompt.txt");
22064        let author = &mut f.runner.state.config.agents[0];
22065        author.command = vec![
22066            "sh".into(),
22067            "-c".into(),
22068            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22069        ];
22070        author.env.insert(
22071            "PROMPT_FILE".into(),
22072            captured.to_string_lossy().into_owned(),
22073        );
22074        std::fs::write(
22075            &f.reply,
22076            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22077        )
22078        .unwrap();
22079        let m = posted(&mut f).await;
22080        let prompt = std::fs::read_to_string(captured).unwrap();
22081        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22082        assert!(!prompt.contains("in English"));
22083        assert_eq!(m.title, "feat(own): ファイルを追加");
22084        assert!(m.body.contains("own.txt を追加します。"));
22085    }
22086
22087    #[tokio::test]
22088    async fn review_fixes_come_after_the_whole_change_summary() {
22089        let mut f = summary_fixture();
22090        f.runner.state.reviews = vec![
22091            round_with_notes(1, Some("handled the PATH case")),
22092            round_with_notes(2, None),
22093        ];
22094        let m = posted(&mut f).await;
22095        let summary = m.body.find("## Summary").expect("summary");
22096        let fixes = m.body.find("## Review rounds").expect("review rounds");
22097        assert!(summary < fixes, "{}", m.body);
22098        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22099        assert!(m.body.contains("## Final verification"), "{}", m.body);
22100    }
22101
22102    #[tokio::test]
22103    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22104        let mut f = summary_fixture();
22105        let first = posted(&mut f).await;
22106        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22107        // The same head reuses the record: a different reply is not read.
22108        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22109        let again = posted(&mut f).await;
22110        assert_eq!((again.title, again.body), (first.title, first.body));
22111
22112        let g = |args: &[&str]| {
22113            let out = std::process::Command::new("git")
22114                .args(args)
22115                .current_dir(&f.repo)
22116                .quiet()
22117                .output()
22118                .expect("spawn git");
22119            assert!(out.status.success(), "git {args:?}");
22120        };
22121        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22122        g(&["add", "-A"]);
22123        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22124        let second = posted(&mut f).await;
22125        let kept = f.runner.state.change_summary.clone().unwrap();
22126        assert_ne!(kept.head, first_head);
22127        assert!(second.body.contains("Not read yet."), "{}", second.body);
22128        assert!(!second.body.contains("so the first summary is read"));
22129        assert_eq!(second.title, "feat(own): other");
22130    }
22131
22132    #[tokio::test]
22133    async fn an_unavailable_summary_is_stated_and_never_invented() {
22134        let mut f = summary_fixture();
22135        std::fs::write(&f.reply, "no title line here").unwrap();
22136        let m = posted(&mut f).await;
22137        assert!(
22138            m.body
22139                .contains("No description of what this change does could be generated"),
22140            "{}",
22141            m.body
22142        );
22143        assert!(m.body.contains("internal reports remain in local records"));
22144        assert!(f.runner.state.change_summary.is_none());
22145        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
22146        // A preview asks nobody and claims nothing.
22147        let winner = f.runner.state.candidates[0].clone();
22148        let preview = f
22149            .runner
22150            .guarded_pr_message(&winner, None, false)
22151            .await
22152            .unwrap()
22153            .unwrap();
22154        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
22155    }
22156
22157    #[test]
22158    fn pr_message_review_single_commit_uses_its_subject() {
22159        let state = review_state(&["feat(nats): per-role user"]);
22160        let m = pr_message(&state, 'A');
22161        assert_eq!(m.title, "feat(nats): per-role user");
22162        assert!(!m.body.contains("Review the work already"), "{}", m.body);
22163        assert!(m.body.contains("## Commits under review"), "{}", m.body);
22164    }
22165
22166    #[test]
22167    fn pr_message_review_multi_commit_takes_the_oldest() {
22168        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
22169        let m = pr_message(&state, 'A');
22170        assert_eq!(m.title, "feat: the change");
22171        for s in ["feat: the change", "fix: typo", "fix: again"] {
22172            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
22173        }
22174    }
22175
22176    #[test]
22177    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
22178        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
22179            let mut state = review_state(&[first, "fix: later fixup"]);
22180            state.candidates[0].branch = "機能/ログイン".to_owned();
22181            let m = pr_message(&state, 'A');
22182            assert!(
22183                m.title.starts_with("chore: land candidate A of run"),
22184                "{}",
22185                m.title
22186            );
22187        }
22188    }
22189
22190    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22191        BranchFacts {
22192            commits: commits
22193                .iter()
22194                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22195                .collect(),
22196            stat: stat.to_owned(),
22197        }
22198    }
22199
22200    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22201        let mut r = review_round(true, 0, 1, 1, true, true);
22202        r.round = round;
22203        r.fix = notes.map(|n| FixRecord {
22204            agent: "fixer".to_owned(),
22205            addressed: Vec::new(),
22206            rejected: Vec::new(),
22207            notes: n.to_owned(),
22208            committed: true,
22209            failed: None,
22210            duration_ms: 0,
22211            continuation: None,
22212        });
22213        r
22214    }
22215
22216    #[test]
22217    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22218        let state = review_state(&["ignored"]);
22219        let f = facts(
22220            &[
22221                (
22222                    "fix(login): resolve PATH on macOS",
22223                    "Login shells skip rc files.",
22224                ),
22225                ("fix: address review", ""),
22226            ],
22227            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22228        );
22229        let m = pr_message_with(&state, 'A', Some(&f));
22230        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22231        assert!(
22232            m.body
22233                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22234            "{}",
22235            m.body
22236        );
22237        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22238        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22239        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22240        for banned in [
22241            "Review the work already",
22242            "no task statement",
22243            "Original task",
22244        ] {
22245            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22246        }
22247        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22248    }
22249
22250    #[test]
22251    fn pr_message_review_truncates_a_huge_first_commit_body() {
22252        let state = review_state(&["ignored"]);
22253        let f = facts(
22254            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22255            "s",
22256        );
22257        let m = pr_message_with(&state, 'A', Some(&f));
22258        assert!(m.body.len() < 30_000, "{}", m.body.len());
22259        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22260        assert!(
22261            m.body.contains("- ... 1 more commit(s)"),
22262            "{}",
22263            m.body.len()
22264        );
22265        assert!(m.body.ends_with("magi:candidate-a\n"));
22266    }
22267
22268    #[test]
22269    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22270        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22271        assert_eq!(m.title, "feat: x");
22272        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22273        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22274        assert!(!m.body.contains("no task statement"), "{}", m.body);
22275    }
22276
22277    #[test]
22278    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22279        let mut state = review_state(&["日本語の件名"]);
22280        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22281        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22282        state.candidates[0].branch = "機能/ログイン".to_owned();
22283        assert!(
22284            pr_message(&state, 'A')
22285                .title
22286                .starts_with("chore: land candidate A")
22287        );
22288    }
22289
22290    #[test]
22291    fn pr_message_review_fixes_survive_a_clean_final_round() {
22292        let mut state = review_state(&["feat: x"]);
22293        state.reviews = vec![
22294            round_with_notes(1, Some("handled the PATH case")),
22295            round_with_notes(2, None),
22296        ];
22297        let body = pr_message(&state, 'A').body;
22298        assert!(
22299            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22300            "{body}"
22301        );
22302        assert!(body.contains("### Round 1"), "{body}");
22303
22304        state.reviews = vec![
22305            round_with_notes(1, Some("first fix")),
22306            round_with_notes(2, Some("")),
22307            round_with_notes(3, Some("second fix")),
22308            round_with_notes(4, None),
22309        ];
22310        let body = pr_message(&state, 'A').body;
22311        assert!(
22312            body.contains("### Round 1") && !body.contains("first fix"),
22313            "{body}"
22314        );
22315        assert!(
22316            body.contains("### Round 3") && !body.contains("second fix"),
22317            "{body}"
22318        );
22319        assert!(body.contains("### Round 2"), "{body}");
22320    }
22321
22322    const FINAL: &str = "abcdef1234567890";
22323
22324    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22325        let mut r = round_with_notes(round, None);
22326        r.verified_head = verified.map(str::to_owned);
22327        r
22328    }
22329
22330    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22331        state.gate = vec![CommandOutcome {
22332            command: "gate".to_owned(),
22333            code: Some(0),
22334            output_tail: String::new(),
22335            duration_ms: 0,
22336            resource_blocked: false,
22337        }];
22338        state.gate_ran = true;
22339        state.gate_head = head.map(str::to_owned);
22340    }
22341
22342    fn body_at(state: &RunState) -> String {
22343        pr_message_raw(
22344            state,
22345            'A',
22346            None,
22347            crate::pr_outcome::FinalHead::At(FINAL),
22348            ChangeNote::NotAsked,
22349        )
22350        .body
22351    }
22352
22353    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22354
22355    #[test]
22356    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22357        let mut state = review_state(&["feat: x"]);
22358        state.reviews = vec![
22359            round_with_notes(1, Some(SANDBOXED)),
22360            passing_round(2, Some(FINAL)),
22361        ];
22362        ok_gate(&mut state, Some(FINAL));
22363        let body = body_at(&state);
22364        assert!(!body.contains("Handled the PATH case."), "{body}");
22365        assert!(!body.contains("could not commit"), "{body}");
22366        assert!(!body.contains("could not run"), "{body}");
22367        assert!(
22368            body.contains("The fix is committed on the branch"),
22369            "{body}"
22370        );
22371        assert!(!body.contains("sandbox"), "{body}");
22372        assert!(
22373            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22374            "{body}"
22375        );
22376        assert!(
22377            body.contains("Gate (`verify.gate`): passed on the final commit"),
22378            "{body}"
22379        );
22380        // The raw report is untouched in the record.
22381        assert!(
22382            state.reviews[0]
22383                .fix
22384                .as_ref()
22385                .unwrap()
22386                .notes
22387                .contains("could not commit")
22388        );
22389    }
22390
22391    #[test]
22392    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22393        let mut state = review_state(&["feat: x"]);
22394        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22395        r1.fix.as_mut().unwrap().committed = false;
22396        let mut r2 = passing_round(2, Some(FINAL));
22397        r2.e2e[0].code = Some(1);
22398        state.reviews = vec![r1, r2];
22399        let body = body_at(&state);
22400        assert!(!body.contains("could not commit"), "{body}");
22401        assert!(!body.contains("could not run"), "{body}");
22402        assert!(body.contains("This round produced no commit"), "{body}");
22403        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22404        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22405        assert!(!body.contains("passed"), "{body}");
22406    }
22407
22408    #[test]
22409    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22410        let mut state = review_state(&["feat: x"]);
22411        state.reviews = vec![
22412            round_with_notes(1, Some(SANDBOXED)),
22413            passing_round(2, Some("0123456789abcdef")),
22414        ];
22415        ok_gate(&mut state, Some("0123456789abcdef"));
22416        let body = body_at(&state);
22417        assert!(!body.contains("HTTP tests could not run"), "{body}");
22418        // The commit claim is still settled by the structured record.
22419        assert!(!body.contains("could not commit"), "{body}");
22420        assert!(
22421            body.contains("passed on an earlier commit (`0123456`)"),
22422            "{body}"
22423        );
22424        assert!(!body.contains("passed on the final commit"), "{body}");
22425
22426        // A gate recorded before `gate_head` existed is "not recorded".
22427        ok_gate(&mut state, None);
22428        let body = body_at(&state);
22429        assert!(
22430            body.contains("passed on a commit that was not recorded"),
22431            "{body}"
22432        );
22433    }
22434
22435    #[test]
22436    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22437        let state = state_with_summary(
22438            "add retries to the client",
22439            "TITLE: feat: retries\n\nDid it.",
22440        );
22441        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22442        assert_eq!(m.title, "feat: retries");
22443        assert!(
22444            m.body.contains("<summary>Original task</summary>"),
22445            "{}",
22446            m.body
22447        );
22448        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22449        assert!(
22450            m.body
22451                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22452            "{}",
22453            m.body
22454        );
22455    }
22456
22457    #[test]
22458    fn pr_message_review_bounds_a_long_english_subject() {
22459        let long = format!("feat: {}", "word ".repeat(100));
22460        let m = pr_message(&review_state(&[&long]), 'A');
22461        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22462        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22463    }
22464
22465    #[test]
22466    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22467        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22468        let m = pr_message(&state, 'A');
22469        assert_eq!(m.title, "add retries");
22470        assert!(m.body.contains("<summary>Original task</summary>"));
22471        assert!(!m.body.contains("Commits under review"));
22472        assert_eq!(landing_subject_source(&state), state.instruction);
22473    }
22474
22475    #[test]
22476    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22477        let state = review_state(&["feat: the change", "fix: typo"]);
22478        let source = landing_subject_source(&state);
22479        assert_eq!(land::merge_subject("", &source), "feat: the change");
22480        assert_eq!(
22481            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22482            "feat: the change"
22483        );
22484        // An operator's rename still wins.
22485        assert_eq!(
22486            land::merge_subject("feat: renamed by hand", &source),
22487            "feat: renamed by hand"
22488        );
22489        let blank = review_state(&["日本語"]);
22490        assert!(
22491            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22492        );
22493    }
22494
22495    #[test]
22496    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22497        let state = review_state(&["feat: the change"]);
22498        let source = landing_subject_source(&state);
22499        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22500        assert_eq!(
22501            land::merge_subject(landing_title(&state, old), &source),
22502            "feat: the change"
22503        );
22504        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22505        let task = state_with_summary("add retries", "");
22506        assert_eq!(landing_title(&task, old), old);
22507    }
22508
22509    #[test]
22510    fn pr_message_describes_the_change_not_the_task() {
22511        let state = state_with_summary(
22512            "今回やってほしいこと: results projector を直す",
22513            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22514        );
22515        let m = pr_message(&state, 'A');
22516        assert_eq!(m.title, "fix(web): batch the runs list reads");
22517        assert!(
22518            m.body.starts_with("## Summary\n\n- reads run.json once"),
22519            "{}",
22520            m.body
22521        );
22522        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22523        let task_at = m.body.find("今回やってほしいこと").unwrap();
22524        let details_at = m.body.find("<details>").unwrap();
22525        assert!(
22526            details_at < task_at,
22527            "the task lives inside <details>: {}",
22528            m.body
22529        );
22530        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22531        assert!(m.body.contains("magi:candidate-a"));
22532    }
22533
22534    #[test]
22535    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22536        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22537        let m = pr_message(&state, 'A');
22538        assert_eq!(m.title, "add retries");
22539        assert!(
22540            m.body.contains("## Summary\n\n- did some things"),
22541            "{}",
22542            m.body
22543        );
22544
22545        let none = RunState::new(
22546            PathBuf::from("/repo"),
22547            "main".to_owned(),
22548            "abc1234".to_owned(),
22549            "add retries".to_owned(),
22550            Config::default(),
22551        );
22552        let m = pr_message(&none, 'A');
22553        assert_eq!(m.title, "add retries");
22554        assert!(!m.body.contains("## Summary"), "{}", m.body);
22555    }
22556
22557    #[test]
22558    fn pr_message_refuses_the_candidate_commit_subject() {
22559        for bad in [
22560            "TITLE: magi: candidate A (uncommitted work)",
22561            "TITLE: chore: stuff (uncommitted work)",
22562            "TITLE:   ",
22563        ] {
22564            let state = state_with_summary("add retries", bad);
22565            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22566        }
22567    }
22568
22569    #[test]
22570    fn pr_message_bounds_a_very_long_task_and_title() {
22571        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22572        let state = state_with_summary(&long, "- nothing");
22573        let m = pr_message(&state, 'A');
22574        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22575        assert!(!m.title.contains('\n'));
22576
22577        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22578        let m = pr_message(&state, 'A');
22579        assert!(m.title.starts_with("feat: "));
22580        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22581        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22582    }
22583
22584    fn long_title_of(instruction: &str) -> String {
22585        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22586    }
22587
22588    #[test]
22589    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22590        let first = "Make the landing path keep a readable title for long tasks";
22591        let line = format!(
22592            "{first}. {}",
22593            "And then keep going with more words ".repeat(20)
22594        );
22595        let t = long_title_of(&line);
22596        assert_eq!(t, first);
22597        assert!(!t.starts_with("chore: land"));
22598    }
22599
22600    #[test]
22601    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22602        let line = "word ".repeat(200);
22603        let t = long_title_of(&line);
22604        assert!(t.ends_with("word..."), "{t}");
22605        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22606    }
22607
22608    #[test]
22609    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22610        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22611            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22612        }
22613    }
22614
22615    #[test]
22616    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22617        let mut state = state_with_summary("再試行を追加する", "- nothing");
22618        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22619        state.config.graph.github_language = "ja".into();
22620        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22621    }
22622
22623    #[test]
22624    fn pr_message_title_limit_is_exact() {
22625        let at = "a".repeat(PR_TITLE_MAX);
22626        assert_eq!(long_title_of(&at), at);
22627        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22628        assert!(over.ends_with("..."), "{over}");
22629        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22630    }
22631
22632    #[test]
22633    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22634        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22635        let t = long_title_of(&line);
22636        assert!(t.ends_with("..."), "{t}");
22637        assert!(t.is_ascii(), "{t}");
22638    }
22639
22640    #[test]
22641    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22642        let line = format!(
22643            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22644            "plus more filler words ".repeat(20)
22645        );
22646        let t = long_title_of(&line);
22647        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22648    }
22649
22650    #[test]
22651    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22652        let line = format!("\n\n# {}", "title words ".repeat(40));
22653        let state = state_with_summary(&line, "- nothing");
22654        let m = pr_message(&state, 'A');
22655        assert!(m.title.starts_with("title words"), "{}", m.title);
22656        assert_eq!(
22657            land::merge_subject(&m.title, &landing_subject_source(&state)),
22658            m.title
22659        );
22660        // An operator's rename wins untouched.
22661        assert_eq!(
22662            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22663            "feat: renamed by hand"
22664        );
22665    }
22666
22667    #[tokio::test]
22668    async fn github_text_rewrite_is_bounded_and_falls_back() {
22669        crate::run::pin_test_home();
22670        for (reply, accepted) in [
22671            (
22672                "TITLE: fix: retries\nAdd retries for failed requests.",
22673                true,
22674            ),
22675            (
22676                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22677                false,
22678            ),
22679            (
22680                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22681                true,
22682            ),
22683            (
22684                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22685                true,
22686            ),
22687            (
22688                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22689                true,
22690            ),
22691            (
22692                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22693                false,
22694            ),
22695            ("TITLE: fix: retries\n", false),
22696        ] {
22697            let dir = tempfile::tempdir().unwrap();
22698            let mut runner = runner_at(RunStatus::Gating);
22699            runner.state = state_with_summary(
22700                "add retries",
22701                "TITLE: fix: retries\n日本語の説明を書きます。",
22702            );
22703            runner.state.repo = dir.path().to_owned();
22704            runner.state.candidates[0].worktree = dir.path().to_owned();
22705            let mut author = spec("alpha");
22706            author.command = vec![
22707                "sh".into(),
22708                "-c".into(),
22709                "printf '%s' \"$REWRITE_REPLY\"".into(),
22710            ];
22711            author.env.insert("REWRITE_REPLY".into(), reply.into());
22712            runner.state.config.agents = vec![author];
22713            seed_public_summary(&mut runner);
22714            let winner = runner.state.candidates[0].clone();
22715            let first = runner
22716                .guarded_pr_message(&winner, None, true)
22717                .await
22718                .unwrap();
22719            let message = if accepted {
22720                first.expect("a passing rewrite never asks")
22721            } else {
22722                // Withheld: the run parks on a question and posts nothing yet.
22723                assert!(first.is_none());
22724                assert!(runner.state.parked);
22725                let store = ask::Questions::open();
22726                let asked: Vec<_> = store
22727                    .list()
22728                    .into_iter()
22729                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22730                    .collect();
22731                assert_eq!(asked.len(), 1);
22732                assert!(!asked[0].detail.contains("secret"));
22733                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22734                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22735                assert_eq!(
22736                    asked[0]
22737                        .choices
22738                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22739                    rewrite.is_some()
22740                );
22741                // Silence: the question lapses and the neutral text is used,
22742                // without a second rewrite call or a second question.
22743                store
22744                    .update(&asked[0].id, |q| {
22745                        q.abandon("test");
22746                        Ok(())
22747                    })
22748                    .unwrap();
22749                let again = runner
22750                    .guarded_pr_message(&winner, None, true)
22751                    .await
22752                    .unwrap()
22753                    .expect("an abandoned question falls back");
22754                let n = store
22755                    .list()
22756                    .into_iter()
22757                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22758                    .count();
22759                assert_eq!(n, 1, "asked once per rejected text");
22760                again
22761            };
22762            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22763            assert!(!message.body.contains("secret"));
22764            assert!(!message.body.contains("prefix"));
22765            assert!(!message.title.contains("secret"));
22766            assert_eq!(
22767                message.body.contains("Add retries for failed requests."),
22768                accepted
22769            );
22770            assert_eq!(
22771                runner.state.seats["github-text-rewrite"].turns, 1,
22772                "only one rewrite invocation"
22773            );
22774            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22775            if accepted {
22776                assert!(runner.state.github_text.is_none());
22777                assert!(!runner.state.parked);
22778                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22779                assert!(message.body.contains("## Final verification"));
22780                assert!(message.body.contains("Evidence: final diff"));
22781                assert!(
22782                    message
22783                        .body
22784                        .contains(&format!("magi:run/{}", runner.state.id))
22785                );
22786            }
22787            if !accepted {
22788                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22789                assert!(
22790                    message
22791                        .body
22792                        .contains(&format!("magi:run/{}", runner.state.id))
22793                );
22794            }
22795        }
22796    }
22797
22798    fn scripted(id: &str, reply: &str) -> AgentSpec {
22799        let mut a = spec(id);
22800        a.command = vec![
22801            "sh".into(),
22802            "-c".into(),
22803            "printf '%s' \"$REWRITE_REPLY\"".into(),
22804        ];
22805        a.env.insert("REWRITE_REPLY".into(), reply.into());
22806        a
22807    }
22808
22809    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22810    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22811
22812    /// Gate tests inject generated public output, not internal reports.
22813    fn seed_public_summary(runner: &mut Runner) {
22814        let repo = &runner.state.repo;
22815        let g = |args: &[&str]| {
22816            let out = std::process::Command::new("git")
22817                .args(args)
22818                .current_dir(repo)
22819                .quiet()
22820                .output()
22821                .unwrap();
22822            assert!(out.status.success(), "git {args:?}");
22823            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22824        };
22825        g(&["init", "-q", "-b", "main"]);
22826        g(&[
22827            "-c",
22828            "user.name=Test",
22829            "-c",
22830            "user.email=test@example.invalid",
22831            "commit",
22832            "-q",
22833            "--allow-empty",
22834            "-m",
22835            "feat: fixture",
22836        ]);
22837        let head = g(&["rev-parse", "HEAD"]);
22838        g(&["update-ref", "refs/remotes/origin/main", &head]);
22839        let evidence = crate::pr_outcome::summary_evidence(
22840            &runner.state,
22841            crate::pr_outcome::FinalHead::At(&head),
22842        );
22843        let generated = &runner.state.candidates[0].summary;
22844        runner.state.change_summary = Some(crate::run::ChangeSummary {
22845            base: head.clone(),
22846            head,
22847            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22848            title: summary_title(generated).unwrap(),
22849            body: summary_without_title(generated),
22850        });
22851    }
22852
22853    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22854        let mut runner = runner_at(RunStatus::Gating);
22855        runner.state = state_with_summary("add retries", JA_SUMMARY);
22856        if review_only {
22857            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22858        }
22859        runner.state.repo = dir.to_owned();
22860        runner.state.candidates[0].worktree = dir.to_owned();
22861        runner.state.config.agents = agents;
22862        seed_public_summary(&mut runner);
22863        runner
22864    }
22865
22866    fn gate_questions(runner: &Runner) -> usize {
22867        ask::Questions::open()
22868            .list()
22869            .into_iter()
22870            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22871            .count()
22872    }
22873
22874    #[tokio::test]
22875    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22876        crate::run::pin_test_home();
22877        let dir = tempfile::tempdir().unwrap();
22878        let mut runner = rewrite_runner(
22879            dir.path(),
22880            true,
22881            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22882        );
22883        runner.state.config.roles.language_judge =
22884            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22885        let winner = runner.state.candidates[0].clone();
22886        let msg = runner
22887            .guarded_pr_message(&winner, None, true)
22888            .await
22889            .unwrap()
22890            .expect("an English rewrite posts without asking");
22891        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22892        assert!(msg.body.contains("<details>kept</details>"));
22893        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22894        assert_eq!(gate_questions(&runner), 0);
22895        assert!(runner.state.github_text.is_none());
22896        assert!(!runner.state.seats.contains_key("impl-A"));
22897        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22898    }
22899
22900    #[tokio::test]
22901    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22902        crate::run::pin_test_home();
22903        let dir = tempfile::tempdir().unwrap();
22904        let mut runner = rewrite_runner(
22905            dir.path(),
22906            false,
22907            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22908        );
22909        runner.state.config.roles.language_judge =
22910            Some(crate::config::AgentChoice::One("beta".into()));
22911        runner.state.seats.insert(
22912            "impl-A".into(),
22913            SeatState::new("impl-A", "alpha", runner.state.seed),
22914        );
22915        let winner = runner.state.candidates[0].clone();
22916        let msg = runner
22917            .guarded_pr_message(&winner, None, true)
22918            .await
22919            .unwrap()
22920            .expect("rewritten");
22921        assert!(msg.body.contains("<details>kept</details>"));
22922        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22923        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22924        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22925    }
22926
22927    #[tokio::test]
22928    async fn rewrite_reuses_a_resumable_implementer_session() {
22929        crate::run::pin_test_home();
22930        let dir = tempfile::tempdir().unwrap();
22931        let mut runner = rewrite_runner(
22932            dir.path(),
22933            false,
22934            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22935        );
22936        runner.state.config.roles.language_judge =
22937            Some(crate::config::AgentChoice::One("beta".into()));
22938        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22939        seat.turns = 1;
22940        runner.state.seats.insert("impl-A".into(), seat);
22941        let winner = runner.state.candidates[0].clone();
22942        runner
22943            .guarded_pr_message(&winner, None, true)
22944            .await
22945            .unwrap()
22946            .expect("rewritten");
22947        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22948        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22949    }
22950
22951    #[tokio::test]
22952    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22953        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22954        // session, was out of quota, and nothing else was asked.
22955        crate::run::pin_test_home();
22956        let dir = tempfile::tempdir().unwrap();
22957        let mut exhausted = spec("alpha");
22958        exhausted.command = vec!["false".into()];
22959        let mut runner = rewrite_runner(
22960            dir.path(),
22961            false,
22962            vec![exhausted, scripted("beta", EN_REWRITE)],
22963        );
22964        // The implementer's own agent is in the role too: it is not asked twice.
22965        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22966            "alpha".into(),
22967            "beta".into(),
22968        ]));
22969        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22970        seat.turns = 1;
22971        runner.state.seats.insert("impl-A".into(), seat);
22972        let winner = runner.state.candidates[0].clone();
22973        let msg = runner
22974            .guarded_pr_message(&winner, None, true)
22975            .await
22976            .unwrap()
22977            .expect("the role rewrites after the implementer fails");
22978        assert!(msg.body.contains("`src/net.rs`"));
22979        assert_eq!(
22980            runner.state.seats["impl-A"].turns, 2,
22981            "the implementer was tried once"
22982        );
22983        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22984        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22985    }
22986
22987    #[tokio::test]
22988    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22989        crate::run::pin_test_home();
22990        let dir = tempfile::tempdir().unwrap();
22991        let mut broken = spec("bad");
22992        broken.command = vec!["false".into()];
22993        let mut runner = rewrite_runner(
22994            dir.path(),
22995            true,
22996            vec![
22997                broken,
22998                scripted("noformat", "Add retries."),
22999                scripted("good", EN_REWRITE),
23000            ],
23001        );
23002        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23003            "bad".into(),
23004            "noformat".into(),
23005            "good".into(),
23006        ]));
23007        let winner = runner.state.candidates[0].clone();
23008        let msg = runner
23009            .guarded_pr_message(&winner, None, true)
23010            .await
23011            .unwrap()
23012            .expect("the third agent rewrites");
23013        assert!(msg.body.contains("`src/net.rs`"));
23014        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23015        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23016    }
23017
23018    #[tokio::test]
23019    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23020        crate::run::pin_test_home();
23021        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23022        for (agents, role, saved) in [
23023            (
23024                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23025                Some("beta"),
23026                true,
23027            ),
23028            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23029            (Vec::new(), None, false),
23030        ] {
23031            let dir = tempfile::tempdir().unwrap();
23032            let mut runner = rewrite_runner(dir.path(), true, agents);
23033            runner.state.config.roles.language_judge =
23034                role.map(|r| crate::config::AgentChoice::One(r.into()));
23035            let winner = runner.state.candidates[0].clone();
23036            let first = runner
23037                .guarded_pr_message(&winner, None, true)
23038                .await
23039                .unwrap();
23040            assert!(first.is_none(), "withheld text parks the run");
23041            assert!(runner.state.parked);
23042            assert_eq!(gate_questions(&runner), 1);
23043            let g = runner.state.github_text.as_ref().unwrap();
23044            assert_eq!(g.rewrite.is_some(), saved);
23045            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23046        }
23047    }
23048
23049    #[tokio::test]
23050    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23051        crate::run::pin_test_home();
23052        let dir = tempfile::tempdir().unwrap();
23053        let mut runner = runner_at(RunStatus::Gating);
23054        runner.state =
23055            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23056        runner.state.repo = dir.path().to_owned();
23057        runner.state.candidates[0].worktree = dir.path().to_owned();
23058        let mut author = spec("alpha");
23059        author.command = vec![
23060            "sh".into(),
23061            "-c".into(),
23062            "printf '%s' \"$REWRITE_REPLY\"".into(),
23063        ];
23064        author.env.insert(
23065            "REWRITE_REPLY".into(),
23066            format!(
23067                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23068                "English sentence. ".repeat(600)
23069            ),
23070        );
23071        runner.state.config.agents = vec![author];
23072        seed_public_summary(&mut runner);
23073        let winner = runner.state.candidates[0].clone();
23074        assert!(
23075            runner
23076                .guarded_pr_message(&winner, None, true)
23077                .await
23078                .unwrap()
23079                .is_none()
23080        );
23081        let record = runner.state.github_text.clone().unwrap();
23082        let rewrite = record.rewrite.clone().unwrap();
23083        assert_eq!(rewrite.failed, ["body-language"]);
23084        assert!(
23085            rewrite
23086                .body
23087                .ends_with(&format!("magi:run/{}", runner.state.id))
23088        );
23089        let store = ask::Questions::open();
23090        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23091        assert!(
23092            q.choices
23093                .contains(&crate::github_text::USE_REWRITE.to_owned())
23094        );
23095        for label in [
23096            "ORIGINAL",
23097            "FALLBACK",
23098            "REWRITE",
23099            "Rules still failed: body-language",
23100            "truncated",
23101        ] {
23102            assert!(q.detail.contains(label), "{label}");
23103        }
23104        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23105            &runner.state,
23106            &crate::github_text::artifact_name(&record.fingerprint),
23107        ))
23108        .unwrap();
23109        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23110        // Restart from serialized state, then retry after a bad owner title.
23111        runner.state =
23112            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23113        store
23114            .update(&q.id, |q| {
23115                q.say("まだ日本語")?;
23116                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23117            })
23118            .unwrap();
23119        assert!(
23120            runner
23121                .guarded_pr_message(&winner, None, true)
23122                .await
23123                .unwrap()
23124                .is_none()
23125        );
23126        let retry = runner.state.github_text.as_ref().unwrap();
23127        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
23128        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
23129        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
23130        assert!(
23131            q.posting_comparison
23132                .as_ref()
23133                .unwrap()
23134                .matches_detail(&q.detail)
23135        );
23136        store
23137            .update(&q.id, |q| {
23138                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23139            })
23140            .unwrap();
23141        let message = runner
23142            .guarded_pr_message(&winner, None, true)
23143            .await
23144            .unwrap()
23145            .unwrap();
23146        assert_eq!(message.title, rewrite.title);
23147        assert_eq!(message.body, rewrite.body);
23148        assert_eq!(
23149            runner.state.github_text.as_ref().unwrap().choice,
23150            Some(GithubTextChoice::Rewrite)
23151        );
23152        let again = runner
23153            .guarded_pr_message(&winner, None, true)
23154            .await
23155            .unwrap()
23156            .unwrap();
23157        assert_eq!(again.title, message.title);
23158        assert_eq!(again.body, message.body);
23159        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23160        assert!(
23161            !runner
23162                .state
23163                .events
23164                .iter()
23165                .any(|e| e.message.contains(&rewrite.title)
23166                    || e.message.contains("English sentence."))
23167        );
23168    }
23169
23170    #[tokio::test]
23171    async fn historical_posting_reply_never_expires_into_fallback() {
23172        crate::run::pin_test_home();
23173        let dir = tempfile::tempdir().unwrap();
23174        let mut runner = runner_at(RunStatus::Gating);
23175        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23176        runner.state.repo = dir.path().to_owned();
23177        runner.state.candidates[0].worktree = dir.path().to_owned();
23178        seed_public_summary(&mut runner);
23179        let winner = runner.state.candidates[0].clone();
23180        assert!(
23181            runner
23182                .guarded_pr_message(&winner, None, true)
23183                .await
23184                .unwrap()
23185                .is_none()
23186        );
23187        let id = runner
23188            .state
23189            .github_text
23190            .as_ref()
23191            .unwrap()
23192            .question
23193            .clone()
23194            .unwrap();
23195        let store = ask::Questions::open();
23196        let mut q = store.get(&id).unwrap();
23197        q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
23198        q.answer(ask::Answer::Choice(
23199            crate::github_text::POST_ORIGINAL.into(),
23200        ))
23201        .unwrap();
23202        q.answer_delivered = true;
23203        store.put(&mut q).unwrap();
23204        let source = std::fs::read(store.path_of(&id)).unwrap();
23205        let migration = crate::run::home().join("notifications/task-chat-migration-v1");
23206        std::fs::create_dir_all(&migration).unwrap();
23207        std::fs::write(
23208            migration.join("state.json"),
23209            serde_json::to_vec(&serde_json::json!({
23210                "schema": 1, "sources": [id], "pending": [id]
23211            }))
23212            .unwrap(),
23213        )
23214        .unwrap();
23215        for _ in 0..2 {
23216            assert!(
23217                runner
23218                    .guarded_pr_message(&winner, None, true)
23219                    .await
23220                    .unwrap()
23221                    .is_none()
23222            );
23223            assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23224            assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23225        }
23226        let home = crate::run::home();
23227        crate::task_chat::sweep(
23228            &crate::queue::Queue::at(home.join("queue")),
23229            &crate::talk::Talks::at(home.join("talks")),
23230            &store,
23231            &home,
23232        );
23233        let outbox = crate::task_inquiry::Outbox::at(&home);
23234        outbox
23235            .update_decision(&id, |d| {
23236                d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
23237                d.source_answer_at = q.answered_at;
23238                Ok(true)
23239            })
23240            .unwrap();
23241        outbox
23242            .settle(
23243                &id,
23244                "owner-chat",
23245                "Owner confirms no publication remains necessary",
23246                None,
23247            )
23248            .unwrap();
23249        assert!(
23250            runner
23251                .guarded_pr_message(&winner, None, true)
23252                .await
23253                .unwrap()
23254                .is_none()
23255        );
23256        assert!(!runner.state.github_text.as_ref().unwrap().resolved);
23257        assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
23258    }
23259
23260    #[tokio::test]
23261    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
23262        crate::run::pin_test_home();
23263        let dir = tempfile::tempdir().unwrap();
23264        let mut runner = runner_at(RunStatus::Gating);
23265        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
23266        runner.state.repo = dir.path().to_owned();
23267        runner.state.candidates[0].worktree = dir.path().to_owned();
23268        seed_public_summary(&mut runner);
23269        let winner = runner.state.candidates[0].clone();
23270        assert!(
23271            runner
23272                .guarded_pr_message(&winner, None, true)
23273                .await
23274                .unwrap()
23275                .is_none()
23276        );
23277        let record = runner.state.github_text.as_ref().unwrap();
23278        assert!(record.rewrite.is_none());
23279        // Records written before the additive field deserialize with no rewrite.
23280        let mut old = serde_json::to_value(record).unwrap();
23281        old.as_object_mut().unwrap().remove("rewrite");
23282        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23283        assert!(old.rewrite.is_none());
23284        let id = record.question.clone().unwrap();
23285        ask::Questions::open()
23286            .update(&id, |q| {
23287                q.choices.push(crate::github_text::USE_REWRITE.into());
23288                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23289            })
23290            .unwrap();
23291        let message = runner
23292            .guarded_pr_message(&winner, None, true)
23293            .await
23294            .unwrap()
23295            .unwrap();
23296        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23297        assert_eq!(
23298            runner.state.github_text.as_ref().unwrap().choice,
23299            Some(GithubTextChoice::Fallback)
23300        );
23301    }
23302
23303    #[tokio::test]
23304    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23305        crate::run::pin_test_home();
23306        let dir = tempfile::tempdir().unwrap();
23307        let mut runner = runner_at(RunStatus::Gating);
23308        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23309        runner.state.repo = dir.path().to_owned();
23310        runner.state.candidates[0].worktree = dir.path().to_owned();
23311        seed_public_summary(&mut runner);
23312        let winner = runner.state.candidates[0].clone();
23313        assert!(
23314            runner
23315                .guarded_pr_message(&winner, None, true)
23316                .await
23317                .unwrap()
23318                .is_none()
23319        );
23320        let store = ask::Questions::open();
23321        let find = |run: &str| {
23322            store
23323                .list()
23324                .into_iter()
23325                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23326                .unwrap()
23327        };
23328        let run = runner.state.id.clone();
23329        let q = find(&run);
23330        let fp = runner
23331            .state
23332            .github_text
23333            .as_ref()
23334            .unwrap()
23335            .fingerprint
23336            .clone();
23337        let name = crate::github_text::artifact_name(&fp);
23338        let path = crate::run::artifact_path(&runner.state, &name);
23339        assert!(q.detail.contains(&path.display().to_string()));
23340        assert!(q.detail.contains("title-language: title"));
23341        let saved = std::fs::read_to_string(&path).expect("artifact written");
23342        assert!(saved.contains("日本語のタイトル"));
23343        assert!(q.cwd.is_none());
23344        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23345        // A bad replacement is refused and asked about once more.
23346        let say = |id: &str, text: &str, choice: &str| {
23347            store
23348                .update(id, |q| {
23349                    q.thread.push(ask::Turn {
23350                        who: ask::Who::Operator,
23351                        body: text.to_owned(),
23352                        at: jiff::Timestamp::now(),
23353                        note: None,
23354                    });
23355                    q.answer(ask::Answer::Choice(choice.to_owned()))
23356                })
23357                .unwrap();
23358        };
23359        say(&q.id, "まだ日本語", "use my text");
23360        assert!(
23361            runner
23362                .guarded_pr_message(&winner, None, true)
23363                .await
23364                .unwrap()
23365                .is_none()
23366        );
23367        let second = find(&run);
23368        assert_ne!(second.id, q.id);
23369        assert!(second.detail.contains("did not pass"));
23370        // The retry still compares the original text; the rejected
23371        // replacement is named by category only.
23372        assert!(second.detail.contains("日本語のタイトル"));
23373        assert!(!second.detail.contains("まだ日本語"));
23374        assert!(second.detail.contains(&path.display().to_string()));
23375        say(&second.id, "fix: retry failed requests", "use my text");
23376        let message = runner
23377            .guarded_pr_message(&winner, None, true)
23378            .await
23379            .unwrap()
23380            .expect("a vetted replacement is posted");
23381        assert_eq!(message.title, "fix: retry failed requests");
23382        assert_eq!(
23383            runner
23384                .state
23385                .github_text
23386                .as_ref()
23387                .unwrap()
23388                .chosen_title
23389                .as_deref(),
23390            Some("fix: retry failed requests")
23391        );
23392        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23393        // Resumed again: the saved decision is applied, nothing is re-asked.
23394        let again = runner
23395            .guarded_pr_message(&winner, None, true)
23396            .await
23397            .unwrap()
23398            .unwrap();
23399        assert_eq!(again.title, message.title);
23400        assert_eq!(
23401            store
23402                .list()
23403                .into_iter()
23404                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23405                .count(),
23406            2
23407        );
23408    }
23409
23410    #[test]
23411    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23412        // What magi itself writes stays English under any configured language,
23413        // so a future localisation of these headings fails here. (The agents'
23414        // own text is also checked by the posting gate.)
23415        let mut state = state_with_summary(
23416            "add retries",
23417            "TITLE: fix(web): batch reads\n- reads run.json once",
23418        );
23419        state.config.graph.language = "ja".to_owned();
23420        let m = pr_message(&state, 'A');
23421        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23422        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23423
23424        // The task is the operator's own text: it goes in untouched, and the
23425        // fallback title (no summary) may be in its language too.
23426        let task = "今回やってほしいこと: results projector を直す";
23427        let mut state = state_with_summary(task, "- no title line");
23428        state.config.graph.language = "ja".to_owned();
23429        let m = pr_message(&state, 'A');
23430        assert_eq!(
23431            m.title,
23432            format!("chore: land candidate A of run {}", state.id)
23433        );
23434        assert!(
23435            m.body.contains(&format!(
23436                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23437            )),
23438            "{}",
23439            m.body
23440        );
23441    }
23442
23443    #[test]
23444    fn pr_message_scrubs_home_paths_and_addresses() {
23445        let state = state_with_summary(
23446            "fix it in /Users/someone/src/x",
23447            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23448        );
23449        let m = pr_message(&state, 'A');
23450        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23451            assert!(!m.body.contains(leak), "{}", m.body);
23452        }
23453        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23454    }
23455
23456    #[test]
23457    fn pr_message_survives_a_task_that_closes_details() {
23458        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23459        let m = pr_message(&state, 'A');
23460        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23461    }
23462
23463    #[test]
23464    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23465        let cmd = manual_merge_command(
23466            MergeStyle::Squash,
23467            Path::new("/repo"),
23468            "b",
23469            "fix: \"quoted\" $(x) `y`\n\nbody",
23470        );
23471        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23472    }
23473
23474    #[test]
23475    fn manual_merge_command_matches_the_configured_style() {
23476        let repo = Path::new("/repo");
23477        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23478
23479        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23480        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23481
23482        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23483        assert_eq!(
23484            squash,
23485            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23486             \"Merge magi run 0832 (candidate A)\""
23487        );
23488
23489        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23490        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23491    }
23492
23493    #[test]
23494    fn a_nudge_gets_a_quarter_of_the_budget() {
23495        // The judge and implement budgets magi ships with.
23496        assert_eq!(retry_budget(secs(1200), true), secs(300));
23497        assert_eq!(retry_budget(secs(3600), true), secs(900));
23498    }
23499
23500    #[test]
23501    fn a_resent_prompt_keeps_the_whole_budget() {
23502        // The seat kept no context, so the retry is the original job again and
23503        // shortening it would only guarantee a second failure.
23504        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23505        assert_eq!(retry_budget(secs(60), false), secs(60));
23506    }
23507
23508    #[test]
23509    fn the_floor_never_exceeds_the_original_budget() {
23510        // A short configured timeout must not be *raised* by the floor: the
23511        // operator asked for a bound, and a retry may not outlast the attempt
23512        // it is retrying.
23513        assert_eq!(retry_budget(secs(60), true), secs(60));
23514        assert_eq!(retry_budget(secs(480), true), secs(120));
23515        assert_eq!(retry_budget(secs(0), true), secs(0));
23516    }
23517
23518    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23519        agent::CommandEvidence {
23520            id: "item1".to_owned(),
23521            description: "cargo test".to_owned(),
23522            exit_code,
23523            result_summary: String::new(),
23524            source: "codex".to_owned(),
23525        }
23526    }
23527
23528    #[test]
23529    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23530        // No evidence is not the same fact as unconfirmed evidence: a
23531        // backend with no adapter, or a reply that ran no commands at all,
23532        // must not be misread as carrying a dangling job.
23533        assert!(!has_unconfirmed_command(&[]));
23534    }
23535
23536    #[test]
23537    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23538        // Deliberately not a check on the exit code's *value*: a fixer
23539        // legitimately runs something that fails mid-iteration before it
23540        // succeeds, and that must never by itself reopen a valid report.
23541        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23542        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23543        assert!(!has_unconfirmed_command(&[
23544            evidence(Some(0)),
23545            evidence(Some(101))
23546        ]));
23547    }
23548
23549    #[test]
23550    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23551        assert!(has_unconfirmed_command(&[
23552            evidence(Some(0)),
23553            evidence(None)
23554        ]));
23555    }
23556
23557    #[test]
23558    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23559        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23560        assert_eq!(
23561            verified_noop_claim(true, &[], text).as_deref(),
23562            Some("already fixed by b32cfc4, on main.")
23563        );
23564    }
23565
23566    #[test]
23567    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23568        // A timeout or a bad exit code reads as the ordinary loss it is,
23569        // whatever the reply's own prose claims.
23570        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23571        assert!(verified_noop_claim(false, &[], text).is_none());
23572    }
23573
23574    #[test]
23575    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23576        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23577        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23578        // A confirmed command alongside the marker is fine.
23579        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23580    }
23581
23582    #[test]
23583    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23584        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23585    }
23586
23587    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23588    /// pair, in order, labelled A, B, C, ...
23589    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23590        runner.state.candidates = shape
23591            .iter()
23592            .enumerate()
23593            .map(|(i, &(empty, verified))| Candidate {
23594                index: i,
23595                label: (b'A' + i as u8) as char,
23596                agent: "sonnet".to_owned(),
23597                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23598                worktree: PathBuf::from(format!("/wt/{i}")),
23599                summary: String::new(),
23600                stat: String::new(),
23601                files: 0,
23602                commits: 0,
23603                empty,
23604                failed: None,
23605                verified_noop: verified.map(str::to_owned),
23606                duration_ms: 0,
23607                folded: false,
23608            })
23609            .collect();
23610    }
23611
23612    #[test]
23613    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23614        ask_test_home();
23615        let mut runner = runner_at(RunStatus::Implementing);
23616        set_candidates(
23617            &mut runner,
23618            &[
23619                (true, Some("already on main at b32cfc4")),
23620                (true, Some("same fix, see the existing test")),
23621            ],
23622        );
23623
23624        runner
23625            .after_implement()
23626            .expect("a verified no-op is not an error");
23627
23628        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23629    }
23630
23631    #[test]
23632    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23633        ask_test_home();
23634        let mut runner = runner_at(RunStatus::Implementing);
23635        // Candidate A declares a verified no-op; candidate B simply wrote
23636        // nothing and said nothing about why. One candidate's claim is not
23637        // the whole run's agreement.
23638        set_candidates(
23639            &mut runner,
23640            &[(true, Some("already on main at b32cfc4")), (true, None)],
23641        );
23642
23643        let err = runner
23644            .after_implement()
23645            .expect_err("an unverified empty candidate must still fail the run");
23646
23647        assert!(
23648            err.to_string().contains("no candidate produced a change"),
23649            "{err}"
23650        );
23651        assert_eq!(runner.state.status, RunStatus::Failed);
23652    }
23653
23654    #[test]
23655    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23656        ask_test_home();
23657        let mut runner = runner_at(RunStatus::Implementing);
23658        set_candidates(&mut runner, &[(true, None), (true, None)]);
23659
23660        let err = runner
23661            .after_implement()
23662            .expect_err("no candidate declared anything; this is an ordinary failure");
23663
23664        assert!(
23665            err.to_string().contains("no candidate produced a change"),
23666            "{err}"
23667        );
23668        assert_eq!(runner.state.status, RunStatus::Failed);
23669    }
23670}
23671
23672/// A graft rollback whose restore could not be confirmed (see
23673/// [`Runner::graft`]).
23674#[derive(Debug)]
23675struct GraftRestoreFailed(String);
23676
23677impl std::fmt::Display for GraftRestoreFailed {
23678    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23679        write!(
23680            f,
23681            "could not restore the winner to its pre-graft commit: {}",
23682            self.0
23683        )
23684    }
23685}
23686
23687impl std::error::Error for GraftRestoreFailed {}
23688
23689/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23690/// not the agent: they never draw on an agent's own timeout, and running out of
23691/// them never reads as an agent failure.
23692#[cfg(test)]
23693mod resource_wait_tests {
23694    use super::*;
23695    use std::collections::BTreeMap;
23696
23697    fn sh(id: &str, script: &str) -> AgentSpec {
23698        AgentSpec {
23699            id: id.to_owned(),
23700            kind: crate::config::AgentKind::Command,
23701            model: None,
23702            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23703            extra_args: Vec::new(),
23704            env: BTreeMap::new(),
23705            prompt_delivery: None,
23706            max_turn_secs: None,
23707        }
23708    }
23709
23710    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23711        SeatJob {
23712            seat: SeatState::new("impl-A", &spec.id, 1),
23713            spec,
23714            cwd: dir.to_path_buf(),
23715            prompt: "do it".to_owned(),
23716            timeout,
23717            allow_write: true,
23718            sessions: false,
23719            artifacts: dir.join("art"),
23720            stem: "impl-A".to_owned(),
23721            handover: None,
23722        }
23723    }
23724
23725    fn state(resource_wait_secs: u64) -> RunState {
23726        let mut config = Config::default();
23727        config.graph.timeout_resource_wait = resource_wait_secs;
23728        RunState::new(
23729            PathBuf::from("/repo"),
23730            "main".to_owned(),
23731            "abc1234".to_owned(),
23732            "task".to_owned(),
23733            config,
23734        )
23735    }
23736
23737    fn hold(cache: &Path) -> crate::cache::Guard {
23738        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23739        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23740            crate::cache::AcquireOutcome::Acquired(g) => g,
23741            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23742        }
23743    }
23744
23745    async fn run(
23746        jobs: Vec<SeatJob>,
23747        sem: Arc<Semaphore>,
23748        cache: Option<&Path>,
23749        st: &mut RunState,
23750    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23751        let prompts = Prompts::default();
23752        let ctx = WaveCtx {
23753            run: "r",
23754            node: "implement",
23755            prompts: &prompts,
23756            cache,
23757            round: None,
23758            carry_seats: false,
23759            pauses: None,
23760        };
23761        wave(jobs, sem, &ctx, st, 0).await
23762    }
23763
23764    #[tokio::test]
23765    async fn verification_cache_wait_preserves_the_command_budget() {
23766        crate::run::pin_test_home();
23767        let tmp = tempfile::tempdir().unwrap();
23768        let cache = tmp.path().join("target");
23769        let guard = hold(&cache);
23770        let release = tokio::spawn(async move {
23771            tokio::time::sleep(Duration::from_millis(100)).await;
23772            guard.release();
23773        });
23774        let mut st = state(10);
23775        let budget = Duration::from_millis(50);
23776        let wait = repository_wait(&st, None);
23777        let (outcomes, _) = with_cache_lease(
23778            &mut st,
23779            Some(&cache),
23780            "gate",
23781            "gate",
23782            tmp.path(),
23783            "abc1234",
23784            budget,
23785            "test gate",
23786            &wait,
23787            |_, available| async move {
23788                assert_eq!(
23789                    available, budget,
23790                    "contention must not shorten verification"
23791                );
23792                (Vec::new(), false, Vec::new())
23793            },
23794        )
23795        .await
23796        .unwrap();
23797        release.await.unwrap();
23798        assert!(outcomes.is_empty(), "{outcomes:?}");
23799        assert!(
23800            st.events
23801                .iter()
23802                .any(|event| event.message.contains("waiting for"))
23803        );
23804        assert!(st.resource_wait.is_none());
23805        // The verification body no longer owns the lease after returning.
23806        hold(&cache).release();
23807    }
23808
23809    #[tokio::test]
23810    async fn verification_cache_wait_expires_without_starting_commands() {
23811        crate::run::pin_test_home();
23812        let tmp = tempfile::tempdir().unwrap();
23813        let cache = tmp.path().join("target");
23814        let _held = hold(&cache);
23815        let mut st = state(0);
23816        let wait = repository_wait(&st, None);
23817        let (outcomes, _) = with_cache_lease(
23818            &mut st,
23819            Some(&cache),
23820            "gate",
23821            "gate",
23822            tmp.path(),
23823            "abc1234",
23824            Duration::from_secs(60),
23825            "test gate",
23826            &wait,
23827            |_, _| async { panic!("commands must not start without the cache lease") },
23828        )
23829        .await
23830        .unwrap();
23831        assert_eq!(outcomes.len(), 1);
23832        assert!(outcomes[0].resource_blocked);
23833        assert!(st.resource_wait.is_none());
23834        assert!(matches!(
23835            st.resource_block.unwrap().cause,
23836            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23837        ));
23838    }
23839
23840    #[tokio::test]
23841    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23842        crate::run::pin_test_home();
23843        let tmp = tempfile::tempdir().unwrap();
23844        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23845        let cache = tmp.path().join("cache");
23846        let mut state = state(21600);
23847        let wait = repository_wait(&state, None);
23848        let (outcomes, _) = with_cache_lease(
23849            &mut state,
23850            Some(&cache),
23851            "gate",
23852            "gate",
23853            tmp.path(),
23854            "head",
23855            Duration::from_secs(60),
23856            "gate",
23857            &wait,
23858            |_, _| async { panic!("freshness failure must not start verification") },
23859        )
23860        .await
23861        .unwrap();
23862        assert_eq!(outcomes.len(), 1);
23863        assert!(outcomes[0].resource_blocked);
23864        let block = state.resource_block.unwrap();
23865        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23866        assert!(!block.description().contains("timed out"));
23867    }
23868
23869    #[tokio::test]
23870    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23871        crate::run::pin_test_home();
23872        let tmp = tempfile::tempdir().unwrap();
23873        let cache = tmp.path().join("cache");
23874        let _held = hold(&cache);
23875        let mut state = state(21600);
23876        let pause = Pause::new();
23877        let interrupt = Pause::new();
23878        let cause = ParkCause::Operator;
23879        pause.park_for(cause.clone());
23880        let prompts = Prompts::default();
23881        let ctx = WaveCtx {
23882            run: "test",
23883            node: "fix",
23884            prompts: &prompts,
23885            cache: Some(&cache),
23886            round: None,
23887            carry_seats: false,
23888            pauses: Some((&pause, &interrupt)),
23889        };
23890        let result = wave(
23891            vec![job(
23892                sh("writer", "exit 99"),
23893                tmp.path(),
23894                Duration::from_secs(1),
23895            )],
23896            Arc::new(Semaphore::new(1)),
23897            &ctx,
23898            &mut state,
23899            0,
23900        )
23901        .await;
23902        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23903        assert_eq!(result[0].1.turns, 0);
23904        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23905        assert_eq!(
23906            state.resource_block.unwrap().cause,
23907            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23908        );
23909        assert!(state.active.is_empty());
23910    }
23911
23912    #[tokio::test]
23913    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23914        crate::run::pin_test_home();
23915        let tmp = tempfile::tempdir().unwrap();
23916        let cache = tmp.path().join("target-a");
23917        let guard = hold(&cache);
23918        // Held for longer than the agent's whole 1s budget.
23919        tokio::spawn(async move {
23920            tokio::time::sleep(Duration::from_millis(1500)).await;
23921            guard.release();
23922        });
23923        let mut st = state(60);
23924        let results = run(
23925            vec![job(
23926                sh("alpha", "echo ok"),
23927                tmp.path(),
23928                Duration::from_secs(1),
23929            )],
23930            Arc::new(Semaphore::new(1)),
23931            Some(&cache),
23932            &mut st,
23933        )
23934        .await;
23935        assert!(
23936            matches!(results[0].2, AgentOutcome::Ok(_)),
23937            "the wait must not be charged to the agent"
23938        );
23939        assert!(st.resource_block.is_none());
23940        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23941        assert!(st.resource_wait.is_none() && st.active.is_empty());
23942    }
23943
23944    #[tokio::test]
23945    #[cfg(unix)]
23946    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23947        use std::os::fd::AsRawFd as _;
23948        crate::run::pin_test_home();
23949        let tmp = tempfile::tempdir().unwrap();
23950        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23951        let common = tmp.path().join(".git");
23952        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23953        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23954        assert_eq!(
23955            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23956            0
23957        );
23958        let mut st = state(1);
23959        st.config.coordination = crate::config::test_coordination();
23960        let mut reader = job(
23961            sh("reader", "echo must-not-run"),
23962            tmp.path(),
23963            Duration::from_secs(30),
23964        );
23965        reader.allow_write = false;
23966        let results = run(
23967            vec![reader.clone()],
23968            Arc::new(Semaphore::new(1)),
23969            None,
23970            &mut st,
23971        )
23972        .await;
23973        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23974        assert_eq!(results[0].1.turns, 0);
23975        assert!(FailClass::of(&results[0].2).is_none());
23976        assert_eq!(
23977            st.resource_block.as_ref().unwrap().kind,
23978            "repository coordination"
23979        );
23980        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23981        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23982        drop(external);
23983        let editor = crate::repo_lock::SeatLease::for_checkout(
23984            tmp.path(),
23985            true,
23986            crate::config::test_coordination().as_ref(),
23987        )
23988        .await
23989        .unwrap();
23990        let pause = Pause::new();
23991        let interrupt = Pause::new();
23992        let prompts = Prompts::default();
23993        let ctx = WaveCtx {
23994            run: "r",
23995            node: "review",
23996            prompts: &prompts,
23997            cache: None,
23998            round: None,
23999            carry_seats: false,
24000            pauses: Some((&pause, &interrupt)),
24001        };
24002        let (results, ()) = tokio::join!(
24003            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24004            async {
24005                tokio::time::sleep(Duration::from_millis(50)).await;
24006                pause.park();
24007            }
24008        );
24009        assert!(
24010            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24011        );
24012        assert_eq!(results[0].1.turns, 0);
24013        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24014        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24015        drop(editor);
24016    }
24017
24018    #[tokio::test]
24019    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24020        crate::run::pin_test_home();
24021        let tmp = tempfile::tempdir().unwrap();
24022        let cache = tmp.path().join("target-b");
24023        let _held = hold(&cache);
24024        let mut st = state(1);
24025        let results = run(
24026            vec![job(
24027                sh("alpha", "echo ok"),
24028                tmp.path(),
24029                Duration::from_secs(1),
24030            )],
24031            Arc::new(Semaphore::new(1)),
24032            Some(&cache),
24033            &mut st,
24034        )
24035        .await;
24036        let (_, seat, out) = &results[0];
24037        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
24038        assert!(wave_blocked(&results));
24039        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
24040        assert_eq!(seat.turns, 0, "nothing was asked");
24041        assert!(
24042            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
24043            "no agent was spawned"
24044        );
24045        assert!(FailClass::of(out).is_none(), "not a failure class");
24046        let block = st.resource_block.as_ref().expect("recorded for the resume");
24047        assert_eq!(block.kind, "shared build cache");
24048        assert!(matches!(
24049            block.cause,
24050            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
24051        ));
24052        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24053        assert!(st.active.is_empty());
24054    }
24055
24056    #[tokio::test]
24057    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
24058        crate::run::pin_test_home();
24059        let tmp = tempfile::tempdir().unwrap();
24060        let mut st = state(1);
24061        let results = run(
24062            vec![job(
24063                sh("alpha", "echo ok"),
24064                tmp.path(),
24065                Duration::from_secs(1),
24066            )],
24067            Arc::new(Semaphore::new(0)),
24068            None,
24069            &mut st,
24070        )
24071        .await;
24072        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24073        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
24074        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
24075    }
24076
24077    #[tokio::test]
24078    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
24079        crate::run::pin_test_home();
24080        let tmp = tempfile::tempdir().unwrap();
24081        let mut st = state(60);
24082        let results = run(
24083            vec![job(
24084                sh("alpha", "sleep 30"),
24085                tmp.path(),
24086                Duration::from_secs(1),
24087            )],
24088            Arc::new(Semaphore::new(1)),
24089            None,
24090            &mut st,
24091        )
24092        .await;
24093        let out = &results[0].2;
24094        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
24095        let class = FailClass::of(out).expect("a failure");
24096        assert_eq!(class, FailClass::Timeout);
24097        assert!(should_hand_over(None, &class), "existing handover policy");
24098        assert!(st.resource_block.is_none());
24099    }
24100}