Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54    Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58    ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62/// How much verification output is kept and fed back to the fixer.
63const OUTPUT_TAIL: usize = 8_000;
64
65/// Bytes of a failing command's output kept in an event, so the reason a run
66/// stopped is readable from the report without opening `run.json`.
67const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
70/// command's pid before releasing the build cache's lease.
71const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
74/// command's pid to actually exit before giving up and releasing anyway.
75///
76/// A timeout means the process was asked to die (`kill_on_drop`,
77/// `start_kill`), not that it already has — on Windows in particular that can
78/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
79/// the instant the command returns would let the very next acquirer (this
80/// run's own next round, another run's verification, the janitor's prune)
81/// start touching the same directory while it might still be writing to it,
82/// so this polls the actual pid — real confirmation, not a fixed guess —
83/// until it is gone or this ceiling is reached. It is still not full
84/// process-tree reaping: a grandchild the timed-out process spawned and that
85/// outlives it independently is invisible to a pid check, and continuing to
86/// observe and collect *that* stays a different piece of work with its own
87/// owner. Set generously because the common case returns early the moment
88/// the pid is confirmed gone, not because every timeout pays this in full.
89const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91/// Consecutive review rounds with no tree progress (see
92/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
93/// instead of spending the rest of the round budget.
94///
95/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
96/// legitimately finds nothing left to change (its previous round's fix already
97/// covered it, and this round's reviewers re-raised only nits) looks the same
98/// as one that is spinning, for exactly one round. Two in a row is where the
99/// two stop being distinguishable, and a review round on this workload has
100/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
101/// third attempt at a tree that has not moved twice running is pure cost.
102/// This does not touch `review_rounds` itself, which stays the operator's
103/// call.
104pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
107/// a base that moved before giving up and leaving the run `Blocked` for a
108/// person.
109///
110/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
111/// that keeps moving faster than a run can catch it is not something more
112/// rebasing fixes, it is a person's call. Not the same *number as*
113/// `land_rounds` - this budget is spent before a pull request exists, land's
114/// after - but bounded for the identical reason, so it uses the same
115/// default. Counted across both call sites in [`Runner::finish_after_tally`]
116/// (once before review, once before the gate), because either one finding
117/// the base still moving is the same signal.
118const BASE_SYNC_ROUNDS: usize = 4;
119
120/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
121/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
122/// reply held no [`FixReport`].
123///
124/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
125/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
126/// "I'll pause here until the `cargo make check` background run reports
127/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
128/// No `FixReport` was ever collected from that seat, and the run moved on to
129/// the next review round regardless.
130///
131/// Bounded independently of `review_rounds` and `graph.retries`: this
132/// recovers one seat's missing report mid-round, not a new round of review or
133/// an ordinary parse retry, and must not itself become the unbounded wait the
134/// rest of this module exists to avoid.
135const MAX_FIX_CONTINUATIONS: usize = 2;
136
137/// One queued agent invocation.
138///
139/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
140/// CLI hung up on its own stream is asked again from the same job rather than
141/// rebuilt from scratch. See [`Runner::resume_undelivered`].
142#[derive(Clone)]
143struct SeatJob {
144    spec: AgentSpec,
145    seat: SeatState,
146    cwd: PathBuf,
147    prompt: String,
148    timeout: Duration,
149    allow_write: bool,
150    sessions: bool,
151    artifacts: PathBuf,
152    stem: String,
153    /// The prompt for a seat that has been handed to another roster agent
154    /// (a fresh session): everything the original seat would have
155    /// remembered. `None` when `prompt` already carries it, as the first
156    /// ranking and the implement prompt do. Never a resume-style prompt.
157    handover: Option<String>,
158}
159
160/// How the graph reads one agent invocation.
161///
162/// Quota is split out from an ordinary failure on purpose: a rate-limited call
163/// is known to fail again if retried now, so the retry loop must not spend an
164/// attempt on it. `Dropped` is split out for the opposite reason: unlike
165/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
166/// error JSON, never the agent's answer — a caller that matched only
167/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
168/// left to read that JSON as if it were usable output. `resume_undelivered`
169/// is the only caller that acts on it; everywhere else it is reported like an
170/// ordinary failure.
171enum AgentOutcome {
172    /// A usable output.
173    Ok(AgentOutput),
174    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
175    Quota(AgentOutput),
176    /// The CLI hung up on its own stream after billed work. See
177    /// [`agent::AgentOutput::work_undelivered`].
178    Dropped(AgentOutput),
179    /// Any other failure: a timeout, a bad exit code, an empty reply.
180    Failed(String),
181    /// The agent was never spawned: a shared resource it needs (the build
182    /// cache's lease) stayed busy past `[graph] timeout_resource_wait`, or the
183    /// wait was abandoned because the run is parking. Not a failure of the
184    /// agent: it must not advance a fallback chain, enter seat history, cost
185    /// quota or spend an attempt. [`RunState::resource_block`] says why.
186    ResourceBlocked(String),
187}
188
189/// A request to park the run at its next node boundary.
190///
191/// Cloning is how the request travels: the loop keeps one handle and hands a
192/// clone to each [`Runner`], and every clone points at the same flag. There
193/// is no channel because there is nothing to send - the only message is
194/// "park", it is idempotent, and a flag cannot be missed by a receiver that
195/// was not listening yet.
196///
197/// The boundary is what makes this cheap. Every node writes the run's state
198/// before the next one starts, and every node skips what is already recorded:
199/// `prep` returns early once candidates exist, `implement` asks only the seats
200/// with nothing on disk, `judge` returns early once judgements exist. So a
201/// parked run resumes into exactly the node it stopped before, and no agent
202/// work is thrown away. Killing the process mid-node, by contrast, loses
203/// whatever the seats in flight had not yet written - which for an implement
204/// wave is an hour of paid work.
205///
206/// A [`Runner`] watches two independent handles of this type - see
207/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
208/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
209/// clone covering the whole daemon's lifetime and is never asked to un-park,
210/// which is correct exactly because nothing is dispatched after it fires.
211/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
212/// that parks for an interrupted task must go on to run other tasks
213/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
214/// reusing the daemon-wide one.
215#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219    /// A pause nobody has asked for yet.
220    #[must_use]
221    pub fn new() -> Self {
222        Self::default()
223    }
224
225    /// Ask the run to park at its next node boundary. Idempotent.
226    pub fn park(&self) {
227        self.0.store(true, Ordering::SeqCst);
228    }
229
230    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
231    /// fold into the run's own `park` event - so an operator reading the run
232    /// later knows this was a deliberate interrupt rather than a shutdown or
233    /// a binary swap. The first reason recorded wins; a park already in
234    /// flight is not relabelled by a second, unrelated request.
235    pub fn park_because(&self, reason: impl Into<String>) {
236        self.park_for(ParkCause::Interrupt {
237            reason: reason.into(),
238        });
239    }
240
241    /// [`Pause::park`], recording the structured cause. The first cause
242    /// recorded wins, like [`Pause::park_because`].
243    pub fn park_for(&self, cause: ParkCause) {
244        let mut reason_guard = self
245            .1
246            .lock()
247            .unwrap_or_else(std::sync::PoisonError::into_inner);
248        if reason_guard.is_none() {
249            *reason_guard = Some(cause);
250        }
251        drop(reason_guard);
252        self.park();
253    }
254
255    /// Has a park been asked for?
256    #[must_use]
257    pub fn parked(&self) -> bool {
258        self.0.load(Ordering::SeqCst)
259    }
260
261    /// Why the park was asked for, when the caller used [`Pause::park_because`].
262    #[must_use]
263    pub fn reason(&self) -> Option<String> {
264        self.cause().map(|c| c.label())
265    }
266
267    /// The structured cause, when one was recorded.
268    #[must_use]
269    pub fn cause(&self) -> Option<ParkCause> {
270        self.1
271            .lock()
272            .unwrap_or_else(std::sync::PoisonError::into_inner)
273            .clone()
274    }
275}
276
277/// Drives one run.
278pub struct Runner {
279    /// Run state; public so the CLI can report on it.
280    pub state: RunState,
281    roles: ResolvedRoles,
282    sem: Arc<Semaphore>,
283    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
284    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
285    /// this is never the same handle as `interrupt`.
286    pause: Pause,
287    /// Set when `magi serve`'s interrupt scheduler wants this specific run
288    /// parked at its next node boundary, to let a task marked
289    /// [`crate::queue::Task::interrupt`] run alone before this one carries
290    /// on. Unlike `pause`, a fresh, unshared handle per run - see
291    /// [`Runner::watch_interrupt`].
292    interrupt: Pause,
293}
294
295/// Where the branch's own commits start: its merge base with the base branch
296/// as the remote has it now, else with the recorded `base_commit`. A branch
297/// rebased onto a base that moved past `base_commit` would otherwise count
298/// the base's commits as its own under `base_commit..branch`.
299async fn review_base(
300    repo: &Path,
301    remote: &str,
302    base_branch: &str,
303    base_commit: &str,
304    branch: &str,
305) -> String {
306    review_base_checked(repo, remote, base_branch, base_commit, branch)
307        .await
308        .0
309}
310
311/// [`review_base`] plus whether the base was read from a freshly fetched
312/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
313/// never older than `base_commit`'s view of the base), but the answer is then
314/// not trusted to rewrite a pull request's title.
315async fn review_base_checked(
316    repo: &Path,
317    remote: &str,
318    base_branch: &str,
319    base_commit: &str,
320    branch: &str,
321) -> (String, bool) {
322    let tracking = format!("{remote}/{base_branch}");
323    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324    if git::rev_exists(repo, &tracking).await
325        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326        && !mb.is_empty()
327    {
328        return (mb, fresh);
329    }
330    let mb = git::merge_base(repo, base_commit, branch)
331        .await
332        .ok()
333        .filter(|mb| !mb.is_empty())
334        .unwrap_or_else(|| base_commit.to_owned());
335    (mb, false)
336}
337
338/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
339/// when git cannot say or finds nothing: a stale list is better than a wrong
340/// or empty one.
341pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342    if !is_review_run(state) {
343        return;
344    }
345    let base = review_base(
346        &state.repo,
347        &state.config.merge.remote,
348        &state.base_branch,
349        &state.base_commit,
350        branch,
351    )
352    .await;
353    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354        && !subjects.is_empty()
355        && state.reviewed_commits.as_ref() != Some(&subjects)
356    {
357        state.reviewed_commits = Some(subjects);
358        state.save().ok();
359    }
360}
361
362/// Subjects of the base's commits between the recorded start and the branch's
363/// merge base: what a stale `base_commit..branch` would have mistaken for the
364/// branch's own work.
365async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366    let (base, trusted) = review_base_checked(
367        &state.repo,
368        &state.config.merge.remote,
369        &state.base_branch,
370        &state.base_commit,
371        branch,
372    )
373    .await;
374    if !trusted {
375        return None;
376    }
377    git::subjects(&state.repo, &state.base_commit, &base)
378        .await
379        .ok()
380}
381
382/// May an adopted pull request's title be replaced with `computed`? Only when
383/// it is empty, magi's own shape, or a base commit's subject that leaked in;
384/// a title a person wrote stays. Never when `computed` is itself a leak.
385pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387    if is_leak(computed) {
388        return false;
389    }
390    let cur = current.trim();
391    cur.is_empty()
392        || cur.starts_with(REVIEW_PROMPT_OPENING)
393        || cur.starts_with("chore: land candidate")
394        || cur.starts_with("magi: candidate")
395        || is_leak(cur)
396}
397
398/// The commit a run branches from: the base branch as the remote has it.
399///
400/// Two failures this replaces. A run used to branch off `HEAD` and so refused
401/// to start on a dirty tree, which made `magi serve` decline every task for as
402/// long as the operator had work in progress - most of the time. Branching off
403/// the *local* base branch fixed that and introduced a worse one: `land` merges
404/// the winner on GitHub, nothing updates the local ref, and the next run
405/// branches off a base missing everything the previous runs landed. Two tasks
406/// in a row from a phone would have had the second silently re-implementing
407/// against stale code and opening a pull request that reverted the first.
408///
409/// Only refs move here - no checkout, no local branch, no merge - so it is safe
410/// with uncommitted work in the tree. A machine with no network still starts:
411/// the fetch may fail and the local tip is used with a warning, because
412/// refusing to run offline is a worse failure than running against a base the
413/// operator can see for themselves.
414///
415/// One function, called by both entry points. Two answers to "where does a run
416/// branch from" is the kind of drift nobody notices until a diff is wrong.
417/// Bring the local `branch` in line with `<remote>/<branch>` before a review
418/// checks it out.
419///
420/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
421/// anything other than plain `git push` from this checkout (a jj colocated
422/// workspace, another clone) moves only the remote-tracking ref - so the local
423/// one can be a stale placeholder. It moves only when local is behind the remote or is an
424/// empty placeholder that diverged from it; unpushed local work is kept, and a real
425/// divergence is refused rather than guessed at.
426async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427    let tracking = format!("{remote}/{branch}");
428    let fetched = git::fetch(repo, remote, branch).await;
429    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430    let local_exists = git::branch_exists(repo, branch).await?;
431    if !fresh {
432        if !local_exists {
433            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434        }
435        tracing::warn!(
436            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437        );
438        return Ok(());
439    }
440    let remote_sha = git::rev_parse(repo, &tracking).await?;
441    if !local_exists {
442        git::git(repo, &["branch", branch, &tracking]).await?;
443        return Ok(());
444    }
445    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447        return Ok(());
448    }
449    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450        // Diverged. `reconcile` settles it only when it can prove nothing is
451        // lost: a local tip that is the remote's change rebased is pushed over
452        // it (lease pinned to the tip read here), a tip whose every extra
453        // commit is empty is a placeholder the remote's work replaced, and
454        // anything else is two different changes - a question for a person.
455        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456            .await?
457        {
458            crate::reconcile::Reconciliation::Pushed => {
459                tracing::warn!(
460                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461                    short(&local_sha),
462                    short(&remote_sha)
463                );
464                return Ok(());
465            }
466            crate::reconcile::Reconciliation::Placeholder => {}
467            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468        }
469    }
470    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471    if !out.ok() {
472        bail!(
473            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474            short(&local_sha),
475            short(&remote_sha),
476            out.stderr
477        );
478    }
479    tracing::warn!(
480        "local `{branch}` was stale: fast-forwarded {} -> {}",
481        short(&local_sha),
482        short(&remote_sha)
483    );
484    Ok(())
485}
486
487async fn resolve_base(
488    repo: &Path,
489    base_branch: &str,
490    remote: &str,
491    coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493    let _lease = crate::repo_lock::SeatLease::operation_with(
494        repo,
495        coordination,
496        &crate::repo_lock::WaitPolicy::current(),
497    )
498    .await?;
499    let tracking = format!("{remote}/{base_branch}");
500    let fetched = git::fetch(repo, remote, base_branch).await;
501    if let Ok(out) = &fetched
502        && out.ok()
503        && git::rev_exists(repo, &tracking).await
504    {
505        return Ok(crate::run::FetchedBase {
506            remote: remote.to_owned(),
507            branch: base_branch.to_owned(),
508            commit: git::rev_parse(repo, &tracking).await?,
509            fetched_at: jiff::Timestamp::now(),
510        });
511    }
512    let why = match &fetched {
513        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514        Ok(_) => format!("{remote} has no {base_branch}"),
515        Err(e) => e.to_string(),
516    };
517    // No fallback to the local branch: it may be behind, and branching off an
518    // old commit is the stale-checkout bug this check exists to prevent.
519    bail!(
520        "cannot read {tracking} ({why}); refusing to branch off the local \
521         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522         base / remote in magi.toml. This is launcher preparation: check the \
523         launcher's Git administration access, network and repository coordination \
524         before retrying; an unattended seat cannot approve those permissions"
525    )
526}
527
528/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
529/// on an early return or a panic.
530///
531/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
532/// manual `magi fix` invocations against the same run are otherwise
533/// invisible to each other and would race to remove and recreate the same
534/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
535/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
536/// which is only ever reclaimed later, out of band, by
537/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
538/// `magi fix` invocation is not necessarily running under either of those, so
539/// nothing would ever sweep a lock a killed or crashed process left behind.
540/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
541/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
542/// lease use: an unreadable or unparsable pid, or a liveness query the
543/// platform cannot answer, reads as alive and the lock is left in place.
544struct FixClaim {
545    path: PathBuf,
546}
547
548impl FixClaim {
549    fn acquire(dir: &Path) -> Result<Self> {
550        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551        let path = dir.join("fix.lock");
552        match Self::create(&path) {
553            Ok(claim) => Ok(claim),
554            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555                if Self::reclaim_if_dead(&path) {
556                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
557                } else {
558                    bail!(
559                        "another repair or `magi fix` is already running for this run ({} exists)",
560                        path.display()
561                    )
562                }
563            }
564            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565        }
566    }
567
568    fn create(path: &Path) -> std::io::Result<Self> {
569        let mut f = std::fs::OpenOptions::new()
570            .write(true)
571            .create_new(true)
572            .open(path)?;
573        use std::io::Write as _;
574        // Read back by `reclaim_if_dead` on a later, stuck invocation.
575        writeln!(f, "{}", std::process::id())?;
576        Ok(Self {
577            path: path.to_owned(),
578        })
579    }
580
581    /// True if the lock named a process confirmed dead, in which case it was
582    /// also removed. Never true on an unreadable file, an unparsable pid, or
583    /// a liveness query the platform cannot answer — see this type's own doc.
584    fn reclaim_if_dead(path: &Path) -> bool {
585        let dead = std::fs::read_to_string(path)
586            .ok()
587            .and_then(|body| body.trim().parse::<u32>().ok())
588            .is_some_and(|pid| !crate::proc::pid_alive(pid));
589        dead && std::fs::remove_file(path).is_ok()
590    }
591}
592
593impl Drop for FixClaim {
594    fn drop(&mut self) {
595        let _ = std::fs::remove_file(&self.path);
596    }
597}
598
599/// Holds the mutable runner as well as its durable lease: cancellation must
600/// clear both copies of ownership, including state a caller may save later.
601struct DriverExecution<'a> {
602    runner: &'a mut Runner,
603    driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607    fn drop(&mut self) {
608        self.driver.release_local(&mut self.runner.state);
609    }
610}
611
612impl Runner {
613    /// Start a fresh run against `repo`.
614    pub async fn start(
615        repo: &Path,
616        instruction: String,
617        config: Config,
618        origin: Origin,
619    ) -> Result<Self> {
620        Self::start_naming(repo, instruction, "", config, origin).await
621    }
622
623    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
624    /// searched for branch and commit references along with the instruction,
625    /// since a task may name the work it is about only in its title.
626    pub async fn start_naming(
627        repo: &Path,
628        instruction: String,
629        also_scan: &str,
630        config: Config,
631        origin: Origin,
632    ) -> Result<Self> {
633        let repo = git::toplevel(repo).await?;
634        let missing = agent::missing_programs(&config.agents);
635        if !missing.is_empty() {
636            bail!(
637                "these agent programs are not on PATH: {}. Fix the roster in \
638                 magi.toml or install them.",
639                missing.join(", ")
640            );
641        }
642        let base_branch =
643            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644                .await?;
645        let fetched_base = resolve_base(
646            &repo,
647            &base_branch,
648            &config.merge.remote,
649            config.coordination.as_ref(),
650        )
651        .await?;
652        let base_commit = fetched_base.commit.clone();
653        let roles = config.resolve_roles()?;
654        let max_parallel = config.graph.max_parallel.max(1);
655        // A task that points at work already in the repository starts from
656        // it; what the repository says about each reference is recorded.
657        let seeds = refs::resolve(
658            &repo,
659            &base_commit,
660            &config.merge.remote,
661            &format!("{also_scan}\n{instruction}"),
662        )
663        .await;
664        refs::plan(&repo, &seeds).await?;
665        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666        // Recorded before the first save, so a crash right after minting
667        // cannot leave a run with no origin. Legibility only: nothing reads it
668        // to decide anything.
669        state.fetched_base = Some(fetched_base);
670        state.origin = Some(origin);
671        for seed in &seeds {
672            state.event(
673                "seed",
674                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675            );
676        }
677        state.seeds = seeds;
678        state.event("start", format!("run {} created", state.id));
679        state.save()?;
680        Ok(Self {
681            state,
682            roles,
683            sem: Arc::new(Semaphore::new(max_parallel)),
684            pause: Pause::new(),
685            interrupt: Pause::new(),
686        })
687    }
688
689    /// Open a review-only run against work that already exists on `branch`.
690    ///
691    /// The expensive half of the graph is the implement wave — measured at
692    /// 111 and 134 internal tool-loop turns on this repository, against a
693    /// handful for a judge or a reviewer. The cheap half is worth running on
694    /// hand-written work too, and there was no way to reach it.
695    ///
696    /// No new state and no schema change are needed: a run with **one** viable
697    /// candidate and a tally already decided degrades `execute` to exactly
698    /// review → gate → merge, because `judge` skips a single-candidate field,
699    /// `deliberate` has fewer than two first choices to reconcile, `vote`
700    /// returns early, `tally` is already present and `fold_losers` has no
701    /// losers. Resuming such a run therefore does the right thing as well.
702    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703        Self::review_taking_over(repo, branch, config, None, origin).await
704    }
705
706    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
707    /// at the same task still has `branch` checked out, its worktree is
708    /// released first if that is safe (see [`crate::handover`]), and the
709    /// review refuses with the reason if it is not. `None` is a hand-run
710    /// review: it has no earlier attempts, so only a worktree of a dead run
711    /// magi recorded itself can be released.
712    pub async fn review_taking_over(
713        repo: &Path,
714        branch: &str,
715        config: Config,
716        takeover: Option<crate::handover::Takeover>,
717        origin: Origin,
718    ) -> Result<Self> {
719        let repo = git::toplevel(repo).await?;
720        let missing = agent::missing_programs(&config.agents);
721        if !missing.is_empty() {
722            bail!(
723                "these agent programs are not on PATH: {}. Fix the roster in \
724                 magi.toml or install them.",
725                missing.join(", ")
726            );
727        }
728        let base_branch =
729            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730                .await?;
731        if base_branch == branch {
732            bail!("`{branch}` is the base branch; there is nothing to review against");
733        }
734        let fetched_base = resolve_base(
735            &repo,
736            &base_branch,
737            &config.merge.remote,
738            config.coordination.as_ref(),
739        )
740        .await?;
741        let base_commit = fetched_base.commit.clone();
742
743        let roles = config.resolve_roles()?;
744        let max_parallel = config.graph.max_parallel.max(1);
745        let mut state = RunState::new(
746            repo.clone(),
747            base_branch,
748            base_commit.clone(),
749            String::new(),
750            config,
751        );
752        state.fetched_base = Some(fetched_base);
753        state.origin = Some(origin);
754
755        // Released before anything else touches the branch: a stale local
756        // branch is moved with `git branch -f`, which git refuses while an
757        // earlier attempt's worktree still has it checked out. Everything
758        // after this point that can fail puts the old run back.
759        // A hand-run review has no task, hence no earlier attempts, but a
760        // worktree of a dead run magi made may still be released.
761        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762            earlier: Vec::new(),
763            home: crate::run::home(),
764            choice: None,
765            task_instruction: None,
766        });
767        let wait = repository_wait(&state, None);
768        let coordination_config = state.config.coordination.clone();
769        let coordination =
770            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771                .await?;
772        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773        if let Some(released) = &released {
774            state.event(
775                "release",
776                format!(
777                    "took `{branch}` over from run {}: its worktree was released: {}",
778                    crate::run::short_of(&released.old_id),
779                    released.audit
780                ),
781            );
782        }
783        // The owner's answer to an earlier divergence question is applied
784        // here: after the release (git will not move a checked-out branch)
785        // and before the sync that would otherwise ask again.
786        if let Some(choice) = takeover.choice.as_ref()
787            && let Err(e) =
788                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789                    .await
790        {
791            if let Some(released) = &released {
792                released.restore(&repo, branch).await;
793            }
794            return Err(e.context("applying the owner's answer about the diverged branch"));
795        }
796        // An earlier attempt of this task that published the task head, and
797        // whose work is the branch under review, hands that head over: the
798        // review run updates the same pull request. Without one (a legacy
799        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
800        // own head and its own open pull request is adopted as before.
801        for id in takeover.earlier.iter().rev() {
802            if let Ok(old) = RunState::load_under(id, &takeover.home)
803                && let Some(head) = old.pr_head.clone()
804                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805            {
806                state.pr_head = Some(head);
807                break;
808            }
809        }
810        drop(coordination);
811        let opened = Self::open_review(
812            &repo,
813            branch,
814            state,
815            roles,
816            max_parallel,
817            base_commit,
818            &takeover,
819        )
820        .await;
821        if opened.is_err()
822            && let Some(released) = &released
823        {
824            let _coordination = crate::repo_lock::SeatLease::operation_with(
825                &repo,
826                coordination_config.as_ref(),
827                &wait,
828            )
829            .await?;
830            released.restore(&repo, branch).await;
831        }
832        opened
833    }
834
835    /// Recover exactly the operator-selected public implementation, without touching old sources.
836    pub async fn recover_pr(
837        repo: &Path,
838        task: &str,
839        selection: crate::pr_recovery::Selection,
840        config: Config,
841    ) -> Result<Self> {
842        selection.check_task(task, &selection.instruction)?;
843        let base_branch =
844            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
845                .await?;
846        let fetched_base = resolve_base(
847            repo,
848            &base_branch,
849            &config.merge.remote,
850            config.coordination.as_ref(),
851        )
852        .await?;
853        let base_commit = fetched_base.commit.clone();
854        selection
855            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
856            .await?;
857        let roles = config.resolve_roles()?;
858        let max_parallel = config.graph.max_parallel.max(1);
859        let mut state = RunState::new(
860            repo.to_path_buf(),
861            base_branch,
862            base_commit.clone(),
863            selection.instruction.clone(),
864            config,
865        );
866        state.fetched_base = Some(fetched_base);
867        state.origin = Some(crate::run::Origin::queue(task));
868        state.pr_head = Some(selection.branch.clone());
869        state.pr_recovery = Some(selection.clone());
870        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
871        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872    }
873
874    async fn open_selected(
875        repo: &Path,
876        state: RunState,
877        roles: ResolvedRoles,
878        max_parallel: usize,
879        base_commit: String,
880        selection: crate::pr_recovery::Selection,
881    ) -> Result<Self> {
882        let mut state = state;
883        let branch = state.branch_for('A');
884        let wait = repository_wait(&state, None);
885        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
886        git::git(repo, &["branch", &branch, &selection.head]).await?;
887        drop(coordination);
888        state.event(
889            "recovery",
890            format!(
891                "selected public implementation {} from {}",
892                selection.head, selection.url
893            ),
894        );
895        let takeover = crate::handover::Takeover {
896            earlier: Vec::new(),
897            home: crate::run::home(),
898            choice: None,
899            task_instruction: Some(selection.instruction.clone()),
900        };
901        state.save()?;
902        let id = state.id.clone();
903        let opened = Self::open_review(
904            repo,
905            &branch,
906            state,
907            roles,
908            max_parallel,
909            base_commit,
910            &takeover,
911        )
912        .await;
913        match opened {
914            Ok(runner) => Ok(runner),
915            Err(e) => {
916                if let Ok(mut partial) = RunState::load(&id) {
917                    partial.status = RunStatus::Blocked;
918                    partial.driver_exited = true;
919                    partial.event(
920                        "recovery",
921                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
922                    );
923                    partial.save()?;
924                }
925                Err(crate::pr_recovery::StartupFailed {
926                    run: id,
927                    cause: format!("{e:#}"),
928                }
929                .into())
930            }
931        }
932    }
933
934    /// What the newest earlier attempt that reviewed `branch` leaves for this
935    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
936    /// no earlier attempts. Reads the earlier record only: ownership of its
937    /// worktree and branch is [`crate::handover`]'s business.
938    async fn inherit(
939        repo: &Path,
940        branch: &str,
941        base_commit: &str,
942        takeover: &crate::handover::Takeover,
943    ) -> Option<crate::recovery::RecoveryRecord> {
944        use crate::recovery::{Relation, assess};
945        let newest = takeover.earlier.last()?;
946        let mut chosen = None;
947        for id in takeover.earlier.iter().rev() {
948            if let Ok(old) = RunState::load_under(id, &takeover.home)
949                && !old.reviews.is_empty()
950                && (old.winner().is_some_and(|w| w.branch == branch)
951                    || old.pr_head.as_deref() == Some(branch))
952            {
953                chosen = Some(old);
954                break;
955            }
956        }
957        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
958            .await
959            .unwrap_or_default();
960        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
961            Some(last) if last.head == tip => Relation::Same,
962            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
963            _ => Relation::Diverged,
964        };
965        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
966        Some(assess(from, chosen.as_ref(), base_commit, relation))
967    }
968
969    /// The half of [`Runner::review_taking_over`] that can fail after an
970    /// earlier attempt's worktree was released.
971    async fn open_review(
972        repo: &Path,
973        branch: &str,
974        mut state: RunState,
975        roles: ResolvedRoles,
976        max_parallel: usize,
977        base_commit: String,
978        takeover: &crate::handover::Takeover,
979    ) -> Result<Self> {
980        let wait = repository_wait(&state, None);
981        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
982        if state.pr_recovery.is_none() {
983            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
984        }
985        // The commit subjects are the closest thing to a task statement that
986        // existing work carries, and the reviewers are told as much.
987        let start = review_base(
988            repo,
989            &state.config.merge.remote,
990            &state.base_branch,
991            &base_commit,
992            branch,
993        )
994        .await;
995        let log = git::log_oneline(repo, &start, branch)
996            .await
997            .unwrap_or_default();
998        let log_text = if log.trim().is_empty() {
999            "(no commit messages)"
1000        } else {
1001            log.trim()
1002        };
1003        let mut instruction = format!(
1004            "Review the work already on branch `{branch}`. There is no task \
1005             statement: what the change claims to do is whatever its commits \
1006             say.\n\n{log_text}"
1007        );
1008        // A retry of a task whose earlier attempt left reviews behind carries
1009        // that attempt's evidence, and the task's own statement with it: the
1010        // completion criteria are the task's, not whatever the commits claim.
1011        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1012            if let Some(task) = takeover
1013                .task_instruction
1014                .as_deref()
1015                .filter(|t| !t.trim().is_empty())
1016                && rec.reviewed_head.is_some()
1017            {
1018                instruction = format!(
1019                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1020                    task.trim()
1021                );
1022                rec.criteria_inherited = true;
1023            }
1024            state.event("recovery", rec.summary());
1025            state.recovery = Some(rec);
1026        }
1027        if let Some(selected) = &state.pr_recovery {
1028            instruction = selected.instruction.clone();
1029        }
1030        state.task_instruction = takeover.task_instruction.clone();
1031        state.instruction = instruction;
1032        state.reviewed_commits = Some(
1033            git::subjects(repo, &start, branch)
1034                .await
1035                .unwrap_or_default(),
1036        );
1037
1038        // An attached worktree, so the fixer's commits land on the branch under
1039        // review rather than on a detached head nobody will look at again.
1040        let worktree = state.worktree_root().join("under-review");
1041        if let Some(parent) = worktree.parent() {
1042            tokio::fs::create_dir_all(parent).await.ok();
1043        }
1044        let path = worktree.to_string_lossy().to_string();
1045        git::git(repo, &["worktree", "add", &path, branch])
1046            .await
1047            .with_context(|| {
1048                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1049            })?;
1050
1051        let setup = match ensure_setup_config(&mut state, repo).await {
1052            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1053            Err(e) => Err(e),
1054        };
1055        if let Err(e) = setup {
1056            git::worktree_remove_clean(repo, &worktree).await.ok();
1057            return Err(e);
1058        }
1059        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1060            .await
1061            .unwrap_or(0);
1062        if commits == 0 {
1063            git::worktree_remove_clean(repo, &worktree).await.ok();
1064            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1065        }
1066        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1067            .await
1068            .map(|f| f.len())
1069            .unwrap_or(0);
1070        if files == 0
1071            && let (Ok(head_tree), Ok(base_tree)) = (
1072                git::tree_of(&worktree, "HEAD").await,
1073                git::tree_of(&worktree, &base_commit).await,
1074            )
1075            && head_tree == base_tree
1076        {
1077            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1078            git::worktree_remove_clean(repo, &worktree).await.ok();
1079            bail!(
1080                "`{branch}` at {} has a tree identical to base {}; this usually means \
1081                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1082                 against `{}/{branch}`) rather than an empty change",
1083                short(&head),
1084                short(&base_commit),
1085                state.config.merge.remote
1086            );
1087        }
1088        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1089            .await
1090            .unwrap_or_default();
1091
1092        state.candidates.push(Candidate {
1093            index: 0,
1094            label: 'A',
1095            // Not an agent id on purpose: nothing in the roster wrote this, and
1096            // the stats tables must not credit anyone with a win for it.
1097            agent: EXISTING_BRANCH.to_owned(),
1098            branch: branch.to_owned(),
1099            worktree,
1100            summary: String::new(),
1101            stat,
1102            files,
1103            commits,
1104            empty: false,
1105            failed: None,
1106            verified_noop: None,
1107            duration_ms: 0,
1108            folded: false,
1109        });
1110        state.tally = Some(Tally {
1111            first_choice: BTreeMap::from([('A', 0)]),
1112            borda: BTreeMap::new(),
1113            winner: 'A',
1114            rankings: 0,
1115            unanimous_initial: false,
1116            deliberated: false,
1117            changed_votes: 0,
1118            unanimous_final: false,
1119            tie_break: None,
1120            // No panel sat, so no quorum applies. Zero judges is the correct
1121            // number for work that never competed, and must not be reported as
1122            // a collapsed panel.
1123            judges: 0,
1124            present: 0,
1125            quorum: 0,
1126            met_quorum: true,
1127            uncontested: Some("review-only run: nothing competed".to_owned()),
1128        });
1129        state.status = RunStatus::Reviewing;
1130        state.prep_complete = Some(true);
1131        state.event(
1132            "start",
1133            format!(
1134                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1135                state.id
1136            ),
1137        );
1138        state.save()?;
1139        Ok(Self {
1140            state,
1141            roles,
1142            sem: Arc::new(Semaphore::new(max_parallel)),
1143            pause: Pause::new(),
1144            interrupt: Pause::new(),
1145        })
1146    }
1147
1148    /// Reopen an existing run.
1149    pub fn resume(id: &str) -> Result<Self> {
1150        let state = RunState::load(id)?;
1151        crate::pr_head::ensure_owner(&state)?;
1152        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1153            bail!(
1154                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1155            );
1156        }
1157        if let Some(to) = &state.released_to {
1158            bail!(
1159                "run {} cannot be resumed: its worktree was released to run {}",
1160                state.short(),
1161                crate::run::short_of(to)
1162            );
1163        }
1164        let roles = state.config.resolve_roles()?;
1165        if let Some(id) = &state.config.graph.selected_implementer
1166            && state.candidates.iter().any(|c| c.agent != *id)
1167        {
1168            bail!(
1169                "replacement implementer pin does not match the saved candidates; task must stay held"
1170            );
1171        }
1172        let max_parallel = state.config.graph.max_parallel.max(1);
1173        Ok(Self {
1174            state,
1175            roles,
1176            sem: Arc::new(Semaphore::new(max_parallel)),
1177            pause: Pause::new(),
1178            interrupt: Pause::new(),
1179        })
1180    }
1181
1182    /// Walk the graph to a terminal state, skipping nodes already recorded.
1183    ///
1184    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1185    /// the phone - ends here, so this is the one place a run that ended
1186    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1187    /// notification centre. Best-effort: see [`crate::notices::raise`].
1188    pub async fn execute(&mut self) -> Result<()> {
1189        // Queue claims cover daemon work; this also serializes manual drivers
1190        // consuming a repair answer, including the new review and decision.
1191        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1192            || self.state.land_approval.is_some()
1193            || self.state.merge_decision.is_some()
1194            || !self.state.merge_repairs.is_empty()
1195            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1196        {
1197            let claim = FixClaim::acquire(&self.state.dir())?;
1198            let disk = RunState::load(&self.state.id)?;
1199            let checkpoint = |s: &RunState| {
1200                (
1201                    s.merge_repairs.len(),
1202                    s.merge_repairs
1203                        .last()
1204                        .map(|r| (r.question.clone(), r.phase)),
1205                    s.land_approval
1206                        .as_ref()
1207                        .map(|a| (a.question.clone(), a.head.clone())),
1208                )
1209            };
1210            if checkpoint(&disk) != checkpoint(&self.state) {
1211                bail!(
1212                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1213                );
1214            }
1215            Some(claim)
1216        } else {
1217            None
1218        };
1219        let mut execution = self.claim_driver()?;
1220        let runner = &mut *execution.runner;
1221        let wait = runner.repository_wait();
1222        // Keep the graph's large future off the callers' stack, including
1223        // standalone task attempts running on a default-sized test thread.
1224        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1225        let result = match result {
1226            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1227                record_repository_block(&mut runner.state, &e);
1228                runner.park_here()?;
1229                Ok(())
1230            }
1231            other => other,
1232        };
1233        execution.driver.finish(&mut runner.state)?;
1234        let ended = if result.is_err() {
1235            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1236        } else {
1237            crate::notices::run_ended(&runner.state)
1238        };
1239        if let Some(notice) = ended {
1240            crate::notices::raise(notice);
1241        }
1242        result
1243    }
1244
1245    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1246        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1247    }
1248
1249    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1250        let wait = self.repository_wait();
1251        coordinate_operation(&mut self.state, cwd, &wait).await
1252    }
1253
1254    /// Validate ownership and persist the driver claim in one critical section.
1255    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1256        let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1257        let disk = RunState::load(&self.state.id)?;
1258        crate::pr_head::ensure_owner(&disk)?;
1259        if disk.liveness(false) == crate::run::Liveness::Live {
1260            bail!("run already has a live driver; wait for its execution to end");
1261        }
1262        if let Some(to) = &disk.released_to {
1263            bail!(
1264                "run {} cannot continue: its worktree was released to run {}",
1265                self.state.short(),
1266                crate::run::short_of(to)
1267            );
1268        }
1269        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1270        let mut execution = DriverExecution {
1271            runner: self,
1272            driver,
1273        };
1274        // Declare after the execution guard: on any claim error this short
1275        // lock must drop before that guard reloads and releases ownership.
1276        let _owner = owner;
1277        let state = &mut execution.runner.state;
1278        // Moving again, so it is no longer parked. Set before the walk rather
1279        // than in `resume`, so every way of re-entering the graph clears it
1280        // and a card cannot claim a run is waiting to be resumed while the
1281        // agents are already working.
1282        state.parked = false;
1283        state.resource_wait = None;
1284        state.resource_block = None;
1285        // Any seat this state still lists as answering belongs to whatever
1286        // process last drove this run — this one included, if it crashed
1287        // mid-wave. Cleared and flushed immediately, before anything else
1288        // runs, so a resume can never show a seat as live when nothing is
1289        // asking it anything yet; the node that actually dispatches the next
1290        // wave repopulates it.
1291        state.clear_active();
1292        // Recorded in the same spot, and flushed together with the clear
1293        // above: this is the pid a reader checks (`RunState::liveness`) when
1294        // no daemon claim exists to answer "is a process still driving this
1295        // run" — a plain `magi run` / `magi review` typed into a terminal
1296        // claims nothing there. Always overwritten, never only-if-absent, so
1297        // a resumed run's stale pid from a previous, possibly-dead process
1298        // can never survive into this one's own report. Unlike
1299        // `clear_active`, this changes on every single `execute()` call, so
1300        // the save below is now unconditional rather than only-if-cleared.
1301        //
1302        // `driver_started_at` is recorded in the same breath, from this same
1303        // pid, so `liveness` can tell a live pid that is genuinely still us
1304        // apart from one the OS has since handed to an unrelated process —
1305        // see that field's own doc for why the pid alone is not enough.
1306        let pid = std::process::id();
1307        state.driver_pid = Some(pid);
1308        state.driver_started_at = crate::proc::process_started_at(pid);
1309        state.driver_exited = false;
1310        state.begin_pass();
1311        execution.driver.bind(state);
1312        state.save()?;
1313        execution.driver.record(state)?;
1314        Ok(execution)
1315    }
1316
1317    async fn execute_graph(&mut self) -> Result<()> {
1318        // Preservation may invalidate an old empty classification. Resolve it
1319        // before any resume shortcut can reuse a verdict or publish a branch.
1320        self.persist_candidates().await?;
1321        if self
1322            .state
1323            .merge_decision
1324            .as_ref()
1325            .is_some_and(|decision| !decision.closed)
1326            && self.state.status != RunStatus::Landing
1327        {
1328            if crate::merge_decision::resumable_hold(&self.state) {
1329                self.state.merge = Some(MergeOutcome {
1330                    mode: MergeMode::Pr,
1331                    ok: false,
1332                    detail: self
1333                        .state
1334                        .merge_decision
1335                        .as_ref()
1336                        .unwrap()
1337                        .target
1338                        .pr
1339                        .clone(),
1340                    empty: false,
1341                });
1342                self.run_land().await?;
1343                self.settle_questions();
1344                return Ok(());
1345            }
1346            bail!(
1347                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1348            );
1349        }
1350        if self.state.config.graph.selected_implementer.is_some()
1351            && self.state.status == RunStatus::Blocked
1352            && self.state.tally.is_none()
1353            && self.state.candidate_preservation.is_empty()
1354        {
1355            bail!(
1356                "selected implementation is held; explicitly release the owning task to retry it"
1357            );
1358        }
1359
1360        if self.state.merge_repairs.last().is_some_and(|r| {
1361            matches!(
1362                r.phase,
1363                crate::run::MergeRepairPhase::Requested
1364                    | crate::run::MergeRepairPhase::Fixing
1365                    | crate::run::MergeRepairPhase::Reviewing
1366            )
1367        }) {
1368            if self.repair_merge_review().await? {
1369                self.run_land().await?;
1370            }
1371            self.settle_questions();
1372            return Ok(());
1373        }
1374        // Publication already passed review and gate. Resume only that node:
1375        // syncing to a newer base here could change the verified commit.
1376        if self.state.pr_publication_pending {
1377            self.merge().await?;
1378            if self.resource_halt()? {
1379                return Ok(());
1380            }
1381            self.state.save()?;
1382            self.settle_questions();
1383            return Ok(());
1384        }
1385        // A run that already lost its quorum never resumes into the verdict
1386        // machinery: `deliberate` and `vote` would otherwise clobber the
1387        // stalled marker back to Voting and the run would keep going past a
1388        // verdict that is no longer trustworthy. Everything already recorded is
1389        // kept, so the run stays resumable (or foldable) for a human to pick up.
1390        //
1391        // On --resume the run gets one chance to repair itself: the seats a
1392        // rate limit took out are re-asked. If their quota has since reset and
1393        // the quorum is restored, the run picks up and finishes; otherwise it
1394        // stays stale and still-resumable for a later retry. If it does not
1395        // recover, the returned status stays `Stalled` and nothing was
1396        // clobbered (the recovery only mutates entries for the lost seats).
1397        if self.state.status == RunStatus::Stalled {
1398            if self.recover_stall().await? {
1399                self.finish_after_tally().await?;
1400            } else {
1401                // Still below quorum: persist the marker and stay resumable.
1402                self.state.save()?;
1403            }
1404            return Ok(());
1405        }
1406        // A run parked inside `land` - watching CI, mid fix-round, or
1407        // waiting on the owner's merge approval - resumes directly into it,
1408        // never back through `prep`. Everything before `merge` already
1409        // concluded; that is the only way `status` reaches `Landing` in the
1410        // first place. Re-walking `review_loop` first would also be actively
1411        // wrong: its own status recomputation (see its doc) treats any
1412        // clean round as reason to set `status` to `Gating`, which would
1413        // clobber this marker before `merge` ever ran, and this run would
1414        // never find its way back into `land` at all.
1415        if self.state.status == RunStatus::Landing {
1416            self.run_land().await?;
1417            // `run_land` may have settled the run right here - CI came back
1418            // green and the PR merged, say - without ever passing back
1419            // through `merge`'s own trailing call. Whatever it left `status`
1420            // as is what this has to read.
1421            self.settle_questions();
1422            return Ok(());
1423        }
1424        // A run parked at `merge` on the owner's word about a withheld PR text
1425        // resumes straight into `merge`. Re-walking the tail would sync to the
1426        // base again (possibly rebasing, possibly spending the fixer) after the
1427        // gate had passed, and then push a tree nobody reviewed or gated.
1428        if self.state.github_text.is_some()
1429            && self.state.merge.is_none()
1430            && self.state.status == RunStatus::Gating
1431        {
1432            self.merge().await?;
1433            if self.resource_halt()? {
1434                return Ok(());
1435            }
1436            self.state.save()?;
1437            return Ok(());
1438        }
1439        if let Err(e) = self.prep().await {
1440            if e.is::<crate::repo_lock::ResourceBlocked>() {
1441                return Err(e);
1442            }
1443            self.state.status = RunStatus::Blocked;
1444            self.state
1445                .prep_failure
1446                .get_or_insert_with(|| format!("{e:#}"));
1447            self.state
1448                .event("prep", format!("preparation stopped: {e:#}"));
1449            self.state.save()?;
1450            return Err(e);
1451        }
1452        if self.park_here()? {
1453            return Ok(());
1454        }
1455        self.advise().await?;
1456        if self.park_here()? {
1457            return Ok(());
1458        }
1459        self.implement().await?;
1460        if self.park_here()? {
1461            return Ok(());
1462        }
1463        // `after_implement` already saved the state and settled any open
1464        // questions when it set this; nothing later in the graph has
1465        // anything to judge.
1466        if self.state.status == RunStatus::VerifiedNoop {
1467            return Ok(());
1468        }
1469        self.judge().await?;
1470        if self.park_here()? {
1471            return Ok(());
1472        }
1473        self.deliberate().await?;
1474        if self.park_here()? {
1475            return Ok(());
1476        }
1477        self.vote().await?;
1478        if self.park_here()? {
1479            return Ok(());
1480        }
1481        self.tally()?;
1482        // A verdict that lost its quorum is not trustworthy: do not review,
1483        // gate, or merge on it. Everything already done is kept, so the run
1484        // stays resumable (or foldable); the human can replace the agent that
1485        // ran out of quota and pick it up.
1486        if self.state.status == RunStatus::Stalled {
1487            // Persist the stalled marker now — the normal end-of-execute save
1488            // below is below this early return, and without it a resumed run
1489            // would reload a pre-tally status and keep going.
1490            self.state.save()?;
1491            return Ok(());
1492        }
1493        self.finish_after_tally().await?;
1494        Ok(())
1495    }
1496
1497    /// A wave could not start its agents because a shared resource stayed busy
1498    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1499    /// resumable. Unstarted seats have no result; completed peers retain
1500    /// their saved candidate state.
1501    fn stop_for_resource(&mut self) -> Result<()> {
1502        // A wait abandoned because the run is parking stays a park.
1503        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1504        {
1505            self.state.status = RunStatus::Blocked;
1506        }
1507        self.state.save()
1508    }
1509
1510    /// Stop walking the graph if a node ended on a resource block.
1511    fn resource_halt(&mut self) -> Result<bool> {
1512        if self.state.resource_block.is_none() {
1513            return Ok(false);
1514        }
1515        self.stop_for_resource()?;
1516        Ok(true)
1517    }
1518
1519    /// Park here if asked to, recording it in the run's own timeline.
1520    ///
1521    /// Returns whether the caller should stop walking the graph. The state is
1522    /// saved either way by the node that just finished; this adds the event so
1523    /// the operator's card says why a run that is neither finished nor moving
1524    /// is sitting where it is.
1525    fn park_here(&mut self) -> Result<bool> {
1526        let parking = self.pause.parked() || self.interrupt.parked();
1527        if !parking && self.state.resource_block.is_some() {
1528            self.stop_for_resource()?;
1529            return Ok(true);
1530        }
1531        // Either handle asking is enough - see `Pause`'s own doc for why
1532        // they are never the same one. `interrupt` is checked second so a
1533        // reason it carries is preferred in the message below over a plain
1534        // shutdown park racing it at the same boundary.
1535        if !self.pause.parked() && !self.interrupt.parked() {
1536            return Ok(false);
1537        }
1538        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1539        let why = match cause.as_ref().map(ParkCause::label) {
1540            Some(reason) => format!(
1541                "parked after `{}` ({reason}) — resume to carry on from here",
1542                self.state.status.as_str()
1543            ),
1544            None => format!(
1545                "parked after `{}` — resume to carry on from here",
1546                self.state.status.as_str()
1547            ),
1548        };
1549        self.state.event("park", why);
1550        self.state
1551            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1552        self.state.parked = true;
1553        self.state.save()?;
1554        Ok(true)
1555    }
1556
1557    /// Only a new explicit operator approval may retry failed preparation.
1558    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1559        if self.state.prep_failure.is_none() {
1560            return Ok(());
1561        }
1562        let failed_at = self
1563            .state
1564            .events
1565            .iter()
1566            .rev()
1567            .find(|event| event.node == "prep")
1568            .map_or(self.state.updated_at, |event| event.at);
1569        anyhow::ensure!(
1570            approval_at > failed_at,
1571            "preparation retry requires a new explicit release after the failure"
1572        );
1573        self.state.prep_failure = None;
1574        self.state.prep_complete = Some(false);
1575        self.state
1576            .event("prep-retry", "operator approved retry of preparation");
1577        Ok(())
1578    }
1579
1580    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1581    pub fn on_pause(&mut self, pause: Pause) {
1582        self.pause = pause;
1583    }
1584
1585    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1586    /// scheduler asking this one run - and no other - to park so a task
1587    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1588    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1589    /// handle.
1590    pub fn watch_interrupt(&mut self, pause: Pause) {
1591        self.interrupt = pause;
1592    }
1593
1594    /// Abandon this run's own open questions, once `status` has actually
1595    /// settled rather than merely paused.
1596    ///
1597    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1598    /// either back up with the candidates, the review round and the seat
1599    /// sessions already on disk, so a question an implementer asked mid-round
1600    /// may still get a real answer read by a real resume. Only the statuses
1601    /// `resumable` excludes are actually final: the run merged, it reached
1602    /// `Ready` with nothing left to do, it failed outright with no
1603    /// established point to continue from, or every candidate agreed, with
1604    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1605    /// every one of those the seat that asked is gone for good, exactly like
1606    /// the run being deleted under `magi run rm` - so the same cleanup
1607    /// applies, worded for what actually happened instead of "the run was
1608    /// deleted".
1609    ///
1610    /// Best-effort and silent on success: called from every place `status`
1611    /// can land on one of those three, including ones a resumed run revisits,
1612    /// so it must cost nothing when there was nothing open to begin with.
1613    fn settle_questions(&mut self) {
1614        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1615            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1616        }
1617    }
1618
1619    /// The tail of the graph after a trustworthy tally: fold losers, review,
1620    /// gate, merge, and persist.
1621    async fn finish_after_tally(&mut self) -> Result<()> {
1622        // Before the losers are folded: their branches are what the graft
1623        // reads from.
1624        self.graft().await?;
1625        if self.resource_halt()? {
1626            return Ok(());
1627        }
1628        self.fold_losers().await?;
1629        // Before review starts, and again right before the gate: a run's
1630        // review rounds can themselves take long enough for the base to move
1631        // a second time, and the gate is the one node whose "green" gets
1632        // acted on.
1633        self.sync_to_base().await?;
1634        if self.state.status == RunStatus::AlreadyInBase {
1635            return Ok(());
1636        }
1637        self.review_loop().await?;
1638        if self.resource_halt()? {
1639            return Ok(());
1640        }
1641        self.sync_to_base().await?;
1642        if self.state.status == RunStatus::AlreadyInBase {
1643            return Ok(());
1644        }
1645        self.gate().await?;
1646        self.merge().await?;
1647        if self.resource_halt()? {
1648            return Ok(());
1649        }
1650        self.state.save()?;
1651        Ok(())
1652    }
1653
1654    // ---------------------------------------------------------------- prep
1655
1656    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1657    /// run with the step and its output in the event; it is never swallowed.
1658    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1659    /// resume starts over instead of finding candidates and missing seats.
1660    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1661        let result = match ensure_setup_config(&mut self.state, repo).await {
1662            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1663            Err(e) => Err(e),
1664        };
1665        if let Err(e) = result {
1666            git::worktree_remove_clean(repo, wt).await.ok();
1667            if in_prep {
1668                self.rollback_prep(repo).await;
1669            }
1670            self.state.status = RunStatus::Blocked;
1671            self.state.event(
1672                "setup",
1673                format!("worktree setup failed in {}: {e:#}", wt.display()),
1674            );
1675            self.state.save()?;
1676            return Err(e);
1677        }
1678        Ok(())
1679    }
1680
1681    /// Undo everything `prep` created, so a resume re-runs it whole.
1682    async fn rollback_prep(&mut self, repo: &Path) {
1683        let root = self.state.worktree_root();
1684        for c in std::mem::take(&mut self.state.candidates) {
1685            if matches!(
1686                git::worktree_remove_clean(repo, &c.worktree).await,
1687                Ok(true)
1688            ) {
1689                git::branch_delete(repo, &c.branch).await.ok();
1690            } else {
1691                self.state.candidates.push(c);
1692            }
1693        }
1694        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1695        for j in 1..=seats {
1696            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1697                .await
1698                .ok();
1699        }
1700        for k in 1..=self.state.config.graph.advisors {
1701            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1702                .await
1703                .ok();
1704        }
1705    }
1706
1707    async fn prep(&mut self) -> Result<()> {
1708        if let Some(reason) = &self.state.prep_failure {
1709            bail!(
1710                "candidate preparation previously failed: {reason}; correct the source before retrying"
1711            );
1712        }
1713        if self
1714            .state
1715            .prep_complete
1716            .unwrap_or(!self.state.candidates.is_empty())
1717        {
1718            return Ok(());
1719        }
1720        let repo = self.state.repo.clone();
1721        // Includes rollback, config, refs, checkout creation and setup. No
1722        // repository mutation may precede the real shared lock acquisition.
1723        let _coordination = self.coordinate(&repo).await?;
1724        self.rollback_prep(&repo).await;
1725        if !self.state.candidates.is_empty() {
1726            bail!(
1727                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1728            );
1729        }
1730        self.state.prep_complete = Some(false);
1731        self.state.status = RunStatus::Prep;
1732        self.state.save()?;
1733        let repo = self.state.repo.clone();
1734        let base = self.state.base_commit.clone();
1735        let plan = refs::plan(&repo, &self.state.seeds).await?;
1736        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1737        let root = self.state.worktree_root();
1738        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1739
1740        // The hook is the write-time half of the blindness contract; the
1741        // presentation filter in `blind` is the half that cannot be bypassed.
1742        let hooks_dir = self.state.dir().join("hooks");
1743        if self.state.config.blind.commit_msg_hook {
1744            std::fs::create_dir_all(&hooks_dir)
1745                .with_context(|| format!("create {}", hooks_dir.display()))?;
1746            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1747            let path = hooks_dir.join("commit-msg");
1748            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1749            make_executable(&path)?;
1750            // Ref-counted rather than a plain idempotent set: with more than
1751            // one run able to be in flight in the same repository at once
1752            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1753            // true?" check cannot tell "another run of mine still needs
1754            // this" from "nobody does", and the run that happens to finish
1755            // first would disable the hook out from under a sibling still
1756            // relying on it.
1757            git::acquire_worktree_config(&repo).await?;
1758            self.state.enabled_worktree_config = true;
1759        }
1760
1761        // `[worktree] setup` hides its products per worktree, which needs the
1762        // same per-worktree config the hook does. Held until fold, like it.
1763        ensure_setup_config(&mut self.state, &repo).await?;
1764
1765        for (index, (spec, label)) in self
1766            .roles
1767            .implementers
1768            .clone()
1769            .into_iter()
1770            .zip(labels)
1771            .enumerate()
1772        {
1773            let branch = self.state.branch_for(label);
1774            let worktree = root.join(format!("cand-{label}"));
1775            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1776            if self.state.config.blind.commit_msg_hook {
1777                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1778            }
1779            git::local_exclude(&worktree, "/.magi/").await?;
1780            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1781                // Nothing of this prep may survive: a resume must start over,
1782                // not find half the candidates and skip the rest.
1783                git::worktree_remove_clean(&repo, &worktree).await.ok();
1784                git::branch_delete(&repo, &branch).await.ok();
1785                self.rollback_prep(&repo).await;
1786                self.state.status = RunStatus::Blocked;
1787                self.state
1788                    .event("prep", format!("worktree setup failed: {e:#}"));
1789                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1790                self.state.save()?;
1791                return Err(e);
1792            }
1793            self.state.candidates.push(Candidate {
1794                index,
1795                label,
1796                agent: spec.id.clone(),
1797                branch,
1798                worktree,
1799                summary: String::new(),
1800                stat: String::new(),
1801                files: 0,
1802                commits: 0,
1803                empty: false,
1804                failed: None,
1805                verified_noop: None,
1806                duration_ms: 0,
1807                folded: false,
1808            });
1809            self.state.save()?;
1810            for pick in &plan.picks {
1811                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1812                {
1813                    self.state.status = RunStatus::Blocked;
1814                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1815                    self.state
1816                        .event("prep", format!("cannot apply referenced commit: {e}"));
1817                    self.state.save()?;
1818                    return Err(e);
1819                }
1820            }
1821        }
1822
1823        for j in 1..=self.roles.judges.len() {
1824            let wt = root.join(format!("judge-{j}"));
1825            if !wt.exists() {
1826                git::worktree_add_detached(&repo, &wt, &base).await?;
1827                self.setup_seat_worktree(&repo, &wt, true).await?;
1828            }
1829        }
1830
1831        // Disposable, detached checkouts for the design-deliberation stage's
1832        // advisor seats — the same shape as the judges' above, at the same
1833        // base commit, since advisors also only ever read. Sized off the
1834        // configured count directly rather than a resolved roster: unlike
1835        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1836        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1837        // `prep` has no `ResolvedRoles` field to read a count from here.
1838        if self.state.config.graph.advise {
1839            for k in 1..=self.state.config.graph.advisors {
1840                let wt = root.join(format!("advisor-{k}"));
1841                if !wt.exists() {
1842                    git::worktree_add_detached(&repo, &wt, &base).await?;
1843                    self.setup_seat_worktree(&repo, &wt, true).await?;
1844                }
1845            }
1846        }
1847
1848        // A judge cannot tell it is looking at its own patch — the seats keep
1849        // separate conversations — but a panel that shares agents with the
1850        // field is less independent than it looks, and that is worth saying out
1851        // loud once per run rather than leaving it in the config.
1852        let authors: Vec<&str> = self
1853            .roles
1854            .implementers
1855            .iter()
1856            .map(|a| a.id.as_str())
1857            .collect();
1858        let overlap: Vec<String> = self
1859            .roles
1860            .judges
1861            .iter()
1862            .enumerate()
1863            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1864            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1865            .collect();
1866        if !overlap.is_empty() {
1867            let note = format!(
1868                "{} also authored a candidate; blind, but the panel is less \
1869                 independent than {} distinct agents would be",
1870                overlap.join(", "),
1871                self.roles.judges.len()
1872            );
1873            self.state.event("prep", note);
1874        }
1875
1876        self.state.event(
1877            "prep",
1878            format!(
1879                "{} candidates, {} judges, base {} ({})",
1880                self.state.candidates.len(),
1881                self.roles.judges.len(),
1882                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1883                self.state.base_branch
1884            ),
1885        );
1886        self.state.prep_complete = Some(true);
1887        self.state.status = RunStatus::Implementing;
1888        self.state.save()?;
1889        Ok(())
1890    }
1891
1892    // -------------------------------------------------------------- advise
1893
1894    /// The design-deliberation stage: independent, read-only advisor seats
1895    /// each sketch a design before any implementer touches the repository,
1896    /// and (when at least one produced a usable proposal) a synthesis seat
1897    /// blends them into a brief `implement` carries in every candidate's
1898    /// prompt.
1899    ///
1900    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1901    /// advisors` is the proposal count. Everything here is best-effort and
1902    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1903    /// that cannot reach quota, or a synthesis seat that produced nothing
1904    /// usable all leave `implement` exactly as it was before this stage
1905    /// existed — the task instruction alone — rather than failing the whole
1906    /// competition over an enrichment stage. Every outcome is still recorded
1907    /// as an event, so a run that got nothing from this stage says why.
1908    ///
1909    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1910    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1911    /// resumed run whose stage failed would re-run it, and re-spend the
1912    /// agent calls, on every reentry before `implement`.
1913    ///
1914    /// Also skipped once any candidate shows implementation progress — the
1915    /// exact predicate `implement` itself uses to decide a candidate is no
1916    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1917    /// is not enough: a run created by an older binary that predates this
1918    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1919    /// an already-`Implementing`-or-later run under this build would
1920    /// otherwise walk straight back through `prep` (a no-op once candidates
1921    /// exist) into this node and spawn every advisor seat against worktrees
1922    /// `prep` never recreated — after implementation has already started,
1923    /// which is exactly the invariant this stage exists to guarantee.
1924    async fn advise(&mut self) -> Result<()> {
1925        let implement_untouched = self
1926            .state
1927            .candidates
1928            .iter()
1929            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1930        if !self.state.config.graph.advise || self.state.advise_attempted {
1931            return Ok(());
1932        }
1933        if !implement_untouched {
1934            self.state.event(
1935                "advise",
1936                "skipping the design-deliberation stage: at least one \
1937                 candidate already shows implementation progress, so this \
1938                 run is past the point the stage exists to run before"
1939                    .to_owned(),
1940            );
1941            self.state.advise_attempted = true;
1942            self.state.save()?;
1943            return Ok(());
1944        }
1945        let run_id = self.state.id.clone();
1946        let prompts = self.state.config.prompts.clone();
1947        let instruction = self.state.instruction.clone();
1948        let language = self.state.config.graph.language.clone();
1949        let root = self.state.worktree_root();
1950        let n = self.state.config.graph.advisors;
1951        let where_recorded = self.state.dir().join("run.json");
1952
1953        let seats = match self.state.config.advisors() {
1954            Ok(seats) if !seats.is_empty() => seats,
1955            Ok(_) => {
1956                self.state.event(
1957                    "advise",
1958                    format!(
1959                        "[graph] advisors is 0; skipping the design-deliberation \
1960                         stage and continuing without a synthesis brief (see {})",
1961                        where_recorded.display()
1962                    ),
1963                );
1964                self.state.advise_attempted = true;
1965                self.state.save()?;
1966                return Ok(());
1967            }
1968            Err(e) => {
1969                self.state.event(
1970                    "advise",
1971                    format!(
1972                        "could not resolve advisor seats ({e:#}); continuing \
1973                         without a design-deliberation brief (see {})",
1974                        where_recorded.display()
1975                    ),
1976                );
1977                self.state.advise_attempted = true;
1978                self.state.save()?;
1979                return Ok(());
1980            }
1981        };
1982
1983        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1984        let artifacts = agent::artifacts_dir(&self.state.dir());
1985        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1986
1987        let mut jobs = Vec::new();
1988        for (i, spec) in seats.iter().cloned().enumerate() {
1989            let seat_key = format!("advisor-{}", i + 1);
1990            let seat = self.seat(&seat_key, &spec.id);
1991            jobs.push(SeatJob {
1992                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1993                spec,
1994                seat,
1995                cwd: worktrees[i % worktrees.len()].clone(),
1996                timeout,
1997                allow_write: false,
1998                sessions: false,
1999                artifacts: artifacts.clone(),
2000                stem: seat_key,
2001                handover: None,
2002            });
2003        }
2004
2005        self.state.event(
2006            "advise",
2007            format!(
2008                "{} advisor seat(s) sketching a design in parallel",
2009                jobs.len()
2010            ),
2011        );
2012        let mut quota_losses = Vec::new();
2013        let cache = self.state.config.cache_dir();
2014        let ctx = WaveCtx {
2015            carry_seats: false,
2016            pauses: Some((&self.pause, &self.interrupt)),
2017            run: &run_id,
2018            node: "advise",
2019            prompts: &prompts,
2020            cache: cache.as_deref(),
2021            round: None,
2022        };
2023        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2024        let results = ask_json_wave::<Proposal>(
2025            jobs,
2026            Arc::clone(&self.sem),
2027            self.state.config.graph.retries,
2028            &advisor_roster,
2029            &ctx,
2030            &mut quota_losses,
2031            &mut self.state,
2032            &|p: &Proposal| p.validate(),
2033        )
2034        .await;
2035        if self.state.resource_block.is_some() {
2036            return self.stop_for_resource();
2037        }
2038        self.state.quota.extend(quota_losses);
2039
2040        let mut records = Vec::with_capacity(results.len());
2041        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2042            let agent_id = seat.agent.clone();
2043            self.state.seats.insert(seat.key.clone(), seat);
2044            match res {
2045                Ok((proposal, out)) => {
2046                    self.state
2047                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2048                    records.push(advise::AdvisorRecord::proposed(
2049                        i + 1,
2050                        agent_id,
2051                        proposal,
2052                        out.duration_ms,
2053                    ));
2054                }
2055                Err(e) => {
2056                    self.state.event(
2057                        "advise",
2058                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2059                    );
2060                    records.push(advise::AdvisorRecord::failed(
2061                        i + 1,
2062                        agent_id,
2063                        e.to_string(),
2064                    ));
2065                }
2066            }
2067        }
2068
2069        let mut advice = advise::Advice {
2070            records,
2071            synthesis: None,
2072        };
2073        if advice.proposals().is_empty() {
2074            self.state.event(
2075                "advise",
2076                "no advisor produced a usable proposal; continuing without a \
2077                 synthesis brief"
2078                    .to_owned(),
2079            );
2080        } else {
2081            match self
2082                .synthesize_brief(
2083                    &advice,
2084                    &instruction,
2085                    &language,
2086                    &worktrees[0],
2087                    &artifacts,
2088                    &run_id,
2089                    &prompts,
2090                    cache.as_deref(),
2091                )
2092                .await
2093            {
2094                Ok(Some(text)) => {
2095                    self.state.event(
2096                        "advise",
2097                        "synthesized a design brief for the implementer".to_owned(),
2098                    );
2099                    advice.synthesis = Some(text);
2100                }
2101                Ok(None) => {
2102                    self.state.event(
2103                        "advise",
2104                        "the synthesis seat produced nothing usable; continuing \
2105                         without a design brief"
2106                            .to_owned(),
2107                    );
2108                }
2109                Err(e) => {
2110                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2111                        return Err(e);
2112                    }
2113                    self.state.event(
2114                        "advise",
2115                        format!("could not synthesize a design brief: {e:#}"),
2116                    );
2117                }
2118            }
2119        }
2120        advise::apply_reflection(&mut advice);
2121
2122        self.state.advice = Some(advice);
2123        self.state.advise_attempted = true;
2124        self.state.save()?;
2125        Ok(())
2126    }
2127
2128    /// The synthesis seat: reads every advisor's proposal and blends them
2129    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2130    /// out of [`Runner::advise`] only for readability — it is not called
2131    /// anywhere else.
2132    ///
2133    /// Picked the same way [`crate::talk`]'s standing conversation and
2134    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2135    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2136    /// order (a claude seat, else the first runnable agent in roster order)
2137    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2138    /// in [`crate::config`] for why a dedicated field exists here at all.
2139    #[allow(clippy::too_many_arguments)]
2140    async fn synthesize_brief(
2141        &mut self,
2142        advice: &advise::Advice,
2143        instruction: &str,
2144        language: &str,
2145        cwd: &Path,
2146        artifacts: &Path,
2147        run_id: &str,
2148        prompts: &Prompts,
2149        cache: Option<&Path>,
2150    ) -> Result<Option<String>> {
2151        let chain = agent::pick_chain(
2152            &self.state.config.agents,
2153            self.state.config.roles.synthesizer.as_ref(),
2154            &agent::installed,
2155            "synthesizer",
2156        )?;
2157        let proposals = advice.proposals();
2158        let mut prompt = prompt::with_overlay(
2159            prompt::synthesize_brief(instruction, &proposals, language),
2160            prompts.overlay("advise"),
2161        );
2162        if cache.is_some() {
2163            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2164            // below — see `prompt::build_cache_note`'s doc for why telling a
2165            // read-only seat to build through the shared cache is exactly how
2166            // a sandbox's write refusal gets misread as a defect.
2167            prompt.push('\n');
2168            prompt.push_str(&prompt::build_cache_note("advise", false));
2169        }
2170        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2171        // Each id is tried once, in order; a quota hit, error or unusable
2172        // answer moves to the next. The seat is single-turn (`sessions:
2173        // false`) and the prompt is the whole context, so a fallback agent
2174        // needs nothing carried over.
2175        let mut last = None;
2176        for (n, spec) in chain.iter().enumerate() {
2177            if n > 0 {
2178                self.state
2179                    .event("advise", format!("synthesis falling back to {}", spec.id));
2180            }
2181            let mut seat = self.seat("advise-synthesis", &spec.id);
2182            let wait = self.repository_wait();
2183            let outcome = wait
2184                .scope(agent::invoke(
2185                    spec,
2186                    &mut seat,
2187                    &Invocation {
2188                        scope: crate::agent::InvocationScope::Repository,
2189                        coordination: self.state.config.coordination.as_ref(),
2190                        fetched_base: None,
2191                        cwd,
2192                        prompt: &prompt,
2193                        timeout,
2194                        allow_write: false,
2195                        unsandboxed: false,
2196                        sessions: false,
2197                        artifacts,
2198                        stem: &if n == 0 {
2199                            "advise-synthesis".to_owned()
2200                        } else {
2201                            format!("advise-synthesis-{}", spec.id)
2202                        },
2203                        run: run_id,
2204                        node: "advise",
2205                        cache_dir: None,
2206                        attachments: &[],
2207                        writable: &[],
2208                    },
2209                ))
2210                .await;
2211            if outcome.is_ok() {
2212                self.state.seats.insert(seat.key.clone(), seat);
2213            }
2214            let advance = agent::chain_advances(&outcome);
2215            last = Some(outcome);
2216            if !advance {
2217                break;
2218            }
2219        }
2220        // Exhausted: the last attempt's result is what a single failed seat
2221        // would have produced.
2222        let out = last.expect("a chain holds at least one agent")?;
2223        if !out.usable() {
2224            return Ok(None);
2225        }
2226        let text =
2227            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2228        Ok((!text.trim().is_empty()).then_some(text))
2229    }
2230
2231    // ----------------------------------------------------------- implement
2232
2233    async fn implement(&mut self) -> Result<()> {
2234        self.persist_candidates().await?;
2235        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2236        // agent files with `magi task add` name the run that paid for it. The
2237        // prompt overlay is cloned alongside it because the waves borrow it
2238        // while `self` is mutably borrowed by the node's own bookkeeping.
2239        let run_id = self.state.id.clone();
2240        let prompts = self.state.config.prompts.clone();
2241        let todo: Vec<usize> = self
2242            .state
2243            .candidates
2244            .iter()
2245            .enumerate()
2246            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2247            .map(|(i, _)| i)
2248            .collect();
2249        if todo.is_empty() {
2250            return self.after_implement();
2251        }
2252        self.state.status = RunStatus::Implementing;
2253
2254        let language = self.state.config.graph.language.clone();
2255        let github_language = self.state.config.graph.github_language.clone();
2256        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2257        let sessions = self.state.config.graph.sessions;
2258        let artifacts = agent::artifacts_dir(&self.state.dir());
2259        // The design-deliberation stage's blended brief, when `advise` found
2260        // one — carried into every implementer's prompt the same way
2261        // regardless of which candidate it is.
2262        let brief = self
2263            .state
2264            .advice
2265            .as_ref()
2266            .and_then(|a| a.synthesis.as_deref())
2267            .map(str::to_owned);
2268        let attachments = self.state.attachments.clone();
2269
2270        let mut jobs = Vec::new();
2271        for &i in &todo {
2272            let (index, label, worktree) = {
2273                let c = &self.state.candidates[i];
2274                (c.index, c.label, c.worktree.clone())
2275            };
2276            let spec = self.roles.implementers[index].clone();
2277            let seat_key = format!("impl-{label}");
2278            let seat = self.seat(&seat_key, &spec.id);
2279            let instruction = seeded_instruction(&self.state);
2280            jobs.push(SeatJob {
2281                spec,
2282                seat,
2283                prompt: prompt::implement_in(
2284                    &instruction,
2285                    &worktree.to_string_lossy(),
2286                    &language,
2287                    &github_language,
2288                    brief.as_deref(),
2289                    &attachments,
2290                ),
2291                cwd: worktree,
2292                timeout,
2293                allow_write: true,
2294                sessions,
2295                artifacts: artifacts.clone(),
2296                stem: format!("impl-{label}"),
2297                handover: None,
2298            });
2299        }
2300
2301        self.state.event(
2302            "implement",
2303            format!("{} candidates in parallel", jobs.len()),
2304        );
2305        // Kept so a seat whose CLI hung up can be asked again from the same
2306        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2307        // can update a seat's own entry once a fallback agent takes it over —
2308        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2309        // whichever agent actually answered, not the one that quota'd out.
2310        let mut sent = jobs.clone();
2311        let cache = self.state.config.cache_dir();
2312        let ctx = WaveCtx {
2313            carry_seats: false,
2314            pauses: Some((&self.pause, &self.interrupt)),
2315            run: &run_id,
2316            node: "implement",
2317            prompts: &prompts,
2318            cache: cache.as_deref(),
2319            round: None,
2320        };
2321        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2322        if !wave_blocked(&results) {
2323            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2324                .await;
2325            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2326                .await?;
2327            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2328                .await;
2329        }
2330        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2331
2332        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2333            && results
2334                .iter()
2335                .any(|(_, _, out)| FailClass::of(out).is_some());
2336        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2337            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2338                continue;
2339            }
2340            let seat_key = seat.key.clone();
2341            // A quota fallback (`resume_seat_handovers`) may have handed this
2342            // seat to a different agent than the one `prep` recorded on the
2343            // candidate; the stats tables and any later fixer-defaults-to-
2344            // winner's-author lookup must credit whoever actually answered —
2345            // unless every fallback also quota'd out, in which case nobody
2346            // actually answered and crediting the last agent tried would
2347            // erase every earlier agent's own quota loss from the stats
2348            // tables instead of just this one seat's.
2349            let agent = seat.agent.clone();
2350            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2351            self.state.seats.insert(seat.key.clone(), seat);
2352            let label = self.state.candidates[i].label;
2353
2354            let (summary, duration, failed, verified_claim) = match out {
2355                AgentOutcome::Ok(o) => {
2356                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2357                    let failed = (!o.usable()).then(|| {
2358                        if o.timed_out {
2359                            "agent timed out".to_owned()
2360                        } else {
2361                            format!(
2362                                "agent exited with {}",
2363                                crate::notices::exit_status(o.exit_code)
2364                            )
2365                        }
2366                    });
2367                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2368                    (text, o.duration_ms, failed, verified_claim)
2369                }
2370                // Left un-resumed by `resume_undelivered` (a dirty tree
2371                // already rescues the work, or there was no session left to
2372                // resume into) — reported like the ordinary failure it is,
2373                // never as if `o.text` (the CLI's raw error JSON) were an
2374                // answer.
2375                AgentOutcome::Dropped(o) => {
2376                    let why = o
2377                        .dropped
2378                        .as_ref()
2379                        .map(|d| d.why.as_str())
2380                        .unwrap_or("the CLI ended the stream without delivering its answer");
2381                    (
2382                        String::new(),
2383                        o.duration_ms,
2384                        Some(format!("the CLI dropped the stream ({why})")),
2385                        None,
2386                    )
2387                }
2388                AgentOutcome::Quota(o) => {
2389                    self.state.quota.push(QuotaLoss {
2390                        seat: seat_key,
2391                        node: "implement".to_owned(),
2392                        at: Timestamp::now(),
2393                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2394                    });
2395                    (
2396                        String::new(),
2397                        o.duration_ms,
2398                        Some("rate limited (quota); produced no change".to_owned()),
2399                        None,
2400                    )
2401                }
2402                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2403                    (String::new(), 0, Some(e), None)
2404                }
2405            };
2406
2407            // Save every completed turn before any fallible Git operation. A failed
2408            // rescue must not erase the reply or prevent collection of other seats.
2409            let c = &mut self.state.candidates[i];
2410            if !exhausted_the_fallback_chain {
2411                c.agent = agent;
2412            }
2413            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2414            c.duration_ms = duration;
2415            c.failed = failed;
2416            c.verified_noop = verified_claim;
2417            self.state
2418                .candidate_preservation
2419                .insert(label, "awaiting persistence".to_owned());
2420        }
2421        self.state.save()?;
2422        self.persist_candidates().await?;
2423
2424        if resource_blocked {
2425            return self.stop_for_resource();
2426        }
2427        if selected_failed {
2428            self.state.status = RunStatus::Blocked;
2429            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2430            self.state.save()?;
2431            bail!(
2432                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2433            );
2434        }
2435        self.after_implement()
2436    }
2437
2438    /// Retry only persistence after a completed implementation turn.
2439    async fn persist_candidates(&mut self) -> Result<()> {
2440        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2441        for label in labels {
2442            let i = self
2443                .state
2444                .candidates
2445                .iter()
2446                .position(|c| c.label == label)
2447                .context("pending candidate is missing")?;
2448            let c = self.state.candidates[i].clone();
2449            let result = async {
2450                let wait = crate::repo_lock::WaitPolicy::new(
2451                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2452                    || false,
2453                );
2454                let _coordination =
2455                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2456                // Best effort evidence requires no index writes. A successful snapshot
2457                // never authorizes removal of a tree whose rescue failed.
2458                if let Err(e) = git::snapshot_uncommitted(
2459                    &c.worktree,
2460                    &self
2461                        .state
2462                        .dir()
2463                        .join("artifacts")
2464                        .join(format!("candidate-{label}-recovery")),
2465                )
2466                .await
2467                {
2468                    self.state.event(
2469                        "preserve",
2470                        format!("candidate {label} snapshot incomplete: {e:#}"),
2471                    );
2472                }
2473                let rescued = git::rescue_commit(
2474                    &c.worktree,
2475                    &format!("magi: candidate {label} (uncommitted work)"),
2476                )
2477                .await?;
2478                self.state.note_withheld("implement", &rescued.withheld);
2479                let base = &self.state.base_commit;
2480                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2481                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2482                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2483                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2484                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2485                if c.empty && commits > 0 && !patch.trim().is_empty()
2486                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2487                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2488                        || self.state.tally.is_some())
2489                {
2490                    // Keep the old classification and pending marker so every
2491                    // retry stops too. Retain decisions as recovery evidence;
2492                    // only a supported recovery operation may replace them.
2493                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2494                }
2495                let c = &mut self.state.candidates[i];
2496                c.commits = commits;
2497                c.stat = stat;
2498                c.files = files;
2499                c.empty = commits == 0 || patch.trim().is_empty();
2500                if !c.empty {
2501                    c.failed = None;
2502                    c.verified_noop = None;
2503                }
2504                self.state.event(
2505                    "preserve",
2506                    format!("candidate {label}: {files} files, {commits} commits"),
2507                );
2508                Ok::<_, anyhow::Error>(())
2509            }
2510            .await;
2511            match result {
2512                Ok(()) => {
2513                    self.state.candidate_preservation.remove(&label);
2514                }
2515                Err(e) => {
2516                    let reason = format!("candidate {label} preservation failed: {e:#}");
2517                    self.state.event("preserve", &reason);
2518                    self.state.candidate_preservation.insert(label, reason);
2519                }
2520            }
2521            if self.state.candidate_preservation.is_empty()
2522                && self.state.status == RunStatus::Blocked
2523                && self.state.candidates.iter().all(|c| c.failed.is_none())
2524            {
2525                // Persist the recovered phase with the cleared pending marker;
2526                // a crash between saves must not turn this into a selected-seat
2527                // failure hold on the next resume.
2528                self.state.status = RunStatus::Implementing;
2529                self.state.event(
2530                    "preserve",
2531                    "candidate persistence recovered; implementation can continue",
2532                );
2533            }
2534            self.state.save()?;
2535        }
2536        if !self.state.candidate_preservation.is_empty() {
2537            self.state.status = RunStatus::Blocked;
2538            self.state.save()?;
2539            bail!(
2540                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2541            );
2542        }
2543        Ok(())
2544    }
2545
2546    /// Ask again, once, for work a CLI did and then failed to hand over.
2547    ///
2548    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2549    /// status with an empty response and a usage report showing output tokens,
2550    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2551    /// seven minutes and 14,267 output tokens that arrived as an empty
2552    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2553    ///
2554    /// Two conditions, and both matter:
2555    ///
2556    /// - **Only when the tree is untouched.** Often the agent has already
2557    ///   written its files and only the closing message was lost; the rescue
2558    ///   commit below picks that up and there is nothing to ask for. Re-asking
2559    ///   then would pay for a second implementation of work already on disk.
2560    /// - **Once.** A CLI that drops one stream can drop the next, and this
2561    ///   node is the most expensive in the graph.
2562    ///
2563    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2564    /// dropped reply still carried its `conversation_id`, so the seat is asked
2565    /// to finish what it was doing rather than sent the whole task again. It
2566    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2567    /// node's - for the same reason a re-ranked judge does: restating finished
2568    /// work is not the work.
2569    ///
2570    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2571    /// same way until it resets, while an abandoned conversation is still
2572    /// there to be picked up.
2573    async fn resume_undelivered(
2574        &mut self,
2575        results: &mut [(usize, SeatState, AgentOutcome)],
2576        sent: &[SeatJob],
2577        prompts: &Prompts,
2578        run_id: &str,
2579    ) {
2580        for (wi, seat, out) in results.iter_mut() {
2581            let Some(dropped) = (match &*out {
2582                AgentOutcome::Dropped(o) => o.dropped.clone(),
2583                _ => None,
2584            }) else {
2585                continue;
2586            };
2587            let Some(job) = sent.get(*wi) else { continue };
2588            // Already on disk? Then only the closing message was lost.
2589            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2590                self.state.event(
2591                    "implement",
2592                    format!(
2593                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2594                         work is in the tree",
2595                        seat.key, dropped.output_tokens, dropped.why
2596                    ),
2597                );
2598                continue;
2599            }
2600            // The re-ask only makes sense as a resume: `resume_after_drop`
2601            // says nothing about the task, trusting the seat to still hold it.
2602            // Without a session to resume — sessions disabled, or this CLI's
2603            // drop shape happened not to carry a session id — that prompt
2604            // would open a brand-new conversation with no context at all,
2605            // which is worse than leaving this as the ordinary failure it
2606            // already is.
2607            if !has_context(&job.spec, seat, job.sessions) {
2608                self.state.event(
2609                    "implement",
2610                    format!(
2611                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2612                         is no session left to resume",
2613                        seat.key, dropped.output_tokens, dropped.why
2614                    ),
2615                );
2616                continue;
2617            }
2618            self.state.event(
2619                "implement",
2620                format!(
2621                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2622                     conversation",
2623                    seat.key, dropped.output_tokens, dropped.why
2624                ),
2625            );
2626            let mut retry = job.clone();
2627            retry.seat = seat.clone();
2628            retry.prompt = prompt::resume_after_drop(&dropped.why);
2629            retry.timeout = retry_budget(job.timeout, true);
2630            retry.stem = format!("{}-resume", job.stem);
2631            let cache = self.state.config.cache_dir();
2632            let ctx = WaveCtx {
2633                carry_seats: false,
2634                pauses: Some((&self.pause, &self.interrupt)),
2635                run: run_id,
2636                node: "implement",
2637                prompts,
2638                cache: cache.as_deref(),
2639                round: None,
2640            };
2641            let (resumed_seat, resumed) =
2642                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2643            *seat = resumed_seat;
2644            *out = resumed;
2645        }
2646    }
2647
2648    /// Fall an implement seat through to the next untried agent in the
2649    /// implementer roster when it lost to quota — or, since the handover was
2650    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2651    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2652    /// quota path itself being unchanged — instead of leaving the
2653    /// seat's loss final the moment one agent's account runs dry.
2654    ///
2655    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2656    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2657    /// truncates to the single slot rotation picked, so a solo task whose one
2658    /// implementer hits quota mid-run used to have nothing else to try. This
2659    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2660    /// unrotated roster — which is the only place the *other* candidates in
2661    /// the machine's roster still exist once `implementers` has been cut down
2662    /// to size.
2663    ///
2664    /// Walks forward from just past the seat's own original position in the
2665    /// roster, never wrapping back to the front: a later candidate slot (say
2666    /// `beta`, the roster's second entry) must fall through to the *next*
2667    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2668    /// almost certainly a different candidate's own agent already — and once
2669    /// the roster's tail is exhausted there is nothing left to fall through
2670    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2671    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2672    /// let this retry that id forever. The loop keeps falling through until
2673    /// an attempt lands something other than `Quota` or the roster's tail
2674    /// runs out of untried ids, at which point the seat is left exactly as
2675    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2676    /// `QuotaLoss` recorded, the candidate failed/empty.
2677    ///
2678    /// `sent` is taken mutably and updated with the fallback agent's spec:
2679    /// `resume_unconfirmed_commands`, which runs after this and also reads
2680    /// `sent`, must see whichever agent actually ended up answering the seat
2681    /// — reading the stale, original spec there would check session
2682    /// eligibility against the wrong CLI and could hand a fallback agent's
2683    /// session id to the agent that just lost the seat to quota.
2684    ///
2685    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2686    /// — `self.seat` only reuses state when the agent id is unchanged, so
2687    /// handing it a different id already gets this for free. Reusing the old
2688    /// seat would resume a different CLI's session as if it were a
2689    /// continuation of this one.
2690    ///
2691    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2692    /// a quota loss cuts an agent off mid-turn, so anything already in the
2693    /// tree is unfinished work, not a completed candidate a re-ask would pay
2694    /// for twice. A dirty tree is rescued into a commit first (the same
2695    /// neutral-identity rescue `implement`'s own outcome loop gives every
2696    /// candidate) so the next agent starts clean.
2697    ///
2698    /// The new agent gets the implementer's full prompt and full
2699    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2700    /// it has no session and no context, and is implementing the task from
2701    /// nothing, unlike a resumed drop which is only restating work already
2702    /// done.
2703    ///
2704    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2705    /// `implement` event, never into `self.state.quota` — that is what
2706    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2707    /// go unspent, and a seat that ultimately recovered on its second or
2708    /// third agent is not the stalled panel that check exists to catch. Only
2709    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2710    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2711    /// outcome loop already has — this helper never pushes to it itself.
2712    async fn resume_seat_handovers(
2713        &mut self,
2714        results: &mut [(usize, SeatState, AgentOutcome)],
2715        sent: &mut [SeatJob],
2716        prompts: &Prompts,
2717        run_id: &str,
2718    ) -> Result<()> {
2719        let instruction = seeded_instruction(&self.state);
2720        let language = self.state.config.graph.language.clone();
2721        let github_language = self.state.config.graph.github_language.clone();
2722        let brief = self
2723            .state
2724            .advice
2725            .as_ref()
2726            .and_then(|a| a.synthesis.as_deref())
2727            .map(str::to_owned);
2728        let attachments = self.state.attachments.clone();
2729        for (wi, seat, out) in results.iter_mut() {
2730            // Who holds the other candidate seats of this wave right now
2731            // (earlier handovers already written back to `sent`).
2732            let others: BTreeSet<String> = sent
2733                .iter()
2734                .enumerate()
2735                .filter(|(j, _)| j != wi)
2736                .map(|(_, j)| j.spec.id.clone())
2737                .collect();
2738            let Some(job) = sent.get_mut(*wi) else {
2739                continue;
2740            };
2741            // Where the seat's own original agent sits in the roster — the
2742            // fallback walk starts just past here, never at the front, so a
2743            // later candidate slot's quota loss does not fall back onto an
2744            // earlier slot's own agent.
2745            let start = self
2746                .roles
2747                .implementer_roster
2748                .iter()
2749                .position(|s| s.id == job.spec.id)
2750                .unwrap_or(0);
2751            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2752            let mut fallback_attempt = 0usize;
2753            let mut prev: Option<FailClass> = None;
2754            while let Some(cur) = FailClass::of(&*out) {
2755                if !should_hand_over(prev.as_ref(), &cur) {
2756                    break;
2757                }
2758                let Some(next) =
2759                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2760                        .cloned()
2761                else {
2762                    break;
2763                };
2764                tried.insert(next.id.clone());
2765                fallback_attempt += 1;
2766
2767                let coordination = self.coordinate(&job.cwd).await?;
2768                if let Ok(r) = git::rescue_commit(
2769                    &job.cwd,
2770                    &format!(
2771                        "magi: candidate {} (uncommitted work before {} fallback)",
2772                        seat.key,
2773                        if cur == FailClass::Quota {
2774                            "quota"
2775                        } else {
2776                            "handover"
2777                        }
2778                    ),
2779                )
2780                .await
2781                {
2782                    self.state.note_withheld("implement", &r.withheld);
2783                }
2784                drop(coordination);
2785
2786                record_handover(
2787                    &mut self.state,
2788                    "implement",
2789                    &seat.key,
2790                    &seat.agent,
2791                    &next.id,
2792                    &cur,
2793                    &fail_reason(&*out),
2794                );
2795                prev = Some(cur.clone());
2796
2797                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2798                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2799                // Kept in sync on `sent` itself, not just the local retry: a
2800                // later helper (`resume_unconfirmed_commands`) reads `sent`
2801                // after this one returns and must see whichever agent is now
2802                // occupying the seat, not the one that just quota'd out —
2803                // otherwise it would judge session/continuation eligibility
2804                // by the wrong CLI and could resend a fallback's session id
2805                // to the agent that lost it the seat in the first place.
2806                job.spec = next.clone();
2807                let mut retry = job.clone();
2808                retry.seat = new_seat;
2809                retry.prompt = prompt::implement_in(
2810                    &instruction,
2811                    &job.cwd.to_string_lossy(),
2812                    &language,
2813                    &github_language,
2814                    brief.as_deref(),
2815                    &attachments,
2816                );
2817                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2818                let cache = self.state.config.cache_dir();
2819                let ctx = WaveCtx {
2820                    carry_seats: false,
2821                    pauses: Some((&self.pause, &self.interrupt)),
2822                    run: run_id,
2823                    node: "implement",
2824                    prompts,
2825                    cache: cache.as_deref(),
2826                    round: None,
2827                };
2828                let (fallback_seat, fallback_out) = run_one(
2829                    retry,
2830                    Arc::clone(&self.sem),
2831                    &ctx,
2832                    &mut self.state,
2833                    fallback_attempt,
2834                )
2835                .await;
2836                *seat = fallback_seat;
2837                *out = fallback_out;
2838            }
2839        }
2840        Ok(())
2841    }
2842
2843    /// Ask an implement seat's own CLI to confirm what it started, once, when
2844    /// its reply reported a command whose completion status it never
2845    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2846    /// that does and does not mean.
2847    ///
2848    /// The completion contract this task asks for, extended to `implement`
2849    /// with the same signal `continue_fix_report` reads for the fixer,
2850    /// rather than a keyword search over the reply or a hard requirement on
2851    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2852    /// a candidate's CLI turn ended cleanly while a test run it had started
2853    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2854    /// named in it at all is untouched by this: `commands` is empty, so
2855    /// there is nothing to be unconfirmed.
2856    ///
2857    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2858    /// this is not about recovering edits that might already be on disk, it
2859    /// is about a result the seat itself never vouched for, which resuming
2860    /// asks for regardless of what the tree already holds. Bounded to one
2861    /// attempt for the same reason `resume_undelivered` is — this is the
2862    /// most expensive node in the graph — and a seat that still cannot
2863    /// confirm on that attempt is left as whatever its (possibly still
2864    /// unconfirmed) reply says; this does not invent a new "failed" reason
2865    /// for a candidate that otherwise produced a real, committed change.
2866    async fn resume_unconfirmed_commands(
2867        &mut self,
2868        results: &mut [(usize, SeatState, AgentOutcome)],
2869        sent: &[SeatJob],
2870        prompts: &Prompts,
2871        run_id: &str,
2872    ) {
2873        for (wi, seat, out) in results.iter_mut() {
2874            let AgentOutcome::Ok(o) = &*out else {
2875                continue;
2876            };
2877            if !has_unconfirmed_command(&o.commands) {
2878                continue;
2879            }
2880            let Some(job) = sent.get(*wi) else { continue };
2881            if !has_context(&job.spec, seat, job.sessions) {
2882                self.state.event(
2883                    "implement",
2884                    format!(
2885                        "{}: the reply named a command whose own CLI never confirmed the exit \
2886                         status of, but there is no session left to resume",
2887                        seat.key
2888                    ),
2889                );
2890                continue;
2891            }
2892            self.state.event(
2893                "implement",
2894                format!(
2895                    "{}: the reply named a command whose own CLI never confirmed the exit \
2896                     status of; resuming the conversation",
2897                    seat.key
2898                ),
2899            );
2900            let mut retry = job.clone();
2901            retry.seat = seat.clone();
2902            retry.prompt = prompt::resume_incomplete(
2903                "a command in your last reply had no confirmed exit status",
2904            );
2905            retry.timeout = retry_budget(job.timeout, true);
2906            retry.stem = format!("{}-confirm", job.stem);
2907            let cache = self.state.config.cache_dir();
2908            let ctx = WaveCtx {
2909                carry_seats: false,
2910                pauses: Some((&self.pause, &self.interrupt)),
2911                run: run_id,
2912                node: "implement",
2913                prompts,
2914                cache: cache.as_deref(),
2915                round: None,
2916            };
2917            let (resumed_seat, resumed) =
2918                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2919            *seat = resumed_seat;
2920            *out = resumed;
2921        }
2922    }
2923
2924    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
2925    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
2926    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
2927    /// that motivated this.
2928    ///
2929    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
2930    /// own nudge loop already covers for judge/review/vote seats, and not a
2931    /// dropped stream, which [`Runner::resume_undelivered`] covers for
2932    /// implement seats: here the CLI turn genuinely finished while the node's
2933    /// own work — the fixer's account of what it did — had not. Gated purely
2934    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
2935    /// reply, never on any wording in it, so a fixer whose valid, first-try
2936    /// `FixReport` happens to mention having waited on a background test is
2937    /// never resumed — the `Ok(report)` branch at the call site returns
2938    /// before this is ever invoked.
2939    ///
2940    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
2941    /// attempt ([`retry_budget`]), nothing attempted once the session is
2942    /// gone, and a quota hit ends the loop immediately rather than retrying a
2943    /// rate limit that fails the same way again.
2944    async fn continue_fix_report(
2945        &mut self,
2946        mut seat: SeatState,
2947        parse_err: String,
2948        job: &SeatJob,
2949        prompts: &Prompts,
2950        run_id: &str,
2951        round: usize,
2952    ) -> (
2953        SeatState,
2954        Option<FixReport>,
2955        Option<String>,
2956        ContinuationRecord,
2957    ) {
2958        let mut last_err = parse_err;
2959        let mut cumulative_wait_ms = 0u64;
2960        let mut attempts = 0usize;
2961        loop {
2962            if !has_context(&job.spec, &seat, job.sessions) {
2963                self.state.event(
2964                    "fix",
2965                    format!(
2966                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
2967                         session left to resume into"
2968                    ),
2969                );
2970                let outcome = if attempts == 0 {
2971                    ContinuationOutcome::NoSession
2972                } else {
2973                    ContinuationOutcome::Exhausted
2974                };
2975                return (
2976                    seat,
2977                    None,
2978                    Some(format!("unparsable fix report: {last_err}")),
2979                    ContinuationRecord {
2980                        attempts,
2981                        cumulative_wait_ms,
2982                        outcome,
2983                    },
2984                );
2985            }
2986            if attempts >= MAX_FIX_CONTINUATIONS {
2987                self.state.event(
2988                    "fix",
2989                    format!(
2990                        "round {round}: fixer's reply still had no adoption report after \
2991                         {attempts} continuation(s) ({last_err}); giving up"
2992                    ),
2993                );
2994                return (
2995                    seat,
2996                    None,
2997                    Some(format!(
2998                        "unparsable fix report after {attempts} continuation(s): {last_err}"
2999                    )),
3000                    ContinuationRecord {
3001                        attempts,
3002                        cumulative_wait_ms,
3003                        outcome: ContinuationOutcome::Exhausted,
3004                    },
3005                );
3006            }
3007            attempts += 1;
3008            self.state.event(
3009                "fix",
3010                format!(
3011                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3012                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3013                ),
3014            );
3015            let mut retry = job.clone();
3016            retry.seat = seat.clone();
3017            retry.prompt = prompt::resume_incomplete(&last_err);
3018            retry.timeout = retry_budget(job.timeout, true);
3019            retry.stem = format!("{}-continue{attempts}", job.stem);
3020            let cache = self.state.config.cache_dir();
3021            let ctx = WaveCtx {
3022                carry_seats: false,
3023                pauses: Some((&self.pause, &self.interrupt)),
3024                run: run_id,
3025                node: "fix",
3026                prompts,
3027                cache: cache.as_deref(),
3028                round: Some(round),
3029            };
3030            let (resumed_seat, resumed_out) = run_one(
3031                retry,
3032                Arc::clone(&self.sem),
3033                &ctx,
3034                &mut self.state,
3035                attempts,
3036            )
3037            .await;
3038            seat = resumed_seat;
3039            match resumed_out {
3040                AgentOutcome::Ok(o) => {
3041                    cumulative_wait_ms += o.duration_ms;
3042                    match verdict::extract_json::<FixReport>(&o.text) {
3043                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3044                            self.state.event(
3045                                "fix",
3046                                format!(
3047                                    "round {round}: fixer's adoption report recovered after \
3048                                     {attempts} continuation(s)"
3049                                ),
3050                            );
3051                            return (
3052                                seat,
3053                                Some(report),
3054                                None,
3055                                ContinuationRecord {
3056                                    attempts,
3057                                    cumulative_wait_ms,
3058                                    outcome: ContinuationOutcome::Resumed,
3059                                },
3060                            );
3061                        }
3062                        // The report parsed, but this same reply's own
3063                        // CommandEvidence — the identical record `state.jobs`
3064                        // renders — names a command whose CLI never
3065                        // confirmed an exit status. Read together, that is
3066                        // not a resolved answer: keep nudging rather than
3067                        // accept a report standing next to a command the
3068                        // seat's own CLI cannot vouch for.
3069                        Ok(_) => {
3070                            last_err = "the reply parsed, but it reported a command whose own CLI \
3071                                 never confirmed an exit status"
3072                                .to_owned();
3073                        }
3074                        Err(e) => last_err = e.to_string(),
3075                    }
3076                }
3077                AgentOutcome::Quota(o) => {
3078                    cumulative_wait_ms += o.duration_ms;
3079                    self.state.quota.push(QuotaLoss {
3080                        seat: seat.key.clone(),
3081                        node: "fix".to_owned(),
3082                        at: Timestamp::now(),
3083                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3084                    });
3085                    self.state.event(
3086                        "fix",
3087                        format!(
3088                            "round {round}: continuation rate limited (quota); not retrying now"
3089                        ),
3090                    );
3091                    return (
3092                        seat,
3093                        None,
3094                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3095                        ContinuationRecord {
3096                            attempts,
3097                            cumulative_wait_ms,
3098                            outcome: ContinuationOutcome::QuotaLost,
3099                        },
3100                    );
3101                }
3102                AgentOutcome::Dropped(o) => {
3103                    cumulative_wait_ms += o.duration_ms;
3104                    let why = o
3105                        .dropped
3106                        .as_ref()
3107                        .map(|d| d.why.as_str())
3108                        .unwrap_or("the CLI ended the stream without delivering its answer");
3109                    last_err = format!("the CLI dropped the stream ({why})");
3110                }
3111                AgentOutcome::Failed(e) => last_err = e,
3112                // The continuation was never started; the report stays lost
3113                // and the caller records that, without any handover.
3114                AgentOutcome::ResourceBlocked(e) => {
3115                    return (
3116                        seat,
3117                        None,
3118                        Some(format!("the continuation was not started: {e}")),
3119                        ContinuationRecord {
3120                            attempts: attempts.saturating_sub(1),
3121                            cumulative_wait_ms,
3122                            outcome: ContinuationOutcome::NoSession,
3123                        },
3124                    );
3125                }
3126            }
3127        }
3128    }
3129
3130    fn after_implement(&mut self) -> Result<()> {
3131        if !self.state.candidate_preservation.is_empty() {
3132            bail!("candidate preservation incomplete; judging is unsafe");
3133        }
3134        // Scan every candidate patch once the set is complete.
3135        if self.state.leaks.is_empty() {
3136            let cfg = self.state.config.blind.clone();
3137            let mut leaks = Vec::new();
3138            for c in &self.state.candidates {
3139                let Some(patch) =
3140                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3141                else {
3142                    continue;
3143                };
3144                leaks.extend(blind::scan(
3145                    &format!("candidate {} patch", c.label),
3146                    &patch,
3147                    &cfg.vendor_tokens,
3148                ));
3149            }
3150            if !leaks.is_empty() {
3151                let summary = leaks
3152                    .iter()
3153                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3154                    .collect::<Vec<_>>()
3155                    .join(", ");
3156                match cfg.on_leak {
3157                    LeakPolicy::Fail => {
3158                        self.state.status = RunStatus::Failed;
3159                        self.state
3160                            .event("blind", format!("vendor text in a patch: {summary}"));
3161                        self.state.leaks = leaks;
3162                        self.state.save()?;
3163                        self.settle_questions();
3164                        bail!(
3165                            "blind.on_leak = \"fail\" and vendor text reached a \
3166                             judged patch: {summary}"
3167                        );
3168                    }
3169                    LeakPolicy::Redact => self.state.event(
3170                        "blind",
3171                        format!("redacting vendor text for judging: {summary}"),
3172                    ),
3173                    LeakPolicy::Warn => self.state.event(
3174                        "blind",
3175                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3176                    ),
3177                }
3178                self.state.leaks = leaks;
3179            }
3180        }
3181
3182        if self.state.viable().is_empty() {
3183            if self.state.all_candidates_verified_noop() {
3184                // Every candidate agreed, with evidence the adoption guard
3185                // accepted, that nothing belongs in this worktree. That is
3186                // not the same fact as a candidate that simply failed to
3187                // write anything, and settling it as an ordinary `Failed`
3188                // (see `SCHEMA`'s doc for schema 10) is what let two of
3189                // task 391f's attempts burn a retry each re-discovering the
3190                // same already-landed fix. Terminal either way, so `judge`
3191                // must never run over an empty candidate set — unlike the
3192                // `Failed` branch below this returns `Ok`, not an error:
3193                // nothing here failed.
3194                self.state.status = RunStatus::VerifiedNoop;
3195                self.state.save()?;
3196                self.settle_questions();
3197                return Ok(());
3198            }
3199            self.state.status = RunStatus::Failed;
3200            self.state.save()?;
3201            self.settle_questions();
3202            bail!("no candidate produced a change; nothing to judge");
3203        }
3204        self.state.status = RunStatus::Judging;
3205        self.state.save()?;
3206        Ok(())
3207    }
3208
3209    // --------------------------------------------------------------- judge
3210
3211    async fn judge(&mut self) -> Result<()> {
3212        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3213        // agent files with `magi task add` name the run that paid for it. The
3214        // prompt overlay is cloned alongside it because the waves borrow it
3215        // while `self` is mutably borrowed by the node's own bookkeeping.
3216        let run_id = self.state.id.clone();
3217        let prompts = self.state.config.prompts.clone();
3218        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3219            return Ok(());
3220        }
3221        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3222        if viable.len() == 1 {
3223            // Recorded so this is a one-time event: `judgements` stays empty
3224            // either way, which without this flag is indistinguishable from
3225            // "not yet judged" on the next reentry — and status is left
3226            // untouched, so a later node's conclusion (e.g. `Blocked` after
3227            // the review budget ran out) survives a resume instead of being
3228            // clobbered back to `Judging` by this node running again.
3229            self.state.judge_skipped = true;
3230            self.state.event(
3231                "judge",
3232                format!(
3233                    "only candidate {} produced a change; judging skipped",
3234                    viable[0].label
3235                ),
3236            );
3237            self.state.save()?;
3238            return Ok(());
3239        }
3240        self.state.status = RunStatus::Judging;
3241
3242        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3243        let language = self.state.config.graph.language.clone();
3244        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3245        let sessions = self.state.config.graph.sessions;
3246        let artifacts = agent::artifacts_dir(&self.state.dir());
3247        let root = self.state.worktree_root();
3248        let base_short = short(&self.state.base_commit);
3249
3250        let mut jobs = Vec::new();
3251        let mut orders = Vec::new();
3252        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3253            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3254            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3255            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3256            let seat_key = format!("judge-{}", j + 1);
3257            let seat = self.seat(&seat_key, &spec.id);
3258            jobs.push(SeatJob {
3259                prompt: prompt::judge(
3260                    &self.state.instruction,
3261                    &views,
3262                    self.roles.judges.len(),
3263                    &base_short,
3264                    &language,
3265                ),
3266                spec,
3267                seat,
3268                cwd: root.join(format!("judge-{}", j + 1)),
3269                timeout,
3270                allow_write: false,
3271                sessions,
3272                artifacts: artifacts.clone(),
3273                stem: format!("judge-{}", j + 1),
3274                handover: None,
3275            });
3276        }
3277
3278        self.state.event(
3279            "judge",
3280            format!(
3281                "{} judges ranking {} candidates blind",
3282                jobs.len(),
3283                viable.len()
3284            ),
3285        );
3286        let labels_for_check = labels.clone();
3287        let mut quota_losses = Vec::new();
3288        let cache = self.state.config.cache_dir();
3289        let ctx = WaveCtx {
3290            carry_seats: false,
3291            pauses: Some((&self.pause, &self.interrupt)),
3292            run: &run_id,
3293            node: "judge",
3294            prompts: &prompts,
3295            cache: cache.as_deref(),
3296            round: None,
3297        };
3298        let results = ask_json_wave::<Ranking>(
3299            jobs,
3300            Arc::clone(&self.sem),
3301            self.state.config.graph.retries,
3302            &self.roles.judge_roster,
3303            &ctx,
3304            &mut quota_losses,
3305            &mut self.state,
3306            &move |r: &Ranking| r.validate(&labels_for_check),
3307        )
3308        .await;
3309        if self.state.resource_block.is_some() {
3310            return self.stop_for_resource();
3311        }
3312        self.state.quota.extend(quota_losses);
3313
3314        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3315            let agent_id = seat.agent.clone();
3316            self.state.seats.insert(seat.key.clone(), seat);
3317            let mut record = Judgement {
3318                judge: j + 1,
3319                seat: format!("judge-{}", j + 1),
3320                agent: agent_id,
3321                ranking: Vec::new(),
3322                reasons: BTreeMap::new(),
3323                confidence: None,
3324                strengths: BTreeMap::new(),
3325                order: orders[j].clone(),
3326                failed: None,
3327                duration_ms: 0,
3328            };
3329            match res {
3330                Ok((ranking, out)) => {
3331                    record.ranking = ranking.normalized();
3332                    record.reasons = ranking.reasons;
3333                    record.strengths = ranking.strengths;
3334                    record.confidence = ranking.confidence;
3335                    record.duration_ms = out.duration_ms;
3336                    self.state.event(
3337                        "judge",
3338                        format!(
3339                            "judge {} ranked {}",
3340                            j + 1,
3341                            record.ranking.iter().collect::<String>()
3342                        ),
3343                    );
3344                }
3345                Err(e) => {
3346                    record.failed = Some(e.to_string());
3347                    self.state
3348                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3349                }
3350            }
3351            self.state.judgements.push(record);
3352            self.state.save()?;
3353        }
3354        Ok(())
3355    }
3356
3357    // ---------------------------------------------------------- deliberate
3358
3359    async fn deliberate(&mut self) -> Result<()> {
3360        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3361        // agent files with `magi task add` name the run that paid for it. The
3362        // prompt overlay is cloned alongside it because the waves borrow it
3363        // while `self` is mutably borrowed by the node's own bookkeeping.
3364        let run_id = self.state.id.clone();
3365        let prompts = self.state.config.prompts.clone();
3366        if !self.state.deliberation.is_empty() {
3367            return Ok(());
3368        }
3369        let tops: Vec<char> = self
3370            .state
3371            .judgements
3372            .iter()
3373            .filter_map(|j| j.ranking.first().copied())
3374            .collect();
3375        let rounds = self.state.config.graph.deliberate_rounds;
3376        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3377            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3378                self.state.event(
3379                    "deliberate",
3380                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3381                );
3382            }
3383            self.state.status = RunStatus::Voting;
3384            self.state.save()?;
3385            return Ok(());
3386        }
3387
3388        self.state.status = RunStatus::Deliberating;
3389        self.state.event(
3390            "deliberate",
3391            format!(
3392                "split: first choices were {} — opening {rounds} round(s)",
3393                tops.iter().collect::<String>()
3394            ),
3395        );
3396
3397        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3398        let language = self.state.config.graph.language.clone();
3399        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3400        let sessions = self.state.config.graph.sessions;
3401        let artifacts = agent::artifacts_dir(&self.state.dir());
3402        let root = self.state.worktree_root();
3403        let base_short = short(&self.state.base_commit);
3404
3405        // Judges argue in sequence so that a turn can answer the one before it;
3406        // that is the difference between deliberation and three parallel
3407        // monologues.
3408        for round in 1..=rounds {
3409            let mut turns: Vec<DeliberationTurn> = Vec::new();
3410            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3411                if self.state.judgements[j].failed.is_some() {
3412                    continue;
3413                }
3414                let seat_key = format!("judge-{}", j + 1);
3415                let spec = self.occupant(&seat_key, spec);
3416                let mut seat = self.seat(&seat_key, &spec.id);
3417                let transcript = self.transcript(&turns, j);
3418                let build = |context: Option<&str>| {
3419                    prompt::deliberate(
3420                        &self.state.instruction,
3421                        context,
3422                        &transcript,
3423                        round,
3424                        rounds,
3425                        &language,
3426                    )
3427                };
3428                let block = self.candidate_block(&viable, &base_short);
3429                let full = build(Some(&block));
3430                let text = if has_context(&spec, &seat, sessions) {
3431                    build(None)
3432                } else {
3433                    full.clone()
3434                };
3435                let job = SeatJob {
3436                    spec,
3437                    seat: seat.clone(),
3438                    prompt: text,
3439                    cwd: root.join(format!("judge-{}", j + 1)),
3440                    timeout,
3441                    allow_write: false,
3442                    sessions,
3443                    artifacts: artifacts.clone(),
3444                    stem: format!("delib-{round}-judge-{}", j + 1),
3445                    handover: Some(full),
3446                };
3447                let cache = self.state.config.cache_dir();
3448                let ctx = WaveCtx {
3449                    carry_seats: false,
3450                    pauses: Some((&self.pause, &self.interrupt)),
3451                    run: &run_id,
3452                    node: "deliberate",
3453                    prompts: &prompts,
3454                    cache: cache.as_deref(),
3455                    round: None,
3456                };
3457                // A turn is never nudged (`retries` 0): a failed seat is
3458                // handed to the next roster agent, which gets the full
3459                // context. An empty answer is a turn, not a failure.
3460                let mut losses = Vec::new();
3461                let mut results = ask_wave_with::<String>(
3462                    vec![job],
3463                    Arc::clone(&self.sem),
3464                    0,
3465                    &self.roles.judge_roster,
3466                    &ctx,
3467                    &mut losses,
3468                    &mut self.state,
3469                    &|text: &str| {
3470                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3471                    },
3472                )
3473                .await;
3474                self.state.quota.extend(losses);
3475                let (updated, res, _) = results.pop().expect("one job in, one result out");
3476                seat = updated;
3477                let agent_id = seat.agent.clone();
3478                self.state.seats.insert(seat.key.clone(), seat);
3479                let body = match res {
3480                    Ok((body, _)) => body,
3481                    // Skip the seat; a CLI's raw error JSON is never read as
3482                    // this judge's position.
3483                    Err(e) => {
3484                        self.state
3485                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3486                        continue;
3487                    }
3488                };
3489                let tentative = verdict::extract_json::<Position>(&body)
3490                    .ok()
3491                    .and_then(|p| p.tentative)
3492                    .and_then(|s| s.trim().chars().next())
3493                    .map(|c| c.to_ascii_uppercase());
3494                self.state.event(
3495                    "deliberate",
3496                    format!(
3497                        "round {round}: judge {} now favours {}",
3498                        j + 1,
3499                        tentative.map_or("—".to_owned(), |c| c.to_string())
3500                    ),
3501                );
3502                turns.push(DeliberationTurn {
3503                    judge: j + 1,
3504                    agent: agent_id,
3505                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3506                    tentative,
3507                });
3508            }
3509            self.state
3510                .deliberation
3511                .push(DeliberationRound { round, turns });
3512            self.state.save()?;
3513        }
3514
3515        self.state.status = RunStatus::Voting;
3516        self.state.save()?;
3517        Ok(())
3518    }
3519
3520    // ---------------------------------------------------------------- vote
3521
3522    async fn vote(&mut self) -> Result<()> {
3523        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3524        // agent files with `magi task add` name the run that paid for it. The
3525        // prompt overlay is cloned alongside it because the waves borrow it
3526        // while `self` is mutably borrowed by the node's own bookkeeping.
3527        let run_id = self.state.id.clone();
3528        let prompts = self.state.config.prompts.clone();
3529        if !self.state.votes.is_empty() {
3530            return Ok(());
3531        }
3532        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3533        if viable.len() == 1 {
3534            return Ok(());
3535        }
3536        self.state.status = RunStatus::Voting;
3537
3538        let language = self.state.config.graph.language.clone();
3539        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3540        let sessions = self.state.config.graph.sessions;
3541        let artifacts = agent::artifacts_dir(&self.state.dir());
3542        let root = self.state.worktree_root();
3543        let base_short = short(&self.state.base_commit);
3544        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3545
3546        let mut jobs = Vec::new();
3547        let mut seats_at = Vec::new();
3548        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3549            if self
3550                .state
3551                .judgements
3552                .get(j)
3553                .is_some_and(|r| r.failed.is_some())
3554            {
3555                continue;
3556            }
3557            let seat_key = format!("judge-{}", j + 1);
3558            let spec = self.occupant(&seat_key, spec);
3559            let seat = self.seat(&seat_key, &spec.id);
3560            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3561            let text = if has_context(&spec, &seat, sessions) {
3562                prompt::final_vote(&viable, &language)
3563            } else {
3564                full.clone()
3565            };
3566            jobs.push(SeatJob {
3567                spec,
3568                seat,
3569                prompt: text,
3570                cwd: root.join(format!("judge-{}", j + 1)),
3571                timeout,
3572                allow_write: false,
3573                sessions,
3574                artifacts: artifacts.clone(),
3575                stem: format!("vote-judge-{}", j + 1),
3576                handover: Some(full),
3577            });
3578            seats_at.push(j);
3579        }
3580
3581        self.state.event(
3582            "vote",
3583            format!(
3584                "collecting {} final votes one by one, privately",
3585                jobs.len()
3586            ),
3587        );
3588        let allowed = viable.clone();
3589        let mut quota_losses = Vec::new();
3590        let cache = self.state.config.cache_dir();
3591        let ctx = WaveCtx {
3592            carry_seats: false,
3593            pauses: Some((&self.pause, &self.interrupt)),
3594            run: &run_id,
3595            node: "vote",
3596            prompts: &prompts,
3597            cache: cache.as_deref(),
3598            round: None,
3599        };
3600        let results = ask_json_wave::<FinalVote>(
3601            jobs,
3602            Arc::clone(&self.sem),
3603            self.state.config.graph.retries,
3604            &self.roles.judge_roster,
3605            &ctx,
3606            &mut quota_losses,
3607            &mut self.state,
3608            &move |v: &FinalVote| match v.label() {
3609                Some(c) if allowed.contains(&c) => Ok(()),
3610                other => bail!("vote {other:?} is not one of {allowed:?}"),
3611            },
3612        )
3613        .await;
3614        if self.state.resource_block.is_some() {
3615            return self.stop_for_resource();
3616        }
3617        self.state.quota.extend(quota_losses);
3618
3619        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3620            let agent_id = seat.agent.clone();
3621            self.state.seats.insert(seat.key.clone(), seat);
3622            let initial = self
3623                .state
3624                .judgements
3625                .get(j)
3626                .and_then(|r| r.ranking.first().copied());
3627            let mut record = VoteRecord {
3628                judge: j + 1,
3629                agent: agent_id,
3630                vote: None,
3631                reason: String::new(),
3632                changed: false,
3633            };
3634            match res {
3635                Ok((v, _)) => {
3636                    record.vote = v.label();
3637                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3638                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3639                    self.state.event(
3640                        "vote",
3641                        format!(
3642                            "judge {} voted {}{}",
3643                            j + 1,
3644                            record.vote.unwrap_or('?'),
3645                            if record.changed { " (changed)" } else { "" }
3646                        ),
3647                    );
3648                }
3649                Err(e) => {
3650                    self.state
3651                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3652                }
3653            }
3654            self.state.votes.push(record);
3655            self.state.save()?;
3656        }
3657        Ok(())
3658    }
3659
3660    // --------------------------------------------------------------- tally
3661
3662    fn tally(&mut self) -> Result<()> {
3663        if self.state.tally.is_some() {
3664            return Ok(());
3665        }
3666        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3667        let tops: Vec<char> = self
3668            .state
3669            .judgements
3670            .iter()
3671            .filter_map(|j| j.ranking.first().copied())
3672            .collect();
3673        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3674
3675        // A judge whose private vote failed still counted once, in the initial
3676        // ranking; using it beats discarding a whole seat.
3677        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3678        let mut cast: Vec<char> = Vec::new();
3679        for (i, j) in self.state.judgements.iter().enumerate() {
3680            let vote = self
3681                .state
3682                .votes
3683                .iter()
3684                .find(|v| v.judge == i + 1)
3685                .and_then(|v| v.vote)
3686                .or_else(|| j.ranking.first().copied());
3687            if let Some(v) = vote {
3688                *first_choice.entry(v).or_insert(0) += 1;
3689                cast.push(v);
3690            }
3691        }
3692
3693        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3694        for j in &self.state.judgements {
3695            let n = j.ranking.len();
3696            for (pos, label) in j.ranking.iter().enumerate() {
3697                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3698            }
3699        }
3700
3701        let best = first_choice.values().copied().max().unwrap_or(0);
3702        let mut leaders: Vec<char> = first_choice
3703            .iter()
3704            .filter(|(_, v)| **v == best)
3705            .map(|(k, _)| *k)
3706            .collect();
3707        let mut tie_break = None;
3708        if leaders.len() > 1 {
3709            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3710            let borda_leaders: Vec<char> = leaders
3711                .iter()
3712                .copied()
3713                .filter(|l| borda[l] == top_borda)
3714                .collect();
3715            tie_break = Some(if borda_leaders.len() == 1 {
3716                format!(
3717                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3718                    leaders.len()
3719                )
3720            } else {
3721                format!(
3722                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3723                    leaders.len()
3724                )
3725            });
3726            leaders = borda_leaders;
3727            leaders.sort_unstable();
3728        }
3729        let winner = *leaders
3730            .first()
3731            .or(viable.first())
3732            .context("no candidate to declare a winner from")?;
3733
3734        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3735        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3736        let deliberated = !self.state.deliberation.is_empty();
3737
3738        // Whose verdict is this? A rate-limited seat is absent even if it
3739        // ranked before the limit hit, so presence is measured against the
3740        // recorded losses, not just "did a ranking ever appear".
3741        let quota_seats: std::collections::BTreeSet<&str> =
3742            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3743        let mut present = 0usize;
3744        for (i, j) in self.state.judgements.iter().enumerate() {
3745            if quota_seats.contains(j.seat.as_str()) {
3746                continue;
3747            }
3748            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3749            let voted = self
3750                .state
3751                .votes
3752                .iter()
3753                .any(|v| v.judge == i + 1 && v.vote.is_some());
3754            if ranked || voted {
3755                present += 1;
3756            }
3757        }
3758        // Strict majority of the configured panel. A bare majority is real
3759        // signal we can act on, while a minority verdict must never stand in
3760        // for a healthy one. A one-candidate run needs no panel at all, and
3761        // `judges` stays `0` rather than the roster size a panel that never
3762        // sat would otherwise be credited with.
3763        let needs_quorum = viable.len() > 1;
3764        let judges_total = if needs_quorum {
3765            self.roles.judges.len()
3766        } else {
3767            0
3768        };
3769        let quorum = if needs_quorum {
3770            judges_total / 2 + 1
3771        } else {
3772            0
3773        };
3774        let met_quorum = !needs_quorum || present >= quorum;
3775        let uncontested = (!needs_quorum).then(|| {
3776            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3777        });
3778
3779        self.state.event(
3780            "tally",
3781            match &uncontested {
3782                Some(reason) => format!("winner {winner} — {reason}"),
3783                None => format!(
3784                    "winner {winner} — votes {} | initial {} | {} changed | \
3785                     {present}/{judges_total} judges{}",
3786                    first_choice
3787                        .iter()
3788                        .map(|(k, v)| format!("{k}:{v}"))
3789                        .collect::<Vec<_>>()
3790                        .join(" "),
3791                    if unanimous_initial {
3792                        "unanimous"
3793                    } else {
3794                        "split"
3795                    },
3796                    changed_votes,
3797                    if met_quorum {
3798                        String::new()
3799                    } else {
3800                        format!(" — below quorum ({quorum} required)")
3801                    },
3802                ),
3803            },
3804        );
3805        if !met_quorum {
3806            self.state.event(
3807                "stall",
3808                format!(
3809                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3810                     the run stops here, resumable"
3811                ),
3812            );
3813        }
3814        self.state.tally = Some(Tally {
3815            first_choice,
3816            borda,
3817            winner,
3818            rankings: tops.len(),
3819            unanimous_initial,
3820            deliberated,
3821            changed_votes,
3822            unanimous_final,
3823            tie_break,
3824            judges: judges_total,
3825            present,
3826            quorum,
3827            met_quorum,
3828            uncontested,
3829        });
3830        self.state.status = if met_quorum {
3831            RunStatus::Reviewing
3832        } else {
3833            RunStatus::Stalled
3834        };
3835        self.state.save()?;
3836        Ok(())
3837    }
3838
3839    // ------------------------------------------------------------- recover
3840
3841    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3842    /// resumed toward completion once the transient cause clears.
3843    ///
3844    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3845    /// it toward the quorum, which is exactly the set of seats whose absence
3846    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3847    /// not depend on which node happened to hit the limit), or an ordinary
3848    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3849    /// seat is never disturbed.
3850    ///
3851    /// A seat that now answers with a usable ranking is "recovered": its
3852    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3853    /// `tally` counts it present again), and its vote re-collected. A seat that
3854    /// still fails keeps its loss and stays absent.
3855    ///
3856    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3857    /// to review/gate/merge), `false` when it is still below quorum (the run
3858    /// stays `Stalled`, still resumable for a later retry).
3859    #[allow(clippy::too_many_lines)]
3860    async fn recover_stall(&mut self) -> Result<bool> {
3861        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3862        // agent files with `magi task add` name the run that paid for it. The
3863        // prompt overlay is cloned alongside it because the waves borrow it
3864        // while `self` is mutably borrowed by the node's own bookkeeping.
3865        let run_id = self.state.id.clone();
3866        let prompts = self.state.config.prompts.clone();
3867        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3868        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3869        // is what keeps a non-quota collapse — or a quota loss recorded at the
3870        // deliberate node — from being a permanent dead-end on `--resume`.
3871        let quota_seats: BTreeSet<&str> =
3872            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3873        let absent: Vec<String> = self
3874            .state
3875            .judgements
3876            .iter()
3877            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3878            .map(|j| j.seat.clone())
3879            .collect();
3880        if absent.is_empty() {
3881            return Ok(false);
3882        }
3883        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3884        if viable.len() <= 1 {
3885            return Ok(false);
3886        }
3887        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3888        let language = self.state.config.graph.language.clone();
3889        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3890        let sessions = self.state.config.graph.sessions;
3891        let artifacts = agent::artifacts_dir(&self.state.dir());
3892        let root = self.state.worktree_root();
3893        let base_short = short(&self.state.base_commit);
3894        let candidates: Vec<Candidate> = viable.clone();
3895
3896        // Map each absent seat key to its 0-based position in `roles.judges`.
3897        let mut positions: Vec<usize> = absent
3898            .iter()
3899            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3900            .collect();
3901        if positions.is_empty() {
3902            return Ok(false);
3903        }
3904        positions.sort_unstable();
3905        positions.dedup();
3906
3907        // Re-rank the lost seats, one blind prompt each.
3908        let mut judge_jobs = Vec::new();
3909        for &j in &positions {
3910            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3911            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3912            let seat_key = format!("judge-{}", j + 1);
3913            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3914            let seat = self.seat(&seat_key, &spec.id);
3915            judge_jobs.push(SeatJob {
3916                spec,
3917                seat,
3918                prompt: prompt::judge(
3919                    &self.state.instruction,
3920                    &views,
3921                    self.roles.judges.len(),
3922                    &base_short,
3923                    &language,
3924                ),
3925                cwd: root.join(seat_key),
3926                timeout,
3927                allow_write: false,
3928                sessions,
3929                artifacts: artifacts.clone(),
3930                stem: format!("judge-{}-recover", j + 1),
3931                handover: None,
3932            });
3933        }
3934
3935        let labels_for_check = labels.clone();
3936        let mut judge_losses = Vec::new();
3937        let retries = self.state.config.graph.retries;
3938        let cache = self.state.config.cache_dir();
3939        let ctx = WaveCtx {
3940            carry_seats: false,
3941            pauses: Some((&self.pause, &self.interrupt)),
3942            run: &run_id,
3943            node: "judge",
3944            prompts: &prompts,
3945            cache: cache.as_deref(),
3946            round: None,
3947        };
3948        let results = ask_json_wave::<Ranking>(
3949            judge_jobs,
3950            Arc::clone(&self.sem),
3951            retries,
3952            &self.roles.judge_roster,
3953            &ctx,
3954            &mut judge_losses,
3955            &mut self.state,
3956            &move |r: &Ranking| r.validate(&labels_for_check),
3957        )
3958        .await;
3959        if self.state.resource_block.is_some() {
3960            self.stop_for_resource()?;
3961            return Ok(false);
3962        }
3963
3964        // Refresh the judgement of every seat that ranked again.
3965        let mut recovered: BTreeSet<usize> = BTreeSet::new();
3966        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3967            let agent_id = seat.agent.clone();
3968            self.state.seats.insert(seat.key.clone(), seat);
3969            let record = &mut self.state.judgements[j];
3970            match res {
3971                Ok((ranking, out)) => {
3972                    record.agent = agent_id;
3973                    record.ranking = ranking.normalized();
3974                    record.reasons = ranking.reasons;
3975                    record.strengths = ranking.strengths;
3976                    record.confidence = ranking.confidence;
3977                    record.failed = None;
3978                    record.duration_ms = out.duration_ms;
3979                    recovered.insert(j);
3980                    self.state.event(
3981                        "recover",
3982                        format!("judge {} ranked again after the limit", j + 1),
3983                    );
3984                }
3985                Err(e) => {
3986                    self.state
3987                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3988                }
3989            }
3990        }
3991
3992        // Re-ask the votes of the seats that recovered a ranking.
3993        let mut vote_jobs = Vec::new();
3994        let mut vote_pos: Vec<usize> = Vec::new();
3995        for &j in &recovered {
3996            let seat_key = format!("judge-{}", j + 1);
3997            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3998            let seat = self.seat(&seat_key, &spec.id);
3999            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4000            let text = if has_context(&spec, &seat, sessions) {
4001                prompt::final_vote(&labels, &language)
4002            } else {
4003                full.clone()
4004            };
4005            vote_jobs.push(SeatJob {
4006                spec,
4007                seat,
4008                prompt: text,
4009                cwd: root.join(seat_key),
4010                timeout,
4011                allow_write: false,
4012                sessions,
4013                artifacts: artifacts.clone(),
4014                stem: format!("vote-judge-{}-recover", j + 1),
4015                handover: Some(full),
4016            });
4017            vote_pos.push(j);
4018        }
4019        let allowed = labels.clone();
4020        let mut vote_losses = Vec::new();
4021        let vote_retries = self.state.config.graph.retries;
4022        let vote_cache = self.state.config.cache_dir();
4023        let ctx = WaveCtx {
4024            carry_seats: false,
4025            pauses: Some((&self.pause, &self.interrupt)),
4026            run: &run_id,
4027            node: "vote",
4028            prompts: &prompts,
4029            cache: vote_cache.as_deref(),
4030            round: None,
4031        };
4032        let votes = ask_json_wave::<FinalVote>(
4033            vote_jobs,
4034            Arc::clone(&self.sem),
4035            vote_retries,
4036            &self.roles.judge_roster,
4037            &ctx,
4038            &mut vote_losses,
4039            &mut self.state,
4040            &move |v: &FinalVote| match v.label() {
4041                Some(c) if allowed.contains(&c) => Ok(()),
4042                other => bail!("vote {other:?} is not one of {allowed:?}"),
4043            },
4044        )
4045        .await;
4046        if self.state.resource_block.is_some() {
4047            self.stop_for_resource()?;
4048            return Ok(false);
4049        }
4050        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4051            let agent_id = seat.agent.clone();
4052            self.state.seats.insert(seat.key.clone(), seat);
4053            match res {
4054                Ok((v, _)) => {
4055                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4056                        rec.vote = v.label();
4057                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4058                    } else {
4059                        self.state.votes.push(VoteRecord {
4060                            judge: j + 1,
4061                            agent: agent_id,
4062                            vote: v.label(),
4063                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4064                            changed: false,
4065                        });
4066                    }
4067                    self.state.event(
4068                        "recover",
4069                        format!("judge {} voted again after the limit", j + 1),
4070                    );
4071                }
4072                Err(e) => {
4073                    self.state
4074                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4075                }
4076            }
4077        }
4078
4079        // A seat that ranked again is present even if its re-vote failed —
4080        // `tally` falls back to the initial ranking's first choice — so clear
4081        // its quota loss. Seats that still fail keep theirs and stay absent.
4082        let recovered_keys: BTreeSet<String> = recovered
4083            .iter()
4084            .map(|&j| format!("judge-{}", j + 1))
4085            .collect();
4086        self.state
4087            .quota
4088            .retain(|q| !recovered_keys.contains(&q.seat));
4089        // A seat that hit the limit again is a fresh loss, not the old one:
4090        // replace the stale entry so the history stays one-per-seat and the
4091        // daemon can tell this attempt's loss from a previous session's.
4092        for loss in judge_losses.into_iter().chain(vote_losses) {
4093            if recovered_keys.contains(&loss.seat) {
4094                continue;
4095            }
4096            self.state.quota.retain(|q| q.seat != loss.seat);
4097            self.state.quota.push(loss);
4098        }
4099
4100        // Recompute the verdict from the refreshed panel.
4101        self.state.tally = None;
4102        self.tally()?;
4103        Ok(self
4104            .state
4105            .tally
4106            .as_ref()
4107            .map(|t| t.met_quorum)
4108            .unwrap_or(false))
4109    }
4110
4111    // ---------------------------------------------------------------- graft
4112
4113    /// Port the strengths the judges attributed to the losing candidates onto
4114    /// the winner, once, between the tally and the review loop.
4115    ///
4116    /// Best-effort by construction: it returns nothing and fails nothing. A
4117    /// skip, a fixer that failed or hit quota, a round that changed nothing
4118    /// and a result over the size limit ([`graft::limit`]) all leave the
4119    /// winner exactly at the commit it had before (`GraftRecord::pre`) and the
4120    /// run carries on as if the phase were off. See [`graft`] for the
4121    /// decisions and `AGENTS.md` for the invariants.
4122    ///
4123    /// The one exception is a rollback that could not be confirmed: the
4124    /// rejected commit may still be the winner's tip, so the run is blocked
4125    /// (resumable; the record stays `Running`, which makes a resume retry the
4126    /// restore) instead of reviewing, gating and merging it.
4127    async fn graft(&mut self) -> Result<()> {
4128        let Err(e) = self.graft_inner().await else {
4129            return Ok(());
4130        };
4131        if e.is::<crate::repo_lock::ResourceBlocked>() {
4132            return Err(e);
4133        }
4134        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4135            self.state.status = RunStatus::Blocked;
4136            self.state.event("graft", format!("blocked: {e:#}"));
4137            self.state.save()?;
4138            return Err(e);
4139        }
4140        // An error after the `Running` record was saved may have left the
4141        // fixer's work in the tree: the record must not stay unsettled
4142        // (a later resume would reset past the review fixes), so confirm the
4143        // restore now, or block.
4144        if let Some(rec) = self.state.graft.clone()
4145            && !rec.outcome.settled()
4146        {
4147            if let Some(winner) = self.state.winner().cloned() {
4148                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4149                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4150                        return Err(re);
4151                    }
4152                    self.state.status = RunStatus::Blocked;
4153                    self.state.event("graft", format!("blocked: {re:#}"));
4154                    self.state.save()?;
4155                    return Err(re);
4156                }
4157            }
4158            if let Some(g) = self.state.graft.as_mut() {
4159                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4160            }
4161        }
4162        self.state.event("graft", format!("skipped: {e:#}"));
4163        if let Err(e) = self.state.save() {
4164            tracing::warn!("save after graft: {e:#}");
4165        }
4166        Ok(())
4167    }
4168
4169    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4170    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4171    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4172        let _coordination = self.coordinate(wt).await?;
4173        crate::recovery::restore_repair(wt, pre)
4174            .await
4175            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4176    }
4177
4178    /// A skip is a run event, recorded once however often the node is re-entered.
4179    fn graft_skipped(&mut self, why: &str) {
4180        let message = format!("skipped: {why}");
4181        if !self
4182            .state
4183            .events
4184            .iter()
4185            .any(|e| e.node == "graft" && e.message == message)
4186        {
4187            self.state.event("graft", message);
4188        }
4189    }
4190
4191    async fn graft_inner(&mut self) -> Result<()> {
4192        // A round that was started and never finished (crash, kill): the
4193        // fixer may have left anything in the tree, so put the winner back.
4194        if let Some(rec) = self.state.graft.clone()
4195            && !rec.outcome.settled()
4196        {
4197            // Reviews exist only if the graft ended without its record being
4198            // settled (a swallowed error): the tree has moved on since, and
4199            // a reset to `pre` would discard the review fixes.
4200            if self.state.reviews.is_empty()
4201                && let Some(winner) = self.state.winner().cloned()
4202            {
4203                self.graft_restore(&winner.worktree, &rec.pre).await?;
4204            }
4205            if rec.outcome == GraftOutcome::WaitingForRestore {
4206                self.state.graft = None;
4207                self.state.save()?;
4208            } else {
4209                if let Some(g) = self.state.graft.as_mut() {
4210                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4211                }
4212                self.state.event(
4213                    "graft",
4214                    "rolled back: the previous round was interrupted; the winner is untouched",
4215                );
4216                self.state.save()?;
4217                return Ok(());
4218            }
4219        }
4220        let plan = match graft::plan(&self.state) {
4221            Ok(plan) => plan,
4222            Err(skip) => {
4223                if skip != graft::Skip::AlreadyDone {
4224                    self.graft_skipped(skip.reason());
4225                    self.state.save()?;
4226                }
4227                return Ok(());
4228            }
4229        };
4230        let Some(winner) = self.state.winner().cloned() else {
4231            return Ok(());
4232        };
4233        let wt = winner.worktree.clone();
4234        if !git::is_clean(&wt).await? {
4235            self.graft_skipped("the winner's worktree has uncommitted changes");
4236            self.state.save()?;
4237            return Ok(());
4238        }
4239        let pre = git::rev_parse(&wt, "HEAD").await?;
4240        let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4241        else {
4242            self.graft_skipped("the winner's diff could not be measured");
4243            self.state.save()?;
4244            return Ok(());
4245        };
4246        let limit = graft::limit(winner_lines);
4247        // Saved before the fixer is called: a stop from here on is repaired
4248        // by the branch above on the next entry.
4249        self.state.graft = Some(GraftRecord {
4250            pre: pre.clone(),
4251            post: None,
4252            lines: None,
4253            limit: Some(limit),
4254            agent: None,
4255            outcome: GraftOutcome::Running,
4256            requested: graft::requested(&plan),
4257            adoption: None,
4258            adoption_error: None,
4259        });
4260        self.state.event(
4261            "graft",
4262            format!(
4263                "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4264                plan.strengths
4265                    .iter()
4266                    .map(|(l, _, _)| l.to_string())
4267                    .collect::<Vec<_>>()
4268                    .join(", "),
4269                plan.winner
4270            ),
4271        );
4272        self.state.save()?;
4273
4274        let fix_prompt = prompt::graft(
4275            &self.state.instruction,
4276            plan.winner,
4277            &plan.strengths,
4278            limit,
4279            &self.state.config.graph.language,
4280        );
4281        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4282        let sessions = self.state.config.graph.sessions;
4283        let artifacts = agent::artifacts_dir(&self.state.dir());
4284        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4285        let (_, seat, out) = self
4286            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4287                SeatJob {
4288                    prompt: fix_prompt.clone(),
4289                    spec,
4290                    seat,
4291                    cwd: wt.clone(),
4292                    timeout,
4293                    allow_write: true,
4294                    sessions,
4295                    artifacts: artifacts.clone(),
4296                    stem: "graft".to_owned(),
4297                    handover: None,
4298                }
4299            })
4300            .await;
4301        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4302            // Preserve the need to restore earlier fallback edits without
4303            // confusing an unstarted seat with an interrupted invocation.
4304            self.state.graft.as_mut().expect("graft record").outcome =
4305                GraftOutcome::WaitingForRestore;
4306            return self.stop_for_resource();
4307        }
4308        let mut why: Option<String> = None;
4309        match &out {
4310            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4311                why = Some("the fixer produced nothing usable".to_owned());
4312            }
4313            AgentOutcome::Ok(_) => {}
4314            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4315            AgentOutcome::Quota(o) => {
4316                self.state.quota.push(QuotaLoss {
4317                    seat: seat.key.clone(),
4318                    node: "graft".to_owned(),
4319                    at: Timestamp::now(),
4320                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4321                });
4322                why = Some("rate limited (quota)".to_owned());
4323            }
4324            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4325                why = Some(format!(
4326                    "the fixer failed: {}",
4327                    e.lines()
4328                        .next()
4329                        .unwrap_or("")
4330                        .chars()
4331                        .take(160)
4332                        .collect::<String>()
4333                ));
4334            }
4335        }
4336        let agent_id = seat.agent.clone();
4337        self.state.seats.insert(seat.key.clone(), seat);
4338
4339        let mut post = None;
4340        let mut lines = None;
4341        let _coordination = self.coordinate(&wt).await?;
4342        if why.is_none() {
4343            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4344            if let Ok(r) = &rescued {
4345                self.state.note_withheld("graft", &r.withheld);
4346            }
4347            let head = git::rev_parse(&wt, "HEAD").await?;
4348            // What is measured must be what gets reviewed and gated: a failed
4349            // rescue, or anything still uncommitted (withheld lockfiles
4350            // aside, which are never part of the tree), rejects the graft.
4351            let dirty = match &rescued {
4352                Ok(r) => git::status_porcelain(&wt)
4353                    .await
4354                    .map(|st| {
4355                        st.lines().any(|l| {
4356                            !r.withheld
4357                                .iter()
4358                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4359                        })
4360                    })
4361                    .unwrap_or(true),
4362                Err(_) => true,
4363            };
4364            if dirty {
4365                why = Some("the fixer left changes that could not be committed".to_owned());
4366            } else if head == pre {
4367                why = Some("the fixer changed nothing".to_owned());
4368            } else {
4369                lines = git::changed_lines(&wt, &pre, &head).await;
4370                if graft::within_budget(lines, limit) {
4371                    post = Some(head);
4372                } else {
4373                    why = Some(format!(
4374                        "{} changed lines against a limit of {limit}",
4375                        lines.map_or_else(
4376                            || "an unmeasurable number of".to_owned(),
4377                            |l| l.to_string()
4378                        )
4379                    ));
4380                }
4381            }
4382        }
4383        let (adoption, adoption_error) = match &out {
4384            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4385            _ => (None, None),
4386        };
4387        let requested = graft::requested(&plan);
4388        let outcome = match why {
4389            None => {
4390                self.state.event(
4391                    "graft",
4392                    format!(
4393                        "applied: {} changed lines (limit {limit})",
4394                        lines.unwrap_or(0)
4395                    ),
4396                );
4397                GraftOutcome::Applied
4398            }
4399            Some(why) => {
4400                // Not settled until the restore is confirmed: the record
4401                // stays `Running`, so a resume retries it.
4402                crate::recovery::restore_repair(&wt, &pre)
4403                    .await
4404                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4405                self.state.event(
4406                    "graft",
4407                    format!("rolled back: {why}; the winner is untouched"),
4408                );
4409                GraftOutcome::RolledBack(why)
4410            }
4411        };
4412        self.state.graft = Some(GraftRecord {
4413            pre,
4414            post,
4415            lines,
4416            limit: Some(limit),
4417            agent: Some(agent_id),
4418            outcome,
4419            requested,
4420            adoption,
4421            adoption_error,
4422        });
4423        self.state.save()?;
4424        Ok(())
4425    }
4426
4427    // ----------------------------------------------------------------- fold
4428
4429    async fn fold_losers(&mut self) -> Result<()> {
4430        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4431            return Ok(());
4432        };
4433        if !self.state.candidate_preservation.is_empty() {
4434            bail!("candidate preservation incomplete; refusing cleanup");
4435        }
4436        let repo = self.state.repo.clone();
4437        let _coordination = self.coordinate(&repo).await?;
4438        let mut folded = Vec::new();
4439        for i in 0..self.state.candidates.len() {
4440            let c = &self.state.candidates[i];
4441            if c.label == winner || c.folded {
4442                continue;
4443            }
4444            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4445            remove_preserved_worktree(&repo, &wt).await?;
4446            if git::branch_exists(&repo, &branch).await?
4447                && !git::branch_delete(&repo, &branch).await?
4448            {
4449                bail!("candidate branch removal refused");
4450            }
4451            self.state.candidates[i].folded = true;
4452            self.state.save()?;
4453            folded.push(label.to_string());
4454        }
4455        // The judges are finished; their checkouts are pure cost from here.
4456        let root = self.state.worktree_root();
4457        for j in 1..=self.roles.judges.len() {
4458            let wt = root.join(format!("judge-{j}"));
4459            if wt.exists() {
4460                remove_preserved_worktree(&repo, &wt).await?;
4461            }
4462        }
4463        // The design-deliberation stage is finished by the time a tally
4464        // exists — same reasoning as the judges above.
4465        if self.state.config.graph.advise {
4466            for k in 1..=self.state.config.graph.advisors {
4467                let wt = root.join(format!("advisor-{k}"));
4468                if wt.exists() {
4469                    remove_preserved_worktree(&repo, &wt).await?;
4470                }
4471            }
4472        }
4473        if !folded.is_empty() {
4474            self.state
4475                .event("fold", format!("folded candidates {}", folded.join(", ")));
4476            self.state.save()?;
4477        }
4478        Ok(())
4479    }
4480
4481    // ------------------------------------------------------------ base sync
4482
4483    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4484    /// anything verifies it.
4485    ///
4486    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4487    /// read whatever is checked out in the winner's worktree. Left alone that
4488    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4489    /// it when the run *branched* - and a run takes long enough that the base
4490    /// has usually moved by the time it gets here. A gate that ran there
4491    /// answers "green on the commit this run started from", not "green on
4492    /// what is about to land", and the difference showed up three times in
4493    /// one day as a green run whose merge would have reverted a file another
4494    /// pull request had already landed.
4495    ///
4496    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4497    /// `land::Step::Rebase` calls, rather than a second implementation of the
4498    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4499    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4500    /// doc warns about ("two answers to a question nobody notices until a
4501    /// diff is wrong").
4502    ///
4503    /// A conflict is not the end of the road: the standing rebase is handed
4504    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4505    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4506    /// land). Once it finishes, review and the gate run as usual on the
4507    /// rebased tree, which is where a breakage the new base caused is caught
4508    /// by the ordinary gate-fix round. magi resolves nothing itself.
4509    ///
4510    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4511    /// `state.base_sync.attempts`, is how many times the base is *rebased
4512    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4513    /// *conflict* was given to a fixer. When the fixer cannot finish the
4514    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4515    /// what was tried (rounds spent, paths still conflicted) and the branch
4516    /// and worktree stay exactly as they were - untouched, for a person to
4517    /// look at - which is also what makes re-entering this function
4518    /// afterwards a no-op instead of a second attempt at the same wall. A
4519    /// push failure ends the same way.
4520    async fn sync_to_base(&mut self) -> Result<()> {
4521        if self.state.status == RunStatus::AlreadyInBase {
4522            return Ok(());
4523        }
4524        let conflicted = self
4525            .state
4526            .base_sync
4527            .as_ref()
4528            .is_some_and(|s| s.conflict.is_some());
4529        let Some(winner) = self.state.winner().cloned() else {
4530            return Ok(());
4531        };
4532
4533        let repo = self.state.repo.clone();
4534        let remote = self.state.config.merge.remote.clone();
4535        let base_branch = self.state.base_branch.clone();
4536        let tracking = format!("{remote}/{base_branch}");
4537
4538        git::fetch(&repo, &remote, &base_branch).await.ok();
4539        // No network, or the remote never had this branch: the run already
4540        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4541        // otherwise), and one that got this far is not blocked by a fetch
4542        // that fails now. It just has no newer tip to compare against.
4543        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4544            return Ok(());
4545        };
4546
4547        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4548        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4549        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4550
4551        // Before any rebase, and before a recorded conflict is honoured: a
4552        // branch whose change reached the base under other commit ids has
4553        // nothing to rebase and nothing to conflict with, and a run that
4554        // already stopped on that phantom conflict recovers here on resume.
4555        // `behind == 0` with head == tip is a branch the base has since taken
4556        // in whole, whether or not a conflict was ever recorded: the ancestry
4557        // proof must still run (`classify` ignores a head still on the start
4558        // commit).
4559        if (behind > 0 || conflicted || head == tip)
4560            && self
4561                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4562                .await?
4563        {
4564            return Ok(());
4565        }
4566        if conflicted {
4567            return Ok(());
4568        }
4569
4570        if behind == 0 {
4571            // A fixer-finished rebase moves the branch ref before the
4572            // winner's worktree is told (`sync_to_head` below). A run that
4573            // died in between resumes here with `behind == 0` and a tree still
4574            // holding the pre-rebase files, which review and the gate would
4575            // then read. That state is exactly: HEAD moved off the tip the
4576            // rebase started from, yet the tree is still identical to that
4577            // tip. A tree with edits of its own differs from it, so nothing
4578            // is thrown away.
4579            if let Some(from) = self
4580                .state
4581                .rebase_fixes
4582                .iter()
4583                .rev()
4584                .find_map(|r| r.from.clone())
4585                && from != head
4586                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4587                    .await
4588                    .is_ok_and(|o| o.ok())
4589            {
4590                git::sync_to_head(&winner.worktree).await?;
4591            }
4592            // An earlier attempt may have rebased the branch locally and died
4593            // before pushing it (only the fresh-rebase arm below pushes).
4594            // Publish it now, so the plain push at PR time is not refused as
4595            // a non-fast-forward. A run already holding a recorded conflict
4596            // never reaches here; that case is out of scope.
4597            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4598            if let Some(why) = &conflict {
4599                self.state.status = RunStatus::Blocked;
4600                self.state.event("land", why.clone());
4601            }
4602            self.state.base_sync = Some(BaseSync {
4603                tip,
4604                behind: 0,
4605                attempts,
4606                conflict,
4607                already_in: None,
4608            });
4609            self.state.save()?;
4610            return Ok(());
4611        }
4612
4613        if attempts >= BASE_SYNC_ROUNDS {
4614            let why = format!(
4615                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4616                 rebase(s); rebasing again would only race it",
4617                winner.branch
4618            );
4619            self.state.status = RunStatus::Blocked;
4620            self.state.base_sync = Some(BaseSync {
4621                tip,
4622                behind,
4623                attempts,
4624                conflict: Some(why.clone()),
4625                already_in: None,
4626            });
4627            self.state.event("land", why);
4628            self.state.save()?;
4629            return Ok(());
4630        }
4631
4632        self.state.event(
4633            "land",
4634            format!(
4635                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4636                winner.branch
4637            ),
4638        );
4639        self.state.save()?;
4640
4641        // The remote's copy of the branch, read now and only if the fetch
4642        // really succeeded (a stale tracking ref must never pin a lease). It is
4643        // pushed over after a rebase only when it is a commit this branch
4644        // already contains, by ancestry or by patch (an earlier rebase of ours
4645        // that never reached the remote): anything else is somebody else's work.
4646        let branch_tracking = format!("{remote}/{}", winner.branch);
4647        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4648        let remote_tip = if self.state.task_pr_branch().is_none()
4649            && matches!(&fetched_branch, Ok(o) if o.ok())
4650        {
4651            git::rev_parse(&repo, &branch_tracking).await.ok()
4652        } else {
4653            None
4654        };
4655        // A remote tip this branch does not contain is somebody else's work:
4656        // rebasing would leave a local tip that can never be pushed. Stop
4657        // before touching anything and say so.
4658        if let Some(theirs) = &remote_tip
4659            && !git::is_ancestor(&repo, theirs, &head).await
4660            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4661                .await
4662                .is_ok_and(|missing| missing.is_empty())
4663        {
4664            let why = format!(
4665                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4666                 them",
4667                short(theirs),
4668                winner.branch
4669            );
4670            self.state.status = RunStatus::Blocked;
4671            self.state.base_sync = Some(BaseSync {
4672                tip,
4673                behind,
4674                attempts,
4675                conflict: Some(why.clone()),
4676                already_in: None,
4677            });
4678            self.state.event("land", why);
4679            self.state.save()?;
4680            return Ok(());
4681        }
4682
4683        let scratch = self.state.dir().join("base-sync");
4684        let rebased = match crate::rebase::rebase_with_fixer(
4685            &mut self.state,
4686            &scratch,
4687            &winner.branch,
4688            &tracking,
4689        )
4690        .await
4691        {
4692            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4693            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4694            Err(e) => Err(e),
4695        };
4696        let attempts = attempts + 1;
4697        match rebased {
4698            Ok(None) => {
4699                // The branch ref moved, but a worktree that already had it
4700                // checked out (the winner's) was not told; sync its index and
4701                // files before anything reads them.
4702                git::sync_to_head(&winner.worktree).await?;
4703                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4704                let mut conflict = None;
4705                if let Some(pinned) = &remote_tip {
4706                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4707                    match pushed {
4708                        Ok(o) if o.ok() => self.state.event(
4709                            "land",
4710                            format!("pushed rebased {} to {remote}", winner.branch),
4711                        ),
4712                        Ok(o) => {
4713                            conflict = Some(format!(
4714                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4715                                winner.branch,
4716                                short(pinned),
4717                                o.stderr.chars().take(600).collect::<String>()
4718                            ));
4719                        }
4720                        Err(e) => {
4721                            conflict = Some(format!(
4722                                "rebased {} locally but could not push it: {e:#}",
4723                                winner.branch
4724                            ));
4725                        }
4726                    }
4727                }
4728                if let Some(why) = &conflict {
4729                    self.state.status = RunStatus::Blocked;
4730                    self.state.event("land", why.clone());
4731                }
4732                self.state.base_sync = Some(BaseSync {
4733                    tip: tip.clone(),
4734                    behind: 0,
4735                    attempts,
4736                    conflict,
4737                    already_in: None,
4738                });
4739                self.state
4740                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4741            }
4742            Ok(Some(conflict)) => {
4743                let why = format!(
4744                    "{} conflicts with {tracking} and did not rebase: {}",
4745                    winner.branch,
4746                    conflict.chars().take(600).collect::<String>()
4747                );
4748                self.state.status = RunStatus::Blocked;
4749                self.state.base_sync = Some(BaseSync {
4750                    tip,
4751                    behind,
4752                    attempts,
4753                    conflict: Some(why.clone()),
4754                    already_in: None,
4755                });
4756                self.state.event("land", why);
4757            }
4758            Err(e) => {
4759                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4760                self.state.status = RunStatus::Blocked;
4761                self.state.base_sync = Some(BaseSync {
4762                    tip,
4763                    behind,
4764                    attempts,
4765                    conflict: Some(why.clone()),
4766                    already_in: None,
4767                });
4768                self.state.event("land", why);
4769            }
4770        }
4771        self.state.save()?;
4772        Ok(())
4773    }
4774
4775    /// Push a branch an earlier attempt rebased locally but never published,
4776    /// pinned to the remote tip read right after a successful fetch. Returns
4777    /// the reason when the run must stop; `None` when there was nothing to do
4778    /// (no remote copy, the same tip, or a remote copy this branch already
4779    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4780    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4781        // Task competitions publish only after the PR text gate, to their
4782        // public head; never resurrect a candidate branch on the remote.
4783        if self.state.task_pr_branch().is_some() {
4784            return None;
4785        }
4786        let repo = self.state.repo.clone();
4787        let remote = self.state.config.merge.remote.clone();
4788        let fetched = git::fetch(&repo, &remote, branch).await;
4789        if !matches!(&fetched, Ok(o) if o.ok()) {
4790            return None;
4791        }
4792        let branch_tracking = format!("{remote}/{branch}");
4793        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4794        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4795            return None;
4796        }
4797        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4798            .await
4799            .is_ok_and(|missing| missing.is_empty())
4800        {
4801            return Some(format!(
4802                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4803                 them",
4804                short(&theirs)
4805            ));
4806        }
4807        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4808            Ok(o) if o.ok() => {
4809                self.state
4810                    .event("land", format!("pushed rebased {branch} to {remote}"));
4811                None
4812            }
4813            Ok(o) => Some(format!(
4814                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4815                 someone may have pushed): {}",
4816                short(&theirs),
4817                o.stderr.chars().take(600).collect::<String>()
4818            )),
4819            Err(e) => Some(format!(
4820                "{branch} is rebased locally but could not be pushed: {e:#}"
4821            )),
4822        }
4823    }
4824
4825    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4826    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4827    /// whether it did.
4828    ///
4829    /// Checked only when the base is ahead of the branch. A failing check is
4830    /// "not proven" - the ordinary rebase path then decides - never a reason to
4831    /// stop the run.
4832    ///
4833    /// The remote copy of the branch is held to the same standard as the local
4834    /// one: if it carries a tip this worktree does not, that tip must itself be
4835    /// proven in the base, or nothing is settled (a pull request would
4836    /// otherwise be closed over commits nobody checked). The pull request is
4837    /// closed *before* the terminal status is saved; if that fails for a
4838    /// reason other than a refusal (no network, a `gh` error) the run is left
4839    /// `Blocked` with the reason as its conflict, which a resume retries -
4840    /// the same recovery a phantom conflict gets.
4841    async fn settle_already_in(
4842        &mut self,
4843        branch: &str,
4844        tip: &str,
4845        head: &str,
4846        attempts: usize,
4847        behind: usize,
4848    ) -> Result<bool> {
4849        let repo = self.state.repo.clone();
4850        let remote = self.state.config.merge.remote.clone();
4851        let start = self.state.base_commit.clone();
4852        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4853            Ok(Some(e)) => e,
4854            Ok(None) => return Ok(false),
4855            Err(e) => {
4856                tracing::warn!("already-in-base check for {branch}: {e:#}");
4857                return Ok(false);
4858            }
4859        };
4860        let mut verified = vec![head.to_owned()];
4861        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4862        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4863        if matches!(&fetched, Ok(o) if o.ok())
4864            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4865            && theirs != head
4866        {
4867            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4868                Ok(Some(_)) => verified.push(theirs),
4869                _ => return Ok(false),
4870            }
4871        }
4872        let base_branch = self.state.base_branch.clone();
4873        let message = format!(
4874            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4875             land",
4876            evidence.names(),
4877            evidence.proof.as_str()
4878        );
4879        let closed =
4880            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4881                .await;
4882        match closed {
4883            Ok(Ok(url)) => self
4884                .state
4885                .event("land", format!("closed {url}: superseded on {base_branch}")),
4886            Ok(Err(why)) => self
4887                .state
4888                .event("land", format!("did not close a pull request: {why}")),
4889            Err(e) => {
4890                let why = format!(
4891                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4892                     not be closed ({e:#}); resume to retry"
4893                );
4894                self.state.status = RunStatus::Blocked;
4895                self.state.base_sync = Some(BaseSync {
4896                    tip: tip.to_owned(),
4897                    behind,
4898                    attempts,
4899                    conflict: Some(why.clone()),
4900                    already_in: None,
4901                });
4902                self.state.event("land", why);
4903                self.state.save()?;
4904                return Ok(true);
4905            }
4906        }
4907        self.state.status = RunStatus::AlreadyInBase;
4908        self.state.base_sync = Some(BaseSync {
4909            tip: tip.to_owned(),
4910            behind,
4911            attempts,
4912            conflict: None,
4913            already_in: Some(evidence),
4914        });
4915        self.state.event("land", message);
4916        self.state.save()?;
4917        self.settle_questions();
4918        Ok(true)
4919    }
4920
4921    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4922    /// last landed the winner on, once it has run, else the commit the run
4923    /// branched from.
4924    ///
4925    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4926    /// and `vote` all happen before there is a winner to rebase, so they
4927    /// compare every candidate against the branch point on purpose, and a
4928    /// base that moves after they are already done cannot change an answer
4929    /// they already gave.
4930    fn landing_base(&self) -> String {
4931        self.state
4932            .base_sync
4933            .as_ref()
4934            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4935    }
4936
4937    // ------------------------------------------------------- operator fix
4938
4939    /// Route specific, already-recorded review findings to a fixer for a
4940    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
4941    /// entry point.
4942    ///
4943    /// Distinct from `review_loop`'s own fix step in three ways: it never
4944    /// runs a reviewer wave, it never spends review-round budget, and what
4945    /// happened is recorded as an [`OperatorFixRequest`] appended to
4946    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
4947    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
4948    /// and vote must never be rewritten to look like a manufactured blocking
4949    /// verdict.
4950    ///
4951    /// Only meaningful once review has actually concluded: `Ready` (handed
4952    /// off with findings still open, or simply concluded clean while minor
4953    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
4954    /// gate failed). Everything else is refused: a run still in progress
4955    /// should simply be resumed, and a `Merged` run's branch has already
4956    /// landed — reopening *this* run's own record cannot change that, so the
4957    /// answer there is a fresh `magi review <branch>`.
4958    ///
4959    /// A real commit here re-verifies through a fresh, ordinary review-only
4960    /// run on the same branch ([`Self::review`]) rather than reopening this
4961    /// run's own `review_loop`: once any round in this run's history went
4962    /// clean, `review_conclusion` treats that as permanent by design (the
4963    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
4964    /// way to force one more genuine reviewer wave out of *this* run without
4965    /// either rewriting history or weakening that guarantee for every other
4966    /// caller. A review-only run costs nothing extra — no implementation, no
4967    /// judging, no vote — and exercises the exact same review → verify →
4968    /// gate → (human) merge path, unmodified.
4969    pub async fn fix_selected(
4970        &mut self,
4971        ids: &[String],
4972        reason: &str,
4973        allow_stale: bool,
4974    ) -> Result<()> {
4975        let reason = reason.trim();
4976        if reason.is_empty() {
4977            bail!("a fix request needs a reason — that is the operator's own record of why");
4978        }
4979        if ids.is_empty() {
4980            bail!("no finding id given");
4981        }
4982        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4983            bail!(
4984                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4985                 has already concluded — can be given a targeted fix. A run still \
4986                 in progress should simply be resumed; a `merged` run's branch has \
4987                 already landed, so its answer is a fresh `magi review <branch>`, \
4988                 not reopening this run's own record",
4989                self.state.id,
4990                self.state.status.as_str()
4991            );
4992        }
4993        let Some(winner) = self.state.winner().cloned() else {
4994            bail!("run {} has no winning candidate to fix", self.state.id);
4995        };
4996        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4997            bail!(
4998                "branch `{}` no longer exists; this run cannot be extended",
4999                winner.branch
5000            );
5001        }
5002        let home = crate::run::home();
5003        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5004            bail!(
5005                "run {} is currently being worked on by another magi process",
5006                self.state.id
5007            );
5008        }
5009        // Held for the rest of this call, including the follow-up review
5010        // below: two `magi fix` invocations against the same run must not
5011        // both reach the worktree manipulation further down, which would
5012        // otherwise race to remove and recreate the same directory — see
5013        // [`FixClaim`]'s own doc.
5014        let _claim = FixClaim::acquire(&self.state.dir())?;
5015
5016        // Resolve every id before spending anything — an unknown id refuses
5017        // the whole request rather than silently dropping it — and dedup
5018        // while keeping the operator's own order.
5019        let mut seen = BTreeSet::new();
5020        let mut findings = Vec::new();
5021        let mut missing = Vec::new();
5022        for id in ids {
5023            if !seen.insert(id.clone()) {
5024                continue;
5025            }
5026            match self.state.finding(id) {
5027                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5028                    id: f.id.clone(),
5029                    severity: f.severity,
5030                    reviewer_vote: rec.vote,
5031                    round: round.round,
5032                    round_head: round.head.clone(),
5033                    reviewer: rec.reviewer,
5034                    agent: rec.agent.clone(),
5035                    file: f.file.clone(),
5036                    line: f.line,
5037                    title: f.title.clone(),
5038                    detail: f.detail.clone(),
5039                    outcome: OperatorFixOutcome::Pending,
5040                }),
5041                None => missing.push(id.clone()),
5042            }
5043        }
5044        if !missing.is_empty() {
5045            bail!(
5046                "unknown finding id(s): {}; nothing was changed",
5047                missing.join(", ")
5048            );
5049        }
5050
5051        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5052        let stale_details: Vec<(String, String)> = findings
5053            .iter()
5054            .filter(|f| f.round_head != head_at_request)
5055            .map(|f| (f.id.clone(), f.round_head.clone()))
5056            .collect();
5057        let stale = !stale_details.is_empty();
5058        if stale && !allow_stale {
5059            bail!(
5060                "the branch has moved since some finding(s) were raised — {} — now \
5061                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5062                stale_details
5063                    .iter()
5064                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5065                    .collect::<Vec<_>>()
5066                    .join(", "),
5067                short(&head_at_request)
5068            );
5069        }
5070
5071        let request = OperatorFixRequest {
5072            requested_at: Timestamp::now(),
5073            reason: reason.to_owned(),
5074            findings,
5075            head_at_request: head_at_request.clone(),
5076            allow_stale,
5077            stale,
5078            fix: None,
5079            result_head: None,
5080            follow_up_review_run: None,
5081        };
5082        self.state.event(
5083            "fix",
5084            format!(
5085                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5086                request.findings.len(),
5087                request
5088                    .findings
5089                    .iter()
5090                    .map(|f| f.id.as_str())
5091                    .collect::<Vec<_>>()
5092                    .join(", "),
5093            ),
5094        );
5095        // Recorded now, before any worktree work or the fixer call itself —
5096        // and re-saved at each checkpoint below: a crash at any point after
5097        // this (mid fixer call, mid follow-up review) must not lose the fact
5098        // that this was requested, for which findings, and why. Everything
5099        // past this point reads and writes through `request_index` rather
5100        // than a local variable, since `request` itself is moved here.
5101        self.state.operator_fixes.push(request);
5102        self.state.save()?;
5103        let request_index = self.state.operator_fixes.len() - 1;
5104
5105        // A fresh, dedicated worktree for this one call, never the winner's
5106        // own worktree in place: that one may already be gone (folded away),
5107        // and reusing it in place would leave the branch checked out there
5108        // when the follow-up review below tries to check it out again. Freed
5109        // immediately after, either way — but only once confirmed clean:
5110        // `worktree_remove` is a `git worktree remove --force`, which would
5111        // otherwise discard uncommitted work left there by the operator or
5112        // another process before this had a chance to even look at it.
5113        let repo = self.state.repo.clone();
5114        let coordination = self.coordinate(&repo).await?;
5115        if winner.worktree.exists() {
5116            // Lockfiles a rescue commit withheld stay untracked on purpose and
5117            // are already recorded; they are not the operator's work to protect.
5118            let dirty = git::git(
5119                &winner.worktree,
5120                &["status", "--porcelain", "--untracked-files=all"],
5121            )
5122            .await?;
5123            let only_withheld = dirty.lines().all(|l| {
5124                l.strip_prefix("?? ")
5125                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5126            });
5127            if !only_withheld {
5128                bail!(
5129                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5130                     discard them first",
5131                    winner.worktree.display()
5132                );
5133            }
5134            git::worktree_remove(&self.state.repo, &winner.worktree)
5135                .await
5136                .ok();
5137        }
5138        let fix_worktree = self.state.worktree_root().join("operator-fix");
5139        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5140        git::git(
5141            &self.state.repo,
5142            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5143        )
5144        .await
5145        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5146        if !git::is_clean(&fix_worktree).await? {
5147            git::worktree_remove(&self.state.repo, &fix_worktree)
5148                .await
5149                .ok();
5150            bail!(
5151                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5152                winner.branch
5153            );
5154        }
5155        let repo_for_setup = self.state.repo.clone();
5156        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5157            Ok(()) => {
5158                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5159            }
5160            Err(e) => Err(e),
5161        };
5162        if let Err(e) = setup {
5163            git::worktree_remove(&self.state.repo, &fix_worktree)
5164                .await
5165                .ok();
5166            return Err(e);
5167        }
5168
5169        let run_id = self.state.id.clone();
5170        let prompts = self.state.config.prompts.clone();
5171        let language = self.state.config.graph.language.clone();
5172        let sessions = self.state.config.graph.sessions;
5173        let artifacts = agent::artifacts_dir(&self.state.dir());
5174        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5175            .findings
5176            .iter()
5177            .map(|f| Finding {
5178                id: f.id.clone(),
5179                severity: f.severity,
5180                file: f.file.clone(),
5181                line: f.line,
5182                title: f.title.clone(),
5183                detail: f.detail.clone(),
5184            })
5185            .collect();
5186        let fix_prompt = prompt::operator_fix_in(
5187            &self.state.instruction,
5188            &finding_list,
5189            reason,
5190            &stale_details,
5191            &head_at_request,
5192            &language,
5193            &self.state.config.graph.github_language,
5194        );
5195        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5196        drop(coordination);
5197        let (job, seat, out) = self
5198            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5199                prompt: fix_prompt.clone(),
5200                spec,
5201                seat,
5202                cwd: fix_worktree.clone(),
5203                timeout,
5204                allow_write: true,
5205                sessions,
5206                artifacts: artifacts.clone(),
5207                stem: "operator-fix".to_owned(),
5208                handover: None,
5209            })
5210            .await;
5211        let agent_id = seat.agent.clone();
5212
5213        let mut fix = FixRecord {
5214            agent: agent_id,
5215            addressed: Vec::new(),
5216            rejected: Vec::new(),
5217            notes: String::new(),
5218            committed: false,
5219            failed: None,
5220            duration_ms: 0,
5221            continuation: None,
5222        };
5223        let mut final_seat = seat.clone();
5224        match out {
5225            AgentOutcome::Ok(o) => {
5226                fix.duration_ms = o.duration_ms;
5227                let parsed = verdict::extract_json::<FixReport>(&o.text);
5228                let incomplete_reason = match &parsed {
5229                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5230                        "the reply parsed, but it reported a command whose own CLI \
5231                         never confirmed an exit status"
5232                            .to_owned(),
5233                    ),
5234                    Ok(_) => None,
5235                    Err(e) => Some(e.to_string()),
5236                };
5237                match incomplete_reason {
5238                    None => {
5239                        let report = parsed.expect("checked Ok above");
5240                        fix.addressed = report.addressed;
5241                        fix.rejected = report.rejected;
5242                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5243                    }
5244                    Some(reason) => {
5245                        let (resumed_seat, resolved, failure, cont) = self
5246                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5247                            .await;
5248                        fix.duration_ms += cont.cumulative_wait_ms;
5249                        fix.continuation = Some(cont);
5250                        final_seat = resumed_seat;
5251                        match resolved {
5252                            Some(report) => {
5253                                fix.addressed = report.addressed;
5254                                fix.rejected = report.rejected;
5255                                fix.notes =
5256                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5257                            }
5258                            None => fix.failed = failure,
5259                        }
5260                    }
5261                }
5262            }
5263            AgentOutcome::Dropped(o) => {
5264                fix.duration_ms = o.duration_ms;
5265                let why = o
5266                    .dropped
5267                    .as_ref()
5268                    .map(|d| d.why.as_str())
5269                    .unwrap_or("the CLI ended the stream without delivering its answer");
5270                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5271            }
5272            AgentOutcome::Quota(o) => {
5273                self.state.quota.push(QuotaLoss {
5274                    seat: final_seat.key.clone(),
5275                    node: "fix".to_owned(),
5276                    at: Timestamp::now(),
5277                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5278                });
5279                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5280            }
5281            AgentOutcome::Failed(e) => fix.failed = Some(e),
5282            AgentOutcome::ResourceBlocked(e) => {
5283                fix.failed = Some(format!("the fixer was not started: {e}"));
5284            }
5285        }
5286        if fix.continuation.is_none() {
5287            fix.continuation = Some(ContinuationRecord::not_needed());
5288        }
5289        self.state.seats.insert(final_seat.key.clone(), final_seat);
5290
5291        let rescue_message = format!(
5292            "magi: operator-selected fix ({}) (uncommitted work)",
5293            self.state.operator_fixes[request_index]
5294                .findings
5295                .iter()
5296                .map(|f| f.id.as_str())
5297                .collect::<Vec<_>>()
5298                .join(", ")
5299        );
5300        let repo = self.state.repo.clone();
5301        let coordination = self.coordinate(&repo).await?;
5302        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5303            self.state.note_withheld("fix", &r.withheld);
5304        }
5305        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5306        fix.committed = after != head_at_request;
5307        git::worktree_remove(&self.state.repo, &fix_worktree)
5308            .await
5309            .ok();
5310        drop(coordination);
5311
5312        self.state.event(
5313            "fix",
5314            match &fix.failed {
5315                Some(reason) => format!(
5316                    "operator fix: adoption report was lost ({reason}); {}",
5317                    if fix.committed {
5318                        "committed"
5319                    } else {
5320                        "NO new commit"
5321                    }
5322                ),
5323                None => format!(
5324                    "operator fix: {} addressed, {} rejected, {}",
5325                    fix.addressed.len(),
5326                    fix.rejected.len(),
5327                    if fix.committed {
5328                        "committed"
5329                    } else {
5330                        "NO new commit"
5331                    }
5332                ),
5333            },
5334        );
5335
5336        // Every selected finding gets an outcome — never left `Pending` once
5337        // the fixer's own turn is over. A report that never came back at all
5338        // marks every one of them `Unreported`, not silently "not addressed":
5339        // quota, a dropped stream, or an exhausted continuation are gaps in
5340        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5341        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5342        for f in &mut self.state.operator_fixes[request_index].findings {
5343            f.outcome = if fix.failed.is_some() {
5344                OperatorFixOutcome::Unreported
5345            } else if fix.addressed.contains(&f.id) {
5346                OperatorFixOutcome::Addressed
5347            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5348                OperatorFixOutcome::Rejected { why: r.why.clone() }
5349            } else {
5350                OperatorFixOutcome::Unreported
5351            };
5352        }
5353
5354        let committed = fix.committed;
5355        if committed {
5356            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5357        }
5358        self.state.operator_fixes[request_index].fix = Some(fix);
5359        // Saved again now that the fixer's own outcome is final, on top of
5360        // the save right after the request was first pushed above.
5361        self.state.save()?;
5362
5363        if committed {
5364            self.state.event(
5365                "fix",
5366                format!(
5367                    "operator fix committed {}; opening a follow-up review-only run",
5368                    short(&after)
5369                ),
5370            );
5371            // The operator asked for the fix, and the follow-up serves whatever
5372            // task the run it follows served.
5373            let origin =
5374                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5375            match Self::review(
5376                &self.state.repo,
5377                &winner.branch,
5378                self.state.config.clone(),
5379                origin,
5380            )
5381            .await
5382            {
5383                Ok(mut follow_up) => {
5384                    follow_up.state.event(
5385                        "start",
5386                        format!(
5387                            "requested by an operator fix on run {} for finding(s) {}",
5388                            self.state.id,
5389                            self.state.operator_fixes[request_index]
5390                                .findings
5391                                .iter()
5392                                .map(|f| f.id.as_str())
5393                                .collect::<Vec<_>>()
5394                                .join(", "),
5395                        ),
5396                    );
5397                    follow_up.state.save()?;
5398                    let follow_up_id = follow_up.state.id.clone();
5399                    // The follow-up is a run like any other: it belongs to the
5400                    // task of the run it follows, or to one filed for it.
5401                    let adopted = match crate::direct::adopt(
5402                        &follow_up.state,
5403                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5404                    ) {
5405                        Ok(a) => a,
5406                        Err(e) => {
5407                            // An ownerless run must not spend agent calls; it
5408                            // stays saved, and `magi run --resume` adopts it.
5409                            self.state.event(
5410                                "fix",
5411                                format!(
5412                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5413                                ),
5414                            );
5415                            self.state.save()?;
5416                            return Ok(());
5417                        }
5418                    };
5419                    let executed = follow_up.execute().await;
5420                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5421                    if let Some(a) = adopted {
5422                        a.finish(&follow_up.state, executed);
5423                    }
5424                    if let Some(e) = failure {
5425                        self.state.event(
5426                            "fix",
5427                            format!(
5428                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5429                            ),
5430                        );
5431                    }
5432                    self.state.operator_fixes[request_index].follow_up_review_run =
5433                        Some(follow_up_id);
5434                }
5435                Err(e) => {
5436                    self.state.event(
5437                        "fix",
5438                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5439                    );
5440                }
5441            }
5442            self.state.save()?;
5443        }
5444
5445        Ok(())
5446    }
5447
5448    // --------------------------------------------------------------- review
5449
5450    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5451    /// names, in order, each at most once, moving on only when the call
5452    /// advances (an error, a quota hit or nothing usable - the same decision
5453    /// point `agent::chain_advances` is for the other chained roles).
5454    ///
5455    /// Each agent gets its own seat from `Runner::seat`: the same agent
5456    /// continues its conversation (the winner's own implementer seat when it
5457    /// is the winner's author, now that the competition is over), another
5458    /// takes a fresh one so the full prompt is sent again. A handover is
5459    /// recorded under `node`, which is what makes the fallback stick for the
5460    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5461    /// is returned, so an exhausted chain reads exactly like a single failed
5462    /// fixer: one quota loss, the same wording, the same refund. An earlier
5463    /// attempt's edits are left in the tree and judged, with the final
5464    /// attempt's, by what git says afterwards.
5465    ///
5466    /// Returns the job that produced the outcome, for `continue_fix_report`.
5467    async fn ask_fixer(
5468        &mut self,
5469        winner: &Candidate,
5470        node: &'static str,
5471        round: Option<usize>,
5472        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5473    ) -> (SeatJob, SeatState, AgentOutcome) {
5474        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5475        self.ask_fixer_among(attempts, node, round, None, build)
5476            .await
5477    }
5478
5479    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5480    /// worktree and a commit: before the next agent is asked after one that
5481    /// did not satisfy the call, the worktree is put back on that commit, so
5482    /// a failed agent's partial edits are not carried into the next one.
5483    async fn ask_fixer_among(
5484        &mut self,
5485        attempts: Vec<(AgentSpec, String)>,
5486        node: &'static str,
5487        round: Option<usize>,
5488        restore: Option<(&Path, &str)>,
5489        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5490    ) -> (SeatJob, SeatState, AgentOutcome) {
5491        let run_id = self.state.id.clone();
5492        let prompts = self.state.config.prompts.clone();
5493        let cache = self.state.config.cache_dir();
5494        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5495        let mut last = None;
5496        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5497            let seat = self.seat(&key, &spec.id);
5498            let mut job = build(spec.clone(), seat);
5499            if i > 0 {
5500                job.stem = format!("{}-{}", job.stem, spec.id);
5501            }
5502            let ctx = WaveCtx {
5503                carry_seats: false,
5504                pauses: Some((&self.pause, &self.interrupt)),
5505                run: &run_id,
5506                node,
5507                prompts: &prompts,
5508                cache: cache.as_deref(),
5509                round,
5510            };
5511            let (seat, out) =
5512                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5513            // A resource block never advances the chain: the chosen agent
5514            // was not asked, so it must be the one asked again on resume.
5515            let advances = match &out {
5516                AgentOutcome::Ok(o) => agent::output_advances(o),
5517                AgentOutcome::ResourceBlocked(_) => false,
5518                _ => true,
5519            };
5520            if let Some(next) = ids.get(i + 1)
5521                && advances
5522            {
5523                self.state.seats.insert(seat.key.clone(), seat.clone());
5524                let class =
5525                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5526                let reason = match &out {
5527                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5528                    other => fail_reason(other),
5529                };
5530                // A failed agent's edits must not reach the next one, and a
5531                // restore that did not work ends the chain: the call fails
5532                // and the caller's own (confirmed) restore decides.
5533                if let Some((wt, rev)) = restore
5534                    && let Err(e) = self.graft_restore(wt, rev).await
5535                {
5536                    let why =
5537                        format!("could not restore the worktree before the next fixer: {e:#}");
5538                    tracing::warn!("{why}");
5539                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5540                        AgentOutcome::ResourceBlocked(why)
5541                    } else {
5542                        AgentOutcome::Failed(why)
5543                    };
5544                    last = Some((job, seat, out));
5545                    break;
5546                }
5547                record_handover(
5548                    &mut self.state,
5549                    node,
5550                    &seat.key,
5551                    &spec.id,
5552                    next,
5553                    &class,
5554                    &reason,
5555                );
5556                continue;
5557            }
5558            last = Some((job, seat, out));
5559            break;
5560        }
5561        last.expect("the fixer chain always has an entry")
5562    }
5563
5564    /// The one repair a verification-only failure earns (see
5565    /// [`crate::recovery`]), before the first review round of a run that took
5566    /// a branch over.
5567    ///
5568    /// Saved as started *before* the fixer is called, so a resume never
5569    /// repairs twice; a start found on re-entry is rolled back to the commit
5570    /// it began from. A failed or unusable fixer leaves the tree as it was and
5571    /// the review goes on over it, so the first round's own verification
5572    /// reports the failure again. The failure is never carried into a round as
5573    /// a result: verification is re-run on whatever commit this leaves.
5574    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5575        use crate::recovery::{Mode, Repair};
5576        if !self.state.reviews.is_empty() {
5577            return Ok(());
5578        }
5579        let Some(mut rec) = self.state.recovery.clone() else {
5580            return Ok(());
5581        };
5582        if rec.mode != Mode::VerifyRepair {
5583            return Ok(());
5584        }
5585        let wt = winner.worktree.clone();
5586        let mut head = git::rev_parse(&wt, "HEAD").await?;
5587        match rec.repair.clone() {
5588            Repair::Done { .. } => return Ok(()),
5589            Repair::WaitingForRestore { before } => {
5590                self.graft_restore(&wt, &before).await?;
5591                head = before;
5592                rec.repair = Repair::NotStarted;
5593                self.state.recovery = Some(rec.clone());
5594                self.state.save()?;
5595            }
5596            Repair::Started { before } => {
5597                // Interrupted: the unfinished edits are not trusted.
5598                // An unconfirmed rollback blocks the run with `Started` kept,
5599                // so a resume retries it; the tree is never reviewed as is.
5600                if let Err(e) = self.graft_restore(&wt, &before).await {
5601                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5602                        return Err(e);
5603                    }
5604                    return self
5605                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5606                        .await;
5607                }
5608                rec.repair = Repair::Done {
5609                    before: before.clone(),
5610                    after: before,
5611                    committed: false,
5612                    failed: Some("interrupted; rolled back".to_owned()),
5613                };
5614                self.state
5615                    .event("recovery", "the repair was interrupted and rolled back");
5616                self.state.recovery = Some(rec);
5617                self.state.save()?;
5618                return Ok(());
5619            }
5620            Repair::NotStarted => {}
5621        }
5622        if crate::recovery::recheck(&mut rec, &head) {
5623            self.state.event("recovery", rec.summary());
5624            self.state.recovery = Some(rec);
5625            self.state.save()?;
5626            return Ok(());
5627        }
5628        rec.repair = Repair::Started {
5629            before: head.clone(),
5630        };
5631        self.state.recovery = Some(rec.clone());
5632        self.state.save()?;
5633
5634        let language = self.state.config.graph.language.clone();
5635        let sessions = self.state.config.graph.sessions;
5636        let artifacts = agent::artifacts_dir(&self.state.dir());
5637        let fix_prompt = prompt::recovery_fix(
5638            &self.state.instruction,
5639            &rec,
5640            self.state.config.graph.review_rounds,
5641            &language,
5642        );
5643        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5644        self.state.event(
5645            "recovery",
5646            format!(
5647                "repair: the fixer starts from the failed verification on {}",
5648                short(&head)
5649            ),
5650        );
5651        let (_job, seat, out) = self
5652            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5653                prompt: fix_prompt.clone(),
5654                spec,
5655                seat,
5656                cwd: wt.clone(),
5657                timeout,
5658                allow_write: true,
5659                sessions,
5660                artifacts: artifacts.clone(),
5661                stem: "fix-recovery".to_owned(),
5662                handover: None,
5663            })
5664            .await;
5665        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5666            rec.repair = Repair::WaitingForRestore { before: head };
5667            self.state.recovery = Some(rec);
5668            return self.stop_for_resource();
5669        }
5670        let failed = match &out {
5671            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5672            AgentOutcome::Quota(o) => {
5673                self.state.quota.push(QuotaLoss {
5674                    seat: seat.key.clone(),
5675                    node: "fix".to_owned(),
5676                    at: Timestamp::now(),
5677                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5678                });
5679                Some("rate limited (quota); the fixer could not run".to_owned())
5680            }
5681            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5682            AgentOutcome::Failed(e) => Some(e.clone()),
5683        };
5684        self.state.seats.insert(seat.key.clone(), seat);
5685        let coordination = self.coordinate(&wt).await?;
5686        let failed = match failed {
5687            // The agent's partial edits are not kept.
5688            Some(why) => {
5689                if let Err(e) = git::restore_to(&wt, &head).await {
5690                    return self
5691                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5692                        .await;
5693                }
5694                Some(why)
5695            }
5696            None => {
5697                if let Ok(r) = git::rescue_commit(
5698                    &wt,
5699                    "magi: recovery repair of the failed verification (uncommitted work)",
5700                )
5701                .await
5702                {
5703                    self.state.note_withheld("fix", &r.withheld);
5704                }
5705                None
5706            }
5707        };
5708        let after = git::rev_parse(&wt, "HEAD").await?;
5709        drop(coordination);
5710        self.state.event(
5711            "recovery",
5712            match &failed {
5713                Some(why) => format!("repair: nothing kept ({why})"),
5714                None if after != head => {
5715                    format!("repair: committed {}", short(&after))
5716                }
5717                None => "repair: the fixer made no commit".to_owned(),
5718            },
5719        );
5720        rec.repair = Repair::Done {
5721            before: head.clone(),
5722            committed: after != head,
5723            after,
5724            failed,
5725        };
5726        self.state.recovery = Some(rec);
5727        self.state.save()?;
5728        Ok(())
5729    }
5730
5731    /// A repair whose rollback could not be confirmed: the run is blocked and
5732    /// resumable, and the unconfirmed state is never reviewed.
5733    async fn stop_recovery(
5734        &mut self,
5735        mut rec: crate::recovery::RecoveryRecord,
5736        head: &str,
5737        why: String,
5738    ) -> Result<()> {
5739        // `Started` stays recorded: a resume retries the restore.
5740        rec.repair = crate::recovery::Repair::Started {
5741            before: head.to_owned(),
5742        };
5743        self.state.recovery = Some(rec);
5744        self.state
5745            .event("recovery", format!("repair could not be undone: {why}"));
5746        self.state.status = RunStatus::Blocked;
5747        self.state.save()?;
5748        bail!("recovery repair could not be rolled back: {why}")
5749    }
5750
5751    async fn review_loop(&mut self) -> Result<()> {
5752        // A base that would not rebase is a person's decision, not a review
5753        // round: nothing here would change the answer, and reviewers and a
5754        // fixer would be spending real budget on a tree that cannot land
5755        // regardless of what they find.
5756        if self
5757            .state
5758            .base_sync
5759            .as_ref()
5760            .is_some_and(|s| s.conflict.is_some())
5761        {
5762            return Ok(());
5763        }
5764        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5765        // agent files with `magi task add` name the run that paid for it. The
5766        // prompt overlay is cloned alongside it because the waves borrow it
5767        // while `self` is mutably borrowed by the node's own bookkeeping.
5768        let run_id = self.state.id.clone();
5769        let prompts = self.state.config.prompts.clone();
5770        let Some(winner) = self.state.winner().cloned() else {
5771            return Ok(());
5772        };
5773        let max_rounds = self.state.config.graph.review_rounds;
5774        // A clean round, an exhausted round budget, or a stalled tree (see
5775        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5776        // they are recorded — recomputed here, not read off `status`, so a
5777        // reentry into a run that already stopped restates the identical
5778        // verdict instead of silently handing back whatever an earlier node
5779        // in this same walk clobbered `status` to (a solo-candidate
5780        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5781        // below runs an empty range once the budget is spent, and would
5782        // otherwise fall through without touching `status` at all.
5783        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5784            // A reentry after a crash between the last round's save and
5785            // `stop_reviewing` reaches the hand-off here, not there.
5786            if status == RunStatus::Gating {
5787                self.record_contested_handoff();
5788            }
5789            self.state.status = status;
5790            self.state.save()?;
5791            return Ok(());
5792        }
5793        self.state.status = RunStatus::Reviewing;
5794        // A last recorded round whose own verification never resolved
5795        // (`ResourceBlocked` — the shared build cache, not the patch) is
5796        // never a concluded round, whatever the round budget says: starting
5797        // a fresh round on top of it would spend a whole new reviewer wave
5798        // re-reading an unchanged patch instead of just retrying the one
5799        // check that actually needs it, and once the budget is spent the
5800        // loop below has nothing left to do at all (its range is empty).
5801        // Retry that check directly instead, exactly the same retry
5802        // `stop_reviewing` already does for its own catch-up case.
5803        if self
5804            .state
5805            .reviews
5806            .last()
5807            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5808        {
5809            let shell = self.state.config.shell();
5810            return self
5811                .stop_reviewing(
5812                    "the last round's own verification never resolved",
5813                    &shell,
5814                    &winner.worktree,
5815                )
5816                .await;
5817        }
5818
5819        self.recovery_repair(&winner).await?;
5820        if self.resource_halt()? {
5821            return Ok(());
5822        }
5823
5824        let repo = self.state.repo.clone();
5825        let root = self.state.worktree_root();
5826        let language = self.state.config.graph.language.clone();
5827        let sessions = self.state.config.graph.sessions;
5828        let artifacts = self.merge_repair_artifacts();
5829        let base = self.landing_base();
5830        let base_short = short(&base);
5831        let reviewers = self.roles.reviewers.clone();
5832        let shell = self.state.config.shell();
5833
5834        for round in (self.state.reviews.len() + 1)..=max_rounds {
5835            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5836            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5837            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5838            // The prior round's own record, already persisted — never a
5839            // hand-carried variable of just its failing output: that is
5840            // exactly what let a round's e2e result drift out of sync with
5841            // which commit it was actually about (see `SCHEMA`'s doc for
5842            // schema 8). Judged against `head`, the commit reviewers are
5843            // about to look at now, so the summary always reads as "an
5844            // earlier head" here — this round's own patch has not been
5845            // checked yet.
5846            let prev_verification = self
5847                .state
5848                .reviews
5849                .last()
5850                .and_then(|r| r.verification_summary(&head));
5851            // Only the first round of a run that took a branch over: later
5852            // rounds have this run's own history.
5853            let prior_section = match (&self.state.recovery, round) {
5854                (Some(rec), 1) if self.state.reviews.is_empty() => {
5855                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5856                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5857                            Some(crate::recovery::head_of(
5858                                &git::diff(&winner.worktree, reviewed, "HEAD")
5859                                    .await
5860                                    .unwrap_or_default(),
5861                                crate::recovery::SINCE_DIFF_MAX,
5862                            ))
5863                        }
5864                        _ => None,
5865                    };
5866                    Some(prompt::recovery_review(rec, since.as_deref()))
5867                }
5868                _ => None,
5869            };
5870
5871            // Each reviewer gets its own detached checkout of exactly this
5872            // commit: nobody can perturb the winner's tree, and the fixer can
5873            // keep working without racing a reviewer.
5874            let mut jobs = Vec::new();
5875            let coordination = self.coordinate(&repo).await?;
5876            for (r, spec) in reviewers.iter().cloned().enumerate() {
5877                let wt = root.join(format!("review-{}", r + 1));
5878                // `clean -fdx` in a reset wipes setup's products, so every
5879                // round sets the seat up again.
5880                if wt.exists() {
5881                    git::reset_detached(&wt, &head).await?;
5882                } else {
5883                    git::worktree_add_detached(&repo, &wt, &head).await?;
5884                }
5885                self.setup_seat_worktree(&repo, &wt, false).await?;
5886                let seat_key = format!("review-{}", r + 1);
5887                // The seat starts the round on whoever answered it last, not
5888                // on the agent the spec names, so a failure is not re-paid.
5889                let spec = pick_start_spec(
5890                    &self.roles.reviewer_roster,
5891                    spec,
5892                    self.state.seat_history.get(&seat_key),
5893                );
5894                let seat = self.seat(&seat_key, &spec.id);
5895                jobs.push(SeatJob {
5896                    prompt: prompt::review(&prompt::ReviewCtx {
5897                        instruction: &self.state.instruction,
5898                        branch: &winner.branch,
5899                        base_short: &base_short,
5900                        stat: &stat,
5901                        patch: &patch,
5902                        verification: prev_verification.as_ref(),
5903                        prior: prior_section.as_deref(),
5904                        reviewers: reviewers.len(),
5905                        round,
5906                        rounds: max_rounds,
5907                        // A review-only run has no rankings, so nothing
5908                        // competed for this patch and the reviewer is told so.
5909                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5910                        lens: Lens::for_seat(r),
5911                        language: &language,
5912                        github_language: &self.state.config.graph.github_language,
5913                    }),
5914                    spec,
5915                    seat,
5916                    cwd: wt,
5917                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5918                    allow_write: false,
5919                    sessions,
5920                    artifacts: artifacts.clone(),
5921                    stem: format!("review-{round}-{}", r + 1),
5922                    handover: None,
5923                });
5924            }
5925
5926            drop(coordination);
5927            self.state.event(
5928                "review",
5929                format!(
5930                    "round {round}: {} reviewers on {}",
5931                    jobs.len(),
5932                    short(&head)
5933                ),
5934            );
5935            let mut quota_losses = Vec::new();
5936            let review_retries = self.state.config.graph.retries;
5937            let review_cache = self.state.config.cache_dir();
5938            let ctx = WaveCtx {
5939                carry_seats: true,
5940                pauses: Some((&self.pause, &self.interrupt)),
5941                run: &run_id,
5942                node: "review",
5943                prompts: &prompts,
5944                cache: review_cache.as_deref(),
5945                round: Some(round),
5946            };
5947            let results = ask_json_wave::<Review>(
5948                jobs,
5949                Arc::clone(&self.sem),
5950                review_retries,
5951                &self.roles.reviewer_roster,
5952                &ctx,
5953                &mut quota_losses,
5954                &mut self.state,
5955                &|_: &Review| Ok(()),
5956            )
5957            .await;
5958            if self.state.resource_block.is_some() {
5959                return self.stop_for_resource();
5960            }
5961            // Counted before the move below: how many of *this* round's
5962            // reviewer seats were lost to their own rate limit, as opposed to
5963            // a crash, a timeout, or unparsable output — see `round_is_clean`.
5964            let round_quota_missing = quota_losses.len();
5965            self.state.quota.extend(quota_losses);
5966
5967            let mut records = Vec::new();
5968            let mut all_findings = Vec::new();
5969            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5970                let agent_id = seat.agent.clone();
5971                self.state.seats.insert(seat.key.clone(), seat);
5972                let mut record = ReviewRecord {
5973                    reviewer: r + 1,
5974                    agent: agent_id,
5975                    summary: String::new(),
5976                    findings: Vec::new(),
5977                    vote: None,
5978                    failed: None,
5979                    duration_ms: 0,
5980                    // Set for both outcomes: `failed: Some(_)` with
5981                    // `attempts > 0` is a seat every retry still lost, not a
5982                    // recovered one — only `failed: None` with `attempts > 0`
5983                    // reads as "answered after a nudge" (see this field's own
5984                    // doc).
5985                    attempts,
5986                };
5987                match res {
5988                    Ok((review, out)) => {
5989                        // Sanitized here, at the point every other piece of
5990                        // agent prose in this file is (candidate summaries,
5991                        // deliberation turns, vote reasons): a reviewer's own
5992                        // words are the one thing about it that could name
5993                        // it, and reconsideration below broadcasts this same
5994                        // summary and these same findings to every other
5995                        // seat on the panel.
5996                        record.summary =
5997                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
5998                        record.vote = Some(review.vote);
5999                        record.duration_ms = out.duration_ms;
6000                        for (n, mut f) in review.findings.into_iter().enumerate() {
6001                            // ids are magi's, never the agent's: the fixer's
6002                            // adoption report is keyed by them.
6003                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6004                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6005                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6006                            // `file` is agent-supplied prose too, never
6007                            // checked against the real tree — the same
6008                            // exposure `title`/`detail` above have, just in
6009                            // a field easy to forget because it looks like a
6010                            // path rather than free text.
6011                            f.file = f
6012                                .file
6013                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6014                            all_findings.push(f.clone());
6015                            record.findings.push(f);
6016                        }
6017                        self.state.event(
6018                            "review",
6019                            format!(
6020                                "round {round}: reviewer {} voted {} with {} finding(s)",
6021                                r + 1,
6022                                review.vote.label(),
6023                                record.findings.len()
6024                            ),
6025                        );
6026                    }
6027                    Err(e) => {
6028                        record.failed = Some(e.to_string());
6029                        self.state.event(
6030                            "review",
6031                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6032                        );
6033                    }
6034                }
6035                records.push(record);
6036            }
6037
6038            // Tally the round's votes and, if they split, spend the one
6039            // round of reconsideration the split -> deliberate -> revote
6040            // shape `judge`/`vote` use for the panel, sized down to what a
6041            // read-only review round can afford: one round, and a revote
6042            // rather than an argument, because the panel already wrote its
6043            // reasoning down as findings the first time around.
6044            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6045            let vote_split =
6046                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6047            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6048            if vote_split {
6049                self.state.event(
6050                    "review",
6051                    format!(
6052                        "round {round}: votes split ({}) — one round of reconsideration",
6053                        initial_votes
6054                            .iter()
6055                            .map(|v| v.label())
6056                            .collect::<Vec<_>>()
6057                            .join(", ")
6058                    ),
6059                );
6060                // Seats read every seat's findings and votes, still numbered
6061                // and never named — the same anonymity `review` itself keeps.
6062                let panel: Vec<ReviewSeatReport<'_>> = records
6063                    .iter()
6064                    .filter_map(|r| {
6065                        r.vote.map(|vote| ReviewSeatReport {
6066                            reviewer: r.reviewer,
6067                            vote,
6068                            summary: &r.summary,
6069                            findings: &r.findings,
6070                        })
6071                    })
6072                    .collect();
6073
6074                let mut jobs = Vec::new();
6075                let mut seats_at = Vec::new();
6076                for (r, spec) in reviewers.iter().cloned().enumerate() {
6077                    // A seat with no initial vote has nothing to reconsider
6078                    // from and stays absent, the same as it stayed absent
6079                    // from `panel` above.
6080                    if records[r].vote.is_none() {
6081                        continue;
6082                    }
6083                    let wt = root.join(format!("review-{}", r + 1));
6084                    let seat_key = format!("review-{}", r + 1);
6085                    let spec = self.occupant(&seat_key, spec);
6086                    let seat = self.seat(&seat_key, &spec.id);
6087                    // A seat with no live session has already forgotten the
6088                    // initial review's prompt — restate the patch it is
6089                    // voting on, the same as `deliberate`/`vote` do for a
6090                    // judge in the same position.
6091                    // The panel already carries this seat's own review and
6092                    // vote, so restating the patch makes the prompt whole for
6093                    // a seat handed to another agent.
6094                    let build = |with_patch: bool| {
6095                        prompt::review_reconsider(&ReviewReconsiderCtx {
6096                            instruction: &self.state.instruction,
6097                            reviewer: r + 1,
6098                            lens: Lens::for_seat(r),
6099                            panel: &panel,
6100                            patch: with_patch.then_some(ReviewPatch {
6101                                branch: &winner.branch,
6102                                base_short: &base_short,
6103                                stat: &stat,
6104                                patch: &patch,
6105                            }),
6106                            round,
6107                            rounds: max_rounds,
6108                            language: &language,
6109                        })
6110                    };
6111                    let full = build(true);
6112                    let prompt = if has_context(&spec, &seat, sessions) {
6113                        build(false)
6114                    } else {
6115                        full.clone()
6116                    };
6117                    jobs.push(SeatJob {
6118                        prompt,
6119                        spec,
6120                        seat,
6121                        cwd: wt,
6122                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6123                        allow_write: false,
6124                        sessions,
6125                        artifacts: artifacts.clone(),
6126                        stem: format!("review-{round}-reconsider-{}", r + 1),
6127                        handover: Some(full),
6128                    });
6129                    seats_at.push(r);
6130                }
6131
6132                let mut recon_quota_losses = Vec::new();
6133                let recon_cache = self.state.config.cache_dir();
6134                let recon_ctx = WaveCtx {
6135                    carry_seats: true,
6136                    pauses: Some((&self.pause, &self.interrupt)),
6137                    run: &run_id,
6138                    node: "review",
6139                    prompts: &prompts,
6140                    cache: recon_cache.as_deref(),
6141                    round: Some(round),
6142                };
6143                let recon_results = ask_json_wave::<ReviewRevote>(
6144                    jobs,
6145                    Arc::clone(&self.sem),
6146                    review_retries,
6147                    &self.roles.reviewer_roster,
6148                    &recon_ctx,
6149                    &mut recon_quota_losses,
6150                    &mut self.state,
6151                    &|_: &ReviewRevote| Ok(()),
6152                )
6153                .await;
6154                if self.state.resource_block.is_some() {
6155                    return self.stop_for_resource();
6156                }
6157                self.state.quota.extend(recon_quota_losses);
6158
6159                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6160                    let agent_id = seat.agent.clone();
6161                    self.state.seats.insert(seat.key.clone(), seat);
6162                    let mut rec = ReviewRevoteRecord {
6163                        reviewer: r + 1,
6164                        agent: agent_id,
6165                        vote: None,
6166                        reason: String::new(),
6167                        failed: None,
6168                    };
6169                    match res {
6170                        Ok((rv, _)) => {
6171                            rec.vote = Some(rv.vote);
6172                            rec.reason =
6173                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6174                            self.state.event(
6175                                "review",
6176                                format!(
6177                                    "round {round}: reviewer {} revoted {}",
6178                                    r + 1,
6179                                    rv.vote.label()
6180                                ),
6181                            );
6182                        }
6183                        Err(e) => {
6184                            rec.failed = Some(e.to_string());
6185                            self.state.event(
6186                                "review",
6187                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6188                            );
6189                        }
6190                    }
6191                    reconsideration.push(rec);
6192                }
6193            } else if initial_votes.len() > 1 {
6194                self.state.event(
6195                    "review",
6196                    format!(
6197                        "round {round}: votes agreed ({}) — no reconsideration",
6198                        initial_votes[0].label()
6199                    ),
6200                );
6201            }
6202
6203            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6204            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6205            // A round that already has a blocking finding and a round left to
6206            // try is going back to the fixer no matter what `verify.e2e`
6207            // says, so running it first only spends the loop's slowest step
6208            // (minutes, for a Rust repo's full test suite) on a head about
6209            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6210            // runs once a round has no blocking findings left (see
6211            // `round_is_clean`, which a deferred — empty — `e2e` can never
6212            // satisfy since `blocking` is nonzero whenever this branch is
6213            // taken), and `stop_reviewing` forces a real run before it will
6214            // ever read a deferred round as green.
6215            let defer_e2e =
6216                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6217            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6218                let reason =
6219                    format!("{blocking} blocking finding(s) already required a fix this round");
6220                self.state.event(
6221                    "verify",
6222                    format!(
6223                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6224                         {}); it will run once a round has none left",
6225                        short(&head)
6226                    ),
6227                );
6228                (Vec::new(), false, true, Some(reason))
6229            } else {
6230                let e2e_commands = self.state.config.verify.e2e.clone();
6231                let cache_dir = self.state.config.cache_dir();
6232                let context = format!("round {round}");
6233                let wait = self.repository_wait();
6234                let (e2e, verify_retried) = with_cache_lease(
6235                    &mut self.state,
6236                    cache_dir.as_deref(),
6237                    "e2e",
6238                    "e2e",
6239                    &winner.worktree,
6240                    &head,
6241                    verify_timeout,
6242                    &context,
6243                    &wait,
6244                    |state, budget| {
6245                        let shell = shell.clone();
6246                        let e2e_commands = e2e_commands.clone();
6247                        let worktree = winner.worktree.clone();
6248                        let context = context.clone();
6249                        async move {
6250                            run_e2e_with_retry(
6251                                state,
6252                                &shell,
6253                                &e2e_commands,
6254                                &worktree,
6255                                budget,
6256                                &context,
6257                            )
6258                            .await
6259                        }
6260                    },
6261                )
6262                .await?;
6263                (e2e, verify_retried, false, None)
6264            };
6265
6266            let expected = records.len();
6267            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6268            let incomplete = answered < expected;
6269            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6270            let policy = self.state.config.graph.incomplete_review;
6271            let clean = round_is_clean(
6272                blocking,
6273                e2e_ok,
6274                answered,
6275                expected,
6276                round_quota_missing,
6277                policy,
6278            );
6279
6280            let mut round_record = ReviewRound {
6281                round,
6282                head: head.clone(),
6283                verified_head: None,
6284                verified_at: None,
6285                reviews: records,
6286                e2e,
6287                verify_retried,
6288                e2e_deferred,
6289                e2e_defer_reason,
6290                fix: None,
6291                blocking,
6292                answered,
6293                expected,
6294                clean,
6295                progressed: false,
6296                vote_split,
6297                reconsideration,
6298                verdict: None,
6299            };
6300            // The final vote per seat is its revote where reconsideration
6301            // ran and answered, its initial vote otherwise — the same
6302            // fallback `tally` uses for a judge whose private vote failed.
6303            round_record.verdict = ReviewVote::worst(
6304                round_record
6305                    .final_votes()
6306                    .into_iter()
6307                    .map(|(_, _, vote)| vote),
6308            );
6309            // Which commit and when magi actually attempted to check —
6310            // known the moment a command was dispatched against `head`,
6311            // whether or not it finished: a resource-blocked attempt still
6312            // targeted a specific commit at a specific time, and leaving
6313            // that unrecorded is exactly what made `verification_summary`
6314            // report a fresh attempt as "commit unknown ... recorded before
6315            // this was tracked", indistinguishable from a genuinely old,
6316            // untracked record. Only a deferred or unconfigured round never
6317            // ran at all and has nothing to record — see
6318            // `ReviewRound::verified_head`'s own doc.
6319            if !matches!(
6320                round_record.e2e_status(),
6321                E2eStatus::Deferred | E2eStatus::NotConfigured
6322            ) {
6323                round_record.verified_head = Some(head.clone());
6324                round_record.verified_at = Some(Timestamp::now());
6325            }
6326            let this_round_verification = round_record.verification_summary(&head);
6327
6328            if incomplete {
6329                let missing: Vec<String> = round_record
6330                    .reviews
6331                    .iter()
6332                    .filter(|r| r.failed.is_some())
6333                    .map(|r| format!("review-{}", r.reviewer))
6334                    .collect();
6335                self.state.event(
6336                    "review",
6337                    format!(
6338                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6339                        missing.join(", ")
6340                    ),
6341                );
6342            }
6343
6344            if clean {
6345                self.state.event(
6346                    "review",
6347                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6348                        format!(
6349                            "round {round}: clean (warn policy, incomplete panel) — no \
6350                             blocking findings from the seats that answered, verification green"
6351                        )
6352                    } else if incomplete {
6353                        format!(
6354                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6355                             quorum) — no blocking findings from the seats that answered, \
6356                             verification green",
6357                            expected - answered
6358                        )
6359                    } else {
6360                        format!("round {round}: clean — no blocking findings, verification green")
6361                    },
6362                );
6363                self.state.reviews.push(round_record);
6364                self.state.status = RunStatus::Gating;
6365                self.state.save()?;
6366                return Ok(());
6367            }
6368
6369            // Nothing was raised and verification passed, but not every seat
6370            // answered and `round_is_clean` still refused to call it clean —
6371            // either a seat is missing for a reason other than its own quota
6372            // (a crash, a timeout, unparsable output — worth another try), or
6373            // every seat that could have answered lost its quota and nobody
6374            // is left to decide on: re-review rather than send the fixer
6375            // after a round with nothing to fix.
6376            if incomplete && blocking == 0 && e2e_ok {
6377                self.state.reviews.push(round_record);
6378                self.state.save()?;
6379                if round == max_rounds {
6380                    self.state.status = RunStatus::Blocked;
6381                    self.state.event(
6382                        "review",
6383                        format!(
6384                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6385                             refusing to call it clean",
6386                            expected - answered
6387                        ),
6388                    );
6389                    return Ok(());
6390                }
6391                continue;
6392            }
6393
6394            // Nothing for the fixer to act on (`blocking == 0`) and the only
6395            // reason this round is not clean is that magi itself never got
6396            // a command to run — the shared build cache, not the patch (see
6397            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6398            // the fixer would invite a change to appease contention that has
6399            // nothing to do with the diff, and would leave this attempt
6400            // sitting in the next round's prompt as if it were about an
6401            // earlier, superseded commit rather than what it actually is:
6402            // the same head, still waiting to be checked. Wait for it the
6403            // same way the final round's own contention is already handled,
6404            // whatever round this happens to be.
6405            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6406                self.state.reviews.push(round_record);
6407                return self
6408                    .stop_reviewing(
6409                        "the round's own verification could not run",
6410                        &shell,
6411                        &winner.worktree,
6412                    )
6413                    .await;
6414            }
6415
6416            if round == max_rounds {
6417                self.state.reviews.push(round_record);
6418                return self
6419                    .stop_reviewing(
6420                        &format!(
6421                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6422                        ),
6423                        &shell,
6424                        &winner.worktree,
6425                    )
6426                    .await;
6427            }
6428
6429            // Fix. The winner's own implementer seat continues its conversation:
6430            // the competition is over, so context is pure benefit now.
6431            let blocking_findings: Vec<_> = all_findings
6432                .iter()
6433                .filter(|f| f.severity.blocks())
6434                .cloned()
6435                .collect();
6436            let fix_prompt = prompt::fix_in(
6437                &self.state.instruction,
6438                &blocking_findings,
6439                this_round_verification.as_ref(),
6440                round,
6441                max_rounds,
6442                &language,
6443                &self.state.config.graph.github_language,
6444            );
6445            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6446            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6447            let (job, seat, out) = self
6448                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6449                    prompt: fix_prompt.clone(),
6450                    spec,
6451                    seat,
6452                    cwd: winner.worktree.clone(),
6453                    timeout,
6454                    allow_write: true,
6455                    sessions,
6456                    artifacts: artifacts.clone(),
6457                    stem: format!("fix-{round}"),
6458                    handover: None,
6459                })
6460                .await;
6461            // Not asked: the round is not recorded (so a resume repeats it),
6462            // no fixer was handed over to, and the seat is left as it was.
6463            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6464                return self.stop_for_resource();
6465            }
6466            let agent_id = seat.agent.clone();
6467
6468            let mut fix = FixRecord {
6469                agent: agent_id,
6470                addressed: Vec::new(),
6471                rejected: Vec::new(),
6472                notes: String::new(),
6473                committed: false,
6474                failed: None,
6475                duration_ms: 0,
6476                continuation: None,
6477            };
6478            let mut continuation = ContinuationRecord::not_needed();
6479            let mut final_seat = seat.clone();
6480            match out {
6481                AgentOutcome::Ok(o) => {
6482                    fix.duration_ms = o.duration_ms;
6483                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6484                    // A parsed report standing next to a command this same
6485                    // reply's own CLI never confirmed the exit status of is
6486                    // not a resolved answer — the identical `CommandEvidence`
6487                    // `state.jobs` renders, read here instead of only on
6488                    // display, per the completion judgment and the shown
6489                    // record needing to agree.
6490                    let incomplete_reason = match &parsed {
6491                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6492                            "the reply parsed, but it reported a command whose own CLI never \
6493                             confirmed an exit status"
6494                                .to_owned(),
6495                        ),
6496                        Ok(_) => None,
6497                        Err(e) => Some(e.to_string()),
6498                    };
6499                    match incomplete_reason {
6500                        None => {
6501                            let report = parsed.expect("checked Ok above");
6502                            fix.addressed = report.addressed;
6503                            fix.rejected = report.rejected;
6504                            fix.notes =
6505                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6506                        }
6507                        Some(reason) => {
6508                            let (resumed_seat, resolved, failure, cont) = self
6509                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6510                                .await;
6511                            fix.duration_ms += cont.cumulative_wait_ms;
6512                            continuation = cont;
6513                            final_seat = resumed_seat;
6514                            match resolved {
6515                                Some(report) => {
6516                                    fix.addressed = report.addressed;
6517                                    fix.rejected = report.rejected;
6518                                    fix.notes = blind::sanitize_prose(
6519                                        &report.notes,
6520                                        &self.state.config.blind,
6521                                    );
6522                                }
6523                                None => fix.failed = failure,
6524                            }
6525                        }
6526                    }
6527                }
6528                // The CLI's raw error JSON is not a fix report to parse.
6529                AgentOutcome::Dropped(o) => {
6530                    fix.duration_ms = o.duration_ms;
6531                    let why = o
6532                        .dropped
6533                        .as_ref()
6534                        .map(|d| d.why.as_str())
6535                        .unwrap_or("the CLI ended the stream without delivering its answer");
6536                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6537                }
6538                AgentOutcome::Quota(o) => {
6539                    self.state.quota.push(QuotaLoss {
6540                        seat: final_seat.key.clone(),
6541                        node: "fix".to_owned(),
6542                        at: Timestamp::now(),
6543                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6544                    });
6545                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6546                }
6547                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6548            }
6549            fix.continuation = Some(continuation);
6550            self.state.seats.insert(final_seat.key.clone(), final_seat);
6551            let coordination = self.coordinate(&winner.worktree).await?;
6552            if let Ok(r) = git::rescue_commit(
6553                &winner.worktree,
6554                &format!("magi: review round {round} fixes (uncommitted work)"),
6555            )
6556            .await
6557            {
6558                self.state.note_withheld("fix", &r.withheld);
6559            }
6560            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6561            fix.committed = after != before;
6562            // Judged by what `git` says moved against base, never by the
6563            // fixer's own `addressed`/`rejected` count — see
6564            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6565            // `patch` snapshot above: swallowing this error would default
6566            // `diff_after` to empty, which almost always differs from a
6567            // non-empty `patch` and reads as "progressed" — exactly backwards
6568            // for a `git` failure the stagnation check cannot see through.
6569            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6570            drop(coordination);
6571            let progressed = diff_after != patch;
6572            let commit_note = if fix.committed {
6573                "committed"
6574            } else {
6575                "NO new commit"
6576            };
6577            let tree_note = if progressed {
6578                "changed vs base"
6579            } else {
6580                "unchanged vs base"
6581            };
6582            self.state.event(
6583                "fix",
6584                match &fix.failed {
6585                    // Distinct on purpose from "0 addressed, 0 rejected": the
6586                    // fixer's own diff still landed (blocking counts do keep
6587                    // falling round over round), only its adoption report did
6588                    // not come back, so this must never read like every
6589                    // finding was reviewed and declined.
6590                    Some(reason) => {
6591                        format!(
6592                            "round {round}: fixer's adoption report was lost ({reason}); \
6593                             {commit_note}, tree {tree_note}"
6594                        )
6595                    }
6596                    None => format!(
6597                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6598                         {tree_note}{}",
6599                        fix.addressed.len(),
6600                        fix.rejected.len(),
6601                        if continuation.outcome == ContinuationOutcome::Resumed {
6602                            format!(
6603                                " (adoption report recovered after {} continuation(s))",
6604                                continuation.attempts
6605                            )
6606                        } else {
6607                            String::new()
6608                        },
6609                    ),
6610                },
6611            );
6612            round_record.fix = Some(fix);
6613            round_record.progressed = progressed;
6614            self.state.reviews.push(round_record);
6615            self.state.save()?;
6616
6617            // The fixer's own report never came back this round, even after
6618            // `continue_fix_report`'s own budget was spent on it — not an
6619            // ordinary "no report" (dropped stream, quota, plain failure),
6620            // which already reads that way and is left to the existing round
6621            // budget. Stopping here, rather than opening another round, is
6622            // what keeps a next reviewer/fixer wave from ever being
6623            // dispatched onto `winner.worktree` while whatever the seat's
6624            // last call may still have running there is unaccounted for: no
6625            // process liveness check exists (and none is being added — see
6626            // AGENTS.md/this task's own scope), so the only way to honour
6627            // "nothing starts before a valid report returns" is to not start
6628            // anything further on this worktree from this run at all.
6629            if matches!(
6630                continuation.outcome,
6631                ContinuationOutcome::Exhausted
6632                    | ContinuationOutcome::QuotaLost
6633                    | ContinuationOutcome::NoSession
6634            ) {
6635                return self
6636                    .stop_reviewing(
6637                        "the fixer's adoption report never came back, even after resuming its \
6638                         own seat; refusing to start another round against the same worktree \
6639                         while that is unresolved",
6640                        &shell,
6641                        &winner.worktree,
6642                    )
6643                    .await;
6644            }
6645
6646            let streak = self
6647                .state
6648                .reviews
6649                .iter()
6650                .rev()
6651                .take_while(|r| !r.progressed)
6652                .count();
6653            if streak >= STAGNANT_LIMIT {
6654                return self
6655                    .stop_reviewing(
6656                        &format!(
6657                            "the tree has not moved against base for {streak} round(s) in a row"
6658                        ),
6659                        &shell,
6660                        &winner.worktree,
6661                    )
6662                    .await;
6663            }
6664        }
6665        Ok(())
6666    }
6667
6668    /// Decide, from the last recorded round's own verification, whether
6669    /// stopping the review loop is a hand-off or a genuine block.
6670    ///
6671    /// Called once the loop has given up trying — the round budget is spent,
6672    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6673    /// findings still open, never while a round is still clean or the
6674    /// incomplete-panel case handled inline above. Gate and e2e are facts
6675    /// about the tree; a lingering review finding is an opinion, and this
6676    /// workload's own `magi stats` puts reviewer precision low enough
6677    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6678    /// must not by itself stand between a green, verified change and the
6679    /// human who decides what to do with it. A red e2e is not an opinion, so
6680    /// that case still blocks, with the failing command and a tail of its
6681    /// output recorded here rather than left in `run.json` for someone to go
6682    /// find.
6683    ///
6684    /// A round that deferred its own e2e (see [`Config::graph`]'s
6685    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6686    /// only because nothing ran, and treating an empty list as a passing one
6687    /// here is exactly the "deferred painted green" bug this function exists
6688    /// to not have. When the last round's own verification never resolved —
6689    /// deferred on purpose, or a real attempt the shared build cache blocked
6690    /// — this makes (or retries) the real run, on the actual worktree this
6691    /// loop is about to stop touching, before deciding anything. A
6692    /// resource-blocked attempt is likewise never read as either green or
6693    /// red: it is evidence about the machine, not the patch (see
6694    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6695    /// blocked cache leaves this call without deciding rather than guessing
6696    /// — the caller retries on a later reentry.
6697    /// Record, once, that the review loop handed off over a blocking finding
6698    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6699    /// `land` asks the owner even with `land_approval` off. Called from every
6700    /// path that concludes `Gating`; a reentry keeps the first record.
6701    fn record_contested_handoff(&mut self) {
6702        if self.state.contested_handoff.is_some() {
6703            return;
6704        }
6705        let Some(contested) = self
6706            .state
6707            .reviews
6708            .last()
6709            .and_then(ReviewRound::contested_handoff)
6710        else {
6711            return;
6712        };
6713        self.state.event(
6714            "review",
6715            format!(
6716                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6717                 wait for the owner's approval",
6718                contested.findings.len(),
6719                contested.rejecters.len()
6720            ),
6721        );
6722        self.state.contested_handoff = Some(contested);
6723    }
6724
6725    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6726        let round_idx = self.state.reviews.len() - 1;
6727        // A deferred round and a resource-blocked one are the same shape
6728        // here: neither has a real result yet, and both get one more
6729        // attempt. Read off `e2e_status` — the single source for this —
6730        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6731        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6732        // still retries instead of being read as a settled result the
6733        // instant it stops being empty.
6734        let needs_catchup_run = matches!(
6735            self.state.reviews[round_idx].e2e_status(),
6736            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6737        );
6738        if needs_catchup_run {
6739            let round = self.state.reviews[round_idx].round;
6740            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6741            let commands = self.state.config.verify.e2e.clone();
6742            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6743            let cache_dir = self.state.config.cache_dir();
6744            let context = format!(
6745                "round {round}: verification unresolved, catching up before the final decision"
6746            );
6747            let wait = self.repository_wait();
6748            let (outcomes, verify_retried) = with_cache_lease(
6749                &mut self.state,
6750                cache_dir.as_deref(),
6751                "e2e",
6752                "e2e",
6753                worktree,
6754                &attempted_head,
6755                timeout,
6756                &context,
6757                &wait,
6758                |state, budget| {
6759                    let shell = shell.to_vec();
6760                    let commands = commands.clone();
6761                    let context = context.clone();
6762                    async move {
6763                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6764                            .await
6765                    }
6766                },
6767            )
6768            .await?;
6769            let last = &mut self.state.reviews[round_idx];
6770            last.e2e = outcomes;
6771            last.verify_retried = verify_retried;
6772            // Always the commit and time this attempt actually targeted,
6773            // whether or not it happens to equal the reviewed `head` and
6774            // whether or not a command finished — see
6775            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6776            // attempt is recorded too, so a later reader sees "attempted
6777            // again at T2" rather than silence.
6778            last.verified_head = Some(attempted_head);
6779            last.verified_at = Some(Timestamp::now());
6780            if verify_inconclusive(&last.e2e) {
6781                // Still not a real result: `e2e_deferred` is left exactly
6782                // as it was, so `needs_catchup_run` above reads
6783                // `ResourceBlocked` (via `e2e_status`, which checks
6784                // `resource_blocked` before `e2e_deferred`) and retries
6785                // again on the next reentry, rather than recording
6786                // contention as a red e2e and blocking the run on it.
6787                self.state.save()?;
6788                return Ok(());
6789            }
6790            last.e2e_deferred = false;
6791        }
6792        let last = &self.state.reviews[round_idx];
6793        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6794
6795        match last.e2e_status() {
6796            E2eStatus::Failed => {
6797                let red: Vec<String> = last
6798                    .e2e
6799                    .iter()
6800                    .filter(|o| !o.ok())
6801                    .map(|o| {
6802                        format!(
6803                            "`{}` -> {:?}\n{}",
6804                            o.command,
6805                            o.code,
6806                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6807                        )
6808                    })
6809                    .collect();
6810                self.state
6811                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6812                self.state.status = RunStatus::Blocked;
6813            }
6814            // `needs_catchup_run` above already retried once this call; if
6815            // it is still blocked, this is magi's own admission it could
6816            // not get a command to run, never a verdict on the patch — the
6817            // run is left exactly where a later reentry can retry again.
6818            E2eStatus::ResourceBlocked => {
6819                self.state.event(
6820                    "review",
6821                    format!(
6822                        "{why}; e2e could not run (shared build cache unavailable); not \
6823                         deciding yet"
6824                    ),
6825                );
6826            }
6827            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6828                self.state.event(
6829                    "review",
6830                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6831                );
6832                self.record_contested_handoff();
6833                self.state.status = RunStatus::Gating;
6834            }
6835        }
6836        self.state.save()?;
6837        Ok(())
6838    }
6839
6840    // ----------------------------------------------------------------- gate
6841
6842    async fn gate(&mut self) -> Result<()> {
6843        // Judged by the review record itself, not by `status`: a solo
6844        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6845        // reentry (see `judge`), and trusting it here is exactly how a run
6846        // that exhausted its review budget got gated and merged a second
6847        // time around. `review_conclusion` recomputes the review loop's own
6848        // verdict from the round records themselves — `Gating` for a clean
6849        // round or a hand-off (see `stop_reviewing`), anything else means the
6850        // loop is still going or genuinely blocked.
6851        // A base the winner could not be replayed onto is a decision, not a
6852        // round: there is no landing tree to gate. Read as its own record for
6853        // the same reason the review verdict is.
6854        if self.state.status == RunStatus::Failed
6855            || self
6856                .state
6857                .base_sync
6858                .as_ref()
6859                .is_some_and(|s| s.conflict.is_some())
6860            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6861                != Some(RunStatus::Gating)
6862        {
6863            return Ok(());
6864        }
6865        if self.state.gate_ran {
6866            // `review_loop` derives its conclusion from the clean review
6867            // record on every reentry and therefore puts a completed run back
6868            // in `Gating`. A recorded gate is a stronger, terminal fact:
6869            // retain its original command output (or lack of any, for a repo
6870            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6871            // doc) and restore `Blocked` on a real failure rather than
6872            // pretending the command is still running or running it a second
6873            // time. `gate_ran == false` remains the only shape — unattempted,
6874            // or a resource-blocked retry — that may still need to execute a
6875            // command.
6876            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6877                self.state.status = RunStatus::Blocked;
6878                self.state.save()?;
6879            }
6880            return Ok(());
6881        }
6882        let Some(winner) = self.state.winner().cloned() else {
6883            return Ok(());
6884        };
6885        self.state.status = RunStatus::Gating;
6886        let mut outcomes = self.run_gate(&winner).await?;
6887        loop {
6888            // A resource-blocked outcome means the gate command never actually
6889            // ran - the shared build cache could not be acquired or confirmed
6890            // fresh in time - which is evidence about the machine, not about
6891            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6892            // Recording it as a red gate would mark a run `Blocked` on nothing
6893            // but contention magi has already logged; leaving `self.state.gate`
6894            // empty and `self.state.gate_ran` false instead keeps the shape
6895            // this function already treats as "still needs to run" (see the
6896            // early-return above), so the next call retries the command
6897            // rather than concluding anything.
6898            if verify_inconclusive(&outcomes) {
6899                self.state.save()?;
6900                return Ok(());
6901            }
6902            if outcomes.iter().all(CommandOutcome::ok) {
6903                break;
6904            }
6905            match self.gate_fix_round(&winner, &outcomes).await? {
6906                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6907                GateFix::Stop => break,
6908                GateFix::Defer => {
6909                    self.state.save()?;
6910                    return Ok(());
6911                }
6912            }
6913        }
6914        let passed = outcomes.iter().all(CommandOutcome::ok);
6915        self.state.gate = outcomes;
6916        self.state.gate_ran = true;
6917        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6918        if !passed {
6919            self.state.status = RunStatus::Blocked;
6920            let spent = self.state.gate_fixes.len();
6921            self.state.event(
6922                "gate",
6923                if spent == 0 {
6924                    "gate failed; not merging".to_owned()
6925                } else {
6926                    format!("gate failed after {spent} gate-fix round(s); not merging")
6927                },
6928            );
6929        }
6930        self.state.save()?;
6931        Ok(())
6932    }
6933
6934    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
6935    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
6936    /// after review is clean and never on a candidate awaiting judging.
6937    ///
6938    /// Never fails the run: a non-zero exit or timeout is a warning and a
6939    /// recorded outcome, and the gate remains the single arbiter. Nothing
6940    /// configured means nothing happens - no event, no commit. `commit_all`
6941    /// commits any leftover change under the neutral identity and returns
6942    /// `false` when the tree is clean, so no empty commit is ever made.
6943    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6944        let commands = state.config.verify.pre_gate.clone();
6945        if commands.is_empty() {
6946            return;
6947        }
6948        let shell = state.config.shell();
6949        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6950        let (outcomes, _) = run_commands(
6951            state,
6952            "pre_gate",
6953            "pre_gate",
6954            0,
6955            &shell,
6956            &commands,
6957            &winner.worktree,
6958            timeout,
6959        )
6960        .await;
6961        for o in &outcomes {
6962            if !o.ok() {
6963                tracing::warn!(
6964                    "pre_gate `{}` failed ({:?}); the gate decides",
6965                    o.command,
6966                    o.code
6967                );
6968            }
6969            state.event(
6970                "pre_gate",
6971                format!(
6972                    "`{}` -> {}",
6973                    o.command,
6974                    if o.ok() {
6975                        "pass".to_owned()
6976                    } else {
6977                        format!(
6978                            "FAIL ({:?})\n{}",
6979                            o.code,
6980                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6981                        )
6982                    }
6983                ),
6984            );
6985        }
6986        state.pre_gate = outcomes;
6987        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6988            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6989                Ok(head) => {
6990                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6991                    state.pre_gate_commit = Some(head);
6992                }
6993                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6994            },
6995            Ok(false) => {}
6996            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6997        }
6998        if let Err(e) = state.save() {
6999            tracing::warn!("could not persist the pre_gate record: {e:#}");
7000        }
7001    }
7002
7003    /// Run `verify.gate` once against the winner's current tree, logging one
7004    /// event per command. Empty when nothing is configured.
7005    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7006        let shell = self.state.config.shell();
7007        let gate_commands = self.state.config.verify.gate.clone();
7008        let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7009        let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7010            Vec::new()
7011        } else {
7012            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7013            let cache_dir = self.state.config.cache_dir();
7014            let wait = self.repository_wait();
7015            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7016            let (outcomes, _) = with_cache_lease(
7017                &mut self.state,
7018                cache_dir.as_deref(),
7019                "gate",
7020                "gate",
7021                &winner.worktree,
7022                &head,
7023                timeout,
7024                "final gate",
7025                &wait,
7026                |state, budget| {
7027                    let shell = shell.clone();
7028                    let gate_commands = gate_commands.clone();
7029                    let winner = winner.clone();
7030                    async move {
7031                        Self::run_pre_gate(state, &winner).await;
7032                        let (outcomes, timed_out_pids) = run_commands(
7033                            state,
7034                            "gate",
7035                            "gate",
7036                            0,
7037                            &shell,
7038                            &gate_commands,
7039                            &winner.worktree,
7040                            budget,
7041                        )
7042                        .await;
7043                        (outcomes, false, timed_out_pids)
7044                    }
7045                },
7046            )
7047            .await?;
7048            outcomes
7049        };
7050        if outcomes.is_empty() {
7051            // Nothing configured to check — distinct from every other
7052            // silence in this run's event log, since an empty `gate` alone
7053            // no longer says whether the gate ran at all (see
7054            // `RunState::gate_ran`'s own doc).
7055            self.state.event(
7056                "gate",
7057                "no gate commands configured; nothing to check, passing",
7058            );
7059        }
7060        for o in &outcomes {
7061            self.state.event(
7062                "gate",
7063                format!(
7064                    "`{}` -> {}",
7065                    o.command,
7066                    if o.ok() {
7067                        "pass".to_owned()
7068                    } else {
7069                        format!(
7070                            "FAIL ({:?})\n{}",
7071                            o.code,
7072                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7073                        )
7074                    }
7075                ),
7076            );
7077        }
7078        Ok(outcomes)
7079    }
7080
7081    /// One bounded fix round for a failing gate.
7082    ///
7083    /// The fixer is told the failure came from the gate itself, not from a
7084    /// reviewer, and is shown the failed commands, their exit codes and a tail
7085    /// of their output - whatever `[verify].gate` holds, nothing here knows
7086    /// what those commands run. Only a normal non-zero exit that printed
7087    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7088    /// command or a full disk says nothing about the code, and a fixer sent
7089    /// after it can only appease the machine. The round is judged by what git
7090    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7091    /// again before the gate does, so a fix cannot trade a green gate for a
7092    /// red e2e unnoticed.
7093    async fn gate_fix_round(
7094        &mut self,
7095        winner: &Candidate,
7096        outcomes: &[CommandOutcome],
7097    ) -> Result<GateFix> {
7098        let cap = self.state.config.graph.gate_fix_rounds;
7099        let spent = self.state.gate_fixes.len();
7100        if spent >= cap {
7101            if cap > 0 {
7102                self.state.event(
7103                    "gate",
7104                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7105                );
7106            }
7107            return Ok(GateFix::Stop);
7108        }
7109        if !gate_fixable(outcomes) {
7110            self.state.event(
7111                "gate",
7112                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7113                 command or similar); not spending a fix round on it",
7114            );
7115            return Ok(GateFix::Stop);
7116        }
7117        let min_free = self.state.config.disk.min_free_bytes;
7118        if min_free > 0 {
7119            match crate::disk::free_bytes(&winner.worktree) {
7120                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7121                Ok(free) => {
7122                    self.state.event(
7123                        "gate",
7124                        format!(
7125                            "only {free} bytes free ({min_free} required by `[disk] \
7126                             min_free_bytes`); not spending a fix round on a failure the disk \
7127                             may explain"
7128                        ),
7129                    );
7130                    return Ok(GateFix::Stop);
7131                }
7132                Err(e) => {
7133                    self.state.event(
7134                        "gate",
7135                        format!("free disk space could not be measured ({e:#}); no fix round"),
7136                    );
7137                    return Ok(GateFix::Stop);
7138                }
7139            }
7140        }
7141
7142        let attempt = spent + 1;
7143        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7144        let base = self.landing_base();
7145        let fix_prompt = prompt::gate_fix_in(
7146            &self.state.instruction,
7147            &failed,
7148            attempt,
7149            cap,
7150            &self.state.config.graph.language,
7151            &self.state.config.graph.github_language,
7152        );
7153        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7154        let sessions = self.state.config.graph.sessions;
7155        let artifacts = self.merge_repair_artifacts();
7156        self.state.event(
7157            "gate",
7158            format!("gate failed; gate-fix round {attempt} of {cap}"),
7159        );
7160        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7161        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7162        let (_, seat, out) = self
7163            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7164                prompt: fix_prompt.clone(),
7165                spec,
7166                seat,
7167                cwd: winner.worktree.clone(),
7168                timeout,
7169                allow_write: true,
7170                sessions,
7171                artifacts: artifacts.clone(),
7172                stem: format!("gate-fix-{attempt}"),
7173                handover: None,
7174            })
7175            .await;
7176        // The fixer was never asked: no round is spent (nothing is recorded)
7177        // and a later reentry asks the same agent again.
7178        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7179            return Ok(GateFix::Defer);
7180        }
7181        let mut record = GateFixRecord {
7182            agent: seat.agent.clone(),
7183            failed,
7184            notes: String::new(),
7185            committed: false,
7186            error: None,
7187        };
7188        match out {
7189            AgentOutcome::Ok(o) => {
7190                // A missing report is not a failed fix: the round is judged
7191                // by the tree below, and the report only carries prose.
7192                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7193                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7194                }
7195            }
7196            AgentOutcome::Dropped(_) => {
7197                record.error = Some("the CLI dropped the stream".to_owned());
7198            }
7199            AgentOutcome::Quota(o) => {
7200                self.state.quota.push(QuotaLoss {
7201                    seat: seat.key.clone(),
7202                    node: "gate-fix".to_owned(),
7203                    at: Timestamp::now(),
7204                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7205                });
7206                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7207            }
7208            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7209        }
7210        self.state.seats.insert(seat.key.clone(), seat);
7211        let coordination = self.coordinate(&winner.worktree).await?;
7212        if let Ok(r) = git::rescue_commit(
7213            &winner.worktree,
7214            &format!("magi: gate fix {attempt} (uncommitted work)"),
7215        )
7216        .await
7217        {
7218            self.state.note_withheld("gate-fix", &r.withheld);
7219        }
7220        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7221        record.committed = after != before;
7222        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7223        let note = record.error.clone();
7224        self.state.gate_fixes.push(record);
7225        self.state.save()?;
7226        if !changed {
7227            self.state.event(
7228                "gate",
7229                match note {
7230                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7231                    None => format!("gate-fix round {attempt}: the tree did not change"),
7232                },
7233            );
7234            return Ok(GateFix::Stop);
7235        }
7236        self.state.event(
7237            "gate",
7238            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7239        );
7240
7241        drop(coordination);
7242        let commands = self.state.config.verify.e2e.clone();
7243        if !commands.is_empty() {
7244            let shell = self.state.config.shell();
7245            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7246            let cache_dir = self.state.config.cache_dir();
7247            let context = format!("gate-fix round {attempt}");
7248            let wait = self.repository_wait();
7249            let (e2e, _) = with_cache_lease(
7250                &mut self.state,
7251                cache_dir.as_deref(),
7252                "e2e",
7253                "e2e",
7254                &winner.worktree,
7255                &after,
7256                timeout,
7257                &context,
7258                &wait,
7259                |state, budget| {
7260                    let shell = shell.clone();
7261                    let commands = commands.clone();
7262                    let context = context.clone();
7263                    let worktree = winner.worktree.clone();
7264                    async move {
7265                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7266                            .await
7267                    }
7268                },
7269            )
7270            .await?;
7271            if verify_inconclusive(&e2e) {
7272                return Ok(GateFix::Defer);
7273            }
7274            if e2e.iter().any(|o| !o.ok()) {
7275                self.state.event(
7276                    "gate",
7277                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7278                );
7279                return Ok(GateFix::Stop);
7280            }
7281        }
7282        Ok(GateFix::Retry)
7283    }
7284
7285    // ---------------------------------------------------------------- merge
7286
7287    /// One read-only rewrite by the summary's author; a text the gate still
7288    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7289    /// applies on their word or on silence. `None` means the run is parked on
7290    /// that question and nothing may be pushed or opened yet.
7291    async fn guarded_pr_message(
7292        &mut self,
7293        winner: &Candidate,
7294        facts: Option<&BranchFacts>,
7295        posting: bool,
7296    ) -> Result<Option<PrMessage>> {
7297        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7298        let head = match final_head.as_deref() {
7299            Some(h) => crate::pr_outcome::FinalHead::At(h),
7300            None => crate::pr_outcome::FinalHead::Unknown,
7301        };
7302        // Only a pull request that is about to be posted pays for a summary.
7303        let summarized = if posting {
7304            Some(self.summarize_change(winner, final_head.as_deref()).await)
7305        } else {
7306            None
7307        };
7308        // A busy shared resource is not a permanent cause: publishing an
7309        // "unavailable" description would stick (a later head move never
7310        // edits the body). Stop before anything is pushed or opened; the
7311        // run is `Blocked` and a resume asks the summary again.
7312        if self.state.resource_block.is_some() {
7313            self.state.event(
7314                "pr-summary",
7315                "a shared resource was busy; nothing was published, resume to summarize again",
7316            );
7317            return Ok(None);
7318        }
7319        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7320            anyhow::bail!(
7321                "the final commit moved during public description generation; retry before publishing"
7322            );
7323        }
7324        let note = match (&summarized, &self.state.change_summary) {
7325            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7326            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7327            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7328            (None, _) => ChangeNote::NotAsked,
7329        };
7330        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7331        if !posting {
7332            let identity = crate::scrub::Identity::current();
7333            return Ok(Some(PrMessage {
7334                title: crate::scrub::scrub(&pr.title, &identity),
7335                body: crate::scrub::scrub(&pr.body, &identity),
7336            }));
7337        }
7338        // This very text was already rejected and asked about: no second
7339        // judge call, rewrite or question, just carry the record on.
7340        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7341        if let Some(known) = self
7342            .state
7343            .github_text
7344            .clone()
7345            .filter(|g| g.fingerprint == raw_fp)
7346        {
7347            return self.settle_withheld(winner, &pr, known).await;
7348        }
7349        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7350        let mut violations =
7351            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7352        let identity = crate::scrub::Identity::current();
7353        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7354            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7355            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7356        {
7357            violations.push(crate::github_text::Violation::SensitiveData);
7358        }
7359        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7360        // only a language violation starts one; sensitive data alone goes
7361        // straight to the owner, with the redacted text as the fallback.
7362        let has_language = violations
7363            .iter()
7364            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7365        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7366            self.state.event(
7367                "github-text",
7368                if has_language {
7369                    format!("description rejected: {violations:?}; requesting one rewrite")
7370                } else {
7371                    format!("description rejected: {violations:?}; asking the owner")
7372                },
7373            );
7374            let mut rewritten = false;
7375            let mut rejected_rewrite = None;
7376            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7377            {
7378                let output = text;
7379                if let Some(title) = summary_title(&output) {
7380                    // Judge the redacted candidate that will actually be posted.
7381                    // Sensitive data alone must not discard a useful rewrite.
7382                    let raw_body = summary_without_title(&output);
7383                    let clean_title = crate::scrub::scrub(&title, &identity);
7384                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7385                    let redacted = clean_title != title || body != raw_body;
7386                    let title = clean_title;
7387                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7388                    if !complete {
7389                        self.state.event(
7390                            "github-text",
7391                            "description rewrite omitted required findings",
7392                        );
7393                    }
7394                    if !body.trim().is_empty() {
7395                        body.push_str("\n\n");
7396                        body.push_str(&public_pr_evidence(&self.state, head));
7397                    }
7398                    let marker = format!("magi:run/{}", self.state.id);
7399                    if !body.trim().is_empty() && !body.contains(&marker) {
7400                        body.push_str(&format!("\n\n{marker}"));
7401                    }
7402                    let rewrite = PrMessage {
7403                        title: title.clone(),
7404                        body: body.clone(),
7405                    };
7406                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7407                    let mut failed = crate::github_text::check_for(
7408                        &self.state.config.graph,
7409                        &title,
7410                        &body,
7411                        redecision,
7412                    );
7413                    if (!crate::github_text::shareable(&title)
7414                        || !crate::github_text::shareable(&body))
7415                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7416                    {
7417                        failed.push(crate::github_text::Violation::SensitiveData);
7418                    }
7419                    if complete
7420                        && !title.trim().is_empty()
7421                        && !body.trim().is_empty()
7422                        && !failed.is_empty()
7423                    {
7424                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7425                            title: title.clone(),
7426                            body: body.clone(),
7427                            failed: failed
7428                                .iter()
7429                                .map(|v| crate::github_text::category(*v).to_owned())
7430                                .collect(),
7431                        });
7432                    }
7433                    if complete
7434                        && !title.trim().is_empty()
7435                        && !raw_body.trim().is_empty()
7436                        && failed.is_empty()
7437                    {
7438                        decision = redecision;
7439                        pr = PrMessage { title, body };
7440                        rewritten = true;
7441                        self.state
7442                            .event("github-text", "description rewrite passed");
7443                        if redacted {
7444                            self.state.event(
7445                                "github-text",
7446                                "sensitive data removed from description rewrite",
7447                            );
7448                        }
7449                    }
7450                }
7451            }
7452            if !rewritten {
7453                self.state.event(
7454                    "github-text",
7455                    "description rewrite unavailable or rejected; asking the owner",
7456                );
7457                if let Some(w) =
7458                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7459                {
7460                    // A different text than the one asked about before: that
7461                    // question no longer describes anything.
7462                    self.retire_github_text_question("the withheld text changed");
7463                    let record = crate::run::GithubTextAsk {
7464                        rewrite: rejected_rewrite,
7465                        fingerprint: raw_fp,
7466                        title: w.title,
7467                        body: w.body,
7468                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7469                        question: None,
7470                        asks: 0,
7471                        resolved: false,
7472                        chosen_title: None,
7473                        choice: None,
7474                    };
7475                    return self.settle_withheld(winner, &pr, record).await;
7476                }
7477            }
7478        }
7479        let (title, body) =
7480            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7481        Ok(Some(PrMessage { title, body }))
7482    }
7483
7484    /// Describe the whole change from its final diff in either run mode.
7485    ///
7486    /// The summary is tied to the final commit and to the merge base of the
7487    /// resolved remote base; a record for any other pair is dropped and the
7488    /// agent asked again, so a description never outlives the diff it was
7489    /// written from. The seat is read-only and the prompt tells it to read the
7490    /// diff and the code around it. `[roles] synthesizer` selects the agents
7491    /// (unset: the default pick), each asked at most once, advancing on an
7492    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7493    /// carries the reason it is unavailable; nothing is invented then.
7494    async fn summarize_change(
7495        &mut self,
7496        winner: &Candidate,
7497        head: Option<&str>,
7498    ) -> std::result::Result<(), String> {
7499        let Some(head) = head else {
7500            return Err("the final commit could not be read".to_owned());
7501        };
7502        let remote = self.state.config.merge.remote.clone();
7503        let tracking = format!("{remote}/{}", self.state.base_branch);
7504        if !git::rev_exists(&winner.worktree, &tracking).await {
7505            return Err(format!("the remote base `{tracking}` could not be read"));
7506        }
7507        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7508            Ok(b) if !b.is_empty() => b,
7509            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7510        };
7511        let evidence = crate::pr_outcome::summary_evidence(
7512            &self.state,
7513            crate::pr_outcome::FinalHead::At(head),
7514        );
7515        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7516        if self
7517            .state
7518            .change_summary
7519            .as_ref()
7520            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7521        {
7522            return Ok(());
7523        }
7524        if self.state.change_summary.take().is_some() {
7525            self.state.event(
7526                "pr-summary",
7527                "the final diff or evidence changed; the earlier change summary was discarded",
7528            );
7529        }
7530        // Save the source snapshot even if no generation agent is available.
7531        if let Err(e) = write_artifact(
7532            &self.state,
7533            &format!("public-description-{fingerprint}.md"),
7534            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7535        ) {
7536            tracing::warn!("could not save public description evidence: {e:#}");
7537        }
7538        let chain = match agent::pick_chain(
7539            &self.state.config.agents,
7540            self.state.config.roles.synthesizer.as_ref(),
7541            &agent::installed,
7542            "synthesizer",
7543        ) {
7544            Ok(chain) => chain,
7545            Err(e) => {
7546                tracing::warn!("no agent for the change summary: {e:#}");
7547                self.state.event("pr-summary", "no summary agent available");
7548                return Err("no agent was available to summarize it".to_owned());
7549            }
7550        };
7551        let subjects = git::subjects(&winner.worktree, &base, head)
7552            .await
7553            .unwrap_or_default();
7554        let stat = git::diff_stat(&winner.worktree, &base, head)
7555            .await
7556            .unwrap_or_default();
7557        let stat: String = stat.chars().take(6_000).collect();
7558        let mut prompt = crate::prompt::change_summary_in(
7559            &base,
7560            head,
7561            &subjects,
7562            stat.trim(),
7563            &self.state.config.graph.github_language,
7564        );
7565        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7566        let sessions = self.state.config.graph.sessions;
7567        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7568        let prompts = self.state.config.prompts.clone();
7569        let cache = self.state.config.cache_dir();
7570        let run = self.state.id.clone();
7571        for (i, spec) in chain.into_iter().enumerate() {
7572            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7573            let job = SeatJob {
7574                spec: spec.clone(),
7575                seat,
7576                cwd: winner.worktree.clone(),
7577                prompt: prompt.clone(),
7578                timeout,
7579                allow_write: false,
7580                sessions,
7581                artifacts: agent::artifacts_dir(&self.state.dir()),
7582                stem: if i == 0 {
7583                    "change-summary".to_owned()
7584                } else {
7585                    format!("change-summary-{}", spec.id)
7586                },
7587                handover: None,
7588            };
7589            let ctx = WaveCtx {
7590                run: &run,
7591                node: "pr-summary",
7592                prompts: &prompts,
7593                cache: cache.as_deref(),
7594                round: None,
7595                carry_seats: false,
7596                pauses: Some((&self.pause, &self.interrupt)),
7597            };
7598            let (seat, outcome) =
7599                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7600            self.state.seats.insert(seat.key.clone(), seat);
7601            let text = match outcome {
7602                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7603                AgentOutcome::ResourceBlocked(why) => {
7604                    // No agent was asked: the run stops resumably before
7605                    // anything is published (see `guarded_pr_message`).
7606                    if self.state.resource_block.is_none() {
7607                        self.state.resource_block = Some(crate::run::ResourceBlock {
7608                            node: "pr-summary".to_owned(),
7609                            kind: "shared resource".to_owned(),
7610                            detail: why,
7611                            at: Timestamp::now(),
7612                        });
7613                    }
7614                    return Err("a shared resource was busy".to_owned());
7615                }
7616                _ => continue,
7617            };
7618            let blind = &self.state.config.blind;
7619            let Some(title) = summary_title(&text) else {
7620                continue;
7621            };
7622            let body: String = summary_without_title(&text)
7623                .lines()
7624                .filter(|l| !l.contains("magi:run/"))
7625                .collect::<Vec<_>>()
7626                .join("\n");
7627            let body = blind::sanitize_prose(body.trim(), blind);
7628            let title = blind::sanitize_prose(&title, blind);
7629            if body.trim().is_empty() || title.trim().is_empty() {
7630                continue;
7631            }
7632            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7633                continue;
7634            }
7635            // The branch must still be the one the agent was shown.
7636            if git::rev_parse(&winner.worktree, "HEAD")
7637                .await
7638                .ok()
7639                .as_deref()
7640                != Some(head)
7641            {
7642                return Err("the branch moved while the summary was written".to_owned());
7643            }
7644            if git::merge_base(&winner.worktree, &tracking, head)
7645                .await
7646                .ok()
7647                .as_deref()
7648                != Some(base.as_str())
7649                || crate::pr_outcome::summary_evidence(
7650                    &self.state,
7651                    crate::pr_outcome::FinalHead::At(head),
7652                ) != evidence
7653            {
7654                return Err("the summary evidence changed while it was written".to_owned());
7655            }
7656            self.state.change_summary = Some(crate::run::ChangeSummary {
7657                head: head.to_owned(),
7658                base,
7659                evidence: fingerprint,
7660                title,
7661                body,
7662            });
7663            self.state.event(
7664                "pr-summary",
7665                format!("change summary written by {} for the final diff", spec.id),
7666            );
7667            if let Err(e) = self.state.save() {
7668                tracing::warn!("could not save the change summary: {e:#}");
7669            }
7670            return Ok(());
7671        }
7672        self.state
7673            .event("pr-summary", "no agent produced a change summary");
7674        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7675    }
7676
7677    /// One read-only rewrite request for a withheld description. The
7678    /// implementer's own conversation is reused only when it exists and can be
7679    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7680    /// pick) writes it in a separate conversation. The prompt carries the whole
7681    /// text, so no agent needs any memory. Each agent is asked at most once and
7682    /// the chain advances on an error, quota, unusable answer or a reply with
7683    /// no `TITLE:` line. Nothing is recorded as a handover.
7684    async fn rewrite_pr_text(
7685        &mut self,
7686        winner: &Candidate,
7687        pr: &PrMessage,
7688        violations: &[crate::github_text::Violation],
7689    ) -> Option<String> {
7690        let prompt = format!(
7691            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7692            pr.title, pr.body
7693        );
7694        let sessions = self.state.config.graph.sessions;
7695        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7696        // (spec, seat, resumed)
7697        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7698        let key = format!("impl-{}", winner.label);
7699        if let Some(spec) = self
7700            .state
7701            .config
7702            .agents
7703            .iter()
7704            .find(|a| a.id == winner.agent)
7705            .cloned()
7706            && let Some(existing) = self.state.seats.get(&key)
7707            && existing.agent == spec.id
7708            && agent::has_session(spec.kind, existing, sessions)
7709        {
7710            attempts.push((spec, existing.clone(), true));
7711        } else {
7712            let chain = agent::pick_chain(
7713                &self.state.config.agents,
7714                self.state.config.roles.language_judge.as_ref(),
7715                &agent::installed,
7716                "language_judge",
7717            );
7718            match chain {
7719                Ok(chain) => {
7720                    for spec in chain {
7721                        let seat = crate::fixer::seat_for(
7722                            &mut self.state,
7723                            "github-text-rewrite",
7724                            &spec.id,
7725                        );
7726                        attempts.push((spec, seat, false));
7727                    }
7728                }
7729                Err(e) => {
7730                    tracing::warn!("no agent for the description rewrite: {e:#}");
7731                    self.state
7732                        .event("github-text", "no rewrite agent available");
7733                    return None;
7734                }
7735            }
7736        }
7737        let prompts = self.state.config.prompts.clone();
7738        let cache = self.state.config.cache_dir();
7739        let run = self.state.id.clone();
7740        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7741            let job = SeatJob {
7742                spec: spec.clone(),
7743                seat,
7744                cwd: winner.worktree.clone(),
7745                prompt: prompt.clone(),
7746                timeout: retry_budget(timeout_full, resumed),
7747                allow_write: false,
7748                sessions,
7749                artifacts: agent::artifacts_dir(&self.state.dir()),
7750                stem: if i == 0 {
7751                    "github-text-rewrite".to_owned()
7752                } else {
7753                    format!("github-text-rewrite-{}", spec.id)
7754                },
7755                handover: None,
7756            };
7757            let ctx = WaveCtx {
7758                run: &run,
7759                node: "github-text",
7760                prompts: &prompts,
7761                cache: cache.as_deref(),
7762                round: None,
7763                carry_seats: false,
7764                pauses: Some((&self.pause, &self.interrupt)),
7765            };
7766            let (seat, outcome) =
7767                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7768            self.state.seats.insert(seat.key.clone(), seat);
7769            match outcome {
7770                AgentOutcome::Ok(o)
7771                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7772                {
7773                    return Some(o.text);
7774                }
7775                AgentOutcome::ResourceBlocked(_) => return None,
7776                _ => {}
7777            }
7778        }
7779        None
7780    }
7781
7782    /// Abandon the standing posting-gate question, if any.
7783    fn retire_github_text_question(&mut self, why: &str) {
7784        let Some(id) = self
7785            .state
7786            .github_text
7787            .as_ref()
7788            .and_then(|g| g.question.clone())
7789        else {
7790            return;
7791        };
7792        let _ = ask::Questions::open().update(&id, |q| {
7793            q.abandon(why);
7794            Ok(())
7795        });
7796    }
7797
7798    /// Carry a withheld text to its end: file the question, park while it is
7799    /// open, and on an answer, silence or timeout pick the text to post. The
7800    /// decision is saved before it is used, so it applies once.
7801    async fn settle_withheld(
7802        &mut self,
7803        winner: &Candidate,
7804        pr: &PrMessage,
7805        mut g: crate::run::GithubTextAsk,
7806    ) -> Result<Option<PrMessage>> {
7807        use crate::github_text as gt;
7808        let store = ask::Questions::open();
7809        let timeout = self.state.config.graph.answer_timeout;
7810        if !g.resolved {
7811            let mut standing = g
7812                .question
7813                .as_ref()
7814                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7815            if let Some(q) = standing.as_ref()
7816                && q.status.open()
7817                && gt::expired(q, timeout)
7818            {
7819                let why = format!("no answer within {}s of asking", timeout.max(1));
7820                standing = store
7821                    .update(&q.id, |q| {
7822                        q.abandon(&why);
7823                        Ok(())
7824                    })
7825                    .ok()
7826                    .map(|(q, ())| q);
7827            }
7828            let mut ask_again: Option<Option<Vec<String>>> = None;
7829            match standing {
7830                // Recorded but never written (a stop between the two saves),
7831                // or removed: ask again without spending the retry.
7832                None if g.asks <= 1 => {
7833                    g.asks = 0;
7834                    ask_again = Some(None);
7835                }
7836                None => {
7837                    g.resolved = true;
7838                    g.chosen_title = None;
7839                    g.choice = Some(GithubTextChoice::Fallback);
7840                }
7841                Some(q) if q.status.open() => {
7842                    self.park_on_github_text(&g)?;
7843                    return Ok(None);
7844                }
7845                Some(q) => match gt::read_reply(&q) {
7846                    gt::Reply::Fallback => {
7847                        g.resolved = true;
7848                        g.chosen_title = None;
7849                        g.choice = Some(GithubTextChoice::Fallback);
7850                    }
7851                    gt::Reply::Original => {
7852                        g.resolved = true;
7853                        g.chosen_title = None;
7854                        g.choice = Some(GithubTextChoice::Original);
7855                    }
7856                    gt::Reply::Rewrite => {
7857                        g.resolved = true;
7858                        g.chosen_title = None;
7859                        g.choice = Some(if g.rewrite.is_some() {
7860                            GithubTextChoice::Rewrite
7861                        } else {
7862                            GithubTextChoice::Fallback
7863                        });
7864                    }
7865                    gt::Reply::Title(text) => {
7866                        let first = text
7867                            .lines()
7868                            .map(str::trim)
7869                            .find(|l| !l.is_empty())
7870                            .unwrap_or("");
7871                        let decision = if first.is_empty() {
7872                            None
7873                        } else {
7874                            let candidate = PrMessage {
7875                                title: first.to_owned(),
7876                                body: String::new(),
7877                            };
7878                            self.judge_pr_language(&winner.worktree, &candidate).await
7879                        };
7880                        match gt::vet_title(
7881                            &text,
7882                            decision,
7883                            self.state.config.graph.github_english(),
7884                        ) {
7885                            Ok(title) => {
7886                                g.resolved = true;
7887                                g.chosen_title = Some(title);
7888                                g.choice = Some(GithubTextChoice::MyTitle);
7889                            }
7890                            Err(cats) if g.asks < 2 => {
7891                                ask_again = Some(Some(
7892                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7893                                ))
7894                            }
7895                            Err(_) => {
7896                                g.resolved = true;
7897                                g.chosen_title = None;
7898                                g.choice = Some(GithubTextChoice::Fallback);
7899                            }
7900                        }
7901                    }
7902                },
7903            }
7904            if let Some(rejected) = ask_again {
7905                // A retry (after a rejected replacement title) is still about
7906                // the original text: the same ORIGINAL / FALLBACK comparison.
7907                let w = gt::Withheld {
7908                    title: g.title,
7909                    body: g.body,
7910                    categories: g.categories.clone(),
7911                };
7912                // Named by the original text's fingerprint alone, so a resume
7913                // rewrites the same file and the detail stays identical.
7914                let artifact = match crate::run::write_artifact(
7915                    &self.state,
7916                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7917                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7918                ) {
7919                    Ok(p) => Some(p),
7920                    Err(e) => {
7921                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7922                        None
7923                    }
7924                };
7925                let (detail, comparison) = gt::question_content(
7926                    &w,
7927                    &pr.title,
7928                    &pr.body,
7929                    &self.state.id,
7930                    rejected.as_deref(),
7931                    artifact.as_deref(),
7932                    g.rewrite.as_ref(),
7933                );
7934                let mut q = ask::Question::new(
7935                    self.state.id.clone(),
7936                    gt::ASK_NODE.to_owned(),
7937                    gt::ASK_SEAT.to_owned(),
7938                    gt::question_summary(&self.state.config.graph.language, &w),
7939                    detail,
7940                    gt::question_choices(&w, g.rewrite.as_ref()),
7941                );
7942                q.posting_comparison = Some(comparison);
7943                q.answer_timeout = timeout;
7944                // Recorded before it is written: a stop in between leaves a
7945                // record naming a question the resume then files again.
7946                g.question = Some(q.id.clone());
7947                g.asks += 1;
7948                self.state.github_text = Some(g.clone());
7949                self.state.event(
7950                    "github-text",
7951                    format!("asking the owner about the withheld text ({})", q.short()),
7952                );
7953                self.state.save()?;
7954                store
7955                    .put(&mut q)
7956                    .context("file the posting-gate question")?;
7957                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7958                    tracing::warn!(
7959                        "could not notify about posting-gate question {}: {e:#}",
7960                        q.short()
7961                    );
7962                }
7963                self.park_on_github_text(&g)?;
7964                return Ok(None);
7965            }
7966        }
7967        let resolved = g.chosen_title.clone();
7968        let original = g.choice == Some(GithubTextChoice::Original);
7969        // Field names only: the event log never records the text.
7970        self.state.event(
7971            "github-text",
7972            if original {
7973                format!(
7974                    "posting the original text for: {}",
7975                    match (g.title, g.body) {
7976                        (true, true) => "title, description",
7977                        (true, false) => "title",
7978                        _ => "description",
7979                    }
7980                )
7981            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7982                "posting the saved rewrite title and description".to_owned()
7983            } else if resolved.is_some() {
7984                "posting the owner's replacement title".to_owned()
7985            } else {
7986                "posting the neutral text for the withheld fields".to_owned()
7987            },
7988        );
7989        self.state.github_text = Some(g.clone());
7990        self.state.save()?;
7991        if g.choice == Some(GithubTextChoice::Rewrite)
7992            && let Some(r) = g.rewrite.as_ref()
7993        {
7994            return Ok(Some(PrMessage {
7995                title: r.title.clone(),
7996                body: r.body.clone(),
7997            }));
7998        }
7999        let w = gt::Withheld {
8000            title: g.title,
8001            body: g.body,
8002            categories: g.categories.clone(),
8003        };
8004        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8005        if let Some(t) = resolved {
8006            title = t;
8007        }
8008        // Every withheld field is now neutral or vetted English.
8009        let english = gt::LanguageDecision {
8010            title_english: true,
8011            body_english: true,
8012            body_complete: true,
8013        };
8014        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8015        if original {
8016            // The owner's explicit choice overrides the gate for the fields it
8017            // was made for; the others keep what the gate produced.
8018            if g.title {
8019                title = pr.title.clone();
8020            }
8021            if g.body {
8022                body = pr.body.clone();
8023            }
8024        }
8025        Ok(Some(PrMessage { title, body }))
8026    }
8027
8028    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8029        self.state.github_text = Some(g.clone());
8030        self.state.end_pass_parked(ParkCause::Question);
8031        self.state.parked = true;
8032        self.state.event(
8033            "github-text",
8034            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8035        );
8036        self.state.save()
8037    }
8038
8039    /// Ask `[roles] language_judge` about this text and record which source
8040    /// decided; the text itself is never recorded.
8041    async fn judge_pr_language(
8042        &mut self,
8043        cwd: &Path,
8044        pr: &PrMessage,
8045    ) -> Option<crate::github_text::LanguageDecision> {
8046        self.state.config.roles.language_judge.as_ref()?;
8047        if !self.state.config.graph.language_gate() {
8048            return None;
8049        }
8050        let decision =
8051            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8052        self.state.event(
8053            "github-text",
8054            if decision.is_some() {
8055                "language decided by the language judge"
8056            } else {
8057                "language judge unavailable; using built-in heuristics"
8058            },
8059        );
8060        decision
8061    }
8062
8063    async fn merge(&mut self) -> Result<()> {
8064        // Same reasoning as `gate`: ask the review and gate records directly
8065        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8066        // skip can rewrite on reentry to something that no longer says
8067        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8068        // so a hand-off (open findings, green verification) reaches merge
8069        // exactly like a genuinely clean round does.
8070        //
8071        // A run resumed mid-`land` never reaches here at all: `execute`
8072        // recognises `RunStatus::Landing` before it even calls `prep`, and
8073        // routes straight to `run_land` instead. That has to happen a level
8074        // up from this function, not with a check in here, because
8075        // `review_loop`'s own status recomputation (see its doc) runs
8076        // *before* `merge` on every reentry and would otherwise overwrite
8077        // the `Landing` marker with `Gating` before this node ever saw it.
8078        if self
8079            .state
8080            .base_sync
8081            .as_ref()
8082            .is_some_and(|s| s.conflict.is_some())
8083            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8084                != Some(RunStatus::Gating)
8085            // `gate_ran == false` is not "passed" - `gate` leaves it false
8086            // both before it has ever run and when its last attempt was
8087            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8088            // permission to merge on nothing but the review record. Only a
8089            // gate that actually ran - zero commands configured and
8090            // vacuously passed, or one or more that all exited 0 - may
8091            // proceed; `RunState::gate_status` is the single place that
8092            // reading is computed.
8093            || !self.state.gate_status().ok()
8094        {
8095            return Ok(());
8096        }
8097        // This node's own record, not `status`: `status == Ready` is not
8098        // unique to the harmless `MergeMode::None` path this line was
8099        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8100        // run's PR was closed without merging — and on that run `mode` is
8101        // still `Pr`, so a reentry that fell through here would push and
8102        // open a second pull request. `self.state.merge` is set exactly once
8103        // this node (or `land`) has already produced a verdict, under every
8104        // mode, which is what "already done" actually means here.
8105        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8106            return Ok(());
8107        }
8108        let Some(winner) = self.state.winner().cloned() else {
8109            return Ok(());
8110        };
8111        let repo = self.state.repo.clone();
8112        let base = self.state.base_branch.clone();
8113        let mode = self.state.config.merge.mode;
8114        let style = self.state.config.merge.style;
8115        let facts = if is_review_run(&self.state) {
8116            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8117            let start = review_base(
8118                &repo,
8119                &self.state.config.merge.remote,
8120                &base,
8121                &self.state.base_commit,
8122                &winner.branch,
8123            )
8124            .await;
8125            branch_facts(&repo, &start, &winner.branch).await
8126        } else {
8127            None
8128        };
8129        // `None` when the base could not be freshly read: then an adopted
8130        // pull request's title is left alone.
8131        let leaked = if is_review_run(&self.state) {
8132            leaked_subjects(&self.state, &winner.branch).await
8133        } else {
8134            Some(Vec::new())
8135        };
8136        // `None`: parked on the owner's word about a withheld title or body.
8137        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8138        // the resume arrives here again.
8139        let Some(pr) = self
8140            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8141            .await?
8142        else {
8143            return Ok(());
8144        };
8145        let message = pr.commit_message();
8146
8147        let outcome = match mode {
8148            MergeMode::None => MergeOutcome {
8149                mode,
8150                ok: true,
8151                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8152                empty: false,
8153            },
8154            MergeMode::Pr | MergeMode::Local
8155                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8156            {
8157                MergeOutcome {
8158                    mode,
8159                    ok: false,
8160                    detail: empty_candidate_detail(&self.state, &base),
8161                    empty: true,
8162                }
8163            }
8164            MergeMode::Local => {
8165                let on = git::current_branch(&repo).await?;
8166                if on.as_deref() != Some(base.as_str()) {
8167                    MergeOutcome {
8168                        mode,
8169                        ok: false,
8170                        detail: format!(
8171                            "{} has {} checked out, not the base branch {base}",
8172                            repo.display(),
8173                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8174                        ),
8175                        empty: false,
8176                    }
8177                } else if !git::is_clean(&repo).await? {
8178                    MergeOutcome {
8179                        mode,
8180                        ok: false,
8181                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8182                        empty: false,
8183                    }
8184                } else {
8185                    let out = match style {
8186                        MergeStyle::Merge => {
8187                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8188                        }
8189                        MergeStyle::Squash => {
8190                            git::merge_squash(&repo, &winner.branch, &message).await?
8191                        }
8192                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8193                    };
8194                    MergeOutcome {
8195                        mode,
8196                        ok: out.ok(),
8197                        detail: if out.ok() { out.stdout } else { out.stderr },
8198                        empty: false,
8199                    }
8200                }
8201            }
8202            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8203                self.state.pr_publication_pending = true;
8204                self.state.save()?;
8205                let out =
8206                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8207                self.state.pr_publication_pending = out.is_err();
8208                MergeOutcome {
8209                    mode,
8210                    ok: out.is_ok(),
8211                    detail: match out {
8212                        Ok(url) => url,
8213                        Err(e) => format!("{e:#}"),
8214                    },
8215                    empty: false,
8216                }
8217            }
8218            MergeMode::Pr => {
8219                let remote = self.state.config.merge.remote.clone();
8220                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8221                if !pushed.ok() {
8222                    MergeOutcome {
8223                        mode,
8224                        ok: false,
8225                        detail: pushed.stderr,
8226                        empty: false,
8227                    }
8228                } else {
8229                    // A retry or resume of a run whose branch already has an
8230                    // open pull request adopts it rather than failing on a
8231                    // duplicate. Only this winner branch into this base:
8232                    // `branch_for` derives the name from the run id, so a
8233                    // different run's pull request never matches.
8234                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8235                    let out = match pr_merge_plan(found) {
8236                        PrPlan::Create => {
8237                            gh_pr_create(
8238                                &winner.worktree,
8239                                &base,
8240                                &winner.branch,
8241                                &pr.title,
8242                                &pr.body,
8243                            )
8244                            .await
8245                        }
8246                        PrPlan::Adopt { url, title } => {
8247                            self.state
8248                                .event("merge", format!("Pr: adopted open pull request {url}"));
8249                            if self.state.github_text.as_ref().is_some_and(|g| {
8250                                g.resolved
8251                                    && g.choice == Some(GithubTextChoice::Rewrite)
8252                                    && g.rewrite.is_some()
8253                            }) {
8254                                // The owner chose the entire pair, even if the
8255                                // existing title already matches or was edited.
8256                                land::set_pr_text_verbatim(
8257                                    &winner.worktree,
8258                                    &url,
8259                                    &pr.title,
8260                                    &pr.body,
8261                                )
8262                                .await
8263                                .map(|()| url)
8264                            } else {
8265                                if title != pr.title
8266                                    && (!is_review_run(&self.state)
8267                                        || leaked
8268                                            .as_deref()
8269                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8270                                    && let Err(e) =
8271                                        if self.state.github_text.as_ref().is_some_and(|g| {
8272                                            g.resolved
8273                                                && g.title
8274                                                && g.choice == Some(GithubTextChoice::Original)
8275                                        }) {
8276                                            // The owner chose to post this title unmodified.
8277                                            land::set_pr_title_verbatim(
8278                                                &winner.worktree,
8279                                                &url,
8280                                                &pr.title,
8281                                            )
8282                                            .await
8283                                        } else {
8284                                            land::set_pr_title(
8285                                                &mut self.state,
8286                                                &winner.worktree,
8287                                                &url,
8288                                                &pr.title,
8289                                            )
8290                                            .await
8291                                        }
8292                                {
8293                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8294                                    self.state
8295                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8296                                }
8297                                Ok(url)
8298                            }
8299                        }
8300                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8301                    };
8302                    match out {
8303                        Ok(url) => MergeOutcome {
8304                            mode,
8305                            ok: true,
8306                            detail: url,
8307                            empty: false,
8308                        },
8309                        Err(e) => MergeOutcome {
8310                            mode,
8311                            ok: false,
8312                            detail: e.to_string(),
8313                            empty: false,
8314                        },
8315                    }
8316                }
8317            }
8318        };
8319
8320        self.state.status = match (mode, outcome.ok) {
8321            (MergeMode::None, _) => RunStatus::Ready,
8322            (_, true) => RunStatus::Merged,
8323            (_, false) => RunStatus::Blocked,
8324        };
8325        self.state.event(
8326            "merge",
8327            format!(
8328                "{:?}: {}",
8329                mode,
8330                outcome.detail.lines().next().unwrap_or("")
8331            ),
8332        );
8333        self.state.merge = Some(outcome);
8334        self.state.save()?;
8335
8336        // The PR is open and the run would historically stop here, leaving the
8337        // operator to watch checks, feed review comments back to a fixer, and
8338        // merge. That was done by hand six times in one session before this
8339        // existed. Opt-in, because merging is the one irreversible thing magi
8340        // can do to a repository.
8341        if self.state.config.graph.land
8342            && mode == MergeMode::Pr
8343            && self.state.status == RunStatus::Merged
8344        {
8345            self.run_land().await?;
8346        }
8347        // `run_land` may have left `status` at `Landing` - still waiting on
8348        // CI or the owner's approval, not actually settled - so this has to
8349        // read whatever `status` ended up as here, not the `Merged` this
8350        // function set a few lines up.
8351        self.settle_questions();
8352        Ok(())
8353    }
8354
8355    /// Enter `land`.
8356    ///
8357    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8358    /// a resumed run's re-entry. `land::land` itself is what serialises the
8359    /// two git-mutating moments inside the loop — the rebase push and
8360    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8361    /// to hold a lock across the whole call, and doing so would serialise
8362    /// this run's CI wait against a *different* run's land-approval resume
8363    /// in the same repository, which is exactly the "must not wait on
8364    /// another task" property the daemon's slot-freeing exists to give.
8365    async fn run_land(&mut self) -> Result<()> {
8366        crate::pr_head::ensure_owner(&self.state)?;
8367        let url = self
8368            .state
8369            .merge
8370            .as_ref()
8371            .map(|m| m.detail.clone())
8372            .unwrap_or_default();
8373        let url = url.lines().next().unwrap_or("").trim().to_owned();
8374        if !url.starts_with("http") {
8375            return Ok(());
8376        }
8377        // A land failure is not a lost run: the work is on a branch and the
8378        // pull request is open, which is exactly where a human takes over.
8379        loop {
8380            match land::land(&mut self.state, &url).await {
8381                Ok(_)
8382                    if self
8383                        .state
8384                        .merge_repairs
8385                        .last()
8386                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8387                {
8388                    if self.repair_merge_review().await? {
8389                        continue;
8390                    }
8391                }
8392                Ok(pr) if self.state.parked => {
8393                    // `land` already saved the parked marker; nothing here
8394                    // overrides `status` back to a terminal value while an
8395                    // approval is still outstanding.
8396                    let _ = pr;
8397                }
8398                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8399                    self.state.status = RunStatus::Superseded;
8400                    self.state.save()?;
8401                }
8402                Ok(pr) => {
8403                    self.state.status = match pr.state {
8404                        land::PrLifecycle::Merged => RunStatus::Merged,
8405                        _ => RunStatus::Blocked,
8406                    };
8407                    // Downstream of a confirmed merge only - see
8408                    // `bump::should_release_bump`'s own doc for why this one
8409                    // check covers all three of `land`'s success paths.
8410                    // Best-effort: the run already landed, so a failure here
8411                    // (the decision call, `gh`, `cargo`) is recorded and never
8412                    // turns a landed run into a failed one.
8413                    if bump::should_release_bump(self.state.status)
8414                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8415                    {
8416                        // The event is the run's own record. Not-eligible cases
8417                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8418                        // `Err` is a bump that was tried and failed:
8419                        // `after_merge` itself raises the operator notice for
8420                        // that, whether or not a release PR exists yet.
8421                        self.state
8422                            .event("bump", format!("release bump skipped: {e:#}"));
8423                    }
8424                    // Independent of the bump, and best-effort in the same way:
8425                    // findings the merge left open become follow-up tasks.
8426                    if self.state.status == RunStatus::Merged {
8427                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8428                    }
8429                    self.state.save()?;
8430                }
8431                Err(e) => {
8432                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8433                        return Err(e);
8434                    }
8435                    self.state.status = RunStatus::Blocked;
8436                    self.state.event("land", format!("gave up: {e}"));
8437                    self.state.save()?;
8438                }
8439            }
8440            break;
8441        }
8442        Ok(())
8443    }
8444
8445    /// One repair cycle stays under the existing driver/task claim. An
8446    /// interrupted fixer is rolled back and held rather than run twice.
8447    async fn repair_merge_review(&mut self) -> Result<bool> {
8448        use crate::run::MergeRepairPhase as Phase;
8449        crate::pr_head::ensure_owner(&self.state)?;
8450        let i = self.state.merge_repairs.len() - 1;
8451        let request = self.state.merge_repairs[i].clone();
8452        if request.phase == Phase::Fixing {
8453            let restored = match self.state.winner().cloned() {
8454                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8455                None => Err(anyhow::anyhow!("repair has no winning branch")),
8456            };
8457            let why = match restored {
8458                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8459                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8460                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8461            };
8462            self.hold_merge_repair(i, why)?;
8463            return Ok(false);
8464        }
8465        let failure = if i >= self.state.config.graph.land_rounds
8466            || self.state.config.graph.review_rounds == 0
8467        {
8468            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8469        } else {
8470            None
8471        };
8472        if let Some(why) = failure {
8473            self.hold_merge_repair(i, why)?;
8474            return Ok(false);
8475        }
8476        if request.phase == Phase::Requested {
8477            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8478                Ok(pr) => pr,
8479                Err(e) => {
8480                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8481                    return Ok(false);
8482                }
8483            };
8484            self.state.merge_repairs[i].phase = Phase::Fixing;
8485            self.state.land_approval = None;
8486            self.state.land_armed_head = None;
8487            self.state.save()?;
8488            // Serialize the exact records, including finding ids/details,
8489            // reconsideration reasons, final votes and verification output.
8490            let evidence = serde_json::to_string_pretty(&request)?;
8491            let reason = format!(
8492                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8493            );
8494            let budget = self.state.config.graph.land_rounds;
8495            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8496            match result {
8497                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8498                Ok(land::Fixed::Failed(why)) => {
8499                    self.hold_merge_repair(
8500                        i,
8501                        format!(
8502                            "repair failed: {why}; inspect the branch and run an explicit review"
8503                        ),
8504                    )?;
8505                    return Ok(false);
8506                }
8507                Err(e) => {
8508                    self.hold_merge_repair(
8509                        i,
8510                        format!(
8511                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8512                        ),
8513                    )?;
8514                    return Ok(false);
8515                }
8516            }
8517            // Even no change gets a real new reviewer wave. The previous
8518            // evidence remains in the request; a former clean/concluded round
8519            // cannot short-circuit the new review or gate.
8520            self.begin_merge_repair_review(i)?;
8521        }
8522        self.review_loop().await?;
8523        self.gate().await?;
8524        if self.resource_halt()? || self.state.parked {
8525            return Ok(false);
8526        }
8527        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8528            != Some(RunStatus::Gating)
8529            || !self.state.gate_status().ok()
8530        {
8531            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8532            return Ok(false);
8533        }
8534        // Review and gate may have added commits after the initial repair.
8535        // Publish exactly their final head to the existing PR, with a lease.
8536        let winner = self
8537            .state
8538            .winner()
8539            .cloned()
8540            .context("repair has no winning branch")?;
8541        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8542        if self.state.gate_head.as_deref() != Some(after.as_str())
8543            || !git::is_clean(&winner.worktree).await?
8544        {
8545            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8546            return Ok(false);
8547        }
8548        let expected = self
8549            .state
8550            .pr_pushed_head
8551            .as_deref()
8552            .unwrap_or(&request.head)
8553            .to_owned();
8554        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8555        // Push the checked commit itself: a local ref moving after the check
8556        // must not change which commit the forge receives.
8557        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8558        let refspec = format!("{after}:refs/heads/{public}");
8559        let push = git::git_raw(
8560            &winner.worktree,
8561            &["push", &lease, &self.state.config.merge.remote, &refspec],
8562        )
8563        .await;
8564        match push {
8565            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8566            Ok(out) => {
8567                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8568                return Ok(false);
8569            }
8570            Err(e) => {
8571                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8572                return Ok(false);
8573            }
8574        }
8575        self.state.merge_repairs[i].phase = Phase::Complete;
8576        self.state.status = RunStatus::Landing;
8577        self.state.event(
8578            "repair",
8579            "new review and gate completed; fresh owner merge decision required",
8580        );
8581        self.state.save()?;
8582        Ok(true)
8583    }
8584
8585    fn merge_repair_artifacts(&self) -> PathBuf {
8586        let artifacts = agent::artifacts_dir(&self.state.dir());
8587        match self
8588            .state
8589            .merge_repairs
8590            .last()
8591            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8592        {
8593            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8594            None => artifacts,
8595        }
8596    }
8597
8598    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8599        self.state.reviews.clear();
8600        self.state.gate.clear();
8601        self.state.gate_ran = false;
8602        self.state.gate_head = None;
8603        self.state.gate_fixes.clear();
8604        self.state.pre_gate.clear();
8605        self.state.pre_gate_commit = None;
8606        self.state.contested_handoff = None;
8607        self.state.land_reruns = None;
8608        self.state.land_approval = None;
8609        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8610        self.state.status = RunStatus::Reviewing;
8611        self.state.save()
8612    }
8613
8614    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8615        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8616        self.state.merge_repairs[index].failure = Some(why.clone());
8617        self.state.status = RunStatus::Blocked;
8618        self.state.event("repair", why);
8619        self.state.save()
8620    }
8621
8622    // -------------------------------------------------------------- helpers
8623
8624    /// Fetch or create a seat, keeping its conversation across nodes.
8625    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8626        if let Some(existing) = self.state.seats.get(key)
8627            && existing.agent == agent
8628        {
8629            return existing.clone();
8630        }
8631        // A seat that changes agent mints its session id from the agent too,
8632        // like a handover: the old agent's uuid is already taken by the CLI.
8633        let fresh = if self.state.seats.contains_key(key) {
8634            handover_seat(key, agent, self.state.next_seat_seed())
8635        } else {
8636            SeatState::new(key, agent, self.state.seed)
8637        };
8638        self.state.seats.insert(key.to_owned(), fresh.clone());
8639        fresh
8640    }
8641
8642    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8643    /// seat to another roster agent, in which case that agent. Nodes that
8644    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8645    /// reconsideration) must keep talking to whoever answered it, not slip
8646    /// back to the agent that failed it.
8647    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8648        match self.state.seats.get(key) {
8649            Some(s) if s.agent != spec.id => {
8650                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8651            }
8652            _ => spec,
8653        }
8654    }
8655
8656    /// A candidate rendered for judging, with the leak policy applied.
8657    fn view(&self, c: &Candidate) -> CandidateView {
8658        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8659            .unwrap_or_default();
8660        let (patch, _) = blind::sanitize_patch(
8661            &format!("candidate {} patch", c.label),
8662            &raw,
8663            &self.state.config.blind,
8664        );
8665        CandidateView {
8666            label: c.label,
8667            branch: c.branch.clone(),
8668            summary: c.summary.clone(),
8669            stat: c.stat.clone(),
8670            patch,
8671        }
8672    }
8673
8674    /// The full candidate set as prompt text, for seats with no live session.
8675    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8676        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8677        prompt::judge(
8678            "(see above)",
8679            &views,
8680            self.roles.judges.len(),
8681            base_short,
8682            "en",
8683        )
8684    }
8685
8686    /// The final-vote prompt with everything a seat that has no session of its
8687    /// own needs: the candidates, the seat's own ranking and reasons, and the
8688    /// anonymised deliberation it took part in (only when there was one, so a
8689    /// handed-over seat never sees more than the seat it replaces did). The
8690    /// `Final vote` heading stays first.
8691    fn vote_prompt_full(
8692        &self,
8693        j: usize,
8694        labels: &[char],
8695        language: &str,
8696        candidates: &[Candidate],
8697        base_short: &str,
8698    ) -> String {
8699        let mut text = format!(
8700            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8701            prompt::final_vote(labels, language),
8702            self.state.instruction,
8703            self.candidate_block(candidates, base_short)
8704        );
8705        if let Some(own) = self
8706            .state
8707            .judgements
8708            .get(j)
8709            .filter(|r| !r.ranking.is_empty())
8710        {
8711            let reasons = own
8712                .reasons
8713                .iter()
8714                .map(|(k, v)| format!("- {k}: {v}"))
8715                .collect::<Vec<_>>()
8716                .join("\n");
8717            text.push_str(&format!(
8718                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8719                own.ranking.iter().collect::<String>(),
8720                if reasons.is_empty() {
8721                    ""
8722                } else {
8723                    ", because:\n"
8724                }
8725            ));
8726        }
8727        if !self.state.deliberation.is_empty() {
8728            text.push_str("\n# What was argued before this vote\n");
8729            for t in self.transcript(&[], j) {
8730                text.push_str(&format!(
8731                    "\n## {}{}\n\n{}\n",
8732                    t.who,
8733                    if t.is_self { " (you)" } else { "" },
8734                    t.body.trim()
8735                ));
8736            }
8737        }
8738        text
8739    }
8740
8741    /// Anonymised transcript for judge `self_idx`.
8742    ///
8743    /// The initial rankings are always the opening statements. Seeding them
8744    /// only when no turn had been taken yet meant every judge after the first
8745    /// argued against a single voice instead of against the actual split — the
8746    /// disagreement is the information, so it is always on the table.
8747    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8748        let mut turns = Vec::new();
8749        for j in &self.state.judgements {
8750            if j.ranking.is_empty() {
8751                continue;
8752            }
8753            let reasons = j
8754                .reasons
8755                .iter()
8756                .map(|(k, v)| format!("- {k}: {v}"))
8757                .collect::<Vec<_>>()
8758                .join("\n");
8759            turns.push(Turn {
8760                who: format!("Judge {} (opening ranking)", j.judge),
8761                is_self: j.judge == self_idx + 1,
8762                body: format!(
8763                    "Ranked {}{}{reasons}",
8764                    j.ranking.iter().collect::<String>(),
8765                    if reasons.is_empty() {
8766                        ""
8767                    } else {
8768                        ", because:\n"
8769                    }
8770                ),
8771            });
8772        }
8773        for t in self
8774            .state
8775            .deliberation
8776            .iter()
8777            .flat_map(|r| r.turns.iter())
8778            .chain(current)
8779        {
8780            turns.push(Turn {
8781                who: format!("Judge {}", t.judge),
8782                is_self: t.judge == self_idx + 1,
8783                body: t.body.clone(),
8784            });
8785        }
8786        turns
8787    }
8788}
8789
8790/// Does this seat still hold the context a follow-up prompt would rely on?
8791fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8792    agent::has_session(spec.kind, seat, sessions)
8793}
8794
8795/// The next entry in `roster` after `start`, never wrapping back to the
8796/// front, whose id is not in `tried` yet.
8797///
8798/// Starts one past `start` rather than at the front of `roster`: `start` is
8799/// the seat's own original position, and a seat whose candidate slot already
8800/// sits on the roster's second entry must fall through to the third next, not
8801/// restart at the first — which is very likely a different candidate's own
8802/// agent already. Never wraps back past `start`, for the same reason: an
8803/// entry earlier in the roster than the seat's own position is almost
8804/// certainly some *other* candidate slot's own agent, and once the tail of
8805/// the roster is exhausted there are no more untried agents for *this* seat
8806/// to fall through to — the caller's fallback chain ends there, exactly as
8807/// "no further untried agents remain in the list for that seat" asks for.
8808///
8809/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8810/// the same id twice (an operator's `roles.implementers` typo, or a
8811/// `[[agents]]` list reused across roles) must not let
8812/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8813/// over `roster` either finds an untried id or runs out, so this always
8814/// terminates regardless of duplicates.
8815fn next_untried_in_roster<'a>(
8816    roster: &'a [AgentSpec],
8817    start: usize,
8818    tried: &BTreeSet<String>,
8819) -> Option<&'a AgentSpec> {
8820    roster
8821        .get(start + 1..)?
8822        .iter()
8823        .find(|s| !tried.contains(&s.id))
8824}
8825
8826/// The successor for a seat, shared by all four seat kinds (implement, judge,
8827/// review, advise). `others` holds the ids that *currently* occupy the other
8828/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8829/// sees them), so roster entries beyond the seat count act as spares: a failed
8830/// seat goes to an agent no other seat holds whenever the roster permits.
8831///
8832/// `carried` is `Some` only for the review loop's carried failure history.
8833/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8834/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8835/// whole roster: untried and not another seat's occupant; (3) the plain walk
8836/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8837/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8838/// a better outcome than an empty seat, and it keeps the answer to "is there
8839/// a successor at all" exactly what it was before occupants were considered,
8840/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8841/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8842/// carried failure is a cheaper bet than doubling an agent on the panel, and
8843/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8844/// the same round are handled one at a time, so a seat later in the batch
8845/// sees an earlier one's new occupant but not yet its own freed agent.
8846fn pick_successor<'a>(
8847    roster: &'a [AgentSpec],
8848    start: usize,
8849    tried: &BTreeSet<String>,
8850    carried: Option<&BTreeSet<String>>,
8851    others: &BTreeSet<String>,
8852) -> Option<&'a AgentSpec> {
8853    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8854    roster
8855        .get(start + 1..)
8856        .and_then(|tail| {
8857            tail.iter()
8858                .filter(free)
8859                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8860        })
8861        .or_else(|| {
8862            carried
8863                .is_some()
8864                .then(|| roster.iter().find(free))
8865                .flatten()
8866        })
8867        .or_else(|| match carried {
8868            Some(c) => next_for_seat(roster, start, tried, c),
8869            None => next_untried_in_roster(roster, start, tried),
8870        })
8871}
8872
8873/// The next agent for a seat that carries its failure history across rounds
8874/// (the review loop). `round_tried` is this round's own bound and starts
8875/// empty every round; `carried_failed` only decides priority.
8876///
8877/// First: an id walking forward from `start`, never wrapping, that is neither
8878/// tried this round nor failed in an earlier one. Only when that is exhausted
8879/// does it rescue: the first roster id (in roster order, so this one *does*
8880/// look before `start`) not yet tried this round, which is by then a carried
8881/// failure. Each id is rescued at most once per round, so it cannot loop.
8882fn next_for_seat<'a>(
8883    roster: &'a [AgentSpec],
8884    start: usize,
8885    round_tried: &BTreeSet<String>,
8886    carried_failed: &BTreeSet<String>,
8887) -> Option<&'a AgentSpec> {
8888    roster
8889        .get(start + 1..)?
8890        .iter()
8891        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8892        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8893}
8894
8895/// Where a reviewer seat starts a round: the agent that last answered it when
8896/// it is still on the roster and not marked failed, else the spec's own agent
8897/// unless it failed, else the next roster agent that has not failed, else the
8898/// spec's own agent again (the whole roster failed). Ids no longer on the
8899/// roster are ignored. An empty roster has no handover, so the spec stands.
8900fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8901    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8902        return spec;
8903    };
8904    let ok = |id: &str| !h.failed.contains(id);
8905    if let Some(last) = h.last_ok.as_deref()
8906        && ok(last)
8907        && let Some(s) = roster.iter().find(|s| s.id == last)
8908    {
8909        return s.clone();
8910    }
8911    if ok(&spec.id) {
8912        return spec;
8913    }
8914    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8915    roster
8916        .get(from + 1..)
8917        .into_iter()
8918        .flatten()
8919        .chain(roster.iter())
8920        .find(|s| ok(&s.id))
8921        .cloned()
8922        .unwrap_or(spec)
8923}
8924
8925/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8926/// seed so a CLI that mints its session id up front (`--session-id`) never
8927/// reuses the uuid the previous agent already opened under the same seat key.
8928pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8929    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8930}
8931
8932/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8933/// so the classifier below and the code that builds the message cannot drift.
8934const TIMED_OUT: &str = "timed out";
8935
8936impl FailClass {
8937    /// `None` for an answer; otherwise how the turn failed.
8938    fn of(out: &AgentOutcome) -> Option<Self> {
8939        match out {
8940            AgentOutcome::Ok(_) => None,
8941            AgentOutcome::Quota(_) => Some(Self::Quota),
8942            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8943            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8944            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8945            // Not a failure of the agent. Callers check for it first; `None`
8946            // here means "no class to hand over on".
8947            AgentOutcome::ResourceBlocked(_) => None,
8948        }
8949    }
8950
8951    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8952    fn stem_word(&self) -> &'static str {
8953        match self {
8954            Self::Quota => "quota",
8955            _ => "handover",
8956        }
8957    }
8958}
8959
8960/// The message's first line with its variable parts removed — digit runs and
8961/// path-like tokens — so "exit code 2" and "exit code 7" read
8962/// as one kind of failure.
8963fn failure_signature(msg: &str) -> String {
8964    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8965    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8966    // the historical machine key while the process diagnosis uses readable prose.
8967    if let Some(code) = line
8968        .strip_prefix("process exited with no usable output (exit code ")
8969        .and_then(|s| s.strip_suffix(')'))
8970        .and_then(|s| s.parse::<i32>().ok())
8971    {
8972        line = format!("exited with some({code}) and no usable output");
8973    } else if matches!(
8974        line.as_str(),
8975        "process exited with no usable output (no exit code reported)"
8976            | "process exited with no usable output (exit code unavailable)"
8977    ) {
8978        line = "exited with none and no usable output".to_owned();
8979    }
8980    let mut out = Vec::new();
8981    for word in line.split_whitespace() {
8982        if word.contains('/') || word.contains('\\') {
8983            out.push("<path>".to_owned());
8984            continue;
8985        }
8986        let mut w = String::new();
8987        let mut in_digits = false;
8988        for c in word.chars() {
8989            if c.is_ascii_digit() {
8990                if !in_digits {
8991                    w.push('#');
8992                }
8993                in_digits = true;
8994            } else {
8995                in_digits = false;
8996                w.push(c);
8997            }
8998        }
8999        out.push(w);
9000    }
9001    out.join(" ").chars().take(120).collect()
9002}
9003
9004/// Whether a seat that just failed with `cur` may go to the next roster agent.
9005/// A quota or a timeout always may. Any other failure may not when the agent
9006/// before it failed the same way: an error the prompt causes would otherwise
9007/// walk the whole roster. `prev` is the class of the immediately preceding
9008/// agent's failure, so a quota or timeout in between breaks the run of
9009/// identical failures by itself.
9010fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9011    match cur {
9012        FailClass::Quota | FailClass::Timeout => true,
9013        FailClass::Other(_) => prev != Some(cur),
9014    }
9015}
9016
9017/// A short human reason for a failed outcome, for the handover record.
9018fn fail_reason(out: &AgentOutcome) -> String {
9019    match out {
9020        AgentOutcome::Ok(_) => String::new(),
9021        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9022        AgentOutcome::Dropped(o) => format!(
9023            "the CLI dropped the stream ({})",
9024            o.dropped
9025                .as_ref()
9026                .map(|d| d.why.as_str())
9027                .unwrap_or("it ended without delivering its answer")
9028        ),
9029        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9030            e.lines().next().unwrap_or("").chars().take(160).collect()
9031        }
9032    }
9033}
9034
9035/// Note one handover in the run: the structured record and, in the timeline,
9036/// the sentence a person reads. A quota keeps the wording it always had.
9037pub(crate) fn record_handover(
9038    state: &mut RunState,
9039    node: &str,
9040    seat: &str,
9041    from: &str,
9042    to: &str,
9043    class: &FailClass,
9044    reason: &str,
9045) {
9046    let message = if *class == FailClass::Quota {
9047        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9048    } else {
9049        format!("{seat}: handed over {from} -> {to} ({reason})")
9050    };
9051    state.event(node, message);
9052    state.handovers.push(Handover {
9053        at: Timestamp::now(),
9054        node: node.to_owned(),
9055        seat: seat.to_owned(),
9056        from: from.to_owned(),
9057        to: to.to_owned(),
9058        reason: reason.to_owned(),
9059    });
9060}
9061
9062/// Did this reply report running a command whose own CLI never confirmed an
9063/// exit status?
9064///
9065/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9066/// command *finished* (see that type's own doc), so this can only be `true`
9067/// for a command whose completion event carried no readable exit code — not
9068/// for one that simply is not mentioned at all. That is the one signal this
9069/// crate can read, from the same record `state.jobs` renders, about a reply
9070/// standing next to work its own CLI cannot vouch for finishing; it is
9071/// deliberately not a check on the exit code's *value* (a fixer legitimately
9072/// runs a command that fails mid-iteration before it succeeds) and not a
9073/// guess at a command still running in the background (which emits no event
9074/// at all, and so leaves no evidence here to find).
9075fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9076    commands.iter().any(|c| c.exit_code.is_none())
9077}
9078
9079/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9080/// trusted as a verified no-op — the adoption guard's own text-level half.
9081///
9082/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9083/// timed out): a marker only earns the benefit of the doubt from a turn the
9084/// CLI itself vouches for finishing properly, the same house style
9085/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9086/// *fix* report to for `commands`. A candidate that timed out, exited
9087/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9088/// is, whatever prose it wrote — this returns `None` before it ever looks at
9089/// `text`. The remaining guards (the tree really is empty, the evidence is
9090/// non-empty) are the caller's: this only reads what the reply *claimed*.
9091fn verified_noop_claim(
9092    usable: bool,
9093    commands: &[agent::CommandEvidence],
9094    text: &str,
9095) -> Option<String> {
9096    (usable && !has_unconfirmed_command(commands))
9097        .then(|| verdict::verified_noop(text))
9098        .flatten()
9099}
9100
9101fn short(commit: &str) -> String {
9102    commit.chars().take(7).collect()
9103}
9104
9105fn make_executable(path: &Path) -> Result<()> {
9106    #[cfg(unix)]
9107    {
9108        use std::os::unix::fs::PermissionsExt as _;
9109        let mut perms = std::fs::metadata(path)?.permissions();
9110        perms.set_mode(0o755);
9111        std::fs::set_permissions(path, perms)?;
9112    }
9113    #[cfg(not(unix))]
9114    {
9115        let _ = path;
9116    }
9117    Ok(())
9118}
9119
9120/// What every seat in one batch shares: where the answers are attributed, the
9121/// prompt overlay they inherit, and the build cache they are told to use.
9122///
9123/// A struct rather than four more parameters: `wave` also needs the run's
9124/// state (to record who is answering right now) and the attempt number, and
9125/// eight positional arguments is both unreadable and a clippy error.
9126struct WaveCtx<'a> {
9127    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9128    /// paid for it.
9129    run: &'a str,
9130    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9131    node: &'a str,
9132    prompts: &'a Prompts,
9133    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9134    cache: Option<&'a Path>,
9135    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9136    /// `JobRecord::round`. `None` for every other node.
9137    round: Option<usize>,
9138    /// Carry each seat's failed-agent history across waves (the review loop
9139    /// only): start-of-round priority and handover choice read
9140    /// [`RunState::seat_history`], and every answer or failure writes it.
9141    carry_seats: bool,
9142    /// The run's park / interrupt requests, so a wait for a shared resource
9143    /// can be abandoned the moment the operator asks to park.
9144    pauses: Option<(&'a Pause, &'a Pause)>,
9145}
9146
9147/// Shared scheduling policy for operations and spawned seat invocations.
9148fn repository_wait(
9149    state: &RunState,
9150    pauses: Option<(&Pause, &Pause)>,
9151) -> crate::repo_lock::WaitPolicy {
9152    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9153    let inherited = crate::repo_lock::WaitPolicy::current();
9154    crate::repo_lock::WaitPolicy::new(
9155        Duration::from_secs(state.config.graph.timeout_resource_wait),
9156        move || {
9157            inherited.cancelled()
9158                || pauses
9159                    .as_ref()
9160                    .is_some_and(|(a, b)| a.parked() || b.parked())
9161        },
9162    )
9163}
9164
9165fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9166    if error.is::<crate::repo_lock::ResourceBlocked>() {
9167        state.resource_block = Some(crate::run::ResourceBlock {
9168            node: "repository".into(),
9169            kind: "repository coordination".into(),
9170            detail: error.to_string(),
9171            at: Timestamp::now(),
9172        });
9173        state.event("resource", error.to_string());
9174    }
9175}
9176
9177async fn coordinate_operation(
9178    state: &mut RunState,
9179    cwd: &Path,
9180    wait: &crate::repo_lock::WaitPolicy,
9181) -> Result<Option<crate::repo_lock::SeatLease>> {
9182    let result =
9183        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9184            .await;
9185    if let Err(e) = &result {
9186        record_repository_block(state, e);
9187    }
9188    result
9189}
9190
9191/// Run one job, honouring the parallelism budget.
9192async fn run_one(
9193    job: SeatJob,
9194    sem: Arc<Semaphore>,
9195    ctx: &WaveCtx<'_>,
9196    state: &mut RunState,
9197    attempt: usize,
9198) -> (SeatState, AgentOutcome) {
9199    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9200        .await
9201        .pop()
9202        .expect("one job in, one result out");
9203    (seat, out)
9204}
9205
9206/// Run every job concurrently, capped by the semaphore, preserving order.
9207///
9208/// Every seat in the batch is recorded into [`RunState::active`] before the
9209/// wave starts and cleared as each answer lands, so the run's own record says
9210/// who is still being waited on rather than only who finished.
9211async fn wave(
9212    jobs: Vec<SeatJob>,
9213    sem: Arc<Semaphore>,
9214    ctx: &WaveCtx<'_>,
9215    state: &mut RunState,
9216    attempt: usize,
9217) -> Vec<(usize, SeatState, AgentOutcome)> {
9218    let WaveCtx {
9219        run,
9220        node,
9221        prompts,
9222        cache,
9223        round,
9224        carry_seats: _,
9225        pauses,
9226    } = *ctx;
9227    for job in &jobs {
9228        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9229    }
9230    if let Err(e) = state.save() {
9231        // A failed persist of "who is answering right now" must not abort the
9232        // wave: the seats are already being asked, and the alternative is
9233        // losing the answers to save a status line nobody may even be
9234        // watching.
9235        tracing::warn!("could not persist in-progress seats: {e:#}");
9236    }
9237    // Hold the shared build cache's lease for the whole batch, not per job:
9238    // several candidates (an implement wave) or a fixer legitimately share
9239    // one cache concurrently within this run, and that stays untouched — a
9240    // single lease taken once for the whole wave and released once it is
9241    // done is what stops a *different* borrower (another run's own wave, its
9242    // e2e/gate, a human's `magi review`) from interleaving a build into the
9243    // same directory while this one is in flight. Best-effort, not
9244    // all-or-nothing: a wave that cannot get the lease within its own
9245    // longest job's budget still runs — an hour of paid implementer calls is
9246    // not thrown away over cache contention — but every write-allowed seat
9247    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9248    // below), the same fallback a read-only seat always gets, rather than
9249    // building into a directory this run was never granted. The identity
9250    // record is still invalidated below either way, so the next tracked
9251    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9252    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9253    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9254    // never borrowed from a job's agent budget (and never charged to it either,
9255    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9256    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9257    let cache_guard = if let Some(cache_dir) = cache {
9258        if jobs_had_a_writer {
9259            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9260            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9261            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9262            {
9263                Ok(Some(g)) => Some(g),
9264                // Parking: leave without spawning anything. The run parks at
9265                // the next boundary and the seats are asked afresh on resume.
9266                Ok(None) => {
9267                    state.resource_block = Some(crate::run::ResourceBlock {
9268                        node: node.to_owned(),
9269                        kind: "shared build cache".to_owned(),
9270                        detail: "the run is parking".to_owned(),
9271                        at: Timestamp::now(),
9272                    });
9273                    return block_wave(
9274                        jobs,
9275                        state,
9276                        node,
9277                        attempt,
9278                        "parked while waiting for the shared build cache",
9279                    );
9280                }
9281                // An unreadable lease is not contention: keep the old
9282                // fallback (no shared cache for this wave's seats).
9283                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9284                Err(e) => {
9285                    // Writing seats may not build into a cache this run does
9286                    // not hold, and an agent that cannot be given one is not
9287                    // asked at all: the run stops resumably instead.
9288                    let why = format!("{e:#}");
9289                    state.resource_block = Some(crate::run::ResourceBlock {
9290                        node: node.to_owned(),
9291                        kind: "shared build cache".to_owned(),
9292                        detail: why.clone(),
9293                        at: Timestamp::now(),
9294                    });
9295                    state.event(
9296                        "resource",
9297                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9298                    );
9299                    return block_wave(jobs, state, node, attempt, &why);
9300                }
9301            }
9302        } else {
9303            None
9304        }
9305    } else {
9306        None
9307    };
9308    // Every job keeps its whole budget: the wait above was not the agent's.
9309    // Restart the seats' clocks so "started" means "being spawned now".
9310    state.seats_spawning(node, attempt);
9311    if let Err(e) = state.save() {
9312        tracing::warn!("could not persist the start of a wave: {e:#}");
9313    }
9314    // Queueing behind this wave's own seats for a parallelism slot is normal
9315    // (each of them is bounded by its own agent timeout); only a wait beyond
9316    // all of them plus the resource limit means the slots never came back.
9317    let slot_limit = resource_limit
9318        + jobs
9319            .iter()
9320            .map(|j| j.timeout)
9321            .max()
9322            .unwrap_or_default()
9323            .saturating_mul(jobs.len() as u32);
9324    let mut set = tokio::task::JoinSet::new();
9325    // Seats announce themselves here the moment they hold a parallelism
9326    // permit, i.e. just before their CLI is spawned.
9327    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9328    let overlay = prompts.overlay(node);
9329    let repository_wait = repository_wait(state, pauses);
9330    for (i, mut job) in jobs.into_iter().enumerate() {
9331        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9332        if cache.is_some() {
9333            job.prompt.push('\n');
9334            job.prompt
9335                .push_str(&prompt::build_cache_note(node, job.allow_write));
9336        }
9337        let sem = Arc::clone(&sem);
9338        let run = run.to_owned();
9339        let node = node.to_owned();
9340        // Only implementers were told about the task's attachments, so only
9341        // their seats get the directory widened for reading.
9342        let attachments = if node == "implement" {
9343            state.attachments.clone()
9344        } else {
9345            Vec::new()
9346        };
9347        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9348        // `prompt::build_cache_note`'s doc for why setting it anyway is
9349        // exactly how a sandboxed reviewer's write refusal got reported as a
9350        // defect in the patch, not a property of its own seat. And a
9351        // write-allowed one is handed it only when the lease above was
9352        // actually acquired: a wave that could not get it (`cache_guard` is
9353        // `None`, see its own comment) must not send seats to build into a
9354        // directory this run does not hold - that is the exact concurrent,
9355        // unmanaged-write race this module exists to prevent, not something
9356        // "proceeding anyway" is allowed to reintroduce.
9357        let cache = cache
9358            .filter(|_| job.allow_write && cache_guard.is_some())
9359            .map(Path::to_path_buf);
9360        let spawned_tx = spawned_tx.clone();
9361        let repository_wait = repository_wait.clone();
9362        let coordination = state.config.coordination.clone();
9363        let fetched_base = state.fetched_base.clone();
9364        set.spawn(async move {
9365            // A wait for a parallelism slot is scheduling, not the agent: it
9366            // is bounded by its own limit and ends as a resource block, never
9367            // as a timeout of the agent that was not started.
9368            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9369                Ok(p) => p,
9370                Err(_) => {
9371                    return (
9372                        i,
9373                        job.seat,
9374                        AgentOutcome::ResourceBlocked(format!(
9375                            "no parallel slot became free within {}s",
9376                            slot_limit.as_secs()
9377                        )),
9378                    );
9379                }
9380            };
9381            spawned_tx.send(job.seat.key.clone()).ok();
9382            let mut seat = job.seat;
9383            let out = repository_wait
9384                .scope(agent::invoke(
9385                    &job.spec,
9386                    &mut seat,
9387                    &Invocation {
9388                        scope: crate::agent::InvocationScope::Repository,
9389                        coordination: coordination.as_ref(),
9390                        fetched_base: fetched_base.as_ref(),
9391                        cwd: &job.cwd,
9392                        prompt: &job.prompt,
9393                        timeout: job.timeout,
9394                        allow_write: job.allow_write,
9395                        unsandboxed: false,
9396                        sessions: job.sessions,
9397                        artifacts: &job.artifacts,
9398                        stem: &job.stem,
9399                        run: &run,
9400                        node: &node,
9401                        cache_dir: cache.as_deref(),
9402                        attachments: &attachments,
9403                        writable: &[],
9404                    },
9405                ))
9406                .await;
9407            let out = match out {
9408                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9409                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9410                // Billed work the CLI failed to hand over is not an ordinary
9411                // failure, but its text is the CLI's raw error JSON, not an
9412                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9413                // read it as one by forgetting to check. `usable()` is always
9414                // false here (dropped implies an empty response), so this has
9415                // to be checked before the catch-all `Failed` below or the
9416                // one shape this exists for is lost with the rest.
9417                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9418                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9419                Ok(o) => AgentOutcome::Failed(format!(
9420                    "process exited with no usable output ({})",
9421                    crate::notices::exit_status(o.exit_code)
9422                )),
9423                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9424                    AgentOutcome::ResourceBlocked(e.to_string())
9425                }
9426                Err(e) => AgentOutcome::Failed(e.to_string()),
9427            };
9428            (i, seat, out)
9429        });
9430    }
9431    drop(spawned_tx);
9432    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9433    loop {
9434        let joined = tokio::select! {
9435            Some(key) = spawned_rx.recv() => {
9436                // The clock of this seat starts now, not when it queued.
9437                state.seat_spawned(&key);
9438                if let Err(e) = state.save() {
9439                    tracing::warn!("could not persist a seat's start: {e:#}");
9440                }
9441                continue;
9442            }
9443            joined = set.join_next() => joined,
9444        };
9445        let Some(joined) = joined else {
9446            break;
9447        };
9448        let (i, seat, out) = match joined {
9449            Ok(v) => v,
9450            // No seat to clear: a panicked task never reported which one it
9451            // was. The defensive sweep below this loop is what stops that
9452            // seat's `active` entry from surviving forever.
9453            Err(e) => {
9454                tracing::error!("agent task panicked: {e}");
9455                continue;
9456            }
9457        };
9458        state.seat_finished(&seat.key);
9459        if let AgentOutcome::ResourceBlocked(why) = &out
9460            && state.resource_block.is_none()
9461        {
9462            state.resource_block = Some(crate::run::ResourceBlock {
9463                node: node.to_owned(),
9464                kind: if why.contains("repository coordination") {
9465                    "repository coordination"
9466                } else {
9467                    "parallel slot"
9468                }
9469                .to_owned(),
9470                detail: why.clone(),
9471                at: Timestamp::now(),
9472            });
9473            state.event("resource", format!("{node}: no agent was started - {why}"));
9474        }
9475        record_jobs(state, node, round, &seat.key, &out);
9476        if let Err(e) = state.save() {
9477            tracing::warn!("could not persist a seat's completion: {e:#}");
9478        }
9479        if collected.len() <= i {
9480            collected.resize_with(i + 1, || None);
9481        }
9482        collected[i] = Some((i, seat, out));
9483    }
9484    // Belt-and-braces for the panic branch above: every seat this exact batch
9485    // started shares this `(node, attempt)` pair, and every seat that finished
9486    // normally already cleared itself, so anything left tagged with it here
9487    // can only be a panicked task's leftover. Cleared unconditionally rather
9488    // than left to read as still answering forever.
9489    if state
9490        .active
9491        .values()
9492        .any(|a| a.node == node && a.attempt == attempt)
9493    {
9494        state
9495            .active
9496            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9497        if let Err(e) = state.save() {
9498            tracing::warn!("could not persist the end of a wave: {e:#}");
9499        }
9500    }
9501    // Whether or not the lease above was actually held, several worktrees
9502    // may just have built into the cache with nothing here able to name one
9503    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9504    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9505    // clean it might not have strictly needed; trusting a stale match would
9506    // cost it a wrong answer.
9507    if let Some(cache_dir) = cache
9508        && jobs_had_a_writer
9509    {
9510        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9511    }
9512    if let Some(guard) = cache_guard {
9513        guard.release();
9514    }
9515    collected.into_iter().flatten().collect()
9516}
9517
9518/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9519/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9520/// this is data collection, not the fix-specific completion contract in
9521/// [`Runner::continue_fix_report`], and applies regardless of which node
9522/// asked.
9523///
9524/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9525/// evidence from; `Failed` does not, and correctly contributes nothing — a
9526/// timeout or crash is not itself evidence about a command the seat may have
9527/// started.
9528fn record_jobs(
9529    state: &mut RunState,
9530    node: &str,
9531    round: Option<usize>,
9532    seat: &str,
9533    out: &AgentOutcome,
9534) {
9535    let commands: &[agent::CommandEvidence] = match out {
9536        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9537        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9538    };
9539    let checked_at = Timestamp::now();
9540    for c in commands {
9541        state.jobs.push(JobRecord {
9542            node: node.to_owned(),
9543            round,
9544            seat: seat.to_owned(),
9545            id: c.id.clone(),
9546            description: c.description.clone(),
9547            checked_at,
9548            status: match c.exit_code {
9549                Some(0) => JobStatus::Completed,
9550                Some(_) => JobStatus::Failed,
9551                None => JobStatus::Unknown,
9552            },
9553            exit_code: c.exit_code,
9554            result_summary: c.result_summary.clone(),
9555            source: c.source.clone(),
9556        });
9557    }
9558}
9559
9560/// Is a review round clean, given how many reviewer seats answered against
9561/// how many the round expected?
9562///
9563/// A seat that never answered (timeout, crash, unparsable output) is not a
9564/// seat that read the patch and found nothing — treating it as such is
9565/// exactly the bug this function exists to close. Under the default `block`
9566/// policy a missing seat can never be clean; `warn` still requires the seats
9567/// that *did* answer to have found nothing blocking and verification to be
9568/// green.
9569///
9570/// `quota_missing` narrows that `block` default for exactly one cause of
9571/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9572/// a session limit lifts by the very next round buys nothing — the seat is
9573/// asked again with the same quota — so once every missing seat is accounted
9574/// for by a quota loss (and at least one seat *did* answer, so a decision has
9575/// something to rest on) the round is decided on the panel that could answer,
9576/// same as `warn` would. A panel that lost every seat to quota is not
9577/// decided here: `answered == 0` falls through to the existing `block`
9578/// fallback so a fully collapsed panel still waits rather than landing on no
9579/// review at all.
9580fn round_is_clean(
9581    blocking: usize,
9582    e2e_ok: bool,
9583    answered: usize,
9584    expected: usize,
9585    quota_missing: usize,
9586    policy: IncompleteReviewPolicy,
9587) -> bool {
9588    if blocking != 0 || !e2e_ok {
9589        return false;
9590    }
9591    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9592        return true;
9593    }
9594    answered > 0 && expected - answered <= quota_missing
9595}
9596
9597/// The review loop's own conclusion, derived entirely from its persisted
9598/// round records and the round budget that produced them — never from
9599/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9600/// recomputes the identical answer regardless of what an earlier node in the
9601/// same walk, or a previous walk, did to `status`.
9602///
9603/// `None` while more rounds remain to try, including when review never ran
9604/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9605/// gone clean, or the budget is spent, or the tree has stopped moving (see
9606/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9607///
9608/// - An incomplete panel that raised nothing is missing input, not a
9609///   verified tree — never a hand-off candidate, whatever verification said
9610///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9611/// - Otherwise, green e2e on the last round hands off (see
9612///   [`Runner::stop_reviewing`]); red e2e blocks.
9613///
9614/// A last round whose own verification is still `ResourceBlocked` — magi
9615/// itself never got a command to run, not evidence the patch is broken —
9616/// is neither: this returns `None` for it too, the same as "more rounds
9617/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9618/// handling of that shape) instead of this cheap recomputation guessing a
9619/// verdict a real attempt never produced.
9620fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9621    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9622        return Some(RunStatus::Gating);
9623    }
9624    let last = reviews.last()?;
9625    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9626    if reviews.len() < max_rounds && !stagnant {
9627        return None;
9628    }
9629    if last.incomplete() && last.blocking == 0 {
9630        return Some(RunStatus::Blocked);
9631    }
9632    if last.e2e_status() == E2eStatus::ResourceBlocked {
9633        return None;
9634    }
9635    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9636        RunStatus::Gating
9637    } else {
9638        RunStatus::Blocked
9639    })
9640}
9641
9642/// How long a re-ask may take, given the budget the first attempt had.
9643///
9644/// A `nudged` retry is a request to restate an answer the seat has already
9645/// worked out: it carries no new work, so it does not deserve the original
9646/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9647/// 133 seconds while a third sat for over ten minutes on a resumed session
9648/// holding 410 KB of prior output - and because the retry had inherited the
9649/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9650/// judging round whose other seats were long finished.
9651///
9652/// A quarter of the budget, with a floor so that a deliberately short timeout
9653/// does not collapse to nothing. A retry that re-sends the whole prompt
9654/// (because the seat kept no context) is the original job again, and keeps the
9655/// original budget.
9656pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9657    if nudged {
9658        (full / 4).max(Duration::from_secs(120)).min(full)
9659    } else {
9660        full
9661    }
9662}
9663
9664/// Run a wave and parse each reply, re-asking the seats whose reply was
9665/// unusable.
9666///
9667/// The re-ask is a nudge rather than the whole prompt again when the seat still
9668/// holds its conversation, which is the difference between a cheap retry and
9669/// paying for the entire candidate set twice.
9670///
9671/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9672/// successor in `roster` is handed to it instead of being re-asked: the
9673/// handover is the retry. A seat with no successor left (a single-agent
9674/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9675/// the asks to one seat in one node number at most
9676/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9677/// is asked once (a dropped stream is nudged first), and only the last agent
9678/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9679/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9680/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9681/// other failure stops the chain when the previous agent failed the same way
9682/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9683/// [`has_context`] is false and the job's own full prompt and full budget are
9684/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9685/// intermediate ones are not losses) and is returned as a failure like any
9686/// other absent seat — the caller decides whether the panel still has a
9687/// quorum. An empty `roster` disables handover: failures are nudged as they
9688/// always were, and a quota is simply lost. A reply that fails to parse or
9689/// validate is the prompt's doing and is only ever nudged, never handed over.
9690///
9691/// The returned [`SeatState`] names the agent that answered (or tried last).
9692#[allow(clippy::too_many_arguments)]
9693async fn ask_json_wave<T>(
9694    jobs: Vec<SeatJob>,
9695    sem: Arc<Semaphore>,
9696    retries: usize,
9697    roster: &[AgentSpec],
9698    ctx: &WaveCtx<'_>,
9699    losses: &mut Vec<QuotaLoss>,
9700    state: &mut RunState,
9701    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9702) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9703where
9704    T: serde::de::DeserializeOwned + Send + 'static,
9705{
9706    ask_wave_with(
9707        jobs,
9708        sem,
9709        retries,
9710        roster,
9711        ctx,
9712        losses,
9713        state,
9714        &|text: &str| {
9715            let v = verdict::extract_json::<T>(text)?;
9716            validate(&v)?;
9717            Ok(v)
9718        },
9719    )
9720    .await
9721}
9722
9723/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9724/// a node whose answer is prose (deliberation) shares the same handover,
9725/// failure classification, quota bookkeeping and bounds instead of a copy.
9726///
9727/// A seat handed to another roster agent is sent the job's `handover` prompt
9728/// (when it has one) rather than `prompt`: the new agent has no session, so a
9729/// resume-style prompt would be incomplete. That holds for the handover ask
9730/// and for every nudge to that agent whose `has_context` is false.
9731#[allow(clippy::too_many_arguments)]
9732async fn ask_wave_with<T>(
9733    jobs: Vec<SeatJob>,
9734    sem: Arc<Semaphore>,
9735    retries: usize,
9736    roster: &[AgentSpec],
9737    ctx: &WaveCtx<'_>,
9738    losses: &mut Vec<QuotaLoss>,
9739    state: &mut RunState,
9740    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9741) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9742where
9743    T: Send + 'static,
9744{
9745    let n = jobs.len();
9746    let originals: Vec<SeatJob> = jobs;
9747    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9748    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9749    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9750    // answer, N once it has gone through N nudges. Read back once this
9751    // returns, so a caller building a history record (`ReviewRecord`) can
9752    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9753    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9754    // field's own doc.
9755    let mut nudges: Vec<usize> = vec![0; n];
9756    // The agent now occupying each seat, the ids it has already been through,
9757    // where in the roster the walk began, the class of the last failure, and
9758    // the stem word of a handover not yet asked (full prompt, full budget).
9759    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9760    let mut tried: Vec<BTreeSet<String>> = specs
9761        .iter()
9762        .map(|s| BTreeSet::from([s.id.clone()]))
9763        .collect();
9764    let starts: Vec<usize> = specs
9765        .iter()
9766        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9767        .collect();
9768    let carry = ctx.carry_seats && !roster.is_empty();
9769    // Carried across rounds: ids that failed the seat earlier, and how the
9770    // last failure went (so a repeat of it is not handed over again).
9771    let carried: Vec<BTreeSet<String>> = originals
9772        .iter()
9773        .map(|j| {
9774            state
9775                .seat_history
9776                .get(&j.seat.key)
9777                .filter(|_| carry)
9778                .map(|h| h.failed.clone())
9779                .unwrap_or_default()
9780        })
9781        .collect();
9782    let mut prev: Vec<Option<FailClass>> = originals
9783        .iter()
9784        .map(|j| {
9785            state
9786                .seat_history
9787                .get(&j.seat.key)
9788                .filter(|_| carry)
9789                .and_then(|h| h.last_fail.clone())
9790        })
9791        .collect();
9792    let next_agent =
9793        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9794            let others: BTreeSet<String> = specs
9795                .iter()
9796                .enumerate()
9797                .filter(|(j, _)| *j != i)
9798                .map(|(_, s)| s.id.clone())
9799                .collect();
9800            pick_successor(
9801                roster,
9802                starts[i],
9803                tried,
9804                carry.then(|| &carried[i]),
9805                &others,
9806            )
9807            .cloned()
9808        };
9809    let mut fresh: Vec<Option<String>> = vec![None; n];
9810    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9811    let mut pending: Vec<usize> = (0..n).collect();
9812
9813    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9814    // asks: an agent with a successor is asked once and handed over, and only
9815    // a seat with no successor spends `retries` nudges on the same agent. This
9816    // only guarantees the loop's own termination whatever those say.
9817    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9818    for round in 0..max_rounds {
9819        if pending.is_empty() {
9820            break;
9821        }
9822        let mut batch = Vec::with_capacity(pending.len());
9823        let mut renudged: Vec<&str> = Vec::new();
9824        for &i in &pending {
9825            let src = &originals[i];
9826            // A seat now held by another agent than the job named has no
9827            // session of its own: it gets the full-context prompt whenever
9828            // it is asked in full (the handover ask, a nudge it cannot
9829            // resume).
9830            let full: &str = match &src.handover {
9831                Some(h) if specs[i].id != src.spec.id => h,
9832                _ => &src.prompt,
9833            };
9834            // The prompt and the budget are one decision: a nudge restates
9835            // finished work, a re-sent prompt redoes it.
9836            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9837                (
9838                    full.to_owned(),
9839                    src.timeout,
9840                    format!("{}-{word}-{}", src.stem, specs[i].id),
9841                )
9842            } else if nudges[i] == 0 {
9843                (full.to_owned(), src.timeout, src.stem.clone())
9844            } else {
9845                renudged.push(src.seat.key.as_str());
9846                let why = done[i]
9847                    .as_ref()
9848                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9849                    .unwrap_or_else(|| "no parsable answer".to_owned());
9850                let nudge = prompt::nudge(&why);
9851                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9852                let prompt = if nudged {
9853                    nudge
9854                } else {
9855                    format!("{full}\n\n---\n\n{nudge}")
9856                };
9857                (
9858                    prompt,
9859                    retry_budget(src.timeout, nudged),
9860                    format!("{}-retry{}", src.stem, nudges[i]),
9861                )
9862            };
9863            batch.push(SeatJob {
9864                spec: specs[i].clone(),
9865                seat: seats[i].clone(),
9866                cwd: src.cwd.clone(),
9867                prompt,
9868                timeout,
9869                allow_write: src.allow_write,
9870                sessions: src.sessions,
9871                artifacts: src.artifacts.clone(),
9872                stem,
9873                handover: None,
9874            });
9875        }
9876
9877        if !renudged.is_empty() {
9878            state.event(
9879                ctx.node,
9880                format!("retry {round}: re-asking {}", renudged.join(", ")),
9881            );
9882        }
9883        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9884        let mut still = Vec::new();
9885        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9886            seats[i] = seat;
9887            // No agent was asked: nothing to hand over, nudge, remember or
9888            // charge. The caller sees `resource_block` and stops the node.
9889            if let AgentOutcome::ResourceBlocked(why) = &out {
9890                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9891                continue;
9892            }
9893            let class = FailClass::of(&out);
9894            // A dropped stream is nudged first (the conversation is still
9895            // there to pick up); only a seat whose nudges are spent hands over.
9896            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9897                && nudges[i] < retries
9898                && !roster.is_empty();
9899            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9900                let next = should_hand_over(prev[i].as_ref(), &cur)
9901                    .then(|| next_agent(i, &tried[i], &specs))
9902                    .flatten();
9903                if carry {
9904                    let h = state
9905                        .seat_history
9906                        .entry(originals[i].seat.key.clone())
9907                        .or_default();
9908                    h.failed.insert(specs[i].id.clone());
9909                    h.last_fail = Some(cur.clone());
9910                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9911                        h.last_ok = None;
9912                    }
9913                    // Saved before the next agent is asked, so a restart in
9914                    // between does not forget who failed.
9915                    if let Err(e) = state.save() {
9916                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9917                    }
9918                }
9919                if let Some(next) = next {
9920                    record_handover(
9921                        state,
9922                        ctx.node,
9923                        &originals[i].seat.key,
9924                        &specs[i].id,
9925                        &next.id,
9926                        &cur,
9927                        &fail_reason(&out),
9928                    );
9929                    tried[i].insert(next.id.clone());
9930                    prev[i] = Some(cur.clone());
9931                    seats[i] =
9932                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9933                    specs[i] = next;
9934                    fresh[i] = Some(cur.stem_word().to_owned());
9935                    nudges[i] = 0;
9936                    done[i] = Some(Err(anyhow::anyhow!(
9937                        "handed over after: {}",
9938                        fail_reason(&out)
9939                    )));
9940                    still.push(i);
9941                    continue;
9942                }
9943            }
9944            if carry
9945                && !nudge_first
9946                && let Some(cur) = class.clone()
9947            {
9948                // The chain ended here (no successor, or a repeated failure).
9949                let h = state
9950                    .seat_history
9951                    .entry(originals[i].seat.key.clone())
9952                    .or_default();
9953                h.failed.insert(specs[i].id.clone());
9954                h.last_fail = Some(cur);
9955            }
9956            let parsed = match out {
9957                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9958                AgentOutcome::Quota(o) => {
9959                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9960                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9961                }
9962                // Not a parseable answer: the nudge loop re-asks it, which is
9963                // exactly what a dropped stream needs. Just don't hand its raw
9964                // error JSON to `extract_json`.
9965                AgentOutcome::Dropped(o) => {
9966                    let why = o
9967                        .dropped
9968                        .as_ref()
9969                        .map(|d| d.why.as_str())
9970                        .unwrap_or("the CLI ended the stream without delivering its answer");
9971                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9972                }
9973                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9974                    Err(anyhow::anyhow!(e))
9975                }
9976            };
9977            let quota = class == Some(FailClass::Quota);
9978            let failed = parsed.is_err();
9979            done[i] = Some(parsed);
9980            if carry && !failed {
9981                let h = state
9982                    .seat_history
9983                    .entry(originals[i].seat.key.clone())
9984                    .or_default();
9985                h.failed.remove(&specs[i].id);
9986                h.last_ok = Some(specs[i].id.clone());
9987                h.last_fail = None;
9988            }
9989            // Do not re-ask a rate-limited seat (quota) — a retry is known to
9990            // fail the same way; and never re-ask a seat that already parsed.
9991            // A failed agent that still has a successor is not re-asked
9992            // either: the handover was its remedy and has just been refused
9993            // (the chain stops on a repeated failure class). A seat with no
9994            // successor left (a single-agent roster, the roster's tail, or an
9995            // empty roster) keeps the same-agent nudge, bounded by `retries`.
9996            let agent_failure = class.is_some()
9997                && !nudge_first
9998                && !roster.is_empty()
9999                && next_agent(i, &tried[i], &specs).is_some();
10000            if failed && !quota && !agent_failure && nudges[i] < retries {
10001                nudges[i] += 1;
10002                still.push(i);
10003            }
10004        }
10005        pending = still;
10006    }
10007
10008    // One loss per seat whose chain ended on a quota: the intermediate ones
10009    // were absorbed by a handover and are not losses.
10010    for (i, q) in last_quota.into_iter().enumerate() {
10011        if let Some(reset) = q {
10012            losses.push(QuotaLoss {
10013                seat: originals[i].seat.key.clone(),
10014                node: ctx.node.to_owned(),
10015                at: Timestamp::now(),
10016                reset,
10017            });
10018        }
10019    }
10020
10021    seats
10022        .into_iter()
10023        .zip(done)
10024        .zip(nudges)
10025        .map(|((seat, res), attempts)| {
10026            (
10027                seat,
10028                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10029                attempts,
10030            )
10031        })
10032        .collect()
10033}
10034
10035/// Did the wave end without asking any agent (see [`block_wave`])?
10036fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10037    results
10038        .iter()
10039        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10040}
10041
10042/// Settle a wave that never spawned anything: every job answers
10043/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10044/// else about the seat (session, history, quota) is touched.
10045fn block_wave(
10046    jobs: Vec<SeatJob>,
10047    state: &mut RunState,
10048    node: &str,
10049    attempt: usize,
10050    why: &str,
10051) -> Vec<(usize, SeatState, AgentOutcome)> {
10052    state.resource_wait = None;
10053    state
10054        .active
10055        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10056    if let Err(e) = state.save() {
10057        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10058    }
10059    jobs.into_iter()
10060        .enumerate()
10061        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10062        .collect()
10063}
10064
10065/// Acquire the shared build cache's lease, waiting out contention within
10066/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10067/// unbounded wait is never acceptable).
10068///
10069/// A first, non-blocking check happens before ever waiting; if it finds the
10070/// lease busy, that fact is logged as a `verify` event *and* flushed with
10071/// [`RunState::save`] immediately — not only once the wait finally succeeds
10072/// or gives up — so a `magi show` run by a different process while this one
10073/// is still waiting reads a `run.json` that says so, rather than whatever it
10074/// looked like before the wait started. The same applies to the terminal
10075/// failure: logged and saved before this returns `Err`, so a caller that
10076/// could not get the lease at all still leaves a legible record of why.
10077async fn acquire_cache_lease(
10078    state: &mut RunState,
10079    cache_dir: &Path,
10080    owner: &crate::cache::Owner,
10081    budget: Duration,
10082    context: &str,
10083    cancel: &(dyn Fn() -> bool + Sync),
10084) -> Result<Option<crate::cache::Guard>> {
10085    let home = crate::run::home();
10086    let started = Instant::now();
10087    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10088        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10089        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10090        Err(e) => {
10091            state.event(
10092                "verify",
10093                format!("{context}: could not check the shared build cache: {e:#}"),
10094            );
10095            if let Err(e2) = state.save() {
10096                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10097            }
10098            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10099        }
10100    };
10101    state.event(
10102        "verify",
10103        format!(
10104            "{context}: waiting for the shared build cache at {} ({})",
10105            cache_dir.display(),
10106            busy.describe()
10107        ),
10108    );
10109    // Seats of this node are waiting, not running: say so, so the countdown a
10110    // reader sees only ever measures an agent that has been spawned.
10111    state.resource_wait = Some(crate::run::ResourceWait {
10112        node: context.to_owned(),
10113        kind: "shared build cache".to_owned(),
10114        holder: busy.describe(),
10115        since: Timestamp::now(),
10116        limit_secs: budget.as_secs(),
10117    });
10118    if let Err(e) = state.save() {
10119        tracing::warn!("could not persist a cache wait: {e:#}");
10120    }
10121    let remaining = budget.saturating_sub(started.elapsed());
10122    let waited = crate::cache::wait_for_unless(
10123        &home,
10124        cache_dir,
10125        owner,
10126        remaining,
10127        Duration::from_secs(5),
10128        cancel,
10129    )
10130    .await;
10131    state.resource_wait = None;
10132    match waited {
10133        Ok(g) => Ok(g),
10134        Err(e) => {
10135            state.event("verify", format!("{context}: {e:#}"));
10136            if let Err(e2) = state.save() {
10137                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10138            }
10139            Err(e)
10140        }
10141    }
10142}
10143
10144/// The lease state could not be read at all (unreadable lease file, a
10145/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10146/// known to be contended, so a wave degrades to running without a shared cache
10147/// instead of stopping on a hold that [`crate::triage`] could never release.
10148#[derive(Debug)]
10149struct CacheCheckError(String);
10150
10151impl std::fmt::Display for CacheCheckError {
10152    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10153        f.write_str(&self.0)
10154    }
10155}
10156
10157impl std::error::Error for CacheCheckError {}
10158
10159/// Run `body` — a verify command batch — while holding the shared build
10160/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10161/// never interleave with another borrower's build against the same
10162/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10163/// timeout, or a human's own `magi review`. See the `cache` module doc for
10164/// why this matters more than Cargo's own per-target locking covers — two
10165/// *different* worktrees building the same package name/version into one
10166/// cache directory is a staleness bug, not a lock contention one.
10167///
10168/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10169/// agent waves do. Repository scheduling also has its own bounded,
10170/// cancellable resource-wait policy. Contention must not consume the
10171/// verification budget: `body` receives the full `budget` after both leases
10172/// are acquired and the cache is fresh. Without a shared cache, only
10173/// repository coordination is acquired before handing over the full budget.
10174///
10175/// A lease that cannot be acquired within the resource wait is reported as a single
10176/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10177/// verification — the same shape a spawn failure already takes in
10178/// [`run_commands`], so a caller need not special-case it.
10179#[allow(clippy::too_many_arguments)]
10180async fn with_cache_lease<'s, F, Fut>(
10181    state: &'s mut RunState,
10182    cache_dir: Option<&Path>,
10183    node: &str,
10184    seat: &str,
10185    worktree: &Path,
10186    head: &str,
10187    budget: Duration,
10188    context: &str,
10189    wait: &crate::repo_lock::WaitPolicy,
10190    body: F,
10191) -> Result<(Vec<CommandOutcome>, bool)>
10192where
10193    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10194    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10195{
10196    // Match editing waves: cache first, repository second. In particular,
10197    // pre_gate must never hold the repository while waiting for a wave's cache.
10198    let started = Instant::now();
10199    let home = crate::run::home();
10200    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10201    let guard = if let Some(cache_dir) = cache_dir {
10202        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10203        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10204            wait.cancelled()
10205        })
10206        .await
10207        .and_then(|g| {
10208            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10209        }) {
10210            Ok(g) => Some(g),
10211            Err(e) => {
10212                return Ok((
10213                    vec![CommandOutcome {
10214                        command: "(waiting for the shared build cache)".into(),
10215                        code: None,
10216                        output_tail: e.to_string(),
10217                        duration_ms: started.elapsed().as_millis() as u64,
10218                        resource_blocked: true,
10219                    }],
10220                    false,
10221                ));
10222            }
10223        }
10224    } else {
10225        None
10226    };
10227    let _coordination = coordinate_operation(state, worktree, wait).await?;
10228    if let Some(cache_dir) = cache_dir {
10229        let identity = crate::cache::Identity::new(worktree, head);
10230        if let Err(e) = crate::cache::ensure_fresh(
10231            &home,
10232            cache_dir,
10233            &identity,
10234            &state.config.verify.cargo_manifests(),
10235        ) {
10236            state.event(
10237                "verify",
10238                format!(
10239                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10240                    worktree.display(),
10241                    short(head)
10242                ),
10243            );
10244            if let Some(guard) = guard {
10245                guard.release();
10246            }
10247            return Ok((
10248                vec![CommandOutcome {
10249                    command: "(confirming the shared build cache is fresh)".into(),
10250                    code: None,
10251                    output_tail: e.to_string(),
10252                    duration_ms: started.elapsed().as_millis() as u64,
10253                    resource_blocked: true,
10254                }],
10255                false,
10256            ));
10257        }
10258    }
10259    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10260    if !timed_out_pids.is_empty() {
10261        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10262    }
10263    // Drop repository coordination before releasing the cache, reversing the
10264    // acquisition order. A cancelled/error future also drops both guards.
10265    drop(_coordination);
10266    if let Some(guard) = guard {
10267        guard.release();
10268    }
10269    Ok((outcomes, retried))
10270}
10271
10272/// Poll `pids` — commands [`run_commands`] reports as still running when its
10273/// own timeout elapsed — until every one is confirmed gone, or
10274/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10275///
10276/// Real confirmation where confirmation is possible, not a substitute for
10277/// full process-tree observation: a grandchild the timed-out process spawned
10278/// and that survives independently of it is invisible to a pid check the
10279/// same way it always was, and continuing to observe and collect *that*
10280/// stays a different piece of work with its own owner. This only narrows a
10281/// fixed blind wait into an actual check of the pids this process does know
10282/// about.
10283async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10284    wait_for_pids_with(
10285        pids,
10286        crate::proc::pid_alive,
10287        LEASE_RELEASE_POLL,
10288        LEASE_RELEASE_MAX_WAIT,
10289    )
10290    .await;
10291}
10292
10293/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10294/// interval and ceiling supplied by the caller, so the polling *logic* -
10295/// returns as soon as every pid reports dead, gives up at the ceiling
10296/// otherwise - is testable on millisecond durations without asking the real
10297/// OS about a pid at all.
10298async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10299    pids: &[u32],
10300    alive: F,
10301    poll: Duration,
10302    max_wait: Duration,
10303) {
10304    let deadline = Instant::now() + max_wait;
10305    loop {
10306        if pids.iter().all(|&pid| !alive(pid)) {
10307            return;
10308        }
10309        if Instant::now() >= deadline {
10310            return;
10311        }
10312        tokio::time::sleep(poll).await;
10313    }
10314}
10315
10316/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10317/// admission that it could not even get a verify command to run, as opposed
10318/// to evidence the command actually produced? A caller that would otherwise
10319/// read a resource-blocked outcome as a red command must check this first:
10320/// see [`Runner::gate`], which retries rather than records `Blocked` when
10321/// this is true.
10322fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10323    outcomes.iter().any(|o| o.resource_blocked)
10324}
10325
10326/// What [`Runner::gate_fix_round`] decided.
10327enum GateFix {
10328    /// The tree changed and `verify.e2e` is still green: run the gate again.
10329    Retry,
10330    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10331    /// last failure stands and the run ends blocked.
10332    Stop,
10333    /// `verify.e2e` could not run after the fix (magi's own contention):
10334    /// decide nothing now, a later reentry retries.
10335    Defer,
10336}
10337
10338/// Is every red command in `outcomes` an ordinary failure the code could
10339/// explain: it ran, exited non-zero, and said something?
10340///
10341/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10342/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10343/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10344/// what the command is stays the gate's business.
10345fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10346    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10347    red.peek().is_some()
10348        && red.all(|o| {
10349            !o.resource_blocked
10350                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10351                && !o.output_tail.trim().is_empty()
10352        })
10353}
10354
10355/// Describe one verify command's outcome for the event log, distinguishing a
10356/// build/link failure — the toolchain never produced a binary to run — from
10357/// an actual test failure, since only the latter is a verdict on the patch.
10358fn e2e_outcome_label(o: &CommandOutcome) -> String {
10359    if o.ok() {
10360        return "pass".to_owned();
10361    }
10362    let reason = if o.build_failed() {
10363        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10364    } else {
10365        format!("FAIL ({:?})", o.code)
10366    };
10367    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10368}
10369
10370/// Run `verify.e2e`, retrying once if the first attempt could not build or
10371/// link — a build/link failure is frequently a race against a shared
10372/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10373/// `verify` event per command, tagged with `context` (normally `"round N"`)
10374/// so the two call sites that need this — the ordinary per-round leg in
10375/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10376/// it will ever call a round green — read identically in the event log.
10377async fn run_e2e_with_retry(
10378    state: &mut RunState,
10379    shell: &[String],
10380    commands: &[String],
10381    worktree: &Path,
10382    timeout: Duration,
10383    context: &str,
10384) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10385    let (mut e2e, mut timed_out_pids) = run_commands(
10386        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10387    )
10388    .await;
10389    for o in &e2e {
10390        state.event(
10391            "verify",
10392            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10393        );
10394    }
10395    // A build/link failure is not a verdict on the patch — it is frequently a
10396    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10397    // one retry before letting a red like that decide the round.
10398    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10399    if verify_retried {
10400        state.event(
10401            "verify",
10402            format!(
10403                "{context}: verify could not build/link, not a test result — retrying once \
10404                 before concluding"
10405            ),
10406        );
10407        let retried = run_commands(
10408            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10409        )
10410        .await;
10411        e2e = retried.0;
10412        // Both attempts' timeouts matter, not just the last one: the first
10413        // attempt's descendants may still be alive alongside the retry's.
10414        timed_out_pids.extend(retried.1);
10415        for o in &e2e {
10416            state.event(
10417                "verify",
10418                format!(
10419                    "{context}: retry `{}` -> {}",
10420                    o.command,
10421                    e2e_outcome_label(o)
10422                ),
10423            );
10424        }
10425    }
10426    (e2e, verify_retried, timed_out_pids)
10427}
10428
10429/// Run configured shell commands in `cwd`, in order. The second element is
10430/// the pid of every command that hit `timeout` and was still running when
10431/// this stopped waiting on it (best-effort: `None` when the platform did not
10432/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10433/// that releases a shared resource afterward needs to know.
10434///
10435/// Records `task` into [`RunState::active`] at every command boundary
10436/// (`RunState::task_command`) and clears it once the whole list has run
10437/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10438/// for minutes with no seat and no output of its own to show for it (see
10439/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10440/// apart from "not yet run" without this), and this is the only place that
10441/// knows which command is running right now and how many are left. Three
10442/// saves per command — start, not per second — matching the same "only at a
10443/// boundary" rule [`wave`] already follows for seats.
10444#[allow(clippy::too_many_arguments)]
10445async fn run_commands(
10446    state: &mut RunState,
10447    node: &str,
10448    task: &str,
10449    attempt: usize,
10450    shell: &[String],
10451    commands: &[String],
10452    cwd: &Path,
10453    timeout: Duration,
10454) -> (Vec<CommandOutcome>, Vec<u32>) {
10455    if commands.is_empty() {
10456        // Nothing to mark as running and nothing to clear — an empty list
10457        // means "not configured", and touching `active` (or the disk) over
10458        // that would be a write for every round of a repo with no
10459        // `verify.e2e` / `verify.gate` commands at all.
10460        return (Vec::new(), Vec::new());
10461    }
10462    let mut out = Vec::new();
10463    let mut timed_out_pids = Vec::new();
10464    let total = commands.len();
10465    for (idx, command) in commands.iter().enumerate() {
10466        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10467        if let Err(e) = state.save() {
10468            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10469        }
10470        let started = Instant::now();
10471        let mut cmd = tokio::process::Command::new(&shell[0]);
10472        cmd.quiet();
10473        cmd.args(&shell[1..])
10474            .arg(command)
10475            .current_dir(cwd)
10476            .stdin(std::process::Stdio::null())
10477            .stdout(std::process::Stdio::piped())
10478            .stderr(std::process::Stdio::piped())
10479            .kill_on_drop(true);
10480        let spawned = cmd.spawn();
10481        let (code, body) = match spawned {
10482            Ok(child) => {
10483                // Captured before the child is consumed below: `kill_on_drop`
10484                // only *asks* the process to die when the timeout branch
10485                // drops it, and the pid is the only way anyone downstream can
10486                // later check whether that request actually took.
10487                let pid = child.id();
10488                match tokio::time::timeout(timeout, child.wait_with_output()).await {
10489                    Ok(Ok(o)) => {
10490                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10491                        body.push_str(&String::from_utf8_lossy(&o.stderr));
10492                        (o.status.code(), body)
10493                    }
10494                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
10495                    Err(_) => {
10496                        if let Some(pid) = pid {
10497                            timed_out_pids.push(pid);
10498                        }
10499                        (None, format!("timed out after {}s", timeout.as_secs()))
10500                    }
10501                }
10502            }
10503            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10504        };
10505        out.push(CommandOutcome {
10506            command: command.clone(),
10507            code,
10508            output_tail: tail(&body, OUTPUT_TAIL),
10509            duration_ms: started.elapsed().as_millis() as u64,
10510            resource_blocked: false,
10511        });
10512    }
10513    state.task_finished(task);
10514    if let Err(e) = state.save() {
10515        tracing::warn!("could not persist the end of {task}: {e:#}");
10516    }
10517    (out, timed_out_pids)
10518}
10519
10520/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10521/// section (`report::run`) and in the `merge` event this node records — for
10522/// the operator to run by hand.
10523///
10524/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10525/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10526/// merge commits", or "require linear history") rejects the push a `--no-ff`
10527/// merge would produce, which is exactly the guidance this function replaces.
10528/// `message`'s first line becomes the squash commit's subject, matching the
10529/// note `report::run` prints alongside this command — see that function for
10530/// why an explicit subject is not optional there.
10531fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10532    let repo = repo.display();
10533    match style {
10534        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10535        MergeStyle::Squash => {
10536            // The subject sits inside double quotes, and a title an agent
10537            // wrote may carry the characters that break out of them.
10538            let subject = message
10539                .lines()
10540                .next()
10541                .unwrap_or(branch)
10542                .replace(['\\', '"', '$', '`'], "");
10543            format!(
10544                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10545            )
10546        }
10547        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10548    }
10549}
10550
10551/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10552/// under the hood, rejects a `title` over 256 characters and the whole
10553/// command fails — no PR at all, for a run whose body was otherwise fine
10554/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10555/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10556/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10557/// this bound. It is a margin, not a guarantee — a title packed
10558/// with multi-unit characters could still in principle land close to the
10559/// edge, but a real task title's occasional emoji or accented letter fits
10560/// comfortably inside it.
10561const PR_TITLE_MAX: usize = 240;
10562
10563/// A pull request title from the opening line of `text`, or `None` when that
10564/// line is not English (GitHub text is) or has no letters.
10565///
10566/// A line within `max` is kept as is. A longer one is cut at the end of its
10567/// first sentence when that falls inside `max`, else at a word boundary with a
10568/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10569/// make every merely-truncated title look non-English). The language check
10570/// runs on the kept text before any mark is added, so only what GitHub will
10571/// show is judged: an English opening followed by non-ASCII far past the cut
10572/// still passes.
10573fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10574    let line = queue::first_line(text)?;
10575    let chars: Vec<char> = line.chars().collect();
10576    let (kept, mark) = if chars.len() <= max {
10577        (line.to_owned(), "")
10578    } else if let Some(end) = sentence_end(&chars, max) {
10579        (chars[..end].iter().collect::<String>(), "")
10580    } else {
10581        let room = max.saturating_sub(3);
10582        // Cut at the last space inside the room; when the char just past the
10583        // room is a space the room already ends on a word.
10584        let cut = if chars[room].is_whitespace() {
10585            room
10586        } else {
10587            chars[..room]
10588                .iter()
10589                .rposition(|c| c.is_whitespace())
10590                .unwrap_or(room)
10591        };
10592        let head: String = chars[..cut].iter().collect();
10593        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10594        (head.to_owned(), "...")
10595    };
10596    if kept.is_empty() {
10597        return None;
10598    }
10599    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10600        return None;
10601    }
10602    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10603        return None;
10604    }
10605    Some(format!("{kept}{mark}"))
10606}
10607
10608/// The char length of the first sentence of `chars` when it ends within `max`
10609/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10610/// abbreviations so `e.g. foo` does not end a title early.
10611fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10612    const MIN: usize = 20;
10613    for i in MIN..max.min(chars.len()) {
10614        if !matches!(chars[i], '.' | '!' | '?') {
10615            continue;
10616        }
10617        let Some(&next) = chars.get(i + 1) else {
10618            continue;
10619        };
10620        if !next.is_whitespace() {
10621            continue;
10622        }
10623        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10624        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10625            continue;
10626        }
10627        let word: String = chars[..i]
10628            .iter()
10629            .rev()
10630            .take_while(|c| !c.is_whitespace())
10631            .collect::<Vec<_>>()
10632            .into_iter()
10633            .rev()
10634            .collect();
10635        let word = word.to_ascii_lowercase();
10636        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10637            continue;
10638        }
10639        let end = chars[..i]
10640            .iter()
10641            .rposition(|c| !c.is_whitespace())
10642            .map_or(i, |p| p + 1);
10643        return Some(end);
10644    }
10645    None
10646}
10647
10648/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10649/// change: a title and a body describing what was *implemented*, not the task
10650/// that asked for it. A task reads as a request; a reader of the merged
10651/// history wants the change.
10652struct PrMessage {
10653    title: String,
10654    body: String,
10655}
10656
10657impl PrMessage {
10658    /// Title, blank line, body. The first line is the squash/merge commit
10659    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10660    /// has to stay one sensible line.
10661    fn commit_message(&self) -> String {
10662        format!("{}\n\n{}", self.title, self.body)
10663    }
10664}
10665
10666/// The text after a leading `TITLE:` (any case) on `line`.
10667fn title_marker(line: &str) -> Option<&str> {
10668    let line = line.trim();
10669    let head = line.get(..6)?;
10670    head.eq_ignore_ascii_case("title:")
10671        .then(|| line[6..].trim())
10672}
10673
10674/// The implementer's own one-line title: the `TITLE:` line the implement
10675/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10676/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10677/// source, and a title that says as much is refused here too.
10678fn summary_title(summary: &str) -> Option<String> {
10679    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10680    let raw = title_marker(first)?;
10681    if raw.is_empty() {
10682        return None;
10683    }
10684    let title = queue::title_from(raw, PR_TITLE_MAX);
10685    let lower = title.to_ascii_lowercase();
10686    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10687        return None;
10688    }
10689    Some(title)
10690}
10691
10692/// How `open_review`'s instruction begins; see [`landing_title`].
10693const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10694
10695/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10696pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10697
10698/// Does this run review work that already existed, rather than implement a
10699/// task? Runs recorded before `reviewed_commits` existed carry only the
10700/// candidate marker.
10701fn is_review_run(state: &RunState) -> bool {
10702    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10703}
10704
10705/// The title of a review-only run: the subject of the oldest commit under
10706/// review. Later commits are usually fixups, and `instruction` is the review
10707/// prompt, which says nothing about the change. GitHub text is English, so a
10708/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10709fn review_title(state: &RunState) -> Option<String> {
10710    english_subject(
10711        state.reviewed_commits.as_ref()?.first()?,
10712        state.config.graph.github_english(),
10713    )
10714}
10715
10716/// `raw` as a pull request title, or `None` when it is blank, not English
10717/// (GitHub text is), or one of magi's own candidate commit subjects.
10718fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10719    let raw = raw.trim();
10720    if raw.is_empty() {
10721        return None;
10722    }
10723    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10724        return None;
10725    }
10726    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10727        return None;
10728    }
10729    let title = queue::title_from(raw, PR_TITLE_MAX);
10730    let lower = title.to_ascii_lowercase();
10731    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10732        return None;
10733    }
10734    Some(title)
10735}
10736
10737/// What a review-only run's branch says about itself, read at the moment the
10738/// pull request is opened.
10739#[derive(Debug, Clone, PartialEq, Eq)]
10740struct BranchFacts {
10741    /// `(subject, body)` of each commit, oldest first.
10742    commits: Vec<(String, String)>,
10743    /// Trimmed `git diff --stat`.
10744    stat: String,
10745}
10746
10747/// Longest diff stat shown: this many file lines plus the summary line.
10748const STAT_FILE_LINES: usize = 25;
10749/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10750const COMMITS_MAX_CHARS: usize = 20_000;
10751
10752/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10753/// say or finds nothing, so the caller falls back to what the run recorded.
10754async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10755    let commits = git::commit_log(repo, base, branch).await.ok()?;
10756    if commits.is_empty() {
10757        return None;
10758    }
10759    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10760    let lines: Vec<&str> = stat.lines().collect();
10761    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10762        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10763        let more = format!(" ... {omitted} more file(s)");
10764        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10765        kept.push(&more);
10766        kept.push(lines[lines.len() - 1]);
10767        kept.join("\n")
10768    } else {
10769        lines.join("\n")
10770    };
10771    Some(BranchFacts { commits, stat })
10772}
10773
10774/// Defang what would break the surrounding markdown: a closing `</details>`
10775/// and a code fence.
10776fn markdown_safe(text: &str) -> String {
10777    text.replace("</details>", "&lt;/details&gt;")
10778        .replace("\x60\x60\x60", "~~~")
10779}
10780
10781fn neutral_title(state: &RunState, winner: char) -> String {
10782    format!(
10783        "chore: land candidate {} of run {}",
10784        winner.to_ascii_uppercase(),
10785        state.id
10786    )
10787}
10788
10789/// The pull request title to hand to `land::merge_subject`. A review-only run
10790/// opened by an earlier build titled its pull request with the review prompt;
10791/// that title is dropped (empty, so the fallback applies) rather than landed.
10792/// Any other title, including an operator's rename, passes through untouched,
10793/// and so does every title of a run that implements a task.
10794pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10795    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10796        ""
10797    } else {
10798        pr_title
10799    }
10800}
10801
10802/// What the squash subject falls back to when the pull request title is empty
10803/// or candidate-shaped: for a review-only run the derived title, never the
10804/// review prompt held in `instruction`.
10805pub fn landing_subject_source(state: &RunState) -> String {
10806    if is_review_run(state) {
10807        let winner = state.candidates.first().map_or('A', |c| c.label);
10808        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10809    }
10810    state.instruction.clone()
10811}
10812
10813/// `summary` without its `TITLE:` line, which the pull request title already
10814/// carries.
10815fn summary_without_title(summary: &str) -> String {
10816    let mut lines = summary.trim().lines().peekable();
10817    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10818        lines.next();
10819    }
10820    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10821}
10822
10823/// The pull request title and body for the winning candidate.
10824///
10825/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
10826/// the task's own opening line via [`queue::title_from`] when there is none.
10827/// `state.instruction` can open with blank lines (`task_text` only rejects a
10828/// body that is blank *entirely*), which `title_from` skips.
10829///
10830/// Publishing uses an independent description generated from the final diff
10831/// and structured evidence. Internal summaries and reports remain local.
10832/// Non-publishing previews may display the original task and candidate summary.
10833/// The footer retains the run and candidate provenance.
10834#[cfg(test)]
10835fn pr_message(state: &RunState, winner: char) -> PrMessage {
10836    pr_message_with(state, winner, None)
10837}
10838
10839/// [`pr_message`] with what the branch of a review-only run says about itself.
10840/// `facts` is ignored for a run that implements a task.
10841#[cfg(test)]
10842fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10843    let raw = pr_message_raw(
10844        state,
10845        winner,
10846        facts,
10847        crate::pr_outcome::FinalHead::Unreported,
10848        ChangeNote::NotAsked,
10849    );
10850    let id = crate::scrub::Identity::current();
10851    PrMessage {
10852        title: crate::scrub::scrub(&raw.title, &id),
10853        body: crate::scrub::scrub(&raw.body, &id),
10854    }
10855}
10856
10857/// What the public description may say about the change itself.
10858#[derive(Debug, Clone, Copy)]
10859enum ChangeNote<'a> {
10860    /// A local preview or non-publishing merge mode: no public generation needed.
10861    NotAsked,
10862    /// A summary written from the final diff.
10863    Summary(&'a crate::run::ChangeSummary),
10864    /// A summary was asked for and could not be produced; the text says why.
10865    Unavailable(&'a str),
10866}
10867
10868fn pr_message_raw(
10869    state: &RunState,
10870    winner: char,
10871    facts: Option<&BranchFacts>,
10872    head: crate::pr_outcome::FinalHead<'_>,
10873    change: ChangeNote<'_>,
10874) -> PrMessage {
10875    if !matches!(change, ChangeNote::NotAsked) {
10876        let (title, mut body) = match change {
10877            ChangeNote::Summary(c) => (
10878                c.title.clone(),
10879                format!("## Summary\n\n{}\n\n", c.body.trim()),
10880            ),
10881            ChangeNote::Unavailable(why) => (
10882                neutral_title(state, winner),
10883                format!(
10884                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10885                ),
10886            ),
10887            ChangeNote::NotAsked => unreachable!(),
10888        };
10889        body.push_str(&public_pr_evidence(state, head));
10890        body.push_str(&format!(
10891            "\n---\nmagi:run/{} magi:candidate-{}\n",
10892            state.id,
10893            winner.to_ascii_lowercase()
10894        ));
10895        return PrMessage { title, body };
10896    }
10897    pr_preview(state, winner, facts, head)
10898}
10899
10900/// Append public facts from the run records after every prose generation,
10901/// including posting-gate rewrites. The agent cannot drop this provenance.
10902fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10903    let mut body = crate::pr_outcome::review_rounds(state);
10904    body.push_str(&crate::pr_outcome::final_verification(state, head));
10905    let open = state.last_round_findings();
10906    if !open.is_empty() {
10907        body.push_str("## Open review findings\n\n");
10908        for f in open {
10909            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10910        }
10911        body.push('\n');
10912    }
10913    if let Some(c) = state.change_summary.as_ref()
10914        && head == crate::pr_outcome::FinalHead::At(&c.head)
10915    {
10916        body.push_str(&format!(
10917            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10918            c.base, c.head
10919        ));
10920    }
10921    body
10922}
10923
10924/// Local-only preview of the operator's task and candidate summary.
10925fn pr_preview(
10926    state: &RunState,
10927    winner: char,
10928    facts: Option<&BranchFacts>,
10929    head: crate::pr_outcome::FinalHead<'_>,
10930) -> PrMessage {
10931    let summary = state
10932        .candidates
10933        .iter()
10934        .find(|c| c.label == winner)
10935        .map(|c| c.summary.as_str())
10936        .unwrap_or_default();
10937    // The fallback is the operator's own words and may not be English; GitHub
10938    // text always is, so a non-English task gets a neutral title instead.
10939    let review = is_review_run(state);
10940    let title = if review {
10941        facts
10942            .and_then(|f| {
10943                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10944            })
10945            .or_else(|| review_title(state))
10946            .or_else(|| {
10947                state
10948                    .candidates
10949                    .iter()
10950                    .find(|c| c.label == winner)
10951                    .filter(|c| !c.branch.starts_with("magi/"))
10952                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10953            })
10954            .unwrap_or_else(|| neutral_title(state, winner))
10955    } else {
10956        summary_title(summary).unwrap_or_else(|| {
10957            english_title(
10958                &state.instruction,
10959                PR_TITLE_MAX,
10960                state.config.graph.github_english(),
10961            )
10962            .unwrap_or_else(|| neutral_title(state, winner))
10963        })
10964    };
10965
10966    let mut body = String::new();
10967    let what = summary_without_title(summary);
10968    if !what.is_empty() {
10969        body.push_str("## Summary\n\n");
10970        body.push_str(&what);
10971        body.push_str("\n\n");
10972    }
10973
10974    // Review history uses structured facts across all rounds, never reports.
10975    body.push_str(&crate::pr_outcome::review_rounds(state));
10976    body.push_str(&crate::pr_outcome::final_verification(state, head));
10977    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10978
10979    let open = state.open_findings();
10980    if !open.is_empty() {
10981        body.push_str("## Open review findings\n\n");
10982        for f in &open {
10983            body.push_str(&format!(
10984                "- `{}` [{:?}] {}\n",
10985                f.id, f.severity, "Context retained in local review records."
10986            ));
10987        }
10988        body.push('\n');
10989    }
10990
10991    if let Some(fix) = fix
10992        && !fix.rejected.is_empty()
10993    {
10994        body.push_str("## Declined by the fixer\n\n");
10995        for r in &fix.rejected {
10996            body.push_str(&format!(
10997                "- `{}`: {}\n",
10998                r.id, "Reason retained in local review records."
10999            ));
11000        }
11001        body.push('\n');
11002    }
11003
11004    if review {
11005        // The review prompt is not the task; list what the branch carries.
11006        body.push_str("## Commits under review\n\n");
11007        if let Some(facts) = facts {
11008            let mut left = COMMITS_MAX_CHARS;
11009            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11010                let mut entry = format!("- {}\n", markdown_safe(subject));
11011                for l in markdown_safe(text).lines() {
11012                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11013                }
11014                if left == 0 {
11015                    body.push_str(&format!(
11016                        "- ... {} more commit(s)\n",
11017                        facts.commits.len() - i
11018                    ));
11019                    break;
11020                }
11021                if entry.len() > left {
11022                    // Even the first commit is cut: one huge body must not
11023                    // push the whole description past GitHub's limit.
11024                    let mut end = left;
11025                    while !entry.is_char_boundary(end) {
11026                        end -= 1;
11027                    }
11028                    entry.truncate(end);
11029                    entry.push_str("\n  ... (truncated)\n");
11030                    left = 0;
11031                } else {
11032                    left -= entry.len();
11033                }
11034                body.push_str(&entry);
11035            }
11036            if !facts.stat.trim().is_empty() {
11037                body.push_str(&format!(
11038                    "\n## Diff stat\n\n```\n{}\n```\n",
11039                    markdown_safe(facts.stat.trim())
11040                ));
11041            }
11042        } else {
11043            match &state.reviewed_commits {
11044                Some(subjects) => {
11045                    for s in subjects {
11046                        body.push_str(&format!("- {}\n", s.trim()));
11047                    }
11048                }
11049                None => {
11050                    // An older run kept only the prompt, with the commit list
11051                    // after its first paragraph.
11052                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11053                    body.push_str(rest.trim());
11054                    body.push('\n');
11055                }
11056            }
11057        }
11058    } else {
11059        let task = state.instruction.trim();
11060        let task = if task.is_empty() {
11061            "(empty task)"
11062        } else {
11063            task
11064        };
11065        body.push_str(&format!(
11066            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11067            task.replace("</details>", "&lt;/details&gt;")
11068        ));
11069    }
11070
11071    body.push_str(&format!(
11072        "\n---\nmagi:run/{} magi:candidate-{}\n",
11073        state.id,
11074        winner.to_ascii_lowercase()
11075    ));
11076
11077    PrMessage { title, body }
11078}
11079
11080/// The task with what the repository says about the existing work it names
11081/// appended, so an implementer knows what it started from and what it must
11082/// not redo. Unchanged when the task names nothing.
11083fn seeded_instruction(state: &RunState) -> String {
11084    match refs::describe(&state.seeds) {
11085        Some(facts) => format!(
11086            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11087             Candidates start from the unmerged branch named above, when there \
11088             is one, and carry any unmerged commit named by sha as a \
11089             cherry-pick. Check that this is what the task meant before \
11090             building on it.",
11091            state.instruction
11092        ),
11093        None => state.instruction.clone(),
11094    }
11095}
11096
11097/// Does the winner have no commits ahead of the base it would land on?
11098/// Any failure to find out reads as "not empty": the merge then behaves as it
11099/// always did rather than refusing on a guess.
11100async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11101    let base = &state.base_branch;
11102    let mut against = base.clone();
11103    if mode == MergeMode::Pr {
11104        // A pull request lands on the remote's base, never the local branch
11105        // of the same name: if that cannot be read, "not empty" is the safe
11106        // answer.
11107        let remote = &state.config.merge.remote;
11108        let tracking = format!("{remote}/{base}");
11109        let fetched = git::fetch(repo, remote, base).await;
11110        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11111            against = tracking;
11112        } else {
11113            return false;
11114        }
11115    }
11116    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11117}
11118
11119/// Why nothing was opened for an empty winner, with what the task's own
11120/// references resolved to.
11121fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11122    let mut detail = format!(
11123        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11124         nothing to open a pull request for"
11125    );
11126    match refs::describe(&state.seeds) {
11127        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11128        None => detail.push_str(
11129            "\nThe task names no existing branch or commit; if it means to land work \
11130             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11131        ),
11132    }
11133    detail
11134}
11135
11136/// What the `Pr` merge does once it knows whether the branch already has an
11137/// open pull request.
11138#[derive(Debug, PartialEq, Eq)]
11139enum PrPlan {
11140    Create,
11141    Adopt { url: String, title: String },
11142    Stop(String),
11143}
11144
11145/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11146/// a failed lookup -> stop with the real reason. Never guesses.
11147fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11148    match found {
11149        Ok(land::OpenPr::None) => PrPlan::Create,
11150        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11151        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11152            "several open pull requests exist for this branch, not picking one: {}",
11153            urls.join(" ")
11154        )),
11155        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11156    }
11157}
11158
11159/// `gh pr create`, returning the PR url.
11160pub(crate) async fn gh_pr_create(
11161    cwd: &Path,
11162    base: &str,
11163    head: &str,
11164    title: &str,
11165    body: &str,
11166) -> Result<String> {
11167    let out = tokio::process::Command::new("gh")
11168        .args([
11169            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11170        ])
11171        .current_dir(cwd)
11172        .quiet()
11173        .stdin(std::process::Stdio::null())
11174        .output()
11175        .await
11176        .context("spawn gh")?;
11177    if out.status.success() {
11178        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11179    } else {
11180        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11181    }
11182}
11183
11184/// Never force deletion of uncommitted work, including unknown legacy trees.
11185async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11186    match std::fs::symlink_metadata(path) {
11187        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11188            // A previous cleanup may have removed the tree before a branch
11189            // deletion or state save failed. Unregister it if still present.
11190            git::worktree_remove_clean(repo, path).await?;
11191            return Ok(());
11192        }
11193        Err(e) => return Err(e.into()),
11194        Ok(_) => {}
11195    }
11196    if !git::is_clean(path).await? {
11197        bail!(
11198            "worktree has uncommitted work; refusing removal: {}",
11199            path.display()
11200        );
11201    }
11202    if !git::worktree_remove_clean(repo, path).await? {
11203        bail!("worktree removal refused: {}", path.display());
11204    }
11205    Ok(())
11206}
11207
11208/// Tear a run's worktrees and branches down.
11209///
11210/// `home` is where the updated `run.json` is saved (via
11211/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11212/// a housekeeping pass already has its own honest `home` handed to it, and
11213/// falling through to the global here would write back through whichever
11214/// directory some other process or test pinned into that `OnceLock` first,
11215/// not the one the caller actually resolved its `runs` and `state` from.
11216pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11217    if crate::pr_recovery::sources_retained(home, &state.id) {
11218        return Ok(Vec::new());
11219    }
11220    if !state.candidate_preservation.is_empty() {
11221        bail!("candidate preservation incomplete; refusing cleanup");
11222    }
11223    let repo = state.repo.clone();
11224    // CLI, Web and automatic cleanup all enter here, independently of
11225    // Runner::fold_losers. Protect deletion and shared config release too.
11226    let wait = repository_wait(state, None);
11227    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11228        Ok(lease) => lease,
11229        Err(e) => {
11230            state.save_under(home)?;
11231            return Err(e);
11232        }
11233    };
11234    let root = state.worktree_root();
11235    let winner = state.tally.as_ref().map(|t| t.winner);
11236    let mut removed = Vec::new();
11237
11238    for i in 0..state.candidates.len() {
11239        let c = state.candidates[i].clone();
11240        let is_winner = Some(c.label) == winner;
11241        if is_winner && !drop_winner {
11242            continue;
11243        }
11244        let existed = c.worktree.exists();
11245        // A vanished directory can still have a Git registration that keeps
11246        // its branch checked out. Remove that registration before the branch.
11247        remove_preserved_worktree(&repo, &c.worktree).await?;
11248        if existed {
11249            removed.push(c.worktree.to_string_lossy().into_owned());
11250        }
11251        // A branch handed to a later run (and its pull request) is not this
11252        // run's to delete.
11253        let handed_over = state.released_branches.contains(&c.branch);
11254        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11255            if !git::branch_delete(&repo, &c.branch).await? {
11256                bail!("candidate branch removal refused");
11257            }
11258            removed.push(c.branch.clone());
11259        }
11260        state.candidates[i].folded = true;
11261    }
11262
11263    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11264        let path = name.path();
11265        let keep = !drop_winner
11266            && winner.is_some_and(|w| {
11267                path.file_name()
11268                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11269            });
11270        if keep {
11271            continue;
11272        }
11273        remove_preserved_worktree(&repo, &path).await?;
11274        removed.push(path.to_string_lossy().into_owned());
11275    }
11276
11277    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11278    // judge worktrees, so once the loop above has cleared all of them out,
11279    // the parent is a bare directory nobody else was ever going to remove -
11280    // git only ever managed what was inside it. Left alone, one of these
11281    // accumulates per fully-folded run; the operator's own machine had 74.
11282    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11283    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11284    // so does anything a run never claimed that happens to share the bay.
11285    remove_if_empty(&root);
11286
11287    if state.enabled_worktree_config && drop_winner {
11288        // A release, not a raw disable: some sibling run in this repository
11289        // may still hold its own reference (see `git::acquire_worktree_config`),
11290        // and only the last release actually turns the setting back off.
11291        git::release_worktree_config(&repo).await.ok();
11292        state.enabled_worktree_config = false;
11293    }
11294    state.save_under(home)?;
11295    Ok(removed)
11296}
11297
11298/// Remove `dir` if it exists and has nothing in it.
11299///
11300/// Best-effort and silent by design: a directory that is not empty (a run
11301/// whose winner is still parked there, a stray file some other process left)
11302/// is exactly the case this must refuse, and a directory that is already gone
11303/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11304/// already refuses a non-empty directory, so the emptiness check below is
11305/// belt, not suspenders - it is what keeps this from ever attempting the
11306/// removal in the case that matters, rather than trusting `remove_dir`'s
11307/// error path to have no side effects if it ever changed.
11308fn remove_if_empty(dir: &Path) {
11309    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11310        std::fs::remove_dir(dir).ok();
11311    }
11312}
11313
11314/// Severity of the worst open finding in the last review round, for reporting.
11315pub fn worst_open(state: &RunState) -> Option<Severity> {
11316    state
11317        .reviews
11318        .last()?
11319        .reviews
11320        .iter()
11321        .flat_map(|r| r.findings.iter())
11322        .map(|f| f.severity)
11323        .max()
11324}
11325
11326/// `[worktree] setup` hides its products per worktree, which needs
11327/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11328/// worktrees, like the hook's; every place that runs setup calls this first.
11329async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11330    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11331        git::acquire_worktree_config(repo).await?;
11332        state.enabled_worktree_config = true;
11333    }
11334    Ok(())
11335}
11336
11337#[cfg(test)]
11338mod tests {
11339    #[tokio::test]
11340    async fn launcher_base_fetch_requires_remote_success() {
11341        let tmp = tempfile::tempdir().unwrap();
11342        let repo = tmp.path().join("repo");
11343        let remote = tmp.path().join("remote");
11344        std::fs::create_dir(&repo).unwrap();
11345        std::fs::create_dir(&remote).unwrap();
11346        git::git(&remote, &["init", "--bare", "-b", "main"])
11347            .await
11348            .unwrap();
11349        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11350        git::git(
11351            &repo,
11352            &[
11353                "-c",
11354                "user.name=Fixture",
11355                "-c",
11356                "user.email=fixture@example.invalid",
11357                "commit",
11358                "--allow-empty",
11359                "-m",
11360                "fixture",
11361            ],
11362        )
11363        .await
11364        .unwrap();
11365        git::git(
11366            &repo,
11367            &["remote", "add", "origin", remote.to_str().unwrap()],
11368        )
11369        .await
11370        .unwrap();
11371        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11372        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11373        assert_eq!(
11374            resolve_base(&repo, "main", "origin", None)
11375                .await
11376                .unwrap()
11377                .commit,
11378            head
11379        );
11380        #[cfg(unix)]
11381        {
11382            let policy: crate::config::Coordination = toml::from_str(
11383                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11384            ).unwrap();
11385            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11386            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11387            let held = crate::repo_lock::SeatLease::operation_with(
11388                &repo,
11389                Some(&policy),
11390                &crate::repo_lock::WaitPolicy::default(),
11391            )
11392            .await
11393            .unwrap();
11394            assert!(held.is_some());
11395            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11396            tokio::pin!(fetch);
11397            assert!(
11398                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11399                    .await
11400                    .is_err()
11401            );
11402            assert!(
11403                !repo.join(".git/FETCH_HEAD").exists(),
11404                "no fetch before admission"
11405            );
11406            drop(held);
11407            let fetched = fetch.await.unwrap();
11408            assert_eq!(fetched.commit, head);
11409            assert_eq!(fetched.remote, "origin");
11410            assert_eq!(fetched.branch, "main");
11411        }
11412        // A surviving tracking ref is not successful fetch evidence.
11413        git::git(
11414            &repo,
11415            &[
11416                "remote",
11417                "set-url",
11418                "origin",
11419                tmp.path().join("missing").to_str().unwrap(),
11420            ],
11421        )
11422        .await
11423        .unwrap();
11424        let error = resolve_base(&repo, "main", "origin", None)
11425            .await
11426            .unwrap_err()
11427            .to_string();
11428        assert!(error.contains("cannot read origin/main"));
11429        assert!(error.contains("refusing to branch off"));
11430    }
11431
11432    #[test]
11433    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11434        let leaked = vec!["chore(deps): update a crate".to_owned()];
11435        let own = "fix(daemon): apply a chosen action";
11436        assert!(should_retitle("", own, &leaked));
11437        assert!(should_retitle(
11438            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11439            own,
11440            &leaked
11441        ));
11442        assert!(should_retitle(
11443            "chore: land candidate A of run 1",
11444            own,
11445            &leaked
11446        ));
11447        assert!(should_retitle(
11448            "magi: candidate A (uncommitted work)",
11449            own,
11450            &leaked
11451        ));
11452        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11453        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11454        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11455    }
11456
11457    #[test]
11458    fn pr_merge_plan_creates_adopts_or_stops() {
11459        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11460        assert_eq!(
11461            pr_merge_plan(Ok(land::OpenPr::One {
11462                url: "u".into(),
11463                title: "t".into()
11464            })),
11465            PrPlan::Adopt {
11466                url: "u".into(),
11467                title: "t".into()
11468            }
11469        );
11470        let PrPlan::Stop(many) =
11471            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11472        else {
11473            panic!("many must stop");
11474        };
11475        assert!(many.contains('a') && many.contains('b'));
11476        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11477            panic!("a failed lookup must stop");
11478        };
11479        assert!(err.contains("bad token"));
11480    }
11481
11482    use super::*;
11483    use crate::run::GateStatus;
11484    use std::collections::BTreeMap;
11485    use std::time::Duration;
11486
11487    fn conductor() -> AgentSpec {
11488        AgentSpec {
11489            id: "conductor".to_owned(),
11490            kind: crate::config::AgentKind::Command,
11491            model: None,
11492            command: vec!["true".to_owned()],
11493            extra_args: Vec::new(),
11494            env: BTreeMap::new(),
11495            prompt_delivery: None,
11496        }
11497    }
11498
11499    fn spec(id: &str) -> AgentSpec {
11500        AgentSpec {
11501            id: id.to_owned(),
11502            kind: crate::config::AgentKind::Command,
11503            model: None,
11504            command: vec!["true".to_owned()],
11505            extra_args: Vec::new(),
11506            env: BTreeMap::new(),
11507            prompt_delivery: None,
11508        }
11509    }
11510
11511    fn ids(xs: &[&str]) -> BTreeSet<String> {
11512        xs.iter().map(|s| (*s).to_owned()).collect()
11513    }
11514
11515    #[test]
11516    fn pick_successor_skips_an_agent_another_seat_holds() {
11517        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
11518        let roster = [spec("a"), spec("b"), spec("c")];
11519        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11520        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11521    }
11522
11523    #[test]
11524    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11525        // The other seat started on c but was handed to d; c is free again.
11526        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11527        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11528        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11529        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11530        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11531    }
11532
11533    #[test]
11534    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11535        let roster = [spec("a"), spec("b")];
11536        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11537        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11538        let carried = ids(&["b"]);
11539        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11540        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11541    }
11542
11543    #[test]
11544    fn pick_successor_returns_none_without_any_untried_successor() {
11545        let roster = [spec("a"), spec("b")];
11546        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11547        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11548        let carried = ids(&["a"]);
11549        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11550    }
11551
11552    #[test]
11553    fn pick_successor_rescue_avoids_another_seats_occupant() {
11554        // b is a carried failure and free; a is held by the other seat.
11555        let roster = [spec("a"), spec("b"), spec("c")];
11556        let carried = ids(&["b", "c"]);
11557        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11558        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11559    }
11560
11561    #[test]
11562    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11563        let roster = [spec("a"), spec("b"), spec("c")];
11564        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11565        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11566    }
11567
11568    #[test]
11569    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11570        let roster = [spec("a"), spec("b"), spec("c")];
11571        let failed = ids(&["a", "b", "c"]);
11572        // Rescue looks at the whole roster, once per id, then runs out.
11573        let mut tried = ids(&["b"]);
11574        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11575        assert_eq!(first.id, "a");
11576        tried.insert(first.id.clone());
11577        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11578        assert_eq!(second.id, "c");
11579        tried.insert(second.id.clone());
11580        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11581    }
11582
11583    #[test]
11584    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11585        let roster = [spec("a"), spec("b")];
11586        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11587        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11588    }
11589
11590    #[test]
11591    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11592        let roster = [spec("a"), spec("b"), spec("c")];
11593        let h = SeatHistory {
11594            failed: ids(&["a"]),
11595            last_ok: Some("b".to_owned()),
11596            last_fail: None,
11597        };
11598        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11599        // A last answerer that left the roster, or later failed, is ignored.
11600        let gone = SeatHistory {
11601            last_ok: Some("zzz".to_owned()),
11602            ..h.clone()
11603        };
11604        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11605        let failed = SeatHistory {
11606            failed: ids(&["a", "b"]),
11607            last_ok: Some("b".to_owned()),
11608            last_fail: None,
11609        };
11610        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11611    }
11612
11613    #[test]
11614    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11615        let first = SeatState::new("review-1", "alpha", 7);
11616        let next = handover_seat("review-1", "gamma", 7);
11617        assert_ne!(first.claude_session, next.claude_session);
11618    }
11619
11620    #[test]
11621    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11622        let mut state = state_with_summary("x", "y");
11623        let a = handover_seat("review-1", "beta", state.next_seat_seed());
11624        let b = handover_seat("review-1", "beta", state.next_seat_seed());
11625        assert_ne!(a.claude_session, b.claude_session);
11626    }
11627
11628    #[test]
11629    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11630        let roster = [spec("a"), spec("b")];
11631        let h = SeatHistory {
11632            failed: ids(&["a", "b"]),
11633            ..SeatHistory::default()
11634        };
11635        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11636        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11637        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11638    }
11639
11640    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
11641    // fallback loop depends on to terminate: it must walk forward from the
11642    // seat's own position, never restart at the front of the roster, and it
11643    // must never hand back an id already tried, however many times that id
11644    // happens to appear.
11645
11646    #[test]
11647    fn failure_signature_ignores_numbers_and_paths() {
11648        assert_eq!(
11649            failure_signature("exited with Some(2) and no usable output"),
11650            failure_signature("exited with Some(137) and no usable output")
11651        );
11652        assert_eq!(
11653            failure_signature("process exited with no usable output (exit code 2)"),
11654            failure_signature("process exited with no usable output (exit code 137)")
11655        );
11656        for (old, new) in [
11657            (
11658                "exited with Some(2) and no usable output",
11659                "process exited with no usable output (exit code 2)",
11660            ),
11661            (
11662                "exited with None and no usable output",
11663                "process exited with no usable output (exit code unavailable)",
11664            ),
11665        ] {
11666            let previous = FailClass::Other(failure_signature(old));
11667            let current = FailClass::Other(failure_signature(new));
11668            assert_eq!(previous, current, "resumption retains the failure class");
11669            assert!(!should_hand_over(Some(&previous), &current));
11670        }
11671        assert_eq!(
11672            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11673            failure_signature("cannot open /var/x.txt: denied")
11674        );
11675        assert_ne!(failure_signature("boom"), failure_signature("bang"));
11676    }
11677
11678    #[test]
11679    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11680        let other = FailClass::Other("x".into());
11681        assert!(should_hand_over(None, &FailClass::Quota));
11682        assert!(should_hand_over(Some(&other), &FailClass::Quota));
11683        assert!(should_hand_over(
11684            Some(&FailClass::Timeout),
11685            &FailClass::Timeout
11686        ));
11687        assert!(should_hand_over(None, &other));
11688        assert!(!should_hand_over(Some(&other), &other));
11689        assert!(should_hand_over(
11690            Some(&other),
11691            &FailClass::Other("y".into())
11692        ));
11693        // A quota or timeout in between ends the run of identical failures.
11694        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11695        assert!(should_hand_over(Some(&FailClass::Quota), &other));
11696    }
11697
11698    #[test]
11699    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11700        let a = SeatState::new("judge-1", "alpha", 7);
11701        let b = handover_seat("judge-1", "beta", 7);
11702        assert_ne!(a.claude_session, b.claude_session);
11703        assert_eq!(b.turns, 0);
11704    }
11705
11706    #[test]
11707    fn a_timeout_is_classified_apart_from_other_failures() {
11708        assert_eq!(
11709            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11710            Some(FailClass::Timeout)
11711        );
11712        assert!(matches!(
11713            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11714            Some(FailClass::Other(_))
11715        ));
11716    }
11717
11718    #[test]
11719    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11720        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11721        let tried = BTreeSet::from(["beta".to_owned()]);
11722        // beta sits at index 1; the next candidate is gamma, never alpha —
11723        // which is very likely a different candidate slot's own agent.
11724        let next = next_untried_in_roster(&roster, 1, &tried);
11725        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11726    }
11727
11728    #[test]
11729    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11730        let roster = vec![spec("alpha"), spec("beta")];
11731        let tried = BTreeSet::from(["beta".to_owned()]);
11732        // beta is the roster's last entry: nothing follows it, and alpha —
11733        // earlier in the roster, almost certainly a different candidate
11734        // slot's own agent — must not be reached by wrapping back to it.
11735        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11736    }
11737
11738    #[test]
11739    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11740        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11741        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11742        // beta (index 1) and gamma (index 2, the only entry after it) have
11743        // both been tried; alpha (index 0) never has, but it comes before
11744        // beta's own position, so there is nothing further for this seat.
11745        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11746    }
11747
11748    #[test]
11749    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11750        let roster = vec![spec("a"), spec("a"), spec("b")];
11751        let tried = BTreeSet::from(["a".to_owned()]);
11752        let next = next_untried_in_roster(&roster, 0, &tried);
11753        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11754    }
11755
11756    #[test]
11757    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11758        let roster = vec![spec("a"), spec("b")];
11759        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11760        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11761    }
11762
11763    #[test]
11764    fn remove_if_empty_only_ever_takes_a_bare_directory() {
11765        let dir = tempfile::tempdir().unwrap();
11766        let bay = dir.path().join("ffff");
11767
11768        // Not there yet: nothing to do, nothing to panic on.
11769        remove_if_empty(&bay);
11770        assert!(!bay.exists());
11771
11772        // Something still inside - the winner's worktree, or a stray file -
11773        // keeps the directory standing.
11774        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11775        remove_if_empty(&bay);
11776        assert!(bay.exists(), "non-empty directory must survive");
11777
11778        // Once the last entry is gone, so is the directory itself.
11779        std::fs::remove_dir(bay.join("cand-A")).unwrap();
11780        remove_if_empty(&bay);
11781        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11782    }
11783
11784    // `round_is_clean` is the exact decision this task fixed: a round with a
11785    // seat that never answered must not read the same as a round every seat
11786    // actually reviewed. These are deterministic and process-free by design —
11787    // the equivalent end-to-end check (a real reviewer timing out under a
11788    // live graph run) is a genuine race against wall-clock contention, and a
11789    // spawn slow enough to blow even a generous budget under a loaded test
11790    // run must not turn this specific regression check flaky.
11791
11792    #[test]
11793    fn a_full_panel_that_found_nothing_is_clean() {
11794        assert!(round_is_clean(
11795            0,
11796            true,
11797            2,
11798            2,
11799            0,
11800            IncompleteReviewPolicy::Block
11801        ));
11802    }
11803
11804    #[test]
11805    fn a_missing_seat_is_never_clean_under_the_default_policy() {
11806        assert!(!round_is_clean(
11807            0,
11808            true,
11809            1,
11810            2,
11811            0,
11812            IncompleteReviewPolicy::Block
11813        ));
11814    }
11815
11816    #[test]
11817    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11818        assert!(!round_is_clean(
11819            1,
11820            true,
11821            1,
11822            2,
11823            0,
11824            IncompleteReviewPolicy::Warn
11825        ));
11826    }
11827
11828    #[test]
11829    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11830        assert!(round_is_clean(
11831            0,
11832            true,
11833            1,
11834            2,
11835            0,
11836            IncompleteReviewPolicy::Warn
11837        ));
11838    }
11839
11840    #[test]
11841    fn a_full_panel_with_an_open_finding_is_not_clean() {
11842        assert!(!round_is_clean(
11843            1,
11844            true,
11845            2,
11846            2,
11847            0,
11848            IncompleteReviewPolicy::Block
11849        ));
11850    }
11851
11852    #[test]
11853    fn a_full_panel_with_a_red_e2e_is_not_clean() {
11854        assert!(!round_is_clean(
11855            0,
11856            false,
11857            2,
11858            2,
11859            0,
11860            IncompleteReviewPolicy::Block
11861        ));
11862    }
11863
11864    // The stall this task closes: under the default `block` policy, a seat
11865    // missing only because it was rate limited must not force a wait for a
11866    // session limit that will not lift by the next round. `round_is_clean`
11867    // is where that quorum carve-out lives; the review loop around it never
11868    // changes what a reviewer's vote or a finding's severity means.
11869
11870    #[test]
11871    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11872        // 1 of 2 answered, and the one missing was quota'd — the exact
11873        // "review-2 rate limited (quota)" shape from the field report.
11874        assert!(round_is_clean(
11875            0,
11876            true,
11877            1,
11878            2,
11879            1,
11880            IncompleteReviewPolicy::Block
11881        ));
11882    }
11883
11884    #[test]
11885    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11886        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
11887        // not a quota loss (`quota_missing` stays 0) — worth another try.
11888        assert!(!round_is_clean(
11889            0,
11890            true,
11891            1,
11892            2,
11893            0,
11894            IncompleteReviewPolicy::Block
11895        ));
11896    }
11897
11898    #[test]
11899    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11900        assert!(!round_is_clean(
11901            1,
11902            true,
11903            1,
11904            2,
11905            1,
11906            IncompleteReviewPolicy::Block
11907        ));
11908        assert!(!round_is_clean(
11909            0,
11910            false,
11911            1,
11912            2,
11913            1,
11914            IncompleteReviewPolicy::Block
11915        ));
11916    }
11917
11918    #[test]
11919    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11920        // Every seat quota'd, nobody answered: there is no panel to decide
11921        // on, so this must fall through to the existing block-and-retry
11922        // fallback rather than call an unreviewed patch clean.
11923        assert!(!round_is_clean(
11924            0,
11925            true,
11926            0,
11927            2,
11928            2,
11929            IncompleteReviewPolicy::Block
11930        ));
11931    }
11932
11933    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11934        CommandOutcome {
11935            command: "test".to_owned(),
11936            code,
11937            output_tail: String::new(),
11938            duration_ms: 0,
11939            resource_blocked,
11940        }
11941    }
11942
11943    #[test]
11944    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11945        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11946        assert!(
11947            !verify_inconclusive(&[outcome(Some(1), false)]),
11948            "an ordinary failure is still evidence about the patch"
11949        );
11950        assert!(verify_inconclusive(&[outcome(None, true)]));
11951        assert!(
11952            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11953            "one inconclusive outcome taints the whole batch"
11954        );
11955        assert!(!verify_inconclusive(&[]));
11956    }
11957
11958    #[tokio::test]
11959    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11960        // Alive for the first two checks, then dead - confirms the loop
11961        // actually re-polls rather than deciding once and sleeping out the
11962        // ceiling regardless.
11963        let calls = std::sync::atomic::AtomicUsize::new(0);
11964        let started = Instant::now();
11965        wait_for_pids_with(
11966            &[123],
11967            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11968            Duration::from_millis(5),
11969            Duration::from_secs(5),
11970        )
11971        .await;
11972        assert!(
11973            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11974            "must keep checking rather than deciding on the first answer"
11975        );
11976        assert!(
11977            started.elapsed() < Duration::from_secs(1),
11978            "must return the moment it is confirmed dead, not wait out the ceiling"
11979        );
11980    }
11981
11982    #[tokio::test]
11983    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11984        let started = Instant::now();
11985        wait_for_pids_with(
11986            &[123],
11987            |_| true, // never reports dead
11988            Duration::from_millis(5),
11989            Duration::from_millis(30),
11990        )
11991        .await;
11992        let elapsed = started.elapsed();
11993        assert!(
11994            elapsed >= Duration::from_millis(30),
11995            "must not give up before its own ceiling: {elapsed:?}"
11996        );
11997        assert!(
11998            elapsed < Duration::from_secs(1),
11999            "must not wait past its own ceiling either: {elapsed:?}"
12000        );
12001    }
12002
12003    #[tokio::test]
12004    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12005        let started = Instant::now();
12006        wait_for_pids_with(
12007            &[],
12008            |_| true,
12009            Duration::from_secs(5),
12010            Duration::from_secs(5),
12011        )
12012        .await;
12013        assert!(
12014            started.elapsed() < Duration::from_millis(200),
12015            "an empty pid list has nothing to confirm"
12016        );
12017    }
12018
12019    // `review_conclusion` is the exact decision the review hand-off task
12020    // fixed: a round budget spent (or a tree that stopped moving) must not
12021    // collapse into `Blocked` regardless of what verification actually
12022    // said. Deterministic and process-free for the same reason the
12023    // `round_is_clean` family above is.
12024    fn review_round(
12025        clean: bool,
12026        blocking: usize,
12027        answered: usize,
12028        expected: usize,
12029        progressed: bool,
12030        e2e_ok: bool,
12031    ) -> ReviewRound {
12032        ReviewRound {
12033            round: 1,
12034            head: "h".to_owned(),
12035            verified_head: None,
12036            verified_at: None,
12037            reviews: Vec::new(),
12038            e2e: vec![CommandOutcome {
12039                command: "test".to_owned(),
12040                code: Some(if e2e_ok { 0 } else { 1 }),
12041                output_tail: String::new(),
12042                duration_ms: 0,
12043                resource_blocked: false,
12044            }],
12045            verify_retried: false,
12046            e2e_deferred: false,
12047            e2e_defer_reason: None,
12048            fix: None,
12049            blocking,
12050            answered,
12051            expected,
12052            clean,
12053            progressed,
12054            vote_split: false,
12055            reconsideration: Vec::new(),
12056            verdict: None,
12057        }
12058    }
12059
12060    #[test]
12061    fn review_conclusion_is_none_when_nothing_has_run() {
12062        assert_eq!(review_conclusion(&[], 3), None);
12063    }
12064
12065    #[test]
12066    fn review_conclusion_is_none_while_rounds_remain() {
12067        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12068        assert_eq!(review_conclusion(&rounds, 3), None);
12069    }
12070
12071    #[test]
12072    fn review_conclusion_is_gating_once_a_round_is_clean() {
12073        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12074        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12075    }
12076
12077    #[test]
12078    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12079        let rounds = vec![
12080            review_round(false, 1, 2, 2, true, true),
12081            review_round(false, 1, 2, 2, true, true),
12082        ];
12083        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12084    }
12085
12086    #[test]
12087    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12088        let rounds = vec![
12089            review_round(false, 1, 2, 2, true, true),
12090            review_round(false, 1, 2, 2, true, false),
12091        ];
12092        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12093    }
12094
12095    #[test]
12096    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12097        // Magi never got a command to run against this round's own head — a
12098        // resource-blocked attempt, not a red one — so this must never
12099        // settle on `Blocked` the way a genuine e2e failure would. `None`
12100        // here is what tells `Runner::review_loop` to retry the check
12101        // itself rather than trust this cheap recomputation with a verdict
12102        // it cannot actually produce.
12103        let mut blocked = review_round(false, 1, 2, 2, true, false);
12104        blocked.e2e[0].resource_blocked = true;
12105        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12106        assert_eq!(review_conclusion(&rounds, 2), None);
12107    }
12108
12109    #[test]
12110    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12111        // Missing input, not a verified tree — never a hand-off candidate.
12112        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12113        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12114    }
12115
12116    #[test]
12117    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12118        let rounds = vec![
12119            review_round(false, 1, 2, 2, false, true),
12120            review_round(false, 1, 2, 2, false, true),
12121        ];
12122        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12123    }
12124
12125    fn secs(n: u64) -> Duration {
12126        Duration::from_secs(n)
12127    }
12128
12129    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12130    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12131    #[tokio::test]
12132    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12133        ask_test_home();
12134        let tmp = tempfile::tempdir().unwrap();
12135        let repo = tmp.path().join("repo");
12136        std::fs::create_dir_all(&repo).unwrap();
12137        init_repo(&repo);
12138        let g = |args: &[&str]| {
12139            let out = std::process::Command::new("git")
12140                .args(args)
12141                .current_dir(&repo)
12142                .quiet()
12143                .output()
12144                .unwrap();
12145            assert!(
12146                out.status.success(),
12147                "{}",
12148                String::from_utf8_lossy(&out.stderr)
12149            );
12150        };
12151        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12152        g(&["checkout", "-b", "historical"]);
12153        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12154        g(&["add", "."]);
12155        g(&["commit", "-m", "old alternative"]);
12156        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12157        g(&["checkout", "-b", "magi/task-selected", &base]);
12158        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12159        g(&["add", "."]);
12160        g(&["commit", "-m", "measurement infrastructure"]);
12161        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12162        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12163        g(&["add", "."]);
12164        g(&["commit", "-m", "dependent implementation"]);
12165        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12166        g(&["checkout", "main"]);
12167        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12168        g(&["add", "."]);
12169        g(&["commit", "-m", "unrelated base change"]);
12170        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12171        let origin = tmp.path().join("origin.git");
12172        g(&[
12173            "clone",
12174            "--bare",
12175            repo.to_str().unwrap(),
12176            origin.to_str().unwrap(),
12177        ]);
12178        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12179        g(&["fetch", "origin"]);
12180        let selection = crate::pr_recovery::Selection {
12181            url: "https://example.test/project/pull/1".into(),
12182            branch: "magi/task-selected".into(),
12183            head: head.clone(),
12184            instruction: "preserve measured behavior".into(),
12185            approved_at: jiff::Timestamp::now(),
12186        };
12187        let mut template = runner_at(RunStatus::Reviewing);
12188        template.state.repo = repo.clone();
12189        template.state.base_commit = current.clone();
12190        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12191        template.state.origin = Some(crate::run::Origin::queue("selected"));
12192        template.state.pr_head = Some(selection.branch.clone());
12193        template.state.pr_recovery = Some(selection.clone());
12194        let mut runner =
12195            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12196                .await
12197                .unwrap();
12198        let wt = runner.state.winner().unwrap().worktree.clone();
12199        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12200        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12201        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12202        assert!(runner.state.seeds.is_empty());
12203        assert_eq!(runner.state.instruction, "preserve measured behavior");
12204        runner.sync_to_base().await.unwrap();
12205        assert_eq!(
12206            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12207            "prerequisite"
12208        );
12209        assert_eq!(
12210            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12211            "measured implementation"
12212        );
12213        assert_eq!(
12214            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12215            "current main change"
12216        );
12217        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12218        assert_eq!(
12219            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12220            head
12221        );
12222
12223        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12224        let mut partial = runner_at(RunStatus::Reviewing);
12225        partial.state.repo = repo.clone();
12226        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12227        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12228            run: Some("exit 1".into()),
12229            ..Default::default()
12230        }];
12231        let selection = runner.state.pr_recovery.clone().unwrap();
12232        partial.state.pr_recovery = Some(selection.clone());
12233        partial.state.pr_head = Some(selection.branch.clone());
12234        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12235        let id = partial.state.id.clone();
12236        let error = match Runner::open_selected(
12237            &repo,
12238            partial.state,
12239            partial.roles,
12240            1,
12241            runner.state.base_commit.clone(),
12242            selection,
12243        )
12244        .await
12245        {
12246            Ok(_) => panic!("setup must fail"),
12247            Err(e) => e,
12248        };
12249        assert_eq!(
12250            error
12251                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12252                .unwrap()
12253                .run,
12254            id
12255        );
12256        let failed = RunState::load(&id).unwrap();
12257        assert_eq!(failed.status, RunStatus::Blocked);
12258        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12259        assert!(Runner::resume(&id).is_err());
12260        runner.state.status = RunStatus::Ready;
12261        runner.state.driver_exited = true;
12262        runner.state.save().unwrap();
12263        let mut retry = runner_at(RunStatus::Reviewing);
12264        retry.state.repo = repo.clone();
12265        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12266        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12267        reselected.approved_at = jiff::Timestamp::now();
12268        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12269        retry.state.pr_head = Some(reselected.branch.clone());
12270        retry.state.pr_recovery = Some(reselected.clone());
12271        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12272            .await
12273            .unwrap();
12274        let restarted = Runner::open_selected(
12275            &repo,
12276            retry.state,
12277            retry.roles,
12278            1,
12279            runner.state.base_commit.clone(),
12280            reselected,
12281        )
12282        .await
12283        .unwrap();
12284        assert_eq!(
12285            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12286                .await
12287                .unwrap(),
12288            head
12289        );
12290        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12291        assert_eq!(
12292            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12293            head
12294        );
12295    }
12296
12297    #[tokio::test]
12298    #[cfg(unix)]
12299    async fn fallback_and_interrupted_restoration_require_coordination() {
12300        use std::os::fd::AsRawFd as _;
12301        ask_test_home();
12302        let tmp = tempfile::tempdir().unwrap();
12303        let mut runner = registered_runner(tmp.path()).await;
12304        runner.state.config.graph.timeout_resource_wait = 1;
12305        let winner = runner.state.candidates[0].clone();
12306        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12307        let leftover = winner.worktree.join("leftover");
12308        std::fs::write(&leftover, "untrusted edits").unwrap();
12309        let next = tmp.path().join("next-fixer");
12310        let mut failed = spec("first");
12311        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12312        let mut successor = spec("second");
12313        successor.command = vec![
12314            "sh".into(),
12315            "-c".into(),
12316            format!("touch '{}'; echo done", next.display()),
12317        ];
12318        let attempts = vec![
12319            (failed.clone(), "first".into()),
12320            (successor.clone(), "second".into()),
12321        ];
12322        let lock_path = runner.state.repo.join(".git/updater.lock");
12323        let external = std::fs::File::create(&lock_path).unwrap();
12324        assert_eq!(
12325            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12326            0
12327        );
12328        let artifacts = tmp.path().join("artifacts");
12329        let build = |spec, seat| SeatJob {
12330            spec,
12331            seat,
12332            cwd: winner.worktree.clone(),
12333            prompt: "fixture".into(),
12334            timeout: Duration::from_secs(10),
12335            allow_write: true,
12336            sessions: false,
12337            artifacts: artifacts.clone(),
12338            stem: "fix".into(),
12339            handover: None,
12340        };
12341        let (_, _, out) = runner
12342            .ask_fixer_among(
12343                attempts.clone(),
12344                "graft",
12345                None,
12346                Some((&winner.worktree, &head)),
12347                build,
12348            )
12349            .await;
12350        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12351        assert!(leftover.exists(), "contention must prevent clean");
12352        assert!(!next.exists());
12353        assert!(runner.state.handovers.is_empty());
12354        // Early interrupted-repair rollback uses the same protected route.
12355        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12356            from_run: "earlier".into(),
12357            generation: 1,
12358            mode: crate::recovery::Mode::VerifyRepair,
12359            reasons: Vec::new(),
12360            reviewed_head: None,
12361            rounds: Vec::new(),
12362            open_findings: Vec::new(),
12363            evidence: None,
12364            criteria_inherited: true,
12365            repair: crate::recovery::Repair::Started {
12366                before: head.clone(),
12367            },
12368        });
12369        assert!(
12370            runner
12371                .recovery_repair(&winner)
12372                .await
12373                .unwrap_err()
12374                .is::<crate::repo_lock::ResourceBlocked>()
12375        );
12376        assert!(matches!(
12377            runner.state.recovery.as_ref().unwrap().repair,
12378            crate::recovery::Repair::Started { .. }
12379        ));
12380        assert!(leftover.exists());
12381        drop(external);
12382        runner.recovery_repair(&winner).await.unwrap();
12383        assert!(!leftover.exists());
12384        assert!(matches!(
12385            runner.state.recovery.as_ref().unwrap().repair,
12386            crate::recovery::Repair::Done { .. }
12387        ));
12388        // A hard lock-open failure also leaves HEAD and untracked work intact.
12389        std::fs::remove_file(&lock_path).unwrap();
12390        std::fs::create_dir(&lock_path).unwrap();
12391        std::fs::write(&leftover, "untrusted edits").unwrap();
12392        let (_, _, out) = runner
12393            .ask_fixer_among(
12394                attempts,
12395                "graft",
12396                None,
12397                Some((&winner.worktree, &head)),
12398                build,
12399            )
12400            .await;
12401        assert!(matches!(out, AgentOutcome::Failed(_)));
12402        assert!(leftover.exists());
12403        assert!(!next.exists());
12404        assert_eq!(
12405            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12406            head
12407        );
12408    }
12409
12410    #[cfg(unix)]
12411    async fn registered_runner(root: &Path) -> Runner {
12412        ask_test_home();
12413        let repo = root.join("repo");
12414        std::fs::create_dir_all(&repo).unwrap();
12415        init_repo(&repo);
12416        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12417        let mut runner = runner_at(RunStatus::Prep);
12418        runner.state.repo = repo.clone();
12419        runner.state.config.coordination = crate::config::test_coordination();
12420        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12421        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12422        runner.state.config.graph.advise = false;
12423        runner.roles.implementers = vec![spec("implement")];
12424        runner.roles.judges = vec![spec("judge")];
12425        runner.prep().await.unwrap();
12426        runner
12427    }
12428
12429    #[tokio::test]
12430    #[cfg(unix)]
12431    async fn parked_implementer_persists_completed_work() {
12432        for registered in [false, true] {
12433            let tmp = tempfile::tempdir().unwrap();
12434            let mut runner = registered_runner(tmp.path()).await;
12435            if !registered {
12436                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12437            }
12438            let marker = tmp.path().join("started");
12439            let release = tmp.path().join("release");
12440            let mut implementer = spec("implement");
12441            implementer.command = vec![
12442                "sh".into(),
12443                "-c".into(),
12444                format!(
12445                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12446                    marker.display(),
12447                    release.display()
12448                ),
12449            ];
12450            runner.roles.implementers = vec![implementer.clone()];
12451            runner.state.config.agents = vec![implementer];
12452            let pause = runner.pause.clone();
12453            let (result, ()) = tokio::join!(runner.implement(), async {
12454                tokio::time::timeout(Duration::from_secs(10), async {
12455                    while !marker.exists() {
12456                        tokio::time::sleep(Duration::from_millis(10)).await;
12457                    }
12458                })
12459                .await
12460                .unwrap();
12461                pause.park();
12462                std::fs::write(&release, "go").unwrap();
12463            });
12464            result.unwrap();
12465            let candidate = &runner.state.candidates[0];
12466            assert!(candidate.commits > 0, "{candidate:?}");
12467            assert!(candidate.summary.contains("saved-result"));
12468            assert_eq!(candidate.files, 1);
12469            let before = candidate.commits;
12470            runner.implement().await.unwrap();
12471            assert_eq!(runner.state.candidates[0].commits, before);
12472        }
12473    }
12474
12475    #[tokio::test]
12476    #[cfg(unix)]
12477    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12478        let tmp = tempfile::tempdir().unwrap();
12479        let mut runner = registered_runner(tmp.path()).await;
12480        let winner = runner.state.candidates[0].clone();
12481        let cache = tmp.path().join("cache");
12482        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12483        runner.state.config.graph.timeout_resource_wait = 0;
12484        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12485        let crate::cache::AcquireOutcome::Acquired(held) =
12486            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12487        else {
12488            panic!("cache must be free")
12489        };
12490        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12491            from_run: "earlier".into(),
12492            generation: 1,
12493            mode: crate::recovery::Mode::VerifyRepair,
12494            reasons: Vec::new(),
12495            reviewed_head: None,
12496            rounds: Vec::new(),
12497            open_findings: Vec::new(),
12498            evidence: None,
12499            criteria_inherited: true,
12500            repair: crate::recovery::Repair::NotStarted,
12501        });
12502        runner.recovery_repair(&winner).await.unwrap();
12503        assert!(matches!(
12504            runner.state.recovery.as_ref().unwrap().repair,
12505            crate::recovery::Repair::WaitingForRestore { .. }
12506        ));
12507        drop(held);
12508        runner.state.resource_block = None;
12509        runner.state.config.graph.timeout_resource_wait = 1;
12510        runner.recovery_repair(&winner).await.unwrap();
12511        assert!(matches!(
12512            runner.state.recovery.as_ref().unwrap().repair,
12513            crate::recovery::Repair::Done { .. }
12514        ));
12515        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12516    }
12517
12518    #[tokio::test]
12519    #[cfg(unix)]
12520    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12521        use std::os::fd::AsRawFd as _;
12522        for graft in [false, true] {
12523            let tmp = tempfile::tempdir().unwrap();
12524            let mut runner = registered_runner(tmp.path()).await;
12525            runner.state.config.graph.timeout_resource_wait = 1;
12526            let winner = runner.state.candidates[0].clone();
12527            let marker = tmp.path().join("fixer-started");
12528            let mut fixer = spec("fixer");
12529            fixer.command = vec![
12530                "sh".into(),
12531                "-c".into(),
12532                format!("echo started >> '{}'; echo done", marker.display()),
12533            ];
12534            runner.roles.fixer = Some(vec![fixer]);
12535            if graft {
12536                let mut loser = winner.clone();
12537                loser.label = 'B';
12538                loser.index = 1;
12539                runner.state.candidates.push(loser);
12540                runner.state.config.graph.graft_runner_up = true;
12541                runner.state.tally = Some(Tally {
12542                    first_choice: BTreeMap::new(),
12543                    borda: BTreeMap::new(),
12544                    winner: winner.label,
12545                    rankings: 1,
12546                    unanimous_initial: true,
12547                    deliberated: false,
12548                    changed_votes: 0,
12549                    unanimous_final: true,
12550                    tie_break: None,
12551                    judges: 1,
12552                    present: 1,
12553                    quorum: 1,
12554                    met_quorum: true,
12555                    uncontested: None,
12556                });
12557                runner.state.judgements.push(crate::run::Judgement {
12558                    judge: 1,
12559                    seat: "judge-1".into(),
12560                    agent: "judge".into(),
12561                    ranking: vec![winner.label, 'B'],
12562                    reasons: BTreeMap::new(),
12563                    confidence: None,
12564                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12565                    order: vec![0, 1],
12566                    failed: None,
12567                    duration_ms: 0,
12568                });
12569            } else {
12570                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12571                    from_run: "earlier".into(),
12572                    generation: 1,
12573                    mode: crate::recovery::Mode::VerifyRepair,
12574                    reasons: Vec::new(),
12575                    reviewed_head: None,
12576                    rounds: Vec::new(),
12577                    open_findings: Vec::new(),
12578                    evidence: None,
12579                    criteria_inherited: true,
12580                    repair: crate::recovery::Repair::NotStarted,
12581                });
12582            }
12583            let reader = crate::repo_lock::SeatLease::for_checkout(
12584                &winner.worktree,
12585                false,
12586                crate::config::test_coordination().as_ref(),
12587            )
12588            .await
12589            .unwrap();
12590            let pause = runner.pause.clone();
12591            let (result, ()) = tokio::join!(
12592                async {
12593                    if graft {
12594                        runner.graft().await
12595                    } else {
12596                        runner.recovery_repair(&winner).await
12597                    }
12598                },
12599                async {
12600                    tokio::time::sleep(Duration::from_millis(100)).await;
12601                    pause.park();
12602                }
12603            );
12604            result.unwrap();
12605            assert!(!marker.exists(), "a blocked seat must not spawn");
12606            // Exercise the persisted state, as a binary replacement/resume does.
12607            runner.state = RunState::load(&runner.state.id).unwrap();
12608            if graft {
12609                assert_eq!(
12610                    runner.state.graft.as_ref().unwrap().outcome,
12611                    GraftOutcome::WaitingForRestore
12612                );
12613            } else {
12614                assert!(matches!(
12615                    runner.state.recovery.as_ref().unwrap().repair,
12616                    crate::recovery::Repair::WaitingForRestore { .. }
12617                ));
12618            }
12619            drop(reader);
12620            runner.pause = Pause::new();
12621            runner.state.resource_block = None;
12622            // Deferred restoration must still respect an external lock owner.
12623            let external =
12624                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12625            assert_eq!(
12626                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12627                0
12628            );
12629            let leftover = winner.worktree.join("leftover");
12630            std::fs::write(&leftover, "earlier fallback edits").unwrap();
12631            let result = if graft {
12632                runner.graft().await
12633            } else {
12634                runner.recovery_repair(&winner).await
12635            };
12636            assert!(
12637                result
12638                    .unwrap_err()
12639                    .is::<crate::repo_lock::ResourceBlocked>()
12640            );
12641            assert!(leftover.exists());
12642            assert!(!marker.exists());
12643            drop(external);
12644            runner.state.resource_block = None;
12645            let result = if graft {
12646                runner.graft().await
12647            } else {
12648                runner.recovery_repair(&winner).await
12649            };
12650            result.unwrap();
12651            assert!(!leftover.exists(), "restore must precede retry");
12652            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12653            if graft {
12654                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12655            } else {
12656                assert!(matches!(
12657                    runner.state.recovery.as_ref().unwrap().repair,
12658                    crate::recovery::Repair::Done { .. }
12659                ));
12660            }
12661        }
12662    }
12663
12664    #[tokio::test]
12665    #[cfg(unix)]
12666    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12667        use std::os::fd::AsRawFd as _;
12668        let tmp = tempfile::tempdir().unwrap();
12669        let mut runner = registered_runner(tmp.path()).await;
12670        runner.state.config.graph.timeout_resource_wait = 1;
12671        let winner = runner.state.candidates[0].clone();
12672        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12673        runner.state.tally = Some(Tally {
12674            first_choice: BTreeMap::new(),
12675            borda: BTreeMap::new(),
12676            winner: winner.label,
12677            rankings: 0,
12678            unanimous_initial: false,
12679            deliberated: false,
12680            changed_votes: 0,
12681            unanimous_final: false,
12682            tie_break: None,
12683            judges: 0,
12684            present: 0,
12685            quorum: 0,
12686            met_quorum: true,
12687            uncontested: Some("fixture".into()),
12688        });
12689        runner.state.graft = Some(GraftRecord {
12690            pre: pre.clone(),
12691            post: None,
12692            lines: None,
12693            limit: None,
12694            agent: None,
12695            outcome: GraftOutcome::Running,
12696            requested: Vec::new(),
12697            adoption: None,
12698            adoption_error: None,
12699        });
12700        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12701        git::commit_all(&winner.worktree, "unfinished graft")
12702            .await
12703            .unwrap();
12704        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12705        let leftover = winner.worktree.join("leftover");
12706        std::fs::write(&leftover, "untrusted edits").unwrap();
12707        let path = runner.state.repo.join(".git/updater.lock");
12708        let external = std::fs::File::create(&path).unwrap();
12709        assert_eq!(
12710            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12711            0
12712        );
12713        assert!(
12714            runner
12715                .graft()
12716                .await
12717                .unwrap_err()
12718                .is::<crate::repo_lock::ResourceBlocked>()
12719        );
12720        assert!(matches!(
12721            runner.state.graft.as_ref().unwrap().outcome,
12722            GraftOutcome::Running
12723        ));
12724        assert_eq!(
12725            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12726            unfinished
12727        );
12728        assert!(leftover.exists());
12729        drop(external);
12730        std::fs::remove_file(&path).unwrap();
12731        std::fs::create_dir(&path).unwrap();
12732        assert!(runner.graft().await.is_err());
12733        assert!(matches!(
12734            runner.state.graft.as_ref().unwrap().outcome,
12735            GraftOutcome::Running
12736        ));
12737        assert_eq!(
12738            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12739            unfinished
12740        );
12741        assert!(leftover.exists());
12742        std::fs::remove_dir(&path).unwrap();
12743        runner.graft().await.unwrap();
12744        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12745        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12746        assert!(!leftover.exists());
12747    }
12748
12749    #[tokio::test]
12750    #[cfg(unix)]
12751    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12752        use std::os::fd::AsRawFd as _;
12753        let tmp = tempfile::tempdir().unwrap();
12754        let mut runner = registered_runner(tmp.path()).await;
12755        runner.state.prep_complete = Some(false);
12756        runner.state.config.graph.timeout_resource_wait = 1;
12757        runner.state.save().unwrap();
12758        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12759        assert_eq!(
12760            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12761            0
12762        );
12763        runner.execute().await.unwrap();
12764        let disk = RunState::load(&runner.state.id).unwrap();
12765        assert!(disk.driver_exited);
12766        assert_eq!(disk.driver_pid, None);
12767        assert_eq!(runner.state.status, RunStatus::Blocked);
12768        assert_eq!(
12769            runner.state.resource_block.as_ref().unwrap().kind,
12770            "repository coordination"
12771        );
12772        assert!(runner.state.prep_failure.is_none());
12773        assert!(runner.state.seats.is_empty());
12774        runner.pause.park();
12775        // Include headroom for Git subprocesses under parallel test load;
12776        // the repository contention deadline itself remains one second.
12777        tokio::time::timeout(Duration::from_secs(10), runner.execute())
12778            .await
12779            .unwrap()
12780            .unwrap();
12781        assert!(runner.state.parked);
12782        assert!(runner.state.prep_failure.is_none());
12783        let disk = RunState::load(&runner.state.id).unwrap();
12784        assert!(disk.driver_exited);
12785        assert_eq!(disk.driver_pid, None);
12786    }
12787
12788    #[tokio::test]
12789    #[cfg(unix)]
12790    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12791        use std::os::fd::AsRawFd as _;
12792        let tmp = tempfile::tempdir().unwrap();
12793        let mut runner = registered_runner(tmp.path()).await;
12794        runner.state.prep_complete = Some(false);
12795        runner.state.config.graph.timeout_resource_wait = 30;
12796        runner.state.save().unwrap();
12797        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12798        assert_eq!(
12799            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12800            0
12801        );
12802        assert!(
12803            tokio::time::timeout(Duration::from_millis(100), runner.execute())
12804                .await
12805                .is_err()
12806        );
12807        let disk = RunState::load(&runner.state.id).unwrap();
12808        assert!(disk.driver_exited);
12809        assert_eq!(disk.driver_pid, None);
12810        assert!(disk.status.resumable());
12811        assert!(runner.state.driver_exited);
12812        assert_eq!(runner.state.driver_pid, None);
12813        runner.state.save().unwrap();
12814        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12815    }
12816
12817    #[tokio::test]
12818    #[cfg(unix)]
12819    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12820        let tmp = tempfile::tempdir().unwrap();
12821        let mut runner = registered_runner(tmp.path()).await;
12822        let winner = runner.state.candidates[0].clone();
12823        let cache = tmp.path().join("cache");
12824        let started = tmp.path().join("pre-gate-started");
12825        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12826        runner.state.config.verify.gate =
12827            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12828        let home = crate::run::home();
12829        let owner =
12830            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12831        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12832            crate::cache::AcquireOutcome::Acquired(g) => g,
12833            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12834        };
12835        let mut gate = Box::pin(runner.run_gate(&winner));
12836        assert!(
12837            tokio::time::timeout(Duration::from_millis(150), &mut gate)
12838                .await
12839                .is_err()
12840        );
12841        assert!(!started.exists(), "pre-gate must wait for the cache too");
12842        // Simulate the wave that already owns the cache entering invoke.
12843        // It must obtain the editor reservation and finish without waiting
12844        // for the gate that is queued behind its cache.
12845        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12846        let editor = crate::repo_lock::SeatLease::checkout_with(
12847            &winner.worktree,
12848            true,
12849            crate::config::test_coordination().as_ref(),
12850            &wait,
12851        )
12852        .await
12853        .unwrap();
12854        drop(editor);
12855        guard.release();
12856        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12857            .await
12858            .unwrap()
12859            .unwrap();
12860        drop(gate);
12861        assert!(outcomes.iter().all(CommandOutcome::ok));
12862        assert!(started.exists());
12863    }
12864
12865    #[tokio::test]
12866    #[cfg(unix)]
12867    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12868        use std::os::fd::AsRawFd as _;
12869
12870        ask_test_home();
12871        let tmp = tempfile::tempdir().unwrap();
12872        let repo = tmp.path().join("repo");
12873        std::fs::create_dir_all(&repo).unwrap();
12874        init_repo(&repo);
12875        let common = repo.join(".git");
12876        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12877        let mut runner = runner_at(RunStatus::Prep);
12878        runner.state.repo = repo.clone();
12879        runner.state.config.coordination = crate::config::test_coordination();
12880        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12881        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12882        runner.state.config.graph.advise = false;
12883        runner.roles.implementers = vec![spec("implement")];
12884        runner.roles.judges = vec![spec("judge")];
12885        runner.prep().await.unwrap();
12886        let winner = runner.state.candidates[0].clone();
12887        // Emulate edits already left by a completed seat. The fixture CLI
12888        // only reports completion; it performs no repository operations.
12889        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12890        let completed = tmp.path().join("seat-completed");
12891        runner.roles.implementers[0].command = vec![
12892            "sh".into(),
12893            "-c".into(),
12894            format!("touch '{}'; echo completed", completed.display()),
12895        ];
12896        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12897        assert_eq!(
12898            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12899            0
12900        );
12901        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12902        let mut implement = Box::pin(runner.implement());
12903        tokio::select! {
12904            result = &mut implement => panic!("rescue must wait: {result:?}"),
12905            result = tokio::time::timeout(Duration::from_secs(10), async {
12906                while !completed.exists() {
12907                    tokio::time::sleep(Duration::from_millis(20)).await;
12908                }
12909            }) => result.unwrap(),
12910        }
12911        assert!(
12912            tokio::time::timeout(Duration::from_millis(200), &mut implement)
12913                .await
12914                .is_err()
12915        );
12916        assert_eq!(
12917            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12918            before
12919        );
12920        assert_eq!(
12921            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12922            0
12923        );
12924        tokio::time::timeout(Duration::from_secs(10), &mut implement)
12925            .await
12926            .unwrap()
12927            .unwrap();
12928        drop(implement);
12929        assert_ne!(
12930            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12931            before
12932        );
12933        assert!(git::is_clean(&winner.worktree).await.unwrap());
12934
12935        let started = tmp.path().join("pre-gate-started");
12936        let proceed = tmp.path().join("pre-gate-proceed");
12937        let checked = tmp.path().join("gate-ran");
12938        runner.state.config.verify.pre_gate = vec![format!(
12939            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12940            started.display(),
12941            proceed.display()
12942        )];
12943        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12944        assert_eq!(
12945            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12946            0
12947        );
12948        let mut gate = Box::pin(runner.run_gate(&winner));
12949        assert!(
12950            tokio::time::timeout(Duration::from_millis(200), &mut gate)
12951                .await
12952                .is_err()
12953        );
12954        assert!(!started.exists());
12955        assert!(!checked.exists());
12956        assert!(git::is_clean(&winner.worktree).await.unwrap());
12957        assert_eq!(
12958            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12959            0
12960        );
12961        tokio::select! {
12962            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12963            result = tokio::time::timeout(Duration::from_secs(10), async {
12964                while !started.exists() {
12965                    tokio::time::sleep(Duration::from_millis(20)).await;
12966                }
12967            }) => result.unwrap(),
12968        }
12969        assert_eq!(
12970            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12971            -1,
12972            "mechanical edits must hold the real lock"
12973        );
12974        std::fs::write(proceed, "ready").unwrap();
12975        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12976            .await
12977            .unwrap()
12978            .unwrap();
12979        drop(gate);
12980        assert!(outcomes.iter().all(CommandOutcome::ok));
12981        assert!(checked.exists());
12982        assert!(runner.state.pre_gate_commit.is_some());
12983        assert!(git::is_clean(&winner.worktree).await.unwrap());
12984        assert_eq!(
12985            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12986            0,
12987            "gate completion must release coordination"
12988        );
12989        drop(outsider);
12990        std::fs::remove_file(common.join("updater.lock")).unwrap();
12991        std::fs::create_dir(common.join("updater.lock")).unwrap();
12992        std::fs::remove_file(&started).unwrap();
12993        std::fs::remove_file(&checked).unwrap();
12994        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12995        let error = runner.run_gate(&winner).await.unwrap_err();
12996        assert!(
12997            error
12998                .to_string()
12999                .contains("acquire mandatory repository lock")
13000        );
13001        assert!(
13002            !started.exists(),
13003            "failed acquisition must prevent pre-gate"
13004        );
13005        assert!(!checked.exists(), "failed acquisition must prevent gate");
13006        assert!(git::is_clean(&winner.worktree).await.unwrap());
13007        assert_eq!(
13008            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13009            head
13010        );
13011    }
13012
13013    #[tokio::test]
13014    #[cfg(unix)]
13015    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13016        use std::os::fd::AsRawFd as _;
13017        use std::time::Duration;
13018
13019        ask_test_home();
13020        let tmp = tempfile::tempdir().unwrap();
13021        let repo = tmp.path().join("repo");
13022        std::fs::create_dir_all(&repo).unwrap();
13023        init_repo(&repo);
13024        let common = repo.join(".git");
13025        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13026        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13027        assert_eq!(
13028            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13029            0
13030        );
13031        let mut runner = runner_at(RunStatus::Prep);
13032        runner.state.repo = repo.clone();
13033        runner.state.config.coordination = crate::config::test_coordination();
13034        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13035        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13036        runner.state.config.graph.advise = true;
13037        runner.state.config.graph.advisors = 1;
13038        runner.roles.implementers = vec![spec("implement")];
13039        runner.roles.judges = vec![spec("judge")];
13040        let root = runner.state.worktree_root();
13041        let started = tmp.path().join("setup-started");
13042        let proceed = tmp.path().join("setup-proceed");
13043        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13044            run: Some(format!(
13045                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13046                started.display(),
13047                proceed.display()
13048            )),
13049            ..Default::default()
13050        }];
13051        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13052        let config = git::git(&repo, &["config", "--local", "--list"])
13053            .await
13054            .unwrap();
13055        let mut prep = Box::pin(runner.prep());
13056        assert!(
13057            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13058                .await
13059                .is_err()
13060        );
13061        assert!(
13062            !root.exists(),
13063            "contention must prevent even checkout creation"
13064        );
13065        assert!(!started.exists(), "contention must prevent setup");
13066        assert_eq!(
13067            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13068            branches
13069        );
13070        assert_eq!(
13071            git::git(&repo, &["config", "--local", "--list"])
13072                .await
13073                .unwrap(),
13074            config
13075        );
13076        drop(lock);
13077        tokio::select! {
13078            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13079            result = tokio::time::timeout(Duration::from_secs(10), async {
13080                while !started.exists() {
13081                    tokio::time::sleep(Duration::from_millis(20)).await;
13082                }
13083            }) => result.unwrap(),
13084        }
13085        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13086        assert_eq!(
13087            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13088            -1
13089        );
13090        assert!(
13091            tokio::time::timeout(
13092                Duration::from_millis(100),
13093                crate::repo_lock::SeatLease::for_checkout(
13094                    &repo,
13095                    true,
13096                    crate::config::test_coordination().as_ref()
13097                )
13098            )
13099            .await
13100            .is_err(),
13101            "preparation must also exclude local editors"
13102        );
13103        std::fs::write(proceed, "ready").unwrap();
13104        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13105            .await
13106            .unwrap()
13107            .unwrap();
13108        drop(prep);
13109        assert_eq!(runner.state.prep_complete, Some(true));
13110        assert!(root.join("judge-1").exists());
13111        assert!(root.join("advisor-1").exists());
13112        assert_eq!(
13113            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13114            0
13115        );
13116    }
13117
13118    #[tokio::test]
13119    #[cfg(unix)]
13120    async fn preparation_lock_failure_prevents_repository_changes() {
13121        ask_test_home();
13122        let tmp = tempfile::tempdir().unwrap();
13123        let repo = tmp.path().join("repo");
13124        std::fs::create_dir_all(&repo).unwrap();
13125        init_repo(&repo);
13126        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13127        // A directory at the required lock path makes opening the real lock
13128        // fail, rather than merely exercising an invalid registration.
13129        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13130        let mut runner = runner_at(RunStatus::Prep);
13131        runner.state.repo = repo.clone();
13132        runner.state.config.coordination = crate::config::test_coordination();
13133        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13134        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13135        runner.roles.implementers = vec![spec("implement")];
13136        let error = runner.prep().await.unwrap_err();
13137        assert!(
13138            error
13139                .to_string()
13140                .contains("acquire mandatory repository lock")
13141        );
13142        assert!(!runner.state.worktree_root().exists());
13143        assert!(runner.state.candidates.is_empty());
13144        assert!(!repo.join(".git/config.worktree").exists());
13145        assert_eq!(
13146            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13147            "* main"
13148        );
13149    }
13150
13151    #[tokio::test]
13152    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13153        ask_test_home();
13154        let tmp = tempfile::tempdir().unwrap();
13155        let repo = tmp.path().join("repo");
13156        std::fs::create_dir_all(&repo).unwrap();
13157        init_repo(&repo);
13158        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13159        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13160        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13161        git::git(&repo, &["add", "."]).await.unwrap();
13162        git::git(&repo, &["commit", "-m", "required seed"])
13163            .await
13164            .unwrap();
13165        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13166        git::git(&repo, &["checkout", "main"]).await.unwrap();
13167        let mut runner = runner_at(RunStatus::Prep);
13168        runner.state.repo = repo.clone();
13169        runner.state.base_commit = base.clone();
13170        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13171        runner.state.config.graph.advise = false;
13172        runner.state.config.blind.commit_msg_hook = false;
13173        runner.state.seeds = vec![refs::Seed {
13174            token: seed.clone(),
13175            sha: seed.clone(),
13176            kind: refs::SeedKind::Unmerged,
13177            branch: false,
13178            detail: String::new(),
13179        }];
13180        runner.roles.implementers = vec![spec("one"), spec("two")];
13181        runner.roles.judges = vec![spec("judge")];
13182        let label = blind::assign_labels(2, runner.state.seed)[0];
13183        let branch = runner.state.branch_for(label);
13184        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13185        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13186            .await
13187            .unwrap();
13188        runner.state.candidates.push(Candidate {
13189            index: 0,
13190            label,
13191            agent: "one".into(),
13192            branch,
13193            worktree,
13194            summary: String::new(),
13195            stat: String::new(),
13196            files: 0,
13197            commits: 0,
13198            empty: false,
13199            failed: None,
13200            verified_noop: None,
13201            duration_ms: 0,
13202            folded: false,
13203        });
13204        runner.state.save().unwrap();
13205        // Reload the checkpoint immediately before its first cherry-pick.
13206        runner.state = RunState::load(&runner.state.id).unwrap();
13207        assert_eq!(runner.state.prep_complete, Some(false));
13208        runner.prep().await.unwrap();
13209        assert_eq!(runner.state.prep_complete, Some(true));
13210        assert_eq!(runner.state.candidates.len(), 2);
13211        for candidate in &runner.state.candidates {
13212            assert_eq!(
13213                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13214                git::tree_of(&repo, &seed).await.unwrap()
13215            );
13216            assert_eq!(
13217                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13218                "required commit"
13219            );
13220        }
13221        assert!(runner.state.worktree_root().join("judge-1").exists());
13222        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13223        runner.prep().await.unwrap();
13224        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13225
13226        runner.state.prep_complete = Some(false);
13227        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13228            run: Some(format!(
13229                "test -f '{}'",
13230                tmp.path().join("dependency-ready").display()
13231            )),
13232            ..Default::default()
13233        }];
13234        assert!(runner.prep().await.is_err());
13235        assert!(runner.state.prep_failure.is_some());
13236        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13237        runner.state = RunState::load(&runner.state.id).unwrap();
13238        assert!(
13239            runner.prep().await.is_err(),
13240            "correcting the cause alone must not auto-retry"
13241        );
13242        let failed_at = runner.state.events.last().unwrap().at;
13243        assert!(
13244            runner.retry_preparation(failed_at).is_err(),
13245            "old approvals cannot unlock the hold"
13246        );
13247        let mut task = crate::queue::Task::new(
13248            "retry".into(),
13249            "retry setup".into(),
13250            repo.clone(),
13251            crate::queue::Source::Human,
13252        );
13253        task.start(runner.state.id.clone());
13254        task.hold_machine(Some("setup failed".into()));
13255        task.release_operator("cli", None);
13256        runner
13257            .retry_preparation(task.resumption.as_ref().unwrap().at)
13258            .unwrap();
13259        runner.prep().await.unwrap();
13260        assert!(runner.state.prep_failure.is_none());
13261        assert_eq!(runner.state.prep_complete, Some(true));
13262        assert_eq!(runner.state.candidates.len(), 2);
13263        assert!(runner.state.worktree_root().join("judge-1").exists());
13264        for candidate in &runner.state.candidates {
13265            assert_eq!(
13266                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13267                git::tree_of(&repo, &seed).await.unwrap()
13268            );
13269        }
13270    }
13271
13272    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13273        let mut heads = Vec::new();
13274        for candidate in candidates {
13275            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13276        }
13277        heads
13278    }
13279
13280    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13281        ask_test_home();
13282        let tmp = tempfile::tempdir().unwrap();
13283        let repo = tmp.path().join("repo");
13284        std::fs::create_dir(&repo).unwrap();
13285        init_repo(&repo);
13286        let mut runner = runner_at(RunStatus::Reviewing);
13287        runner.state.repo = repo.clone();
13288        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13289        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13290        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13291            let branch = format!("candidate-{label}");
13292            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13293            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13294                .await
13295                .unwrap();
13296            if label == 'B' {
13297                std::fs::write(worktree.join("change"), "winner").unwrap();
13298                git::rescue_commit(&worktree, "candidate change")
13299                    .await
13300                    .unwrap();
13301            }
13302            runner.state.candidates.push(Candidate {
13303                index,
13304                label,
13305                agent: "alpha".into(),
13306                branch,
13307                worktree,
13308                summary: "completed reply".into(),
13309                stat: String::new(),
13310                files: usize::from(label == 'B'),
13311                commits: usize::from(label == 'B'),
13312                empty: label == 'A',
13313                failed: None,
13314                verified_noop: None,
13315                duration_ms: 1,
13316                folded: false,
13317            });
13318        }
13319        runner.state.judge_skipped = true;
13320        runner.state.tally = Some(Tally {
13321            first_choice: BTreeMap::new(),
13322            borda: BTreeMap::new(),
13323            winner: 'B',
13324            rankings: 0,
13325            unanimous_initial: false,
13326            deliberated: false,
13327            changed_votes: 0,
13328            unanimous_final: false,
13329            tie_break: None,
13330            judges: 0,
13331            present: 0,
13332            quorum: 0,
13333            met_quorum: true,
13334            uncontested: Some("only B produced a change".into()),
13335        });
13336        runner.state.save().unwrap();
13337        (tmp, runner)
13338    }
13339
13340    #[tokio::test]
13341    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13342        let (_tmp, mut runner) = preservation_resume_fixture().await;
13343        // Schema 15 already confirmed the unfolded no-change result.
13344        runner.implement().await.unwrap();
13345        assert!(runner.state.candidate_preservation.is_empty());
13346        assert!(
13347            !runner
13348                .state
13349                .dir()
13350                .join("artifacts/candidate-A-recovery")
13351                .exists()
13352        );
13353        runner.fold_losers().await.unwrap();
13354        assert!(!runner.state.candidates[0].worktree.exists());
13355        for schema in [14, crate::run::SCHEMA] {
13356            runner.state.schema = schema;
13357            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13358            runner.state.save().unwrap();
13359            runner.state = RunState::load(&runner.state.id).unwrap();
13360            runner.implement().await.unwrap();
13361            assert!(runner.state.candidate_preservation.is_empty());
13362            assert!(runner.state.candidates[0].folded);
13363            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13364        }
13365    }
13366
13367    #[tokio::test]
13368    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13369        let (_tmp, mut runner) = preservation_resume_fixture().await;
13370        let wt = runner.state.candidates[0].worktree.clone();
13371        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13372        runner.state.schema = 14;
13373        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13374        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13375        let tally = serde_json::to_value(&runner.state.tally).unwrap();
13376        for _ in 0..2 {
13377            assert!(runner.implement().await.is_err());
13378            assert_eq!(runner.state.status, RunStatus::Blocked);
13379            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13380            assert!(
13381                runner.state.candidates[0].empty,
13382                "old verdict must remain unusable"
13383            );
13384            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13385            assert!(runner.state.judge_skipped);
13386            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13387            assert!(
13388                crate::run::read_artifact(&runner.state, "cand-A.patch")
13389                    .unwrap()
13390                    .contains("owned edits")
13391            );
13392            assert!(wt.exists());
13393            assert_eq!(
13394                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13395                    .await
13396                    .unwrap(),
13397                1
13398            );
13399            runner.state = RunState::load(&runner.state.id).unwrap();
13400        }
13401    }
13402
13403    #[tokio::test]
13404    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13405        let (_tmp, mut runner) = preservation_resume_fixture().await;
13406        let wt = runner.state.candidates[0].worktree.clone();
13407        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13408        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13409        let index_lock = PathBuf::from(
13410            git::git(
13411                &wt,
13412                &[
13413                    "rev-parse",
13414                    "--path-format=absolute",
13415                    "--git-path",
13416                    "index.lock",
13417                ],
13418            )
13419            .await
13420            .unwrap(),
13421        );
13422        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13423        runner.state.schema = 14;
13424        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13425        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13426        assert!(runner.implement().await.is_err());
13427        assert!(runner.after_implement().is_err());
13428        assert!(runner.fold_losers().await.is_err());
13429        assert_eq!(
13430            std::fs::read_to_string(wt.join("recovered")).unwrap(),
13431            "failed seat edits"
13432        );
13433        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13434        runner.state = RunState::load(&runner.state.id).unwrap();
13435        std::fs::remove_file(index_lock).unwrap();
13436        // Recovery finds changes that the old uncontested result never judged.
13437        assert!(runner.implement().await.is_err());
13438        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13439        assert!(runner.state.candidates[0].empty);
13440        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13441        assert!(
13442            crate::run::read_artifact(&runner.state, "cand-A.patch")
13443                .unwrap()
13444                .contains("failed seat edits")
13445        );
13446        assert!(wt.exists());
13447    }
13448
13449    #[tokio::test]
13450    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13451        let (_tmp, mut runner) = preservation_resume_fixture().await;
13452        let repo = runner.state.repo.clone();
13453        let common = git::git(
13454            &repo,
13455            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13456        )
13457        .await
13458        .unwrap();
13459        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13460        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13461        assert!(runner.fold_losers().await.is_err());
13462        assert!(!runner.state.candidates[0].worktree.exists());
13463        assert!(!runner.state.candidates[0].folded);
13464        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13465        runner.state = RunState::load(&runner.state.id).unwrap();
13466        std::fs::remove_file(branch_lock).unwrap();
13467        let judge = runner.state.worktree_root().join("judge-1");
13468        git::worktree_add_detached(&repo, &judge, "main")
13469            .await
13470            .unwrap();
13471        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13472        runner.roles.judges = vec![spec("judge")];
13473        assert!(runner.fold_losers().await.is_err());
13474        runner.state = RunState::load(&runner.state.id).unwrap();
13475        assert!(
13476            runner.state.candidates[0].folded,
13477            "candidate progress saved before judge failure"
13478        );
13479        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13480        assert!(judge.join("uncommitted").exists());
13481        std::fs::remove_file(judge.join("uncommitted")).unwrap();
13482        runner.fold_losers().await.unwrap();
13483        assert!(!judge.exists());
13484        runner.fold_losers().await.unwrap();
13485    }
13486
13487    #[tokio::test]
13488    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13489        let (_tmp, mut runner) = preservation_resume_fixture().await;
13490        runner.state.candidates.retain(|c| c.label == 'B');
13491        runner.state.tally = None;
13492        runner.state.judge_skipped = false;
13493        runner.state.prep_complete = Some(true);
13494        runner.state.config.graph.advise = false;
13495        runner.state.config.graph.selected_implementer = Some("alpha".into());
13496        let wt = runner.state.candidates[0].worktree.clone();
13497        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13498        runner
13499            .state
13500            .candidate_preservation
13501            .insert('B', "awaiting persistence".into());
13502        let index_lock = git::git(
13503            &wt,
13504            &[
13505                "rev-parse",
13506                "--path-format=absolute",
13507                "--git-path",
13508                "index.lock",
13509            ],
13510        )
13511        .await
13512        .unwrap();
13513        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13514        runner.state.save().unwrap();
13515        assert!(runner.execute_graph().await.is_err());
13516        assert_eq!(runner.state.status, RunStatus::Blocked);
13517        assert!(runner.state.candidate_preservation.contains_key(&'B'));
13518        std::fs::remove_file(index_lock).unwrap();
13519        // Park at the next graph boundary: this tests the real selected-seat
13520        // hold guard without dispatching reviewers or publishing anything.
13521        for _ in 0..2 {
13522            runner.state = RunState::load(&runner.state.id).unwrap();
13523            runner.pause.park();
13524            runner.execute_graph().await.unwrap();
13525            assert_eq!(runner.state.status, RunStatus::Implementing);
13526            assert!(runner.state.candidate_preservation.is_empty());
13527            assert_eq!(runner.state.candidates[0].commits, 2);
13528            assert!(runner.state.tally.is_none());
13529        }
13530        runner.implement().await.unwrap();
13531        assert_eq!(runner.state.status, RunStatus::Judging);
13532        // A genuine selected-implementer hold must still require release.
13533        runner.state.status = RunStatus::Blocked;
13534        runner.state.candidates[0].failed = Some("implementation failed".into());
13535        runner.state.save().unwrap();
13536        let error = runner.execute_graph().await.unwrap_err();
13537        assert!(
13538            error
13539                .to_string()
13540                .contains("selected implementation is held")
13541        );
13542    }
13543
13544    #[tokio::test]
13545    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13546        let (_tmp, mut runner) = preservation_resume_fixture().await;
13547        let repo = runner.state.repo.clone();
13548        let candidate = runner.state.candidates[0].clone();
13549        // Remove only the disposable fixture directory, leaving Git's linked
13550        // worktree registration and branch intact, as in a vanished checkout.
13551        let registered = git::worktree_holding(&repo, &candidate.branch)
13552            .await
13553            .unwrap()
13554            .unwrap();
13555        assert_eq!(
13556            registered.canonicalize().unwrap(),
13557            candidate.worktree.canonicalize().unwrap()
13558        );
13559        std::fs::remove_dir_all(&candidate.worktree).unwrap();
13560        assert_eq!(
13561            git::worktree_holding(&repo, &candidate.branch)
13562                .await
13563                .unwrap(),
13564            Some(registered)
13565        );
13566        assert!(
13567            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13568            "the stale registration initially keeps the branch checked out"
13569        );
13570        runner.fold_losers().await.unwrap();
13571        assert!(runner.state.candidates[0].folded);
13572        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13573        assert_eq!(
13574            git::worktree_holding(&repo, &candidate.branch)
13575                .await
13576                .unwrap(),
13577            None
13578        );
13579        runner.state = RunState::load(&runner.state.id).unwrap();
13580        runner.fold_losers().await.unwrap();
13581    }
13582
13583    #[tokio::test]
13584    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13585        let (_tmp, mut runner) = preservation_resume_fixture().await;
13586        let repo = runner.state.repo.clone();
13587        let candidate = runner.state.candidates[0].clone();
13588        let winner = runner.state.candidates[1].clone();
13589        let registered = git::worktree_holding(&repo, &candidate.branch)
13590            .await
13591            .unwrap()
13592            .unwrap();
13593        assert_eq!(
13594            registered.canonicalize().unwrap(),
13595            candidate.worktree.canonicalize().unwrap()
13596        );
13597        std::fs::remove_dir_all(&candidate.worktree).unwrap();
13598        assert_eq!(
13599            git::worktree_holding(&repo, &candidate.branch)
13600                .await
13601                .unwrap(),
13602            Some(registered)
13603        );
13604        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13605
13606        for _ in 0..2 {
13607            fold_run(&mut runner.state, false, &crate::run::home())
13608                .await
13609                .unwrap();
13610            runner.state = RunState::load(&runner.state.id).unwrap();
13611            assert!(runner.state.candidates[0].folded);
13612            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13613            assert_eq!(
13614                git::worktree_holding(&repo, &candidate.branch)
13615                    .await
13616                    .unwrap(),
13617                None
13618            );
13619            assert!(!runner.state.candidates[1].folded);
13620            assert!(winner.worktree.exists());
13621            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13622        }
13623    }
13624
13625    #[tokio::test]
13626    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13627        ask_test_home();
13628        let tmp = tempfile::tempdir().unwrap();
13629        let repo = tmp.path().join("repo");
13630        std::fs::create_dir(&repo).unwrap();
13631        init_repo(&repo);
13632        let mut runner = runner_at(RunStatus::Implementing);
13633        runner.state.repo = repo.clone();
13634        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13635        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13636        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13637            let branch = format!("candidate-{label}");
13638            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13639            git::worktree_add_branch(&repo, &wt, &branch, "main")
13640                .await
13641                .unwrap();
13642            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13643            git::git(&wt, &["add", "README.md"]).await.unwrap();
13644            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13645            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13646            runner.state.candidates.push(Candidate {
13647                index,
13648                label,
13649                agent: "alpha".to_owned(),
13650                branch,
13651                worktree: wt,
13652                summary: "completed implementation".to_owned(),
13653                stat: String::new(),
13654                files: 0,
13655                commits: 0,
13656                empty: false,
13657                failed: None,
13658                verified_noop: None,
13659                duration_ms: 42,
13660                folded: false,
13661            });
13662            runner
13663                .state
13664                .candidate_preservation
13665                .insert(label, "awaiting persistence".to_owned());
13666        }
13667        let wt = runner.state.candidates[0].worktree.clone();
13668        let index_lock = git::git(
13669            &wt,
13670            &[
13671                "rev-parse",
13672                "--path-format=absolute",
13673                "--git-path",
13674                "index.lock",
13675            ],
13676        )
13677        .await
13678        .unwrap();
13679        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13680        runner.state.save().unwrap();
13681        assert!(runner.persist_candidates().await.is_err());
13682        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13683        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13684        assert_eq!(
13685            runner.state.candidates[1].commits, 1,
13686            "other results are still collected"
13687        );
13688        assert!(
13689            !runner.state.candidates[0].empty,
13690            "failed rescue is not no change"
13691        );
13692        assert_eq!(
13693            runner.state.candidates[0].summary,
13694            "completed implementation"
13695        );
13696        assert!(runner.after_implement().is_err());
13697        let home = tmp.path().join("fold-home");
13698        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13699        assert!(wt.join("new-file").exists());
13700        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13701        assert!(recovery.join("staged.patch").exists());
13702        assert!(recovery.join("unstaged.patch").exists());
13703        assert_eq!(
13704            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13705            "untracked edits\n"
13706        );
13707        runner.state.begin_pass();
13708        runner.state.end_pass_parked(ParkCause::Upgrade {
13709            from: "1.0".into(),
13710            to: Some("1.1".into()),
13711        });
13712        runner.state.save().unwrap();
13713        runner.state = RunState::load(&runner.state.id).unwrap();
13714        assert!(matches!(
13715            runner.state.passes.last().unwrap().end,
13716            Some(crate::run::PassEnd::Parked {
13717                cause: ParkCause::Upgrade { .. }
13718            })
13719        ));
13720        std::fs::remove_file(index_lock).unwrap();
13721        // No implementer is configured: a persistence retry cannot dispatch a seat.
13722        runner.implement().await.unwrap();
13723        assert!(runner.state.candidate_preservation.is_empty());
13724        assert!(
13725            runner
13726                .state
13727                .candidates
13728                .iter()
13729                .all(|c| c.commits == 1 && !c.empty)
13730        );
13731        assert!(git::is_clean(&wt).await.unwrap());
13732        assert!(
13733            crate::run::read_artifact(&runner.state, "cand-A.patch")
13734                .unwrap()
13735                .contains("later tracked edits")
13736        );
13737    }
13738
13739    #[tokio::test]
13740    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13741        ask_test_home();
13742        let tmp = tempfile::tempdir().unwrap();
13743        let repo = tmp.path().join("repo");
13744        std::fs::create_dir(&repo).unwrap();
13745        init_repo(&repo);
13746        let wt = tmp.path().join("candidate");
13747        git::worktree_add_branch(&repo, &wt, "candidate", "main")
13748            .await
13749            .unwrap();
13750        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13751        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13752        assert!(wt.join("uncommitted").exists());
13753        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13754        std::fs::remove_file(wt.join("uncommitted")).unwrap();
13755        let mut runner = runner_at(RunStatus::Implementing);
13756        runner.state.repo = repo.clone();
13757        runner.state.base_commit = "missing-base".to_owned();
13758        runner.state.candidates.push(Candidate {
13759            index: 0,
13760            label: 'A',
13761            agent: "alpha".into(),
13762            branch: "candidate".into(),
13763            worktree: wt.clone(),
13764            summary: "completed".into(),
13765            stat: String::new(),
13766            files: 0,
13767            commits: 0,
13768            empty: false,
13769            failed: None,
13770            verified_noop: None,
13771            duration_ms: 1,
13772            folded: false,
13773        });
13774        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13775        assert!(
13776            fold_run(&mut runner.state, true, &tmp.path().join("home"))
13777                .await
13778                .is_err()
13779        );
13780        assert!(!runner.state.candidates[0].folded);
13781        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13782        std::fs::remove_file(wt.join("uncommitted")).unwrap();
13783        runner
13784            .state
13785            .candidate_preservation
13786            .insert('A', "pending".into());
13787        assert!(runner.persist_candidates().await.is_err());
13788        assert!(!runner.state.candidates[0].empty);
13789        assert!(!runner.state.candidates[0].folded);
13790        assert!(wt.exists());
13791        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13792        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13793        std::fs::create_dir_all(&patch_path).unwrap();
13794        assert!(
13795            runner.persist_candidates().await.is_err(),
13796            "artifact failure stays pending"
13797        );
13798        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13799        assert!(!runner.state.candidates[0].empty);
13800        std::fs::remove_dir(patch_path).unwrap();
13801        runner.persist_candidates().await.unwrap();
13802        assert!(
13803            runner.state.candidates[0].empty,
13804            "only successful inspection confirms no change"
13805        );
13806    }
13807
13808    fn init_repo(dir: &Path) {
13809        let run = |args: &[&str]| {
13810            let out = std::process::Command::new("git")
13811                .args(args)
13812                .current_dir(dir)
13813                .quiet()
13814                .output()
13815                .expect("spawn git");
13816            assert!(
13817                out.status.success(),
13818                "git {args:?} failed: {}",
13819                String::from_utf8_lossy(&out.stderr)
13820            );
13821        };
13822        run(&["init", "-b", "main"]);
13823        run(&["config", "user.name", "magi test"]);
13824        run(&["config", "user.email", "magi@example.com"]);
13825        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13826        run(&["add", "-A"]);
13827        run(&["commit", "-m", "init"]);
13828    }
13829
13830    // `settle_questions` is what closes the ghost the phone showed: a run's
13831    // seat asked something, the run then ended, and nothing was left to
13832    // abandon the question it left `open`. `HOME` is a process-wide
13833    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
13834    // first time it runs in the binary — every test below still reaches the
13835    // same directory whichever call won, and each gets its own run id from
13836    // `RunState::new`, so they never collide there.
13837    fn ask_test_home() {
13838        crate::run::pin_test_home();
13839    }
13840
13841    /// A minimal, git-free `Runner` at a given status — `settle_questions`
13842    /// reads nothing else off it.
13843    fn runner_at(status: RunStatus) -> Runner {
13844        let mut state = RunState::new(
13845            PathBuf::from("/nonexistent/repo"),
13846            "main".to_owned(),
13847            "deadbeef".to_owned(),
13848            "task".to_owned(),
13849            Config::default(),
13850        );
13851        // These fixtures share one home and save concurrently. Short clock
13852        // IDs can collide, making an ownership test read another test's run.
13853        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13854        state.id = format!(
13855            "20000101-000000-{:04x}",
13856            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13857        );
13858        state.status = status;
13859        Runner {
13860            state,
13861            roles: ResolvedRoles {
13862                implementers: Vec::new(),
13863                judges: Vec::new(),
13864                reviewers: Vec::new(),
13865                fixer: None,
13866                conductor: conductor(),
13867                implementer_roster: Vec::new(),
13868                judge_roster: Vec::new(),
13869                reviewer_roster: Vec::new(),
13870            },
13871            sem: Arc::new(Semaphore::new(1)),
13872            pause: Pause::new(),
13873            interrupt: Pause::new(),
13874        }
13875    }
13876
13877    #[test]
13878    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13879        crate::run::pin_test_home();
13880        let mut runner = runner_at(RunStatus::Gating);
13881        runner.state.save().unwrap();
13882        // run.json can commit, but the subsequent lease metadata write fails.
13883        // Cleanup must run after releasing the short ownership lock.
13884        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13885        assert!(runner.claim_driver().is_err());
13886        assert!(runner.state.driver_exited);
13887        assert_eq!(runner.state.driver_pid, None);
13888        let disk = RunState::load(&runner.state.id).unwrap();
13889        assert!(disk.driver_exited);
13890        assert_eq!(disk.driver_pid, None);
13891        assert_eq!(disk.status, RunStatus::Gating);
13892    }
13893
13894    #[tokio::test]
13895    async fn an_early_graph_error_releases_its_driver() {
13896        crate::run::pin_test_home();
13897        let mut runner = runner_at(RunStatus::Blocked);
13898        runner.state.config.graph.selected_implementer = Some("selected".into());
13899        runner.state.save().unwrap();
13900        assert!(runner.execute().await.is_err());
13901        let disk = RunState::load(&runner.state.id).unwrap();
13902        assert!(disk.driver_exited);
13903        assert_eq!(disk.driver_pid, None);
13904        assert_eq!(disk.status, RunStatus::Blocked);
13905    }
13906
13907    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13908        crate::run::MergeRepair {
13909            question: "repair-question".into(),
13910            head: "old-head".into(),
13911            pr: "https://github.com/o/r/pull/42".into(),
13912            reviews: vec![review_round(false, 1, 1, 1, false, true)],
13913            gate: vec![outcome(Some(0), false)],
13914            gate_head: Some("old-head".into()),
13915            gate_fixes: Vec::new(),
13916            pre_gate: Vec::new(),
13917            phase,
13918            failure: None,
13919        }
13920    }
13921
13922    #[tokio::test]
13923    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13924        crate::run::pin_test_home();
13925        let mut runner = runner_at(RunStatus::Landing);
13926        runner.state.merge_repairs.push(merge_repair_request(
13927            crate::run::MergeRepairPhase::Requested,
13928        ));
13929        runner.state.save().unwrap();
13930        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13931        assert!(
13932            runner
13933                .execute()
13934                .await
13935                .unwrap_err()
13936                .to_string()
13937                .contains("already running")
13938        );
13939        assert_eq!(
13940            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13941            crate::run::MergeRepairPhase::Requested
13942        );
13943        drop(claim);
13944        let mut disk = runner.state.clone();
13945        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13946        disk.save().unwrap();
13947        assert!(
13948            runner
13949                .execute()
13950                .await
13951                .unwrap_err()
13952                .to_string()
13953                .contains("decision changed")
13954        );
13955        assert_eq!(
13956            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13957            crate::run::MergeRepairPhase::Complete
13958        );
13959    }
13960
13961    #[test]
13962    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13963        crate::run::pin_test_home();
13964        let mut runner = runner_at(RunStatus::Landing);
13965        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13966        runner.state.reviews = request.reviews.clone();
13967        runner.state.gate = request.gate.clone();
13968        runner.state.gate_ran = true;
13969        runner.state.land_approval = Some(crate::run::LandApproval {
13970            question: "old".into(),
13971            head: "old-head".into(),
13972        });
13973        runner.state.merge_repairs.push(request);
13974        runner.begin_merge_repair_review(0).unwrap();
13975        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13976        assert!(!runner.state.gate_status().ok());
13977        assert!(runner.state.land_approval.is_none());
13978        let disk = RunState::load(&runner.state.id).unwrap();
13979        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13980        assert!(disk.merge_repairs[0].gate[0].ok());
13981        assert_eq!(
13982            disk.merge_repairs[0].phase,
13983            crate::run::MergeRepairPhase::Reviewing
13984        );
13985        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13986        assert!(disk.reviews.is_empty());
13987    }
13988
13989    #[tokio::test]
13990    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13991        crate::run::pin_test_home();
13992        for phase in [
13993            crate::run::MergeRepairPhase::Fixing,
13994            crate::run::MergeRepairPhase::Requested,
13995        ] {
13996            let mut runner = runner_at(RunStatus::Landing);
13997            runner.state.config.graph.land_rounds = 0;
13998            runner.state.merge_repairs.push(merge_repair_request(phase));
13999            runner.state.save().unwrap();
14000            assert!(!runner.repair_merge_review().await.unwrap());
14001            assert_eq!(runner.state.status, RunStatus::Blocked);
14002            assert_eq!(
14003                runner.state.merge_repairs[0].phase,
14004                crate::run::MergeRepairPhase::Failed
14005            );
14006            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14007            assert!(
14008                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14009                    "interrupted"
14010                } else {
14011                    "budget exhausted"
14012                })
14013            );
14014            assert!(runner.state.merge.is_none());
14015        }
14016    }
14017
14018    /// `park_here` folding in the reason `Pause::park_because` recorded -
14019    /// this is what lets an operator reading a run's events tell an
14020    /// interrupt-driven park from an ordinary shutdown park.
14021    #[test]
14022    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14023        crate::run::pin_test_home();
14024        let mut runner = runner_at(RunStatus::Implementing);
14025        let interrupt = Pause::new();
14026        runner.watch_interrupt(interrupt.clone());
14027
14028        interrupt.park_because("task a1b2 asked to run first");
14029
14030        assert!(runner.park_here().expect("park_here"));
14031        assert!(runner.state.parked);
14032        let last = runner.state.events.last().expect("a park event");
14033        assert_eq!(last.node, "park");
14034        assert!(
14035            last.message.contains("task a1b2 asked to run first"),
14036            "expected the interrupt reason in {:?}",
14037            last.message
14038        );
14039    }
14040
14041    /// The cause is stored on the pass when it parks, and the first cause a
14042    /// shared pause was given is the one that stays.
14043    #[test]
14044    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14045        crate::run::pin_test_home();
14046        let mut runner = runner_at(RunStatus::Implementing);
14047        runner.state.begin_pass();
14048        let shutdown = Pause::new();
14049        runner.on_pause(shutdown.clone());
14050        let upgrade = ParkCause::Upgrade {
14051            from: "0.121.0".to_owned(),
14052            to: Some("0.121.1".to_owned()),
14053        };
14054        shutdown.park_for(upgrade.clone());
14055        shutdown.park_for(ParkCause::Operator);
14056
14057        assert!(runner.park_here().expect("park_here"));
14058        assert_eq!(
14059            runner.state.passes[0].end,
14060            Some(crate::run::PassEnd::Parked { cause: upgrade })
14061        );
14062        let last = runner.state.events.last().expect("a park event");
14063        assert!(last.message.contains("0.121.0"), "{}", last.message);
14064    }
14065
14066    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14067    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14068    /// cleared) must not make a *different* run - one only watching its own,
14069    /// unshared interrupt `Pause` - see itself as parked. If a future change
14070    /// ever collapsed these back into one handle, the interrupt scheduler
14071    /// would park every run for the rest of the daemon's life, not just the
14072    /// one it meant to interrupt.
14073    #[test]
14074    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14075        crate::run::pin_test_home();
14076        let mut runner = runner_at(RunStatus::Implementing);
14077        let shutdown = Pause::new();
14078        runner.on_pause(shutdown.clone());
14079        let interrupt = Pause::new();
14080        runner.watch_interrupt(interrupt.clone());
14081
14082        // Nobody has asked for anything yet.
14083        assert!(!runner.park_here().expect("park_here"));
14084        assert!(!runner.state.parked);
14085
14086        // Only the interrupt handle fires; the shutdown handle stays clear.
14087        interrupt.park_because("test");
14088        assert!(!shutdown.parked());
14089        assert!(runner.park_here().expect("park_here"));
14090    }
14091
14092    /// The property every prior attempt at this feature failed to pin down:
14093    /// asking a run to park while one of its nodes has a real, in-flight
14094    /// async operation running (an agent call, in production) must not cut
14095    /// that operation short. `park_here` is only ever consulted *between*
14096    /// `execute`'s node calls - see its own doc - so nothing inside a node
14097    /// can observe a park request until the node itself returns. This proves
14098    /// that structurally, with real `tokio` concurrency and a channel
14099    /// handshake (never a sleep, which would only prove "usually", not
14100    /// "cannot"): the "node" below reports that it has genuinely started,
14101    /// and only then is the park requested; the node still has to be told to
14102    /// finish before `park_here` is ever called, exactly mirroring every
14103    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14104    /// in `execute`.
14105    #[tokio::test]
14106    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14107        crate::run::pin_test_home();
14108        let mut runner = runner_at(RunStatus::Implementing);
14109        let interrupt = Pause::new();
14110        runner.watch_interrupt(interrupt.clone());
14111
14112        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14113        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14114
14115        // Stands in for one node's in-flight agent call: it proves it has
14116        // genuinely started, then blocks - exactly as a spawned CLI process
14117        // does - until told to finish.
14118        let node = async move {
14119            started_tx.send(()).expect("send started");
14120            finish_rx.await.expect("recv finish");
14121            "node finished"
14122        };
14123
14124        let interrupter = async move {
14125            started_rx.await.expect("recv started");
14126            // The call is now genuinely in flight. Ask it to park.
14127            interrupt.park_because("higher-priority task waiting");
14128            // Nothing the node does can observe this yet - there is no
14129            // check inside it, by construction - so let the executor run
14130            // anything pending and then let the node finish on its own.
14131            tokio::task::yield_now().await;
14132            finish_tx.send(()).expect("send finish");
14133        };
14134
14135        let (node_result, ()) = tokio::join!(node, interrupter);
14136        assert_eq!(
14137            node_result, "node finished",
14138            "the in-flight call ran to completion"
14139        );
14140
14141        // Only now, at the boundary the real `execute` would check right
14142        // after this node, does the park take effect.
14143        assert!(runner.park_here().expect("park_here"));
14144        assert!(runner.state.parked);
14145    }
14146
14147    /// A run parked mid-competition carries every field it had accumulated
14148    /// through the exact same disk round-trip an ordinary resume uses -
14149    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14150    /// Nothing about parking for an interrupt is a special case of that path;
14151    /// this is what proves it rather than assuming it.
14152    #[test]
14153    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14154        crate::run::pin_test_home();
14155        let mut runner = runner_at(RunStatus::Judging);
14156        // `Runner::resume` re-resolves roles from the saved config, which
14157        // refuses an empty roster - give it the same minimal one `conductor`
14158        // itself uses.
14159        runner.state.config.agents = vec![conductor()];
14160        runner.state.candidates = vec![Candidate {
14161            index: 0,
14162            label: 'A',
14163            agent: "alpha".to_owned(),
14164            branch: "magi/x/A".to_owned(),
14165            worktree: PathBuf::from("/nonexistent/worktree"),
14166            summary: "did the thing".to_owned(),
14167            stat: "1 file changed".to_owned(),
14168            files: 1,
14169            commits: 1,
14170            empty: false,
14171            failed: None,
14172            verified_noop: None,
14173            duration_ms: 1234,
14174            folded: false,
14175        }];
14176        let run_id = runner.state.id.clone();
14177
14178        let interrupt = Pause::new();
14179        runner.watch_interrupt(interrupt.clone());
14180        interrupt.park_because("task c3d4 asked to run first");
14181        assert!(runner.park_here().expect("park_here"));
14182
14183        let resumed = Runner::resume(&run_id).expect("resume");
14184        assert_eq!(resumed.state.candidates.len(), 1);
14185        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14186        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14187        assert_eq!(resumed.state.status, runner.state.status);
14188        assert!(
14189            resumed.state.parked,
14190            "still parked until `execute` actually walks the graph again"
14191        );
14192        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14193    }
14194
14195    /// A fresh open question on `run`, stored and handed back for assertions.
14196    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14197        let mut q = ask::Question::new(
14198            run.to_owned(),
14199            "implement".to_owned(),
14200            "impl-A".to_owned(),
14201            "Which storage backend should the cache use?".to_owned(),
14202            String::new(),
14203            vec!["SQLite".to_owned(), "Redis".to_owned()],
14204        );
14205        store.put(&mut q).unwrap();
14206        q
14207    }
14208
14209    #[test]
14210    fn a_failed_runs_open_question_is_abandoned() {
14211        ask_test_home();
14212        let store = ask::Questions::open();
14213        let mut runner = runner_at(RunStatus::Failed);
14214        let run = runner.state.id.clone();
14215        let q = ask_open_question(&store, &run);
14216
14217        runner.settle_questions();
14218
14219        let back = store.get(&q.id).unwrap();
14220        assert!(
14221            !back.status.open(),
14222            "the seat that asked died with the run; nobody is left to read an answer"
14223        );
14224        assert!(
14225            back.detail.contains(&run) && back.detail.contains("failed"),
14226            "the reason names what the run became, not just that it is gone: {}",
14227            back.detail
14228        );
14229    }
14230
14231    #[test]
14232    fn a_merged_runs_open_question_is_abandoned_too() {
14233        ask_test_home();
14234        let store = ask::Questions::open();
14235        // A run that finishes cleanly still leaves nobody to read an answer -
14236        // this is not only a failure-path cleanup.
14237        for status in [RunStatus::Merged, RunStatus::Ready] {
14238            let mut runner = runner_at(status);
14239            let run = runner.state.id.clone();
14240            let q = ask_open_question(&store, &run);
14241
14242            runner.settle_questions();
14243
14244            let back = store.get(&q.id).unwrap();
14245            assert!(
14246                !back.status.open(),
14247                "{status:?} run's question must not outlive the run"
14248            );
14249        }
14250    }
14251
14252    #[test]
14253    fn a_still_resumable_runs_open_question_is_left_alone() {
14254        ask_test_home();
14255        let store = ask::Questions::open();
14256        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14257        // review round and the seat sessions are all still on disk — so a
14258        // question asked mid-round may yet get a real answer from a real
14259        // resume. Sweeping it here would be exactly the failure mode this
14260        // whole feature exists to avoid on the other side.
14261        for status in [RunStatus::Blocked, RunStatus::Stalled] {
14262            let mut runner = runner_at(status);
14263            let run = runner.state.id.clone();
14264            let q = ask_open_question(&store, &run);
14265
14266            runner.settle_questions();
14267
14268            let back = store.get(&q.id).unwrap();
14269            assert!(
14270                back.status.open(),
14271                "{status:?} is still alive; the question must still be waiting"
14272            );
14273        }
14274    }
14275
14276    #[test]
14277    fn settle_questions_never_touches_an_already_answered_question() {
14278        ask_test_home();
14279        let store = ask::Questions::open();
14280        let mut runner = runner_at(RunStatus::Failed);
14281        let run = runner.state.id.clone();
14282        let mut q = ask_open_question(&store, &run);
14283        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14284            .unwrap();
14285        store.put(&mut q).unwrap();
14286
14287        // Called twice, the way a crash-recovered daemon reclaim and the
14288        // graph's own cleanup both can for the same run — `abandon_for_run`
14289        // only ever touches what is still open, so this must be inert both
14290        // times, not merely the second.
14291        runner.settle_questions();
14292        runner.settle_questions();
14293
14294        let back = store.get(&q.id).unwrap();
14295        assert_eq!(
14296            back.status,
14297            ask::QuestionStatus::Answered,
14298            "a real answer is a decision on record, never overwritten by a sweep"
14299        );
14300    }
14301
14302    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
14303    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
14304    /// without merging, whose winner is still the operator's answer to read.
14305    /// Nothing previously called `fold_run` itself with a real `tally`, so
14306    /// this is the first test to pin down the one distinction the whole
14307    /// automatic-fold feature depends on: the winner's worktree and branch
14308    /// must survive, everything else sharing the run's worktree bay - a
14309    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
14310    /// second sweep treats every non-winner directory under the bay alike -
14311    /// must not.
14312    #[tokio::test]
14313    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14314        crate::run::pin_test_home();
14315        let tmp = tempfile::tempdir().expect("tempdir");
14316        let repo = tmp.path().join("repo");
14317        std::fs::create_dir_all(&repo).unwrap();
14318        init_repo(&repo);
14319
14320        let mut config = Config::default();
14321        config.graph.worktree_root = Some(tmp.path().join("wt"));
14322
14323        let mut state = RunState::new(
14324            repo.clone(),
14325            "main".to_owned(),
14326            "deadbeef".to_owned(),
14327            "task".to_owned(),
14328            config,
14329        );
14330        let root = state.worktree_root();
14331        let wt_a = root.join("cand-A");
14332        let wt_b = root.join("cand-B");
14333        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14334            .await
14335            .expect("worktree A");
14336        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14337            .await
14338            .expect("worktree B");
14339
14340        state.candidates = vec![
14341            Candidate {
14342                index: 0,
14343                label: 'A',
14344                agent: "alpha".to_owned(),
14345                branch: "magi/x/A".to_owned(),
14346                worktree: wt_a.clone(),
14347                summary: String::new(),
14348                stat: String::new(),
14349                files: 0,
14350                commits: 0,
14351                empty: false,
14352                failed: None,
14353                verified_noop: None,
14354                duration_ms: 0,
14355                folded: false,
14356            },
14357            Candidate {
14358                index: 1,
14359                label: 'B',
14360                agent: "beta".to_owned(),
14361                branch: "magi/x/B".to_owned(),
14362                worktree: wt_b.clone(),
14363                summary: String::new(),
14364                stat: String::new(),
14365                files: 0,
14366                commits: 0,
14367                empty: false,
14368                failed: None,
14369                verified_noop: None,
14370                duration_ms: 0,
14371                folded: false,
14372            },
14373        ];
14374        state.tally = Some(Tally {
14375            first_choice: BTreeMap::from([('A', 1)]),
14376            borda: BTreeMap::new(),
14377            winner: 'A',
14378            rankings: 1,
14379            unanimous_initial: true,
14380            deliberated: false,
14381            changed_votes: 0,
14382            unanimous_final: true,
14383            tie_break: None,
14384            judges: 1,
14385            present: 1,
14386            quorum: 1,
14387            met_quorum: true,
14388            uncontested: None,
14389        });
14390        state.status = RunStatus::Ready;
14391
14392        fold_run(&mut state, false, &crate::run::home())
14393            .await
14394            .expect("fold_run");
14395
14396        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14397        assert!(
14398            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14399            "the unmerged winner's branch survives"
14400        );
14401        assert!(
14402            !state.candidates[0].folded,
14403            "the winner is not marked folded"
14404        );
14405
14406        assert!(!wt_b.exists(), "the loser's worktree is removed");
14407        assert!(
14408            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14409            "the loser's branch is removed"
14410        );
14411        assert!(state.candidates[1].folded, "the loser is marked folded");
14412    }
14413
14414    #[tokio::test]
14415    #[cfg(unix)]
14416    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14417        use std::os::fd::AsRawFd as _;
14418        use std::time::Duration;
14419
14420        let tmp = tempfile::tempdir().unwrap();
14421        let repo = tmp.path().join("repo");
14422        std::fs::create_dir_all(&repo).unwrap();
14423        init_repo(&repo);
14424        let common = repo.join(".git");
14425        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14426        let home = tmp.path().join("home");
14427        let mut config = Config::default();
14428        config.coordination = crate::config::test_coordination();
14429        config.graph.worktree_root = Some(tmp.path().join("wt"));
14430        config.graph.timeout_resource_wait = 1;
14431        let mut state = RunState::new(
14432            repo.clone(),
14433            "main".into(),
14434            "deadbeef".into(),
14435            "task".into(),
14436            config,
14437        );
14438        let root = state.worktree_root();
14439        let candidate = root.join("cand-A");
14440        let reviewer = root.join("review-1");
14441        let branch = state.branch_for('A');
14442        git::worktree_add_branch(&repo, &candidate, &branch, "main")
14443            .await
14444            .unwrap();
14445        git::worktree_add_detached(&repo, &reviewer, "main")
14446            .await
14447            .unwrap();
14448        git::acquire_worktree_config(&repo).await.unwrap();
14449        state.enabled_worktree_config = true;
14450        state.status = RunStatus::Ready;
14451        state.candidates.push(Candidate {
14452            index: 0,
14453            label: 'A',
14454            agent: "implement".into(),
14455            branch: branch.clone(),
14456            worktree: candidate.clone(),
14457            summary: String::new(),
14458            stat: String::new(),
14459            files: 0,
14460            commits: 0,
14461            empty: false,
14462            failed: None,
14463            verified_noop: None,
14464            duration_ms: 0,
14465            folded: false,
14466        });
14467        state.save_under(&home).unwrap();
14468        let config_before = std::fs::read(common.join("config")).unwrap();
14469        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14470            .await
14471            .unwrap();
14472        // A separate OS handle bypasses the launcher's local gate, exactly as
14473        // an external updater does, and must still prevent all folding work.
14474        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14475        assert_eq!(
14476            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14477            0
14478        );
14479        let mut fold = Box::pin(fold_run(&mut state, true, &home));
14480        assert!(
14481            tokio::time::timeout(Duration::from_millis(200), &mut fold)
14482                .await
14483                .is_err()
14484        );
14485        assert!(candidate.exists());
14486        assert!(reviewer.exists());
14487        assert!(git::branch_exists(&repo, &branch).await.unwrap());
14488        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14489        assert_eq!(
14490            git::git(&repo, &["worktree", "list", "--porcelain"])
14491                .await
14492                .unwrap(),
14493            worktrees_before
14494        );
14495        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14496            .await
14497            .unwrap()
14498            .unwrap_err();
14499        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14500        drop(fold);
14501        assert!(candidate.exists() && reviewer.exists());
14502        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14503        assert_eq!(
14504            state.resource_block.as_ref().unwrap().kind,
14505            "repository coordination"
14506        );
14507        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14508        drop(external);
14509        let mut fold = Box::pin(fold_run(&mut state, true, &home));
14510        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14511            .await
14512            .unwrap()
14513            .unwrap();
14514        drop(fold);
14515        assert!(removed.contains(&branch));
14516        assert!(!root.exists());
14517        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14518        assert!(state.candidates[0].folded);
14519        assert!(!state.enabled_worktree_config);
14520        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14521        let enabled = git::git_raw(
14522            &repo,
14523            &["config", "--local", "--get", "extensions.worktreeConfig"],
14524        )
14525        .await
14526        .unwrap();
14527        assert!(!enabled.ok(), "the last config reference was released");
14528    }
14529
14530    /// A branch handed to a later run is that run's (and its pull request's):
14531    /// folding the run that released it must not delete it.
14532    #[tokio::test]
14533    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14534        let tmp = tempfile::tempdir().expect("tempdir");
14535        let repo = tmp.path().join("repo");
14536        std::fs::create_dir_all(&repo).unwrap();
14537        init_repo(&repo);
14538        let home = tmp.path().join("home");
14539
14540        let mut config = Config::default();
14541        config.graph.worktree_root = Some(tmp.path().join("wt"));
14542        let mut state = RunState::new(
14543            repo.clone(),
14544            "main".to_owned(),
14545            "deadbeef".to_owned(),
14546            "task".to_owned(),
14547            config,
14548        );
14549        // The worktree is already gone (released); the branch survives.
14550        git::git(&repo, &["branch", "magi/x/A", "main"])
14551            .await
14552            .expect("branch");
14553        state.candidates = vec![Candidate {
14554            index: 0,
14555            label: 'A',
14556            agent: "alpha".to_owned(),
14557            branch: "magi/x/A".to_owned(),
14558            worktree: state.worktree_root().join("cand-A"),
14559            summary: String::new(),
14560            stat: String::new(),
14561            files: 0,
14562            commits: 0,
14563            empty: false,
14564            failed: None,
14565            verified_noop: None,
14566            duration_ms: 0,
14567            folded: true,
14568        }];
14569        state.released_to = Some("20260901-000000-new1".to_owned());
14570        state.released_branches = vec!["magi/x/A".to_owned()];
14571
14572        fold_run(&mut state, true, &home).await.expect("fold_run");
14573
14574        assert!(
14575            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14576            "the handed-over branch survives a fold"
14577        );
14578    }
14579
14580    /// A winner with nothing ahead of the base is caught before `gh` is ever
14581    /// asked for a pull request, and the message carries what the task's
14582    /// references resolved to.
14583    #[tokio::test]
14584    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14585        let tmp = tempfile::tempdir().expect("tempdir");
14586        let repo = tmp.path().join("repo");
14587        std::fs::create_dir_all(&repo).unwrap();
14588        init_repo(&repo);
14589        let run = |args: &[&str]| {
14590            let out = std::process::Command::new("git")
14591                .quiet()
14592                .args(args)
14593                .current_dir(&repo)
14594                .output()
14595                .expect("spawn git");
14596            assert!(out.status.success(), "git {args:?}");
14597        };
14598        run(&["branch", "magi/x/A"]);
14599        run(&["checkout", "-q", "-b", "magi/x/B"]);
14600        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14601        run(&["add", "-A"]);
14602        run(&["commit", "-q", "-m", "work"]);
14603        run(&["checkout", "-q", "main"]);
14604
14605        // A pull request is compared against the remote's base, so the
14606        // fixture needs one. Before it exists nothing can be read, and the
14607        // answer must be "not empty".
14608        let probe = RunState::new(
14609            repo.clone(),
14610            "main".to_owned(),
14611            "deadbeef".to_owned(),
14612            "task".to_owned(),
14613            Config::default(),
14614        );
14615        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14616        let bare = tmp.path().join("origin.git");
14617        let out = std::process::Command::new("git")
14618            .quiet()
14619            .args(["init", "-q", "--bare"])
14620            .arg(&bare)
14621            .output()
14622            .expect("spawn git");
14623        assert!(out.status.success(), "git init --bare");
14624        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14625        run(&["push", "-q", "origin", "main"]);
14626
14627        let mut state = RunState::new(
14628            repo.clone(),
14629            "main".to_owned(),
14630            "deadbeef".to_owned(),
14631            "task".to_owned(),
14632            Config::default(),
14633        );
14634        state.seeds = vec![refs::Seed {
14635            token: "magi/27b2/A".to_owned(),
14636            kind: refs::SeedKind::Unresolved,
14637            sha: String::new(),
14638            branch: true,
14639            detail: "no branch or commit named magi/27b2/A".to_owned(),
14640        }];
14641
14642        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14643        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14644        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14645        let detail = empty_candidate_detail(&state, "main");
14646        assert!(detail.starts_with("empty candidate"), "{detail}");
14647        assert!(detail.contains("magi/27b2/A"), "{detail}");
14648    }
14649
14650    /// `status == Ready` used to be read as "this is the harmless
14651    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
14652    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
14653    /// run's PR was closed without merging — and reentering `merge` with
14654    /// `mode` still `Pr` does not know the difference, so it pushed and
14655    /// opened a second pull request. `mode == Local` reproduces the same
14656    /// blind spot without a network call: reentry must not attempt another
14657    /// git merge once this node has already recorded an outcome.
14658    #[tokio::test]
14659    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14660        let tmp = tempfile::tempdir().expect("tempdir");
14661        let repo = tmp.path().join("repo");
14662        std::fs::create_dir_all(&repo).unwrap();
14663        init_repo(&repo);
14664
14665        let mut config = Config::default();
14666        config.merge.mode = MergeMode::Local;
14667
14668        let mut state = RunState::new(
14669            repo.clone(),
14670            "main".to_owned(),
14671            "deadbeef".to_owned(),
14672            "task".to_owned(),
14673            config,
14674        );
14675        state.candidates = vec![Candidate {
14676            index: 0,
14677            label: 'A',
14678            agent: "alpha".to_owned(),
14679            branch: "does-not-exist".to_owned(),
14680            worktree: repo.clone(),
14681            summary: String::new(),
14682            stat: String::new(),
14683            files: 0,
14684            commits: 0,
14685            empty: false,
14686            failed: None,
14687            verified_noop: None,
14688            duration_ms: 0,
14689            folded: false,
14690        }];
14691        state.tally = Some(Tally {
14692            first_choice: BTreeMap::from([('A', 1)]),
14693            borda: BTreeMap::new(),
14694            winner: 'A',
14695            rankings: 1,
14696            unanimous_initial: true,
14697            deliberated: false,
14698            changed_votes: 0,
14699            unanimous_final: true,
14700            tie_break: None,
14701            judges: 0,
14702            present: 0,
14703            quorum: 0,
14704            met_quorum: true,
14705            uncontested: Some("only candidate A produced a change".to_owned()),
14706        });
14707        state.reviews = vec![ReviewRound {
14708            round: 1,
14709            head: "deadbeef".to_owned(),
14710            verified_head: None,
14711            verified_at: None,
14712            reviews: Vec::new(),
14713            e2e: Vec::new(),
14714            fix: None,
14715            blocking: 0,
14716            answered: 0,
14717            expected: 0,
14718            clean: true,
14719            verify_retried: false,
14720            e2e_deferred: false,
14721            e2e_defer_reason: None,
14722            progressed: false,
14723            vote_split: false,
14724            reconsideration: Vec::new(),
14725            verdict: None,
14726        }];
14727        state.gate = vec![CommandOutcome {
14728            command: "test".to_owned(),
14729            code: Some(0),
14730            output_tail: String::new(),
14731            duration_ms: 0,
14732            resource_blocked: false,
14733        }];
14734        state.gate_ran = true;
14735        // Reached its conclusion already — e.g. `land` closing the PR without
14736        // merging it, which (like the honest `MergeMode::None` path) leaves
14737        // `status` at `Ready`. The recorded outcome is what actually marks
14738        // this node done.
14739        state.status = RunStatus::Ready;
14740        state.merge = Some(MergeOutcome {
14741            mode: MergeMode::Local,
14742            ok: false,
14743            detail: "already concluded".to_owned(),
14744            empty: false,
14745        });
14746
14747        let mut runner = Runner {
14748            state,
14749            roles: ResolvedRoles {
14750                implementers: Vec::new(),
14751                judges: Vec::new(),
14752                reviewers: Vec::new(),
14753                fixer: None,
14754                conductor: conductor(),
14755                implementer_roster: Vec::new(),
14756                judge_roster: Vec::new(),
14757                reviewer_roster: Vec::new(),
14758            },
14759            sem: Arc::new(Semaphore::new(1)),
14760            pause: Pause::new(),
14761            interrupt: Pause::new(),
14762        };
14763
14764        runner.merge().await.expect("merge");
14765
14766        assert_eq!(
14767            runner.state.status,
14768            RunStatus::Ready,
14769            "a concluded run's status must not change on reentry"
14770        );
14771        assert_eq!(
14772            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14773            Some("already concluded"),
14774            "merge must not run again once the node already recorded an outcome"
14775        );
14776    }
14777
14778    /// `gate` leaves `state.gate_ran` false both before it has ever run and
14779    /// when its last attempt was resource-blocked (the shared build cache
14780    /// could not be acquired or confirmed fresh in time - see
14781    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
14782    /// `Vec` this also leaves behind used to read as "nothing failed" and let
14783    /// a run merge a tree the gate never actually checked - exactly the case
14784    /// a contended cache produces on every retry until it clears. `merge`
14785    /// must refuse until `gate` has actually recorded an attempt.
14786    #[tokio::test]
14787    async fn merge_refuses_a_gate_that_has_not_actually_run() {
14788        let tmp = tempfile::tempdir().expect("tempdir");
14789        let repo = tmp.path().join("repo");
14790        std::fs::create_dir_all(&repo).unwrap();
14791        init_repo(&repo);
14792
14793        let mut config = Config::default();
14794        config.merge.mode = MergeMode::Local;
14795
14796        let mut state = RunState::new(
14797            repo.clone(),
14798            "main".to_owned(),
14799            "deadbeef".to_owned(),
14800            "task".to_owned(),
14801            config,
14802        );
14803        state.candidates = vec![Candidate {
14804            index: 0,
14805            label: 'A',
14806            agent: "alpha".to_owned(),
14807            branch: "does-not-exist".to_owned(),
14808            worktree: repo.clone(),
14809            summary: String::new(),
14810            stat: String::new(),
14811            files: 0,
14812            commits: 0,
14813            empty: false,
14814            failed: None,
14815            verified_noop: None,
14816            duration_ms: 0,
14817            folded: false,
14818        }];
14819        state.tally = Some(Tally {
14820            first_choice: BTreeMap::from([('A', 1)]),
14821            borda: BTreeMap::new(),
14822            winner: 'A',
14823            rankings: 1,
14824            unanimous_initial: true,
14825            deliberated: false,
14826            changed_votes: 0,
14827            unanimous_final: true,
14828            tie_break: None,
14829            judges: 0,
14830            present: 0,
14831            quorum: 0,
14832            met_quorum: true,
14833            uncontested: Some("only candidate A produced a change".to_owned()),
14834        });
14835        state.reviews = vec![ReviewRound {
14836            round: 1,
14837            head: "deadbeef".to_owned(),
14838            verified_head: None,
14839            verified_at: None,
14840            reviews: Vec::new(),
14841            e2e: Vec::new(),
14842            fix: None,
14843            blocking: 0,
14844            answered: 0,
14845            expected: 0,
14846            clean: true,
14847            verify_retried: false,
14848            e2e_deferred: false,
14849            e2e_defer_reason: None,
14850            progressed: false,
14851            vote_split: false,
14852            reconsideration: Vec::new(),
14853            verdict: None,
14854        }];
14855        // The point: `gate` has not recorded anything yet.
14856        state.gate = Vec::new();
14857        state.gate_ran = false;
14858        state.status = RunStatus::Gating;
14859
14860        let mut runner = Runner {
14861            state,
14862            roles: ResolvedRoles {
14863                implementers: Vec::new(),
14864                judges: Vec::new(),
14865                reviewers: Vec::new(),
14866                fixer: None,
14867                conductor: conductor(),
14868                implementer_roster: Vec::new(),
14869                judge_roster: Vec::new(),
14870                reviewer_roster: Vec::new(),
14871            },
14872            sem: Arc::new(Semaphore::new(1)),
14873            pause: Pause::new(),
14874            interrupt: Pause::new(),
14875        };
14876
14877        runner.merge().await.expect("merge");
14878
14879        assert!(
14880            runner.state.merge.is_none(),
14881            "an empty gate must never be read as a passing one: {:?}",
14882            runner.state.merge
14883        );
14884    }
14885
14886    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
14887    /// commands configured and `merge.mode` is `none` (a review-only run).
14888    /// `gate` must still record a real attempt — zero commands, vacuously
14889    /// passed — rather than leaving `state.gate` empty in a way `merge`
14890    /// cannot tell apart from "never ran"; otherwise the run reaches
14891    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
14892    #[tokio::test]
14893    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14894        let tmp = tempfile::tempdir().expect("tempdir");
14895        let repo = tmp.path().join("repo");
14896        std::fs::create_dir_all(&repo).unwrap();
14897        init_repo(&repo);
14898
14899        // Default config: `verify.gate` empty, `merge.mode` is `none`.
14900        let config = Config::default();
14901
14902        let mut state = RunState::new(
14903            repo.clone(),
14904            "main".to_owned(),
14905            "deadbeef".to_owned(),
14906            "task".to_owned(),
14907            config,
14908        );
14909        state.candidates = vec![Candidate {
14910            index: 0,
14911            label: 'A',
14912            agent: "alpha".to_owned(),
14913            branch: "does-not-exist".to_owned(),
14914            worktree: repo.clone(),
14915            summary: String::new(),
14916            stat: String::new(),
14917            files: 0,
14918            commits: 0,
14919            empty: false,
14920            failed: None,
14921            verified_noop: None,
14922            duration_ms: 0,
14923            folded: false,
14924        }];
14925        state.tally = Some(Tally {
14926            first_choice: BTreeMap::from([('A', 1)]),
14927            borda: BTreeMap::new(),
14928            winner: 'A',
14929            rankings: 1,
14930            unanimous_initial: true,
14931            deliberated: false,
14932            changed_votes: 0,
14933            unanimous_final: true,
14934            tie_break: None,
14935            judges: 0,
14936            present: 0,
14937            quorum: 0,
14938            met_quorum: true,
14939            uncontested: Some("only candidate A produced a change".to_owned()),
14940        });
14941        state.reviews = vec![ReviewRound {
14942            round: 1,
14943            head: "deadbeef".to_owned(),
14944            verified_head: None,
14945            verified_at: None,
14946            reviews: Vec::new(),
14947            e2e: Vec::new(),
14948            fix: None,
14949            blocking: 0,
14950            answered: 0,
14951            expected: 0,
14952            clean: true,
14953            verify_retried: false,
14954            e2e_deferred: false,
14955            e2e_defer_reason: None,
14956            progressed: false,
14957            vote_split: false,
14958            reconsideration: Vec::new(),
14959            verdict: None,
14960        }];
14961
14962        let mut runner = Runner {
14963            state,
14964            roles: ResolvedRoles {
14965                implementers: Vec::new(),
14966                judges: Vec::new(),
14967                reviewers: Vec::new(),
14968                fixer: None,
14969                conductor: conductor(),
14970                implementer_roster: Vec::new(),
14971                judge_roster: Vec::new(),
14972                reviewer_roster: Vec::new(),
14973            },
14974            sem: Arc::new(Semaphore::new(1)),
14975            pause: Pause::new(),
14976            interrupt: Pause::new(),
14977        };
14978
14979        runner.gate().await.expect("gate");
14980        assert!(
14981            runner.state.gate_ran,
14982            "zero configured commands is still a real attempt, not an unrun gate"
14983        );
14984        assert!(runner.state.gate.is_empty());
14985        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
14986        assert_ne!(
14987            runner.state.status,
14988            RunStatus::Blocked,
14989            "a gate with nothing to check must not read as failed"
14990        );
14991
14992        runner.merge().await.expect("merge");
14993        assert_eq!(
14994            runner.state.status,
14995            RunStatus::Ready,
14996            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
14997        );
14998    }
14999
15000    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15001    /// `verify.gate` (so the e2e leg and the final gate never build against
15002    /// different directories). With zero `verify.gate` commands but a
15003    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15004    /// that lease before discovering it had nothing to run - so a repo with
15005    /// no gate commands could come back `resource_blocked` (and therefore
15006    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15007    /// cache, exactly the contention this run's own zero commands could
15008    /// never have touched. `gate` must recognise there is nothing to check
15009    /// before it ever asks for the lease.
15010    #[tokio::test]
15011    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15012        crate::run::pin_test_home();
15013        let home = crate::run::home();
15014
15015        let tmp = tempfile::tempdir().expect("tempdir");
15016        let repo = tmp.path().join("repo");
15017        std::fs::create_dir_all(&repo).unwrap();
15018        init_repo(&repo);
15019        // Unique to this test, so holding its lease cannot collide with
15020        // another test sharing the same process-wide `home`.
15021        let cache_dir = tmp.path().join("target");
15022
15023        let mut config = Config::default();
15024        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15025        // `verify.gate` stays empty (the default). Bounded so a regression
15026        // that does start waiting fails the test in seconds, not hangs it.
15027        config.graph.timeout_verify = Some(2);
15028        config.graph.timeout_resource_wait = 2;
15029
15030        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15031        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15032            .expect("no io error acquiring directly")
15033        {
15034            crate::cache::AcquireOutcome::Acquired(g) => g,
15035            crate::cache::AcquireOutcome::Busy(b) => {
15036                panic!("expected the direct acquire to win the lease first: {b:?}")
15037            }
15038        };
15039
15040        let mut state = RunState::new(
15041            repo.clone(),
15042            "main".to_owned(),
15043            "deadbeef".to_owned(),
15044            "task".to_owned(),
15045            config,
15046        );
15047        state.candidates = vec![Candidate {
15048            index: 0,
15049            label: 'A',
15050            agent: "alpha".to_owned(),
15051            branch: "does-not-exist".to_owned(),
15052            worktree: repo.clone(),
15053            summary: String::new(),
15054            stat: String::new(),
15055            files: 0,
15056            commits: 0,
15057            empty: false,
15058            failed: None,
15059            verified_noop: None,
15060            duration_ms: 0,
15061            folded: false,
15062        }];
15063        state.tally = Some(Tally {
15064            first_choice: BTreeMap::from([('A', 1)]),
15065            borda: BTreeMap::new(),
15066            winner: 'A',
15067            rankings: 1,
15068            unanimous_initial: true,
15069            deliberated: false,
15070            changed_votes: 0,
15071            unanimous_final: true,
15072            tie_break: None,
15073            judges: 0,
15074            present: 0,
15075            quorum: 0,
15076            met_quorum: true,
15077            uncontested: Some("only candidate A produced a change".to_owned()),
15078        });
15079        state.reviews = vec![ReviewRound {
15080            round: 1,
15081            head: "deadbeef".to_owned(),
15082            verified_head: None,
15083            verified_at: None,
15084            reviews: Vec::new(),
15085            e2e: Vec::new(),
15086            fix: None,
15087            blocking: 0,
15088            answered: 0,
15089            expected: 0,
15090            clean: true,
15091            verify_retried: false,
15092            e2e_deferred: false,
15093            e2e_defer_reason: None,
15094            progressed: false,
15095            vote_split: false,
15096            reconsideration: Vec::new(),
15097            verdict: None,
15098        }];
15099
15100        let mut runner = Runner {
15101            state,
15102            roles: ResolvedRoles {
15103                implementers: Vec::new(),
15104                judges: Vec::new(),
15105                reviewers: Vec::new(),
15106                fixer: None,
15107                conductor: conductor(),
15108                implementer_roster: Vec::new(),
15109                judge_roster: Vec::new(),
15110                reviewer_roster: Vec::new(),
15111            },
15112            sem: Arc::new(Semaphore::new(1)),
15113            pause: Pause::new(),
15114            interrupt: Pause::new(),
15115        };
15116
15117        let started = std::time::Instant::now();
15118        runner.gate().await.expect("gate");
15119        assert!(
15120            started.elapsed() < Duration::from_secs(1),
15121            "a gate with nothing to run must never wait on a lease it never needed"
15122        );
15123        assert!(
15124            runner.state.gate_ran,
15125            "zero commands is still a real, immediate attempt"
15126        );
15127        assert!(runner.state.gate.is_empty());
15128        assert_ne!(
15129            runner.state.status,
15130            RunStatus::Blocked,
15131            "must not read as resource-blocked on a lease it never asked for"
15132        );
15133    }
15134
15135    /// The addendum's second gap: a `verify.gate` command running for real
15136    /// wall-clock time had nothing at all to show for it in `active` before
15137    /// `run_commands` learned to record it — a run could sit in `Gating` for
15138    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15139    /// what was actually happening. Proven with a genuinely still-running
15140    /// command, not just a before/after check on the final state: a poller
15141    /// task reads the same `run.json` `gate()` is writing, the same way the
15142    /// phone or `magi show` would, while the shell command is still blocked
15143    /// on its own release marker.
15144    #[tokio::test]
15145    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15146        crate::run::pin_test_home();
15147
15148        let tmp = tempfile::tempdir().expect("tempdir");
15149        let repo = tmp.path().join("repo");
15150        std::fs::create_dir_all(&repo).unwrap();
15151        init_repo(&repo);
15152
15153        let mut config = Config::default();
15154        config.verify.gate = vec![
15155            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15156             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15157                .to_owned(),
15158        ];
15159
15160        let mut state = RunState::new(
15161            repo.clone(),
15162            "main".to_owned(),
15163            "deadbeef".to_owned(),
15164            "task".to_owned(),
15165            config,
15166        );
15167        let run_id = state.id.clone();
15168        state.candidates = vec![Candidate {
15169            index: 0,
15170            label: 'A',
15171            agent: "alpha".to_owned(),
15172            branch: "does-not-exist".to_owned(),
15173            worktree: repo.clone(),
15174            summary: String::new(),
15175            stat: String::new(),
15176            files: 0,
15177            commits: 0,
15178            empty: false,
15179            failed: None,
15180            verified_noop: None,
15181            duration_ms: 0,
15182            folded: false,
15183        }];
15184        state.tally = Some(Tally {
15185            first_choice: BTreeMap::from([('A', 1)]),
15186            borda: BTreeMap::new(),
15187            winner: 'A',
15188            rankings: 1,
15189            unanimous_initial: true,
15190            deliberated: false,
15191            changed_votes: 0,
15192            unanimous_final: true,
15193            tie_break: None,
15194            judges: 0,
15195            present: 0,
15196            quorum: 0,
15197            met_quorum: true,
15198            uncontested: Some("only candidate A produced a change".to_owned()),
15199        });
15200        state.reviews = vec![ReviewRound {
15201            round: 1,
15202            head: "deadbeef".to_owned(),
15203            verified_head: None,
15204            verified_at: None,
15205            reviews: Vec::new(),
15206            e2e: Vec::new(),
15207            fix: None,
15208            blocking: 0,
15209            answered: 0,
15210            expected: 0,
15211            clean: true,
15212            verify_retried: false,
15213            e2e_deferred: false,
15214            e2e_defer_reason: None,
15215            progressed: false,
15216            vote_split: false,
15217            reconsideration: Vec::new(),
15218            verdict: None,
15219        }];
15220
15221        let mut runner = Runner {
15222            state,
15223            roles: ResolvedRoles {
15224                implementers: Vec::new(),
15225                judges: Vec::new(),
15226                reviewers: Vec::new(),
15227                fixer: None,
15228                conductor: conductor(),
15229                implementer_roster: Vec::new(),
15230                judge_roster: Vec::new(),
15231                reviewer_roster: Vec::new(),
15232            },
15233            sem: Arc::new(Semaphore::new(1)),
15234            pause: Pause::new(),
15235            interrupt: Pause::new(),
15236        };
15237
15238        let started_marker = repo.join("started.marker");
15239        let release_marker = repo.join("release.marker");
15240        let poller = tokio::spawn(async move {
15241            // Bounded so a regression that never records the task entry
15242            // fails this test in seconds instead of hanging the suite —
15243            // the same shape `a_park_requested_while_a_seat_is_mid_call_
15244            // does_not_cut_it_short` uses for the same reason.
15245            for _ in 0..100 {
15246                if started_marker.exists()
15247                    && let Ok(s) = crate::run::RunState::load(&run_id)
15248                    && let Some(a) = s.active.get("gate")
15249                {
15250                    std::fs::write(&release_marker, b"go").expect("release marker");
15251                    return Some(a.clone());
15252                }
15253                tokio::time::sleep(Duration::from_millis(50)).await;
15254            }
15255            None
15256        });
15257
15258        runner.gate().await.expect("gate");
15259        let captured = poller.await.expect("poller task");
15260        let captured = captured.expect(
15261            "the poller never saw a `gate` task entry in run.json while the command was \
15262             still blocked on its own release marker",
15263        );
15264
15265        assert_eq!(captured.task.as_deref(), Some("gate"));
15266        assert_eq!(captured.node, "gate");
15267        assert_eq!(captured.index, Some(1));
15268        assert_eq!(captured.total, Some(1));
15269        assert!(
15270            captured
15271                .command
15272                .as_deref()
15273                .is_some_and(|c| c.contains("started.marker")),
15274            "{captured:?}"
15275        );
15276
15277        assert!(
15278            runner.state.active.is_empty(),
15279            "the entry must be cleared once the command actually finished: {:?}",
15280            runner.state.active
15281        );
15282        assert!(runner.state.gate_ran);
15283        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15284    }
15285
15286    /// The hand-off over a blocking finding a reviewer rejected on leaves a
15287    /// record for `land`; one with only a Minor, or no reject, leaves none.
15288    #[tokio::test]
15289    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15290        use crate::verdict::{Finding, ReviewVote, Severity};
15291        crate::run::pin_test_home();
15292        let tmp = tempfile::tempdir().expect("tempdir");
15293        let repo = tmp.path().join("repo");
15294        std::fs::create_dir_all(&repo).unwrap();
15295        init_repo(&repo);
15296
15297        for (severity, vote, expect) in [
15298            (Severity::Major, ReviewVote::Reject, true),
15299            (Severity::Minor, ReviewVote::Reject, false),
15300            (Severity::Major, ReviewVote::Approve, false),
15301        ] {
15302            let mut round = review_round(false, 1, 1, 1, false, true);
15303            round.reviews = vec![ReviewRecord {
15304                reviewer: 1,
15305                agent: "alpha".to_owned(),
15306                summary: String::new(),
15307                findings: vec![Finding {
15308                    id: "R1-1-1".to_owned(),
15309                    severity,
15310                    file: None,
15311                    line: None,
15312                    title: "t".to_owned(),
15313                    detail: String::new(),
15314                }],
15315                vote: Some(vote),
15316                failed: None,
15317                duration_ms: 0,
15318                attempts: 0,
15319            }];
15320            let mut state = RunState::new(
15321                repo.clone(),
15322                "main".to_owned(),
15323                "deadbeef".to_owned(),
15324                "task".to_owned(),
15325                Config::default(),
15326            );
15327            state.reviews = vec![round];
15328            let mut runner = Runner {
15329                state,
15330                roles: ResolvedRoles {
15331                    implementers: Vec::new(),
15332                    judges: Vec::new(),
15333                    reviewers: Vec::new(),
15334                    fixer: None,
15335                    conductor: conductor(),
15336                    implementer_roster: Vec::new(),
15337                    judge_roster: Vec::new(),
15338                    reviewer_roster: Vec::new(),
15339                },
15340                sem: Arc::new(Semaphore::new(1)),
15341                pause: Pause::new(),
15342                interrupt: Pause::new(),
15343            };
15344            let shell = runner.state.config.shell();
15345            runner
15346                .stop_reviewing("round budget spent", &shell, &repo)
15347                .await
15348                .expect("stop_reviewing");
15349            assert_eq!(runner.state.status, RunStatus::Gating);
15350            assert_eq!(
15351                runner.state.contested_handoff.is_some(),
15352                expect,
15353                "{severity:?} + {vote:?}"
15354            );
15355        }
15356    }
15357
15358    /// The shape the incident this whole fix responds to actually had: the
15359    /// round budget spent, the last round's own e2e blocked on the shared
15360    /// build cache (held here by a live pid — this test process — exactly
15361    /// `cache`'s own unit tests' pattern for "another owner, still alive"
15362    /// without forking a process). `stop_reviewing` must retry it — not
15363    /// silently leave the round looking untouched (the catch-up-only half of
15364    /// the bug), and not read the contention as a red `e2e` and block the
15365    /// run on it (the other half). Called directly, the same way
15366    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
15367    /// above exercises `gate`, so this never needs a real cargo build to
15368    /// reach: the lease is never released, so `with_cache_lease` never gets
15369    /// past acquiring it into anything that would need a real workspace.
15370    #[tokio::test]
15371    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15372        crate::run::pin_test_home();
15373        let home = crate::run::home();
15374
15375        let tmp = tempfile::tempdir().expect("tempdir");
15376        let repo = tmp.path().join("repo");
15377        std::fs::create_dir_all(&repo).unwrap();
15378        init_repo(&repo);
15379        let head = crate::git::rev_parse(&repo, "HEAD")
15380            .await
15381            .expect("rev-parse");
15382        // Unique to this test, so holding its lease cannot collide with
15383        // another test sharing the same process-wide `home`.
15384        let cache_dir = tmp.path().join("target");
15385
15386        let mut config = Config::default();
15387        config.verify.e2e = vec![format!(
15388            "CARGO_TARGET_DIR='{}' test -f README.md",
15389            cache_dir.display()
15390        )];
15391        config.graph.review_rounds = 1;
15392        // Bounded so a regression that does start waiting fails the test in
15393        // seconds, not hangs it.
15394        config.graph.timeout_verify = Some(2);
15395        config.graph.timeout_resource_wait = 2;
15396
15397        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15398        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15399            .expect("no io error acquiring directly")
15400        {
15401            crate::cache::AcquireOutcome::Acquired(g) => g,
15402            crate::cache::AcquireOutcome::Busy(b) => {
15403                panic!("expected the direct acquire to win the lease first: {b:?}")
15404            }
15405        };
15406
15407        let mut state = RunState::new(
15408            repo.clone(),
15409            "main".to_owned(),
15410            head.clone(),
15411            "task".to_owned(),
15412            config,
15413        );
15414        state.candidates = vec![Candidate {
15415            index: 0,
15416            label: 'A',
15417            agent: "alpha".to_owned(),
15418            branch: "does-not-exist".to_owned(),
15419            worktree: repo.clone(),
15420            summary: String::new(),
15421            stat: String::new(),
15422            files: 0,
15423            commits: 0,
15424            empty: false,
15425            failed: None,
15426            verified_noop: None,
15427            duration_ms: 0,
15428            folded: false,
15429        }];
15430        state.tally = Some(Tally {
15431            first_choice: BTreeMap::from([('A', 1)]),
15432            borda: BTreeMap::new(),
15433            winner: 'A',
15434            rankings: 1,
15435            unanimous_initial: true,
15436            deliberated: false,
15437            changed_votes: 0,
15438            unanimous_final: true,
15439            tie_break: None,
15440            judges: 0,
15441            present: 0,
15442            quorum: 0,
15443            met_quorum: true,
15444            uncontested: Some("only candidate A produced a change".to_owned()),
15445        });
15446        // The round budget's last round, deferred: `needs_catchup_run`'s
15447        // other trigger. `stop_reviewing`'s retry machinery must treat this
15448        // exactly like a resource-blocked attempt once it actually runs.
15449        state.reviews = vec![ReviewRound {
15450            round: 1,
15451            head: head.clone(),
15452            verified_head: None,
15453            verified_at: None,
15454            reviews: Vec::new(),
15455            e2e: Vec::new(),
15456            fix: None,
15457            blocking: 1,
15458            answered: 1,
15459            expected: 1,
15460            clean: false,
15461            verify_retried: false,
15462            e2e_deferred: true,
15463            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15464            progressed: false,
15465            vote_split: false,
15466            reconsideration: Vec::new(),
15467            verdict: None,
15468        }];
15469
15470        let mut runner = Runner {
15471            state,
15472            roles: ResolvedRoles {
15473                implementers: Vec::new(),
15474                judges: Vec::new(),
15475                reviewers: Vec::new(),
15476                fixer: None,
15477                conductor: conductor(),
15478                implementer_roster: Vec::new(),
15479                judge_roster: Vec::new(),
15480                reviewer_roster: Vec::new(),
15481            },
15482            sem: Arc::new(Semaphore::new(1)),
15483            pause: Pause::new(),
15484            interrupt: Pause::new(),
15485        };
15486
15487        let shell = runner.state.config.shell();
15488        runner
15489            .stop_reviewing("round budget spent", &shell, &repo)
15490            .await
15491            .expect("stop_reviewing");
15492
15493        let last = runner.state.reviews.last().expect("round record");
15494        assert_eq!(
15495            last.e2e_status(),
15496            E2eStatus::ResourceBlocked,
15497            "the shared cache is still held; the attempt must read as blocked, not deferred or \
15498             failed: {last:?}"
15499        );
15500        assert_eq!(
15501            last.verified_head.as_deref(),
15502            Some(head.as_str()),
15503            "which commit this attempt targeted is known even though nothing finished checking \
15504             it"
15505        );
15506        let first_attempt_at = last
15507            .verified_at
15508            .expect("when this attempt ran is known too");
15509        assert_ne!(
15510            runner.state.status,
15511            RunStatus::Blocked,
15512            "contention is evidence about the machine, not the patch — it must not settle the \
15513             run as blocked: {:?}",
15514            runner.state.status
15515        );
15516        assert!(
15517            !runner
15518                .state
15519                .events
15520                .iter()
15521                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15522            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15523            runner.state.events
15524        );
15525
15526        // The cache is still held: a later reentry must retry the same
15527        // round's verification again — not leave it looking exactly as
15528        // untouched as the first blocked attempt, which is indistinguishable
15529        // from never having tried again at all.
15530        runner
15531            .stop_reviewing("round budget spent", &shell, &repo)
15532            .await
15533            .expect("stop_reviewing retry");
15534        assert_eq!(
15535            runner.state.reviews.len(),
15536            1,
15537            "no new round was started: {:?}",
15538            runner.state.reviews
15539        );
15540        let last = runner.state.reviews.last().expect("round record");
15541        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15542        assert!(
15543            last.verified_at.expect("still known") > first_attempt_at,
15544            "a second reentry must be a fresh attempt, not a stale copy of the first"
15545        );
15546        assert_ne!(runner.state.status, RunStatus::Blocked);
15547
15548        held.release();
15549    }
15550
15551    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
15552    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
15553    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
15554    /// own top-of-function fast path (`review_conclusion`) correctly reads
15555    /// this shape as `None` rather than guessing `Blocked`, and the loop's
15556    /// own `for` range is empty once the round budget is spent, so
15557    /// `review_loop` must retry the check itself rather than silently doing
15558    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
15559    /// above exercises directly, but through `review_loop`'s own entry point
15560    /// this time, proving the wiring between the two rather than just the
15561    /// retry logic in isolation.
15562    #[tokio::test]
15563    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15564        crate::run::pin_test_home();
15565        let home = crate::run::home();
15566
15567        let tmp = tempfile::tempdir().expect("tempdir");
15568        let repo = tmp.path().join("repo");
15569        std::fs::create_dir_all(&repo).unwrap();
15570        init_repo(&repo);
15571        let head = crate::git::rev_parse(&repo, "HEAD")
15572            .await
15573            .expect("rev-parse");
15574        let cache_dir = tmp.path().join("target");
15575
15576        let mut config = Config::default();
15577        config.verify.e2e = vec![format!(
15578            "CARGO_TARGET_DIR='{}' test -f README.md",
15579            cache_dir.display()
15580        )];
15581        config.graph.review_rounds = 1;
15582        config.graph.timeout_verify = Some(2);
15583        config.graph.timeout_resource_wait = 2;
15584
15585        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15586        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15587            .expect("no io error acquiring directly")
15588        {
15589            crate::cache::AcquireOutcome::Acquired(g) => g,
15590            crate::cache::AcquireOutcome::Busy(b) => {
15591                panic!("expected the direct acquire to win the lease first: {b:?}")
15592            }
15593        };
15594
15595        let mut state = RunState::new(
15596            repo.clone(),
15597            "main".to_owned(),
15598            head.clone(),
15599            "task".to_owned(),
15600            config,
15601        );
15602        state.candidates = vec![Candidate {
15603            index: 0,
15604            label: 'A',
15605            agent: "alpha".to_owned(),
15606            branch: "does-not-exist".to_owned(),
15607            worktree: repo.clone(),
15608            summary: String::new(),
15609            stat: String::new(),
15610            files: 0,
15611            commits: 0,
15612            empty: false,
15613            failed: None,
15614            verified_noop: None,
15615            duration_ms: 0,
15616            folded: false,
15617        }];
15618        state.tally = Some(Tally {
15619            first_choice: BTreeMap::from([('A', 1)]),
15620            borda: BTreeMap::new(),
15621            winner: 'A',
15622            rankings: 1,
15623            unanimous_initial: true,
15624            deliberated: false,
15625            changed_votes: 0,
15626            unanimous_final: true,
15627            tie_break: None,
15628            judges: 0,
15629            present: 0,
15630            quorum: 0,
15631            met_quorum: true,
15632            uncontested: Some("only candidate A produced a change".to_owned()),
15633        });
15634        // The exact shape a prior process's `stop_reviewing` would have left
15635        // on disk: the round budget's last round, a real attempt already
15636        // made and already resource-blocked.
15637        state.reviews = vec![ReviewRound {
15638            round: 1,
15639            head: head.clone(),
15640            verified_head: Some(head.clone()),
15641            verified_at: Some(jiff::Timestamp::now()),
15642            reviews: Vec::new(),
15643            e2e: vec![CommandOutcome {
15644                command: format!(
15645                    "CARGO_TARGET_DIR='{}' test -f README.md",
15646                    cache_dir.display()
15647                ),
15648                code: None,
15649                output_tail: "waiting for the shared build cache".to_owned(),
15650                duration_ms: 0,
15651                resource_blocked: true,
15652            }],
15653            fix: None,
15654            blocking: 1,
15655            answered: 1,
15656            expected: 1,
15657            clean: false,
15658            verify_retried: false,
15659            e2e_deferred: false,
15660            e2e_defer_reason: None,
15661            progressed: false,
15662            vote_split: false,
15663            reconsideration: Vec::new(),
15664            verdict: None,
15665        }];
15666
15667        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15668        let mut runner = Runner {
15669            state,
15670            roles: ResolvedRoles {
15671                implementers: Vec::new(),
15672                judges: Vec::new(),
15673                reviewers: Vec::new(),
15674                fixer: None,
15675                conductor: conductor(),
15676                implementer_roster: Vec::new(),
15677                judge_roster: Vec::new(),
15678                reviewer_roster: Vec::new(),
15679            },
15680            sem: Arc::new(Semaphore::new(1)),
15681            pause: Pause::new(),
15682            interrupt: Pause::new(),
15683        };
15684
15685        // The lease is still held throughout, so this reentry's own retry is
15686        // also contended — proving `review_loop` actually tried again (not
15687        // that it happened to succeed) is what the timestamp comparison
15688        // below is for.
15689        runner.review_loop().await.expect("review_loop");
15690
15691        assert_eq!(
15692            runner.state.reviews.len(),
15693            1,
15694            "no new round was started on top of the unresolved one: {:?}",
15695            runner.state.reviews
15696        );
15697        let last = &runner.state.reviews[0];
15698        assert_eq!(
15699            last.e2e_status(),
15700            E2eStatus::ResourceBlocked,
15701            "still contended: {last:?}"
15702        );
15703        assert!(
15704            last.verified_at.expect("still known") > first_attempt_at,
15705            "review_loop must have actually retried the check, not left it exactly as found"
15706        );
15707        assert_ne!(
15708            runner.state.status,
15709            RunStatus::Blocked,
15710            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15711            runner.state.status
15712        );
15713
15714        held.release();
15715    }
15716
15717    #[tokio::test]
15718    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15719        crate::run::pin_test_home();
15720        let tmp = tempfile::tempdir().expect("tempdir");
15721        let repo = tmp.path().join("repo");
15722        std::fs::create_dir_all(&repo).unwrap();
15723        init_repo(&repo);
15724
15725        let mut config = Config::default();
15726        config.merge.mode = MergeMode::Pr;
15727        config.graph.land = true;
15728        config.graph.land_approval = false;
15729
15730        let mut state = RunState::new(
15731            repo.clone(),
15732            "main".to_owned(),
15733            "deadbeef".to_owned(),
15734            "task".to_owned(),
15735            config,
15736        );
15737        state.candidates = vec![Candidate {
15738            index: 0,
15739            label: 'A',
15740            agent: "alpha".to_owned(),
15741            branch: "does-not-exist".to_owned(),
15742            worktree: repo.clone(),
15743            summary: String::new(),
15744            stat: String::new(),
15745            files: 0,
15746            commits: 0,
15747            empty: false,
15748            failed: None,
15749            verified_noop: None,
15750            duration_ms: 0,
15751            folded: false,
15752        }];
15753        state.tally = Some(Tally {
15754            first_choice: BTreeMap::from([('A', 1)]),
15755            borda: BTreeMap::new(),
15756            winner: 'A',
15757            rankings: 1,
15758            unanimous_initial: true,
15759            deliberated: false,
15760            changed_votes: 0,
15761            unanimous_final: true,
15762            tie_break: None,
15763            judges: 0,
15764            present: 0,
15765            quorum: 0,
15766            met_quorum: true,
15767            uncontested: Some("only candidate A produced a change".to_owned()),
15768        });
15769        state.reviews = vec![ReviewRound {
15770            round: 1,
15771            head: "deadbeef".to_owned(),
15772            verified_head: None,
15773            verified_at: None,
15774            reviews: Vec::new(),
15775            e2e: Vec::new(),
15776            fix: None,
15777            blocking: 0,
15778            answered: 0,
15779            expected: 0,
15780            clean: true,
15781            verify_retried: false,
15782            e2e_deferred: false,
15783            e2e_defer_reason: None,
15784            progressed: false,
15785            vote_split: false,
15786            reconsideration: Vec::new(),
15787            verdict: None,
15788        }];
15789        state.gate = vec![CommandOutcome {
15790            command: "test".to_owned(),
15791            code: Some(0),
15792            output_tail: String::new(),
15793            duration_ms: 0,
15794            resource_blocked: false,
15795        }];
15796        state.gate_ran = true;
15797        // A first pass through `merge` already pushed and opened this pull
15798        // request; `status` is `Landing` because a previous call into `land`
15799        // parked or was interrupted before it reached a terminal outcome.
15800        state.status = RunStatus::Landing;
15801        state.merge = Some(MergeOutcome {
15802            mode: MergeMode::Pr,
15803            ok: true,
15804            detail: "https://example.invalid/x/y/pull/1".to_owned(),
15805            empty: false,
15806        });
15807
15808        // The Landing-resume shortcut calls `run_land` directly rather than
15809        // through `merge`, which is exactly the call site that used to skip
15810        // `settle_questions` - see the fixture below.
15811        ask_test_home();
15812        // A real resume loads a persisted run; driver ownership is checked
15813        // against that record before the Landing shortcut is entered.
15814        state.save().expect("save landing run");
15815        let store = ask::Questions::open();
15816        let q = ask_open_question(&store, &state.id);
15817
15818        let mut runner = Runner {
15819            state,
15820            roles: ResolvedRoles {
15821                implementers: Vec::new(),
15822                judges: Vec::new(),
15823                reviewers: Vec::new(),
15824                fixer: None,
15825                conductor: conductor(),
15826                implementer_roster: Vec::new(),
15827                judge_roster: Vec::new(),
15828                reviewer_roster: Vec::new(),
15829            },
15830            sem: Arc::new(Semaphore::new(1)),
15831            pause: Pause::new(),
15832            interrupt: Pause::new(),
15833        };
15834
15835        // `execute`, not `merge` directly: the Landing-resume shortcut lives
15836        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
15837        // exactly because `review_loop` would otherwise clobber the marker
15838        // first.
15839        runner.execute().await.expect("execute");
15840
15841        assert_eq!(
15842            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15843            Some("https://example.invalid/x/y/pull/1"),
15844            "reentry must not push again or open a second pull request over the \
15845             one `land` is already watching"
15846        );
15847        assert_ne!(
15848            runner.state.status,
15849            RunStatus::Landing,
15850            "land could not actually reach the fake pull request, so it must \
15851             have given up rather than left the run silently parked forever"
15852        );
15853        // `land` could not reach the fake pull request, so it gave up into
15854        // `Blocked` - still resumable, so the question must not have been
15855        // swept just because this branch now also calls `settle_questions`.
15856        assert_eq!(runner.state.status, RunStatus::Blocked);
15857        assert!(
15858            store.get(&q.id).unwrap().status.open(),
15859            "Blocked is still alive; settle_questions must have been a no-op here"
15860        );
15861    }
15862
15863    fn state_with_round(round: ReviewRound) -> RunState {
15864        let mut s = RunState::new(
15865            PathBuf::from("/repo"),
15866            "main".to_owned(),
15867            "abc1234".to_owned(),
15868            "add retries".to_owned(),
15869            Config::default(),
15870        );
15871        s.reviews = vec![round];
15872        s
15873    }
15874
15875    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15876        crate::verdict::Finding {
15877            id: id.to_owned(),
15878            severity,
15879            file: None,
15880            line: None,
15881            title: title.to_owned(),
15882            detail: String::new(),
15883        }
15884    }
15885
15886    #[test]
15887    fn pr_body_names_open_findings_and_declined_ones() {
15888        let round = ReviewRound {
15889            round: 2,
15890            head: "deadbee".to_owned(),
15891            verified_head: None,
15892            verified_at: None,
15893            reviews: vec![ReviewRecord {
15894                attempts: 0,
15895                reviewer: 1,
15896                agent: "alpha".to_owned(),
15897                summary: String::new(),
15898                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15899                vote: None,
15900                failed: None,
15901                duration_ms: 0,
15902            }],
15903            e2e: vec![CommandOutcome {
15904                command: "cargo test".to_owned(),
15905                code: Some(0),
15906                output_tail: String::new(),
15907                duration_ms: 0,
15908                resource_blocked: false,
15909            }],
15910            verify_retried: false,
15911            e2e_deferred: false,
15912            e2e_defer_reason: None,
15913            fix: Some(FixRecord {
15914                agent: "alpha".to_owned(),
15915                addressed: Vec::new(),
15916                rejected: vec![crate::verdict::Rejection {
15917                    id: "R1-1-1".to_owned(),
15918                    why: "not reachable from any caller".to_owned(),
15919                }],
15920                notes: String::new(),
15921                committed: true,
15922                failed: None,
15923                duration_ms: 0,
15924                continuation: None,
15925            }),
15926            blocking: 0,
15927            answered: 1,
15928            expected: 1,
15929            clean: false,
15930            progressed: true,
15931            vote_split: false,
15932            reconsideration: Vec::new(),
15933            verdict: None,
15934        };
15935        let state = state_with_round(round);
15936        let body = pr_message(&state, 'A').body;
15937
15938        assert!(body.contains("add retries"), "the task must still be there");
15939        assert!(body.contains("R2-1-1"), "{body}");
15940        assert!(!body.contains("unused import"), "{body}");
15941        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15942        assert!(
15943            !body.contains("not reachable from any caller"),
15944            "the reason it was declined: {body}"
15945        );
15946    }
15947
15948    #[test]
15949    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15950        let round = ReviewRound {
15951            round: 1,
15952            head: "deadbee".to_owned(),
15953            verified_head: None,
15954            verified_at: None,
15955            reviews: vec![ReviewRecord {
15956                attempts: 0,
15957                reviewer: 1,
15958                agent: "alpha".to_owned(),
15959                summary: String::new(),
15960                findings: Vec::new(),
15961                vote: None,
15962                failed: None,
15963                duration_ms: 0,
15964            }],
15965            e2e: Vec::new(),
15966            verify_retried: false,
15967            e2e_deferred: false,
15968            e2e_defer_reason: None,
15969            fix: None,
15970            blocking: 0,
15971            answered: 1,
15972            expected: 1,
15973            clean: true,
15974            progressed: false,
15975            vote_split: false,
15976            reconsideration: Vec::new(),
15977            verdict: None,
15978        };
15979        let state = state_with_round(round);
15980        let body = pr_message(&state, 'A').body;
15981        assert!(!body.contains("Open review findings"), "{body}");
15982        assert!(!body.contains("Declined"), "{body}");
15983    }
15984
15985    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15986        let mut state = RunState::new(
15987            PathBuf::from("/repo"),
15988            "main".to_owned(),
15989            "abc1234".to_owned(),
15990            instruction.to_owned(),
15991            Config::default(),
15992        );
15993        state.candidates.push(Candidate {
15994            index: 0,
15995            label: 'A',
15996            agent: "alpha".to_owned(),
15997            branch: "magi/x/A".to_owned(),
15998            worktree: PathBuf::from("/wt"),
15999            summary: summary.to_owned(),
16000            stat: String::new(),
16001            files: 1,
16002            commits: 1,
16003            empty: false,
16004            failed: None,
16005            verified_noop: None,
16006            folded: false,
16007            duration_ms: 0,
16008        });
16009        state
16010    }
16011
16012    fn review_state(subjects: &[&str]) -> RunState {
16013        let mut state = state_with_summary(
16014            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16015            "",
16016        );
16017        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16018        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16019        state
16020    }
16021
16022    /// A branch rebased onto a main that moved past the recorded
16023    /// `base_commit` is titled from its own first commit, never main's.
16024    #[tokio::test]
16025    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16026        ask_test_home();
16027        let tmp = tempfile::tempdir().unwrap();
16028        let repo = tmp.path().join("repo");
16029        std::fs::create_dir_all(&repo).unwrap();
16030        init_repo(&repo);
16031        let origin = tmp.path().join("origin.git");
16032        let g = |dir: &Path, args: &[&str]| {
16033            let out = std::process::Command::new("git")
16034                .args(args)
16035                .current_dir(dir)
16036                .quiet()
16037                .output()
16038                .expect("spawn git");
16039            assert!(
16040                out.status.success(),
16041                "git {args:?}: {}",
16042                String::from_utf8_lossy(&out.stderr)
16043            );
16044        };
16045        g(
16046            tmp.path(),
16047            &[
16048                "clone",
16049                "--bare",
16050                "-q",
16051                repo.to_str().unwrap(),
16052                origin.to_str().unwrap(),
16053            ],
16054        );
16055        g(
16056            &repo,
16057            &["remote", "add", "origin", origin.to_str().unwrap()],
16058        );
16059        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16060
16061        // Main moves on; the branch is built on top of the new main.
16062        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16063        g(&repo, &["add", "-A"]);
16064        g(
16065            &repo,
16066            &["commit", "-q", "-m", "chore(deps): update a crate"],
16067        );
16068        g(&repo, &["push", "-q", "origin", "main"]);
16069        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16070        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16071        g(&repo, &["add", "-A"]);
16072        g(
16073            &repo,
16074            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16075        );
16076        g(&repo, &["checkout", "-q", "main"]);
16077
16078        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16079        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16080        // Without a readable tracking ref the recorded base's merge base is used.
16081        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16082        assert_eq!(fallback, c1);
16083
16084        let mut state = review_state(&[
16085            "chore(deps): update a crate",
16086            "fix(daemon): apply a chosen action",
16087        ]);
16088        state.repo = repo.clone();
16089        state.base_branch = "main".to_owned();
16090        state.base_commit = c1;
16091        refresh_reviewed_commits(&mut state, "feat/own").await;
16092        assert_eq!(
16093            state.reviewed_commits,
16094            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16095        );
16096        assert_eq!(
16097            review_title(&state).as_deref(),
16098            Some("fix(daemon): apply a chosen action")
16099        );
16100        assert_eq!(
16101            leaked_subjects(&state, "feat/own").await,
16102            Some(vec!["chore(deps): update a crate".to_owned()])
16103        );
16104        // A stale tracking ref is still used when the fetch fails, but the
16105        // leak list is withheld.
16106        state.config.merge.remote = "nowhere".to_owned();
16107        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16108    }
16109
16110    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16111    /// with `origin/main` one commit behind it, and a summarizing agent that
16112    /// prints whatever `reply` holds.
16113    struct SummaryFixture {
16114        _tmp: tempfile::TempDir,
16115        repo: PathBuf,
16116        reply: PathBuf,
16117        runner: Runner,
16118    }
16119
16120    fn summary_fixture() -> SummaryFixture {
16121        ask_test_home();
16122        let tmp = tempfile::tempdir().unwrap();
16123        let repo = tmp.path().join("repo");
16124        std::fs::create_dir_all(&repo).unwrap();
16125        init_repo(&repo);
16126        let origin = tmp.path().join("origin.git");
16127        let g = |args: &[&str]| {
16128            let out = std::process::Command::new("git")
16129                .args(args)
16130                .current_dir(&repo)
16131                .quiet()
16132                .output()
16133                .expect("spawn git");
16134            assert!(
16135                out.status.success(),
16136                "git {args:?}: {}",
16137                String::from_utf8_lossy(&out.stderr)
16138            );
16139        };
16140        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16141        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16142        g(&["push", "-q", "origin", "main"]);
16143        g(&["checkout", "-q", "-b", "feat/own"]);
16144        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16145        g(&["add", "-A"]);
16146        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16147        let reply = tmp.path().join("reply.txt");
16148        std::fs::write(
16149            &reply,
16150            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16151        )
16152        .unwrap();
16153        let mut runner = runner_at(RunStatus::Gating);
16154        let mut state = review_state(&["feat(own): add the own file"]);
16155        state.repo = repo.clone();
16156        state.base_branch = "main".to_owned();
16157        state.candidates[0].branch = "feat/own".to_owned();
16158        state.candidates[0].worktree = repo.clone();
16159        let mut author = spec("alpha");
16160        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16161        author
16162            .env
16163            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16164        state.config.agents = vec![author];
16165        runner.state = state;
16166        SummaryFixture {
16167            _tmp: tmp,
16168            repo,
16169            reply,
16170            runner,
16171        }
16172    }
16173
16174    async fn posted(f: &mut SummaryFixture) -> PrMessage {
16175        let winner = f.runner.state.candidates[0].clone();
16176        f.runner
16177            .guarded_pr_message(&winner, None, true)
16178            .await
16179            .unwrap()
16180            .expect("an English description never parks")
16181    }
16182
16183    fn clean_round_with_minor_findings() -> ReviewRound {
16184        let mut round = passing_round(2, None);
16185        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16186        round.reviews = vec![ReviewRecord {
16187            attempts: 0,
16188            reviewer: 1,
16189            agent: "alpha".into(),
16190            summary: "内部レビュー".into(),
16191            findings: vec![
16192                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16193                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16194            ],
16195            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16196            failed: None,
16197            duration_ms: 0,
16198        }];
16199        round
16200    }
16201
16202    #[tokio::test]
16203    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16204        for review_only in [false, true] {
16205            let mut f = summary_fixture();
16206            if !review_only {
16207                f.runner.state.candidates[0].agent = "alpha".into();
16208            }
16209            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16210            assert!(f.runner.state.reviews[0].clean);
16211            assert!(f.runner.state.reviews[0].fix.is_none());
16212            assert!(f.runner.state.open_findings().is_empty());
16213            assert_eq!(
16214                crate::pr_outcome::required_finding_ids(&f.runner.state),
16215                ["R2-1-1", "R2-1-2"]
16216            );
16217            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16218            let evidence = crate::pr_outcome::summary_evidence(
16219                &f.runner.state,
16220                crate::pr_outcome::FinalHead::At(&head),
16221            );
16222            assert!(
16223                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16224            );
16225            // The ordinary reply omits both IDs: no cached description is adopted.
16226            let omitted = posted(&mut f).await;
16227            assert!(omitted.body.contains("No description"));
16228            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16229            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16230            assert!(f.runner.state.change_summary.is_none());
16231            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16232            let described = posted(&mut f).await;
16233            assert!(described.body.contains("improve documentation for callers"));
16234            assert!(described.body.contains("remaining edge case"));
16235            assert!(described.body.contains("## Open review findings"));
16236            assert!(!described.body.contains("未修正"));
16237        }
16238    }
16239
16240    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16241        let mut f = summary_fixture();
16242        if !review_only {
16243            f.runner.state.candidates[0].agent = "alpha".into();
16244        }
16245        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16246        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16247        fixed
16248            .fix
16249            .as_mut()
16250            .unwrap()
16251            .rejected
16252            .push(crate::verdict::Rejection {
16253                id: "R1-1-1".into(),
16254                why: "呼び出されないコードのため辞退".into(),
16255            });
16256        let mut clean = clean_round_with_minor_findings();
16257        clean.verified_head = Some(head.clone());
16258        f.runner.state.reviews = vec![fixed, clean];
16259        ok_gate(&mut f.runner.state, Some(&head));
16260        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16261        f.runner.state.config.agents.push(scripted("beta", reply));
16262        f.runner.state.config.roles.language_judge =
16263            Some(crate::config::AgentChoice::One("beta".into()));
16264        f
16265    }
16266
16267    #[tokio::test]
16268    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16269        for review_only in [false, true] {
16270            let mut f = rewrite_fixture_with_findings(review_only,
16271                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16272            let m = posted(&mut f).await;
16273            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16274            let expected =
16275                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16276            assert!(m.body.contains(&expected));
16277            assert!(m.body.contains("## Review rounds"));
16278            assert!(
16279                m.body
16280                    .contains("Gate (`verify.gate`): passed on the final commit")
16281            );
16282            assert!(
16283                m.body
16284                    .contains("Tests (`verify.e2e`): passed on the final commit")
16285            );
16286            let c = f.runner.state.change_summary.as_ref().unwrap();
16287            assert!(
16288                m.body
16289                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16290            );
16291            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16292            assert!(crate::pr_outcome::covers_required_findings(
16293                &f.runner.state,
16294                &m.body
16295            ));
16296            assert!(m.body.contains("documentation remains incomplete"));
16297            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16298            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16299            assert!(f.runner.state.github_text.is_none());
16300        }
16301    }
16302
16303    #[tokio::test]
16304    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16305        for review_only in [false, true] {
16306            for reply in [
16307                "TITLE: fix: public description\nA short English description.",
16308                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16309            ] {
16310                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16311                let winner = f.runner.state.candidates[0].clone();
16312                let posted = f
16313                    .runner
16314                    .guarded_pr_message(&winner, None, true)
16315                    .await
16316                    .unwrap();
16317                assert!(posted.is_none());
16318                assert!(f.runner.state.parked);
16319                let withheld = f.runner.state.github_text.as_ref().unwrap();
16320                assert!(
16321                    withheld.rewrite.is_none(),
16322                    "an incomplete rewrite must not be selectable"
16323                );
16324                let q = ask::Questions::open()
16325                    .get(withheld.question.as_ref().unwrap())
16326                    .unwrap();
16327                assert!(
16328                    !q.choices
16329                        .contains(&crate::github_text::USE_REWRITE.to_owned())
16330                );
16331                assert!(
16332                    q.detail.contains("R1-1-1")
16333                        && q.detail.contains("R2-1-1")
16334                        && q.detail.contains("R2-1-2")
16335                );
16336            }
16337        }
16338    }
16339
16340    #[tokio::test]
16341    async fn japanese_reports_stay_local_in_both_public_run_modes() {
16342        for review_only in [false, true] {
16343            let mut f = summary_fixture();
16344            if !review_only {
16345                f.runner.state.candidates[0].agent = "alpha".into();
16346            }
16347            f.runner.state.instruction = "内部タスクの説明".into();
16348            f.runner.state.candidates[0].summary = "内部の候補報告".into();
16349            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16350            f.runner.state.reviews = vec![
16351                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16352                round_with_notes(2, Some("二回目の内部修正報告です。")),
16353                passing_round(3, Some(&head)),
16354            ];
16355            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16356            ok_gate(&mut f.runner.state, Some(&head));
16357            let captured = f.repo.join("public-prompt.txt");
16358            let author = &mut f.runner.state.config.agents[0];
16359            author.command = vec![
16360                "sh".into(),
16361                "-c".into(),
16362                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16363            ];
16364            author.env.insert(
16365                "PROMPT_FILE".into(),
16366                captured.to_string_lossy().into_owned(),
16367            );
16368            let m = posted(&mut f).await;
16369            assert!(m.body.contains("Adds `own.txt`"));
16370            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16371            assert!(m.body.contains("R1-1-1"));
16372            assert!(
16373                m.body
16374                    .contains("Gate (`verify.gate`): passed on the final commit")
16375            );
16376            assert!(
16377                m.body
16378                    .contains("Tests (`verify.e2e`): passed on the final commit")
16379            );
16380            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16381            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16382            let prompt = std::fs::read_to_string(captured).unwrap();
16383            assert!(prompt.contains(&format!(
16384                "git diff {} {head}",
16385                f.runner.state.change_summary.as_ref().unwrap().base
16386            )));
16387            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16388            assert_eq!(
16389                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16390                "二回目の内部修正報告です。"
16391            );
16392        }
16393    }
16394
16395    #[tokio::test]
16396    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16397        for review_only in [false, true] {
16398            let mut f = summary_fixture();
16399            if !review_only {
16400                f.runner.state.candidates[0].agent = "alpha".into();
16401            }
16402            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16403            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16404            std::fs::write(&f.reply, "no usable description").unwrap();
16405            let failed = posted(&mut f).await;
16406            assert!(
16407                failed
16408                    .body
16409                    .contains("No description of what this change does could be generated")
16410            );
16411            assert!(failed.body.is_ascii());
16412            assert!(f.runner.state.github_text.is_none());
16413            assert!(f.runner.state.change_summary.is_none());
16414            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16415            let evidence = crate::pr_outcome::summary_evidence(
16416                &f.runner.state,
16417                crate::pr_outcome::FinalHead::At(&head),
16418            );
16419            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16420            assert!(
16421                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16422                    .exists()
16423            );
16424            f.runner.state =
16425                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16426            std::fs::write(
16427                &f.reply,
16428                "TITLE: feat(own): add the own file\nAdds the committed own file.",
16429            )
16430            .unwrap();
16431            let retried = posted(&mut f).await;
16432            assert!(retried.body.contains("Adds the committed own file."));
16433            assert_eq!(
16434                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16435                "日本語の修正報告"
16436            );
16437        }
16438    }
16439
16440    #[tokio::test]
16441    async fn same_head_verification_and_findings_invalidate_public_prose() {
16442        let mut f = summary_fixture();
16443        posted(&mut f).await;
16444        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16445        let original = f
16446            .runner
16447            .state
16448            .change_summary
16449            .as_ref()
16450            .unwrap()
16451            .evidence
16452            .clone();
16453        ok_gate(&mut f.runner.state, Some(&head));
16454        std::fs::write(
16455            &f.reply,
16456            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16457        )
16458        .unwrap();
16459        let verified = posted(&mut f).await;
16460        assert!(verified.body.contains("now verified"));
16461        assert_ne!(
16462            f.runner.state.change_summary.as_ref().unwrap().evidence,
16463            original
16464        );
16465        let mut round = round_with_notes(1, Some("内部報告"));
16466        round.reviews.push(ReviewRecord {
16467            attempts: 0,
16468            reviewer: 1,
16469            agent: "alpha".into(),
16470            summary: "内部レビュー".into(),
16471            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16472            vote: None,
16473            failed: None,
16474            duration_ms: 0,
16475        });
16476        round
16477            .fix
16478            .as_mut()
16479            .unwrap()
16480            .rejected
16481            .push(crate::verdict::Rejection {
16482                id: "R1-1-1".into(),
16483                why: "呼び出し元から到達できない".into(),
16484            });
16485        f.runner.state.reviews = vec![round];
16486        // A description omitting the required ID is unusable, even with a title.
16487        let omitted = posted(&mut f).await;
16488        assert!(omitted.body.contains("No description"));
16489        assert!(omitted.body.contains("R1-1-1"));
16490        assert!(!omitted.body.contains("到達"));
16491        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16492        let described = posted(&mut f).await;
16493        assert!(described.body.contains("unreachable from callers"));
16494        assert!(described.body.contains("R1-1-1"));
16495        assert!(!described.body.contains("到達"));
16496        let evidence = f
16497            .runner
16498            .state
16499            .change_summary
16500            .as_ref()
16501            .unwrap()
16502            .evidence
16503            .clone();
16504        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16505        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16506        assert!(posted(&mut f).await.body.contains("rationale updated"));
16507        assert_ne!(
16508            f.runner.state.change_summary.as_ref().unwrap().evidence,
16509            evidence
16510        );
16511        // Old serialized summaries have no evidence fingerprint and must regenerate.
16512        f.runner
16513            .state
16514            .change_summary
16515            .as_mut()
16516            .unwrap()
16517            .evidence
16518            .clear();
16519        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16520        assert!(posted(&mut f).await.body.contains("current specification"));
16521    }
16522
16523    #[tokio::test]
16524    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16525        let mut f = summary_fixture();
16526        let m = posted(&mut f).await;
16527        assert_eq!(m.title, "feat(own): add the own file");
16528        assert!(
16529            m.body
16530                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16531            "{}",
16532            m.body
16533        );
16534        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16535        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16536        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16537        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16538        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16539        let kept = f.runner.state.change_summary.as_ref().unwrap();
16540        assert_eq!(
16541            (kept.head.as_str(), kept.base.as_str()),
16542            (head.as_str(), base.as_str())
16543        );
16544        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16545    }
16546
16547    #[tokio::test]
16548    async fn review_only_summary_receives_the_configured_github_language() {
16549        let mut f = summary_fixture();
16550        f.runner.state.config.graph.github_language = "ja".into();
16551        let captured = f.repo.join("summary-prompt.txt");
16552        let author = &mut f.runner.state.config.agents[0];
16553        author.command = vec![
16554            "sh".into(),
16555            "-c".into(),
16556            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16557        ];
16558        author.env.insert(
16559            "PROMPT_FILE".into(),
16560            captured.to_string_lossy().into_owned(),
16561        );
16562        std::fs::write(
16563            &f.reply,
16564            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16565        )
16566        .unwrap();
16567        let m = posted(&mut f).await;
16568        let prompt = std::fs::read_to_string(captured).unwrap();
16569        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16570        assert!(!prompt.contains("in English"));
16571        assert_eq!(m.title, "feat(own): ファイルを追加");
16572        assert!(m.body.contains("own.txt を追加します。"));
16573    }
16574
16575    #[tokio::test]
16576    async fn review_fixes_come_after_the_whole_change_summary() {
16577        let mut f = summary_fixture();
16578        f.runner.state.reviews = vec![
16579            round_with_notes(1, Some("handled the PATH case")),
16580            round_with_notes(2, None),
16581        ];
16582        let m = posted(&mut f).await;
16583        let summary = m.body.find("## Summary").expect("summary");
16584        let fixes = m.body.find("## Review rounds").expect("review rounds");
16585        assert!(summary < fixes, "{}", m.body);
16586        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16587        assert!(m.body.contains("## Final verification"), "{}", m.body);
16588    }
16589
16590    #[tokio::test]
16591    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16592        let mut f = summary_fixture();
16593        let first = posted(&mut f).await;
16594        let first_head = f.runner.state.change_summary.clone().unwrap().head;
16595        // The same head reuses the record: a different reply is not read.
16596        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16597        let again = posted(&mut f).await;
16598        assert_eq!((again.title, again.body), (first.title, first.body));
16599
16600        let g = |args: &[&str]| {
16601            let out = std::process::Command::new("git")
16602                .args(args)
16603                .current_dir(&f.repo)
16604                .quiet()
16605                .output()
16606                .expect("spawn git");
16607            assert!(out.status.success(), "git {args:?}");
16608        };
16609        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16610        g(&["add", "-A"]);
16611        g(&["commit", "-q", "-m", "fix(own): handle more"]);
16612        let second = posted(&mut f).await;
16613        let kept = f.runner.state.change_summary.clone().unwrap();
16614        assert_ne!(kept.head, first_head);
16615        assert!(second.body.contains("Not read yet."), "{}", second.body);
16616        assert!(!second.body.contains("so the first summary is read"));
16617        assert_eq!(second.title, "feat(own): other");
16618    }
16619
16620    #[tokio::test]
16621    async fn an_unavailable_summary_is_stated_and_never_invented() {
16622        let mut f = summary_fixture();
16623        std::fs::write(&f.reply, "no title line here").unwrap();
16624        let m = posted(&mut f).await;
16625        assert!(
16626            m.body
16627                .contains("No description of what this change does could be generated"),
16628            "{}",
16629            m.body
16630        );
16631        assert!(m.body.contains("internal reports remain in local records"));
16632        assert!(f.runner.state.change_summary.is_none());
16633        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16634        // A preview asks nobody and claims nothing.
16635        let winner = f.runner.state.candidates[0].clone();
16636        let preview = f
16637            .runner
16638            .guarded_pr_message(&winner, None, false)
16639            .await
16640            .unwrap()
16641            .unwrap();
16642        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16643    }
16644
16645    #[test]
16646    fn pr_message_review_single_commit_uses_its_subject() {
16647        let state = review_state(&["feat(nats): per-role user"]);
16648        let m = pr_message(&state, 'A');
16649        assert_eq!(m.title, "feat(nats): per-role user");
16650        assert!(!m.body.contains("Review the work already"), "{}", m.body);
16651        assert!(m.body.contains("## Commits under review"), "{}", m.body);
16652    }
16653
16654    #[test]
16655    fn pr_message_review_multi_commit_takes_the_oldest() {
16656        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16657        let m = pr_message(&state, 'A');
16658        assert_eq!(m.title, "feat: the change");
16659        for s in ["feat: the change", "fix: typo", "fix: again"] {
16660            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16661        }
16662    }
16663
16664    #[test]
16665    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16666        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16667            let mut state = review_state(&[first, "fix: later fixup"]);
16668            state.candidates[0].branch = "機能/ログイン".to_owned();
16669            let m = pr_message(&state, 'A');
16670            assert!(
16671                m.title.starts_with("chore: land candidate A of run"),
16672                "{}",
16673                m.title
16674            );
16675        }
16676    }
16677
16678    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16679        BranchFacts {
16680            commits: commits
16681                .iter()
16682                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16683                .collect(),
16684            stat: stat.to_owned(),
16685        }
16686    }
16687
16688    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16689        let mut r = review_round(true, 0, 1, 1, true, true);
16690        r.round = round;
16691        r.fix = notes.map(|n| FixRecord {
16692            agent: "fixer".to_owned(),
16693            addressed: Vec::new(),
16694            rejected: Vec::new(),
16695            notes: n.to_owned(),
16696            committed: true,
16697            failed: None,
16698            duration_ms: 0,
16699            continuation: None,
16700        });
16701        r
16702    }
16703
16704    #[test]
16705    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16706        let state = review_state(&["ignored"]);
16707        let f = facts(
16708            &[
16709                (
16710                    "fix(login): resolve PATH on macOS",
16711                    "Login shells skip rc files.",
16712                ),
16713                ("fix: address review", ""),
16714            ],
16715            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16716        );
16717        let m = pr_message_with(&state, 'A', Some(&f));
16718        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16719        assert!(
16720            m.body
16721                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
16722            "{}",
16723            m.body
16724        );
16725        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16726        assert!(m.body.contains("## Diff stat"), "{}", m.body);
16727        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16728        for banned in [
16729            "Review the work already",
16730            "no task statement",
16731            "Original task",
16732        ] {
16733            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16734        }
16735        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16736    }
16737
16738    #[test]
16739    fn pr_message_review_truncates_a_huge_first_commit_body() {
16740        let state = review_state(&["ignored"]);
16741        let f = facts(
16742            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16743            "s",
16744        );
16745        let m = pr_message_with(&state, 'A', Some(&f));
16746        assert!(m.body.len() < 30_000, "{}", m.body.len());
16747        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16748        assert!(
16749            m.body.contains("- ... 1 more commit(s)"),
16750            "{}",
16751            m.body.len()
16752        );
16753        assert!(m.body.ends_with("magi:candidate-a\n"));
16754    }
16755
16756    #[test]
16757    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16758        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16759        assert_eq!(m.title, "feat: x");
16760        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16761        assert!(!m.body.contains("Diff stat"), "{}", m.body);
16762        assert!(!m.body.contains("no task statement"), "{}", m.body);
16763    }
16764
16765    #[test]
16766    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16767        let mut state = review_state(&["日本語の件名"]);
16768        state.candidates[0].branch = "fix/macos-login-path".to_owned();
16769        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16770        state.candidates[0].branch = "機能/ログイン".to_owned();
16771        assert!(
16772            pr_message(&state, 'A')
16773                .title
16774                .starts_with("chore: land candidate A")
16775        );
16776    }
16777
16778    #[test]
16779    fn pr_message_review_fixes_survive_a_clean_final_round() {
16780        let mut state = review_state(&["feat: x"]);
16781        state.reviews = vec![
16782            round_with_notes(1, Some("handled the PATH case")),
16783            round_with_notes(2, None),
16784        ];
16785        let body = pr_message(&state, 'A').body;
16786        assert!(
16787            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16788            "{body}"
16789        );
16790        assert!(body.contains("### Round 1"), "{body}");
16791
16792        state.reviews = vec![
16793            round_with_notes(1, Some("first fix")),
16794            round_with_notes(2, Some("")),
16795            round_with_notes(3, Some("second fix")),
16796            round_with_notes(4, None),
16797        ];
16798        let body = pr_message(&state, 'A').body;
16799        assert!(
16800            body.contains("### Round 1") && !body.contains("first fix"),
16801            "{body}"
16802        );
16803        assert!(
16804            body.contains("### Round 3") && !body.contains("second fix"),
16805            "{body}"
16806        );
16807        assert!(body.contains("### Round 2"), "{body}");
16808    }
16809
16810    const FINAL: &str = "abcdef1234567890";
16811
16812    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16813        let mut r = round_with_notes(round, None);
16814        r.verified_head = verified.map(str::to_owned);
16815        r
16816    }
16817
16818    fn ok_gate(state: &mut RunState, head: Option<&str>) {
16819        state.gate = vec![CommandOutcome {
16820            command: "gate".to_owned(),
16821            code: Some(0),
16822            output_tail: String::new(),
16823            duration_ms: 0,
16824            resource_blocked: false,
16825        }];
16826        state.gate_ran = true;
16827        state.gate_head = head.map(str::to_owned);
16828    }
16829
16830    fn body_at(state: &RunState) -> String {
16831        pr_message_raw(
16832            state,
16833            'A',
16834            None,
16835            crate::pr_outcome::FinalHead::At(FINAL),
16836            ChangeNote::NotAsked,
16837        )
16838        .body
16839    }
16840
16841    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16842
16843    #[test]
16844    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16845        let mut state = review_state(&["feat: x"]);
16846        state.reviews = vec![
16847            round_with_notes(1, Some(SANDBOXED)),
16848            passing_round(2, Some(FINAL)),
16849        ];
16850        ok_gate(&mut state, Some(FINAL));
16851        let body = body_at(&state);
16852        assert!(!body.contains("Handled the PATH case."), "{body}");
16853        assert!(!body.contains("could not commit"), "{body}");
16854        assert!(!body.contains("could not run"), "{body}");
16855        assert!(
16856            body.contains("The fix is committed on the branch"),
16857            "{body}"
16858        );
16859        assert!(!body.contains("sandbox"), "{body}");
16860        assert!(
16861            body.contains("Tests (`verify.e2e`): passed on the final commit"),
16862            "{body}"
16863        );
16864        assert!(
16865            body.contains("Gate (`verify.gate`): passed on the final commit"),
16866            "{body}"
16867        );
16868        // The raw report is untouched in the record.
16869        assert!(
16870            state.reviews[0]
16871                .fix
16872                .as_ref()
16873                .unwrap()
16874                .notes
16875                .contains("could not commit")
16876        );
16877    }
16878
16879    #[test]
16880    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16881        let mut state = review_state(&["feat: x"]);
16882        let mut r1 = round_with_notes(1, Some(SANDBOXED));
16883        r1.fix.as_mut().unwrap().committed = false;
16884        let mut r2 = passing_round(2, Some(FINAL));
16885        r2.e2e[0].code = Some(1);
16886        state.reviews = vec![r1, r2];
16887        let body = body_at(&state);
16888        assert!(!body.contains("could not commit"), "{body}");
16889        assert!(!body.contains("could not run"), "{body}");
16890        assert!(body.contains("This round produced no commit"), "{body}");
16891        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16892        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16893        assert!(!body.contains("passed"), "{body}");
16894    }
16895
16896    #[test]
16897    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16898        let mut state = review_state(&["feat: x"]);
16899        state.reviews = vec![
16900            round_with_notes(1, Some(SANDBOXED)),
16901            passing_round(2, Some("0123456789abcdef")),
16902        ];
16903        ok_gate(&mut state, Some("0123456789abcdef"));
16904        let body = body_at(&state);
16905        assert!(!body.contains("HTTP tests could not run"), "{body}");
16906        // The commit claim is still settled by the structured record.
16907        assert!(!body.contains("could not commit"), "{body}");
16908        assert!(
16909            body.contains("passed on an earlier commit (`0123456`)"),
16910            "{body}"
16911        );
16912        assert!(!body.contains("passed on the final commit"), "{body}");
16913
16914        // A gate recorded before `gate_head` existed is "not recorded".
16915        ok_gate(&mut state, None);
16916        let body = body_at(&state);
16917        assert!(
16918            body.contains("passed on a commit that was not recorded"),
16919            "{body}"
16920        );
16921    }
16922
16923    #[test]
16924    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16925        let state = state_with_summary(
16926            "add retries to the client",
16927            "TITLE: feat: retries\n\nDid it.",
16928        );
16929        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16930        assert_eq!(m.title, "feat: retries");
16931        assert!(
16932            m.body.contains("<summary>Original task</summary>"),
16933            "{}",
16934            m.body
16935        );
16936        assert!(!m.body.contains("Commits under review"), "{}", m.body);
16937        assert!(
16938            m.body
16939                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16940            "{}",
16941            m.body
16942        );
16943    }
16944
16945    #[test]
16946    fn pr_message_review_bounds_a_long_english_subject() {
16947        let long = format!("feat: {}", "word ".repeat(100));
16948        let m = pr_message(&review_state(&[&long]), 'A');
16949        assert!(m.title.starts_with("feat: word"), "{}", m.title);
16950        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16951    }
16952
16953    #[test]
16954    fn pr_message_implementation_run_is_unchanged_by_review_support() {
16955        let state = state_with_summary("add retries\n\ndetails", "- did some things");
16956        let m = pr_message(&state, 'A');
16957        assert_eq!(m.title, "add retries");
16958        assert!(m.body.contains("<summary>Original task</summary>"));
16959        assert!(!m.body.contains("Commits under review"));
16960        assert_eq!(landing_subject_source(&state), state.instruction);
16961    }
16962
16963    #[test]
16964    fn review_run_squash_subject_is_the_change_not_the_prompt() {
16965        let state = review_state(&["feat: the change", "fix: typo"]);
16966        let source = landing_subject_source(&state);
16967        assert_eq!(land::merge_subject("", &source), "feat: the change");
16968        assert_eq!(
16969            land::merge_subject("magi: candidate A (uncommitted work)", &source),
16970            "feat: the change"
16971        );
16972        // An operator's rename still wins.
16973        assert_eq!(
16974            land::merge_subject("feat: renamed by hand", &source),
16975            "feat: renamed by hand"
16976        );
16977        let blank = review_state(&["日本語"]);
16978        assert!(
16979            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16980        );
16981    }
16982
16983    #[test]
16984    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16985        let state = review_state(&["feat: the change"]);
16986        let source = landing_subject_source(&state);
16987        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16988        assert_eq!(
16989            land::merge_subject(landing_title(&state, old), &source),
16990            "feat: the change"
16991        );
16992        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16993        let task = state_with_summary("add retries", "");
16994        assert_eq!(landing_title(&task, old), old);
16995    }
16996
16997    #[test]
16998    fn pr_message_describes_the_change_not_the_task() {
16999        let state = state_with_summary(
17000            "今回やってほしいこと: results projector を直す",
17001            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17002        );
17003        let m = pr_message(&state, 'A');
17004        assert_eq!(m.title, "fix(web): batch the runs list reads");
17005        assert!(
17006            m.body.starts_with("## Summary\n\n- reads run.json once"),
17007            "{}",
17008            m.body
17009        );
17010        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17011        let task_at = m.body.find("今回やってほしいこと").unwrap();
17012        let details_at = m.body.find("<details>").unwrap();
17013        assert!(
17014            details_at < task_at,
17015            "the task lives inside <details>: {}",
17016            m.body
17017        );
17018        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17019        assert!(m.body.contains("magi:candidate-a"));
17020    }
17021
17022    #[test]
17023    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17024        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17025        let m = pr_message(&state, 'A');
17026        assert_eq!(m.title, "add retries");
17027        assert!(
17028            m.body.contains("## Summary\n\n- did some things"),
17029            "{}",
17030            m.body
17031        );
17032
17033        let none = RunState::new(
17034            PathBuf::from("/repo"),
17035            "main".to_owned(),
17036            "abc1234".to_owned(),
17037            "add retries".to_owned(),
17038            Config::default(),
17039        );
17040        let m = pr_message(&none, 'A');
17041        assert_eq!(m.title, "add retries");
17042        assert!(!m.body.contains("## Summary"), "{}", m.body);
17043    }
17044
17045    #[test]
17046    fn pr_message_refuses_the_candidate_commit_subject() {
17047        for bad in [
17048            "TITLE: magi: candidate A (uncommitted work)",
17049            "TITLE: chore: stuff (uncommitted work)",
17050            "TITLE:   ",
17051        ] {
17052            let state = state_with_summary("add retries", bad);
17053            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17054        }
17055    }
17056
17057    #[test]
17058    fn pr_message_bounds_a_very_long_task_and_title() {
17059        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17060        let state = state_with_summary(&long, "- nothing");
17061        let m = pr_message(&state, 'A');
17062        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17063        assert!(!m.title.contains('\n'));
17064
17065        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17066        let m = pr_message(&state, 'A');
17067        assert!(m.title.starts_with("feat: "));
17068        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17069        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17070    }
17071
17072    fn long_title_of(instruction: &str) -> String {
17073        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17074    }
17075
17076    #[test]
17077    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17078        let first = "Make the landing path keep a readable title for long tasks";
17079        let line = format!(
17080            "{first}. {}",
17081            "And then keep going with more words ".repeat(20)
17082        );
17083        let t = long_title_of(&line);
17084        assert_eq!(t, first);
17085        assert!(!t.starts_with("chore: land"));
17086    }
17087
17088    #[test]
17089    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17090        let line = "word ".repeat(200);
17091        let t = long_title_of(&line);
17092        assert!(t.ends_with("word..."), "{t}");
17093        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17094    }
17095
17096    #[test]
17097    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17098        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17099            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17100        }
17101    }
17102
17103    #[test]
17104    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17105        let mut state = state_with_summary("再試行を追加する", "- nothing");
17106        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17107        state.config.graph.github_language = "ja".into();
17108        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17109    }
17110
17111    #[test]
17112    fn pr_message_title_limit_is_exact() {
17113        let at = "a".repeat(PR_TITLE_MAX);
17114        assert_eq!(long_title_of(&at), at);
17115        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17116        assert!(over.ends_with("..."), "{over}");
17117        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17118    }
17119
17120    #[test]
17121    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17122        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17123        let t = long_title_of(&line);
17124        assert!(t.ends_with("..."), "{t}");
17125        assert!(t.is_ascii(), "{t}");
17126    }
17127
17128    #[test]
17129    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17130        let line = format!(
17131            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17132            "plus more filler words ".repeat(20)
17133        );
17134        let t = long_title_of(&line);
17135        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17136    }
17137
17138    #[test]
17139    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17140        let line = format!("\n\n# {}", "title words ".repeat(40));
17141        let state = state_with_summary(&line, "- nothing");
17142        let m = pr_message(&state, 'A');
17143        assert!(m.title.starts_with("title words"), "{}", m.title);
17144        assert_eq!(
17145            land::merge_subject(&m.title, &landing_subject_source(&state)),
17146            m.title
17147        );
17148        // An operator's rename wins untouched.
17149        assert_eq!(
17150            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17151            "feat: renamed by hand"
17152        );
17153    }
17154
17155    #[tokio::test]
17156    async fn github_text_rewrite_is_bounded_and_falls_back() {
17157        crate::run::pin_test_home();
17158        for (reply, accepted) in [
17159            (
17160                "TITLE: fix: retries\nAdd retries for failed requests.",
17161                true,
17162            ),
17163            (
17164                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17165                false,
17166            ),
17167            (
17168                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17169                true,
17170            ),
17171            (
17172                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17173                true,
17174            ),
17175            (
17176                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17177                true,
17178            ),
17179            (
17180                "TITLE: fix: retries\n日本語の説明です。 token=secret",
17181                false,
17182            ),
17183            ("TITLE: fix: retries\n", false),
17184        ] {
17185            let dir = tempfile::tempdir().unwrap();
17186            let mut runner = runner_at(RunStatus::Gating);
17187            runner.state = state_with_summary(
17188                "add retries",
17189                "TITLE: fix: retries\n日本語の説明を書きます。",
17190            );
17191            runner.state.repo = dir.path().to_owned();
17192            runner.state.candidates[0].worktree = dir.path().to_owned();
17193            let mut author = spec("alpha");
17194            author.command = vec![
17195                "sh".into(),
17196                "-c".into(),
17197                "printf '%s' \"$REWRITE_REPLY\"".into(),
17198            ];
17199            author.env.insert("REWRITE_REPLY".into(), reply.into());
17200            runner.state.config.agents = vec![author];
17201            seed_public_summary(&mut runner);
17202            let winner = runner.state.candidates[0].clone();
17203            let first = runner
17204                .guarded_pr_message(&winner, None, true)
17205                .await
17206                .unwrap();
17207            let message = if accepted {
17208                first.expect("a passing rewrite never asks")
17209            } else {
17210                // Withheld: the run parks on a question and posts nothing yet.
17211                assert!(first.is_none());
17212                assert!(runner.state.parked);
17213                let store = ask::Questions::open();
17214                let asked: Vec<_> = store
17215                    .list()
17216                    .into_iter()
17217                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17218                    .collect();
17219                assert_eq!(asked.len(), 1);
17220                assert!(!asked[0].detail.contains("secret"));
17221                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17222                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17223                assert_eq!(
17224                    asked[0]
17225                        .choices
17226                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
17227                    rewrite.is_some()
17228                );
17229                // Silence: the question lapses and the neutral text is used,
17230                // without a second rewrite call or a second question.
17231                store
17232                    .update(&asked[0].id, |q| {
17233                        q.abandon("test");
17234                        Ok(())
17235                    })
17236                    .unwrap();
17237                let again = runner
17238                    .guarded_pr_message(&winner, None, true)
17239                    .await
17240                    .unwrap()
17241                    .expect("an abandoned question falls back");
17242                let n = store
17243                    .list()
17244                    .into_iter()
17245                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17246                    .count();
17247                assert_eq!(n, 1, "asked once per rejected text");
17248                again
17249            };
17250            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17251            assert!(!message.body.contains("secret"));
17252            assert!(!message.body.contains("prefix"));
17253            assert!(!message.title.contains("secret"));
17254            assert_eq!(
17255                message.body.contains("Add retries for failed requests."),
17256                accepted
17257            );
17258            assert_eq!(
17259                runner.state.seats["github-text-rewrite"].turns, 1,
17260                "only one rewrite invocation"
17261            );
17262            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17263            if accepted {
17264                assert!(runner.state.github_text.is_none());
17265                assert!(!runner.state.parked);
17266                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17267                assert!(message.body.contains("## Final verification"));
17268                assert!(message.body.contains("Evidence: final diff"));
17269                assert!(
17270                    message
17271                        .body
17272                        .contains(&format!("magi:run/{}", runner.state.id))
17273                );
17274            }
17275            if !accepted {
17276                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17277                assert!(
17278                    message
17279                        .body
17280                        .contains(&format!("magi:run/{}", runner.state.id))
17281                );
17282            }
17283        }
17284    }
17285
17286    fn scripted(id: &str, reply: &str) -> AgentSpec {
17287        let mut a = spec(id);
17288        a.command = vec![
17289            "sh".into(),
17290            "-c".into(),
17291            "printf '%s' \"$REWRITE_REPLY\"".into(),
17292        ];
17293        a.env.insert("REWRITE_REPLY".into(), reply.into());
17294        a
17295    }
17296
17297    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17298    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17299
17300    /// Gate tests inject generated public output, not internal reports.
17301    fn seed_public_summary(runner: &mut Runner) {
17302        let repo = &runner.state.repo;
17303        let g = |args: &[&str]| {
17304            let out = std::process::Command::new("git")
17305                .args(args)
17306                .current_dir(repo)
17307                .quiet()
17308                .output()
17309                .unwrap();
17310            assert!(out.status.success(), "git {args:?}");
17311            String::from_utf8_lossy(&out.stdout).trim().to_owned()
17312        };
17313        g(&["init", "-q", "-b", "main"]);
17314        g(&[
17315            "-c",
17316            "user.name=Test",
17317            "-c",
17318            "user.email=test@example.invalid",
17319            "commit",
17320            "-q",
17321            "--allow-empty",
17322            "-m",
17323            "feat: fixture",
17324        ]);
17325        let head = g(&["rev-parse", "HEAD"]);
17326        g(&["update-ref", "refs/remotes/origin/main", &head]);
17327        let evidence = crate::pr_outcome::summary_evidence(
17328            &runner.state,
17329            crate::pr_outcome::FinalHead::At(&head),
17330        );
17331        let generated = &runner.state.candidates[0].summary;
17332        runner.state.change_summary = Some(crate::run::ChangeSummary {
17333            base: head.clone(),
17334            head,
17335            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17336            title: summary_title(generated).unwrap(),
17337            body: summary_without_title(generated),
17338        });
17339    }
17340
17341    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17342        let mut runner = runner_at(RunStatus::Gating);
17343        runner.state = state_with_summary("add retries", JA_SUMMARY);
17344        if review_only {
17345            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17346        }
17347        runner.state.repo = dir.to_owned();
17348        runner.state.candidates[0].worktree = dir.to_owned();
17349        runner.state.config.agents = agents;
17350        seed_public_summary(&mut runner);
17351        runner
17352    }
17353
17354    fn gate_questions(runner: &Runner) -> usize {
17355        ask::Questions::open()
17356            .list()
17357            .into_iter()
17358            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17359            .count()
17360    }
17361
17362    #[tokio::test]
17363    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17364        crate::run::pin_test_home();
17365        let dir = tempfile::tempdir().unwrap();
17366        let mut runner = rewrite_runner(
17367            dir.path(),
17368            true,
17369            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17370        );
17371        runner.state.config.roles.language_judge =
17372            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17373        let winner = runner.state.candidates[0].clone();
17374        let msg = runner
17375            .guarded_pr_message(&winner, None, true)
17376            .await
17377            .unwrap()
17378            .expect("an English rewrite posts without asking");
17379        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17380        assert!(msg.body.contains("<details>kept</details>"));
17381        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17382        assert_eq!(gate_questions(&runner), 0);
17383        assert!(runner.state.github_text.is_none());
17384        assert!(!runner.state.seats.contains_key("impl-A"));
17385        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17386    }
17387
17388    #[tokio::test]
17389    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17390        crate::run::pin_test_home();
17391        let dir = tempfile::tempdir().unwrap();
17392        let mut runner = rewrite_runner(
17393            dir.path(),
17394            false,
17395            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17396        );
17397        runner.state.config.roles.language_judge =
17398            Some(crate::config::AgentChoice::One("beta".into()));
17399        runner.state.seats.insert(
17400            "impl-A".into(),
17401            SeatState::new("impl-A", "alpha", runner.state.seed),
17402        );
17403        let winner = runner.state.candidates[0].clone();
17404        let msg = runner
17405            .guarded_pr_message(&winner, None, true)
17406            .await
17407            .unwrap()
17408            .expect("rewritten");
17409        assert!(msg.body.contains("<details>kept</details>"));
17410        assert_eq!(runner.state.seats["impl-A"].turns, 0);
17411        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17412        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17413    }
17414
17415    #[tokio::test]
17416    async fn rewrite_reuses_a_resumable_implementer_session() {
17417        crate::run::pin_test_home();
17418        let dir = tempfile::tempdir().unwrap();
17419        let mut runner = rewrite_runner(
17420            dir.path(),
17421            false,
17422            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17423        );
17424        runner.state.config.roles.language_judge =
17425            Some(crate::config::AgentChoice::One("beta".into()));
17426        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17427        seat.turns = 1;
17428        runner.state.seats.insert("impl-A".into(), seat);
17429        let winner = runner.state.candidates[0].clone();
17430        runner
17431            .guarded_pr_message(&winner, None, true)
17432            .await
17433            .unwrap()
17434            .expect("rewritten");
17435        assert_eq!(runner.state.seats["impl-A"].turns, 2);
17436        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17437    }
17438
17439    #[tokio::test]
17440    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17441        crate::run::pin_test_home();
17442        let dir = tempfile::tempdir().unwrap();
17443        let mut broken = spec("bad");
17444        broken.command = vec!["false".into()];
17445        let mut runner = rewrite_runner(
17446            dir.path(),
17447            true,
17448            vec![
17449                broken,
17450                scripted("noformat", "Add retries."),
17451                scripted("good", EN_REWRITE),
17452            ],
17453        );
17454        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17455            "bad".into(),
17456            "noformat".into(),
17457            "good".into(),
17458        ]));
17459        let winner = runner.state.candidates[0].clone();
17460        let msg = runner
17461            .guarded_pr_message(&winner, None, true)
17462            .await
17463            .unwrap()
17464            .expect("the third agent rewrites");
17465        assert!(msg.body.contains("`src/net.rs`"));
17466        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17467        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17468    }
17469
17470    #[tokio::test]
17471    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17472        crate::run::pin_test_home();
17473        // (agents, role): still Japanese; no TITLE; nothing resolvable.
17474        for (agents, role, saved) in [
17475            (
17476                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17477                Some("beta"),
17478                true,
17479            ),
17480            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17481            (Vec::new(), None, false),
17482        ] {
17483            let dir = tempfile::tempdir().unwrap();
17484            let mut runner = rewrite_runner(dir.path(), true, agents);
17485            runner.state.config.roles.language_judge =
17486                role.map(|r| crate::config::AgentChoice::One(r.into()));
17487            let winner = runner.state.candidates[0].clone();
17488            let first = runner
17489                .guarded_pr_message(&winner, None, true)
17490                .await
17491                .unwrap();
17492            assert!(first.is_none(), "withheld text parks the run");
17493            assert!(runner.state.parked);
17494            assert_eq!(gate_questions(&runner), 1);
17495            let g = runner.state.github_text.as_ref().unwrap();
17496            assert_eq!(g.rewrite.is_some(), saved);
17497            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17498        }
17499    }
17500
17501    #[tokio::test]
17502    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17503        crate::run::pin_test_home();
17504        let dir = tempfile::tempdir().unwrap();
17505        let mut runner = runner_at(RunStatus::Gating);
17506        runner.state =
17507            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17508        runner.state.repo = dir.path().to_owned();
17509        runner.state.candidates[0].worktree = dir.path().to_owned();
17510        let mut author = spec("alpha");
17511        author.command = vec![
17512            "sh".into(),
17513            "-c".into(),
17514            "printf '%s' \"$REWRITE_REPLY\"".into(),
17515        ];
17516        author.env.insert(
17517            "REWRITE_REPLY".into(),
17518            format!(
17519                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17520                "English sentence. ".repeat(600)
17521            ),
17522        );
17523        runner.state.config.agents = vec![author];
17524        seed_public_summary(&mut runner);
17525        let winner = runner.state.candidates[0].clone();
17526        assert!(
17527            runner
17528                .guarded_pr_message(&winner, None, true)
17529                .await
17530                .unwrap()
17531                .is_none()
17532        );
17533        let record = runner.state.github_text.clone().unwrap();
17534        let rewrite = record.rewrite.clone().unwrap();
17535        assert_eq!(rewrite.failed, ["body-language"]);
17536        assert!(
17537            rewrite
17538                .body
17539                .ends_with(&format!("magi:run/{}", runner.state.id))
17540        );
17541        let store = ask::Questions::open();
17542        let q = store.get(record.question.as_ref().unwrap()).unwrap();
17543        assert!(
17544            q.choices
17545                .contains(&crate::github_text::USE_REWRITE.to_owned())
17546        );
17547        for label in [
17548            "ORIGINAL",
17549            "FALLBACK",
17550            "REWRITE",
17551            "Rules still failed: body-language",
17552            "truncated",
17553        ] {
17554            assert!(q.detail.contains(label), "{label}");
17555        }
17556        let artifact = std::fs::read_to_string(crate::run::artifact_path(
17557            &runner.state,
17558            &crate::github_text::artifact_name(&record.fingerprint),
17559        ))
17560        .unwrap();
17561        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17562        // Restart from serialized state, then retry after a bad owner title.
17563        runner.state =
17564            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17565        store
17566            .update(&q.id, |q| {
17567                q.say("まだ日本語")?;
17568                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17569            })
17570            .unwrap();
17571        assert!(
17572            runner
17573                .guarded_pr_message(&winner, None, true)
17574                .await
17575                .unwrap()
17576                .is_none()
17577        );
17578        let retry = runner.state.github_text.as_ref().unwrap();
17579        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17580        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17581        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17582        assert!(
17583            q.posting_comparison
17584                .as_ref()
17585                .unwrap()
17586                .matches_detail(&q.detail)
17587        );
17588        store
17589            .update(&q.id, |q| {
17590                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17591            })
17592            .unwrap();
17593        let message = runner
17594            .guarded_pr_message(&winner, None, true)
17595            .await
17596            .unwrap()
17597            .unwrap();
17598        assert_eq!(message.title, rewrite.title);
17599        assert_eq!(message.body, rewrite.body);
17600        assert_eq!(
17601            runner.state.github_text.as_ref().unwrap().choice,
17602            Some(GithubTextChoice::Rewrite)
17603        );
17604        let again = runner
17605            .guarded_pr_message(&winner, None, true)
17606            .await
17607            .unwrap()
17608            .unwrap();
17609        assert_eq!(again.title, message.title);
17610        assert_eq!(again.body, message.body);
17611        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17612        assert!(
17613            !runner
17614                .state
17615                .events
17616                .iter()
17617                .any(|e| e.message.contains(&rewrite.title)
17618                    || e.message.contains("English sentence."))
17619        );
17620    }
17621
17622    #[tokio::test]
17623    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17624        crate::run::pin_test_home();
17625        let dir = tempfile::tempdir().unwrap();
17626        let mut runner = runner_at(RunStatus::Gating);
17627        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17628        runner.state.repo = dir.path().to_owned();
17629        runner.state.candidates[0].worktree = dir.path().to_owned();
17630        seed_public_summary(&mut runner);
17631        let winner = runner.state.candidates[0].clone();
17632        assert!(
17633            runner
17634                .guarded_pr_message(&winner, None, true)
17635                .await
17636                .unwrap()
17637                .is_none()
17638        );
17639        let record = runner.state.github_text.as_ref().unwrap();
17640        assert!(record.rewrite.is_none());
17641        // Records written before the additive field deserialize with no rewrite.
17642        let mut old = serde_json::to_value(record).unwrap();
17643        old.as_object_mut().unwrap().remove("rewrite");
17644        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17645        assert!(old.rewrite.is_none());
17646        let id = record.question.clone().unwrap();
17647        ask::Questions::open()
17648            .update(&id, |q| {
17649                q.choices.push(crate::github_text::USE_REWRITE.into());
17650                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17651            })
17652            .unwrap();
17653        let message = runner
17654            .guarded_pr_message(&winner, None, true)
17655            .await
17656            .unwrap()
17657            .unwrap();
17658        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17659        assert_eq!(
17660            runner.state.github_text.as_ref().unwrap().choice,
17661            Some(GithubTextChoice::Fallback)
17662        );
17663    }
17664
17665    #[tokio::test]
17666    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17667        crate::run::pin_test_home();
17668        let dir = tempfile::tempdir().unwrap();
17669        let mut runner = runner_at(RunStatus::Gating);
17670        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17671        runner.state.repo = dir.path().to_owned();
17672        runner.state.candidates[0].worktree = dir.path().to_owned();
17673        seed_public_summary(&mut runner);
17674        let winner = runner.state.candidates[0].clone();
17675        assert!(
17676            runner
17677                .guarded_pr_message(&winner, None, true)
17678                .await
17679                .unwrap()
17680                .is_none()
17681        );
17682        let store = ask::Questions::open();
17683        let find = |run: &str| {
17684            store
17685                .list()
17686                .into_iter()
17687                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17688                .unwrap()
17689        };
17690        let run = runner.state.id.clone();
17691        let q = find(&run);
17692        let fp = runner
17693            .state
17694            .github_text
17695            .as_ref()
17696            .unwrap()
17697            .fingerprint
17698            .clone();
17699        let name = crate::github_text::artifact_name(&fp);
17700        let path = crate::run::artifact_path(&runner.state, &name);
17701        assert!(q.detail.contains(&path.display().to_string()));
17702        assert!(q.detail.contains("title-language: title"));
17703        let saved = std::fs::read_to_string(&path).expect("artifact written");
17704        assert!(saved.contains("日本語のタイトル"));
17705        assert!(q.cwd.is_none());
17706        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17707        // A bad replacement is refused and asked about once more.
17708        let say = |id: &str, text: &str, choice: &str| {
17709            store
17710                .update(id, |q| {
17711                    q.thread.push(ask::Turn {
17712                        who: ask::Who::Operator,
17713                        body: text.to_owned(),
17714                        at: jiff::Timestamp::now(),
17715                        note: None,
17716                    });
17717                    q.answer(ask::Answer::Choice(choice.to_owned()))
17718                })
17719                .unwrap();
17720        };
17721        say(&q.id, "まだ日本語", "use my text");
17722        assert!(
17723            runner
17724                .guarded_pr_message(&winner, None, true)
17725                .await
17726                .unwrap()
17727                .is_none()
17728        );
17729        let second = find(&run);
17730        assert_ne!(second.id, q.id);
17731        assert!(second.detail.contains("did not pass"));
17732        // The retry still compares the original text; the rejected
17733        // replacement is named by category only.
17734        assert!(second.detail.contains("日本語のタイトル"));
17735        assert!(!second.detail.contains("まだ日本語"));
17736        assert!(second.detail.contains(&path.display().to_string()));
17737        say(&second.id, "fix: retry failed requests", "use my text");
17738        let message = runner
17739            .guarded_pr_message(&winner, None, true)
17740            .await
17741            .unwrap()
17742            .expect("a vetted replacement is posted");
17743        assert_eq!(message.title, "fix: retry failed requests");
17744        assert_eq!(
17745            runner
17746                .state
17747                .github_text
17748                .as_ref()
17749                .unwrap()
17750                .chosen_title
17751                .as_deref(),
17752            Some("fix: retry failed requests")
17753        );
17754        assert!(runner.state.github_text.as_ref().unwrap().resolved);
17755        // Resumed again: the saved decision is applied, nothing is re-asked.
17756        let again = runner
17757            .guarded_pr_message(&winner, None, true)
17758            .await
17759            .unwrap()
17760            .unwrap();
17761        assert_eq!(again.title, message.title);
17762        assert_eq!(
17763            store
17764                .list()
17765                .into_iter()
17766                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17767                .count(),
17768            2
17769        );
17770    }
17771
17772    #[test]
17773    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17774        // What magi itself writes stays English under any configured language,
17775        // so a future localisation of these headings fails here. (The agents'
17776        // own text is also checked by the posting gate.)
17777        let mut state = state_with_summary(
17778            "add retries",
17779            "TITLE: fix(web): batch reads\n- reads run.json once",
17780        );
17781        state.config.graph.language = "ja".to_owned();
17782        let m = pr_message(&state, 'A');
17783        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17784        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17785
17786        // The task is the operator's own text: it goes in untouched, and the
17787        // fallback title (no summary) may be in its language too.
17788        let task = "今回やってほしいこと: results projector を直す";
17789        let mut state = state_with_summary(task, "- no title line");
17790        state.config.graph.language = "ja".to_owned();
17791        let m = pr_message(&state, 'A');
17792        assert_eq!(
17793            m.title,
17794            format!("chore: land candidate A of run {}", state.id)
17795        );
17796        assert!(
17797            m.body.contains(&format!(
17798                "<summary>Original task</summary>\n\n{task}\n\n</details>"
17799            )),
17800            "{}",
17801            m.body
17802        );
17803    }
17804
17805    #[test]
17806    fn pr_message_scrubs_home_paths_and_addresses() {
17807        let state = state_with_summary(
17808            "fix it in /Users/someone/src/x",
17809            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17810        );
17811        let m = pr_message(&state, 'A');
17812        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17813            assert!(!m.body.contains(leak), "{}", m.body);
17814        }
17815        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17816    }
17817
17818    #[test]
17819    fn pr_message_survives_a_task_that_closes_details() {
17820        let state = state_with_summary("a </details> b", "TITLE: fix: x");
17821        let m = pr_message(&state, 'A');
17822        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17823    }
17824
17825    #[test]
17826    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17827        let cmd = manual_merge_command(
17828            MergeStyle::Squash,
17829            Path::new("/repo"),
17830            "b",
17831            "fix: \"quoted\" $(x) `y`\n\nbody",
17832        );
17833        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17834    }
17835
17836    #[test]
17837    fn manual_merge_command_matches_the_configured_style() {
17838        let repo = Path::new("/repo");
17839        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17840
17841        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17842        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17843
17844        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17845        assert_eq!(
17846            squash,
17847            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17848             \"Merge magi run 0832 (candidate A)\""
17849        );
17850
17851        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17852        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17853    }
17854
17855    #[test]
17856    fn a_nudge_gets_a_quarter_of_the_budget() {
17857        // The judge and implement budgets magi ships with.
17858        assert_eq!(retry_budget(secs(1200), true), secs(300));
17859        assert_eq!(retry_budget(secs(3600), true), secs(900));
17860    }
17861
17862    #[test]
17863    fn a_resent_prompt_keeps_the_whole_budget() {
17864        // The seat kept no context, so the retry is the original job again and
17865        // shortening it would only guarantee a second failure.
17866        assert_eq!(retry_budget(secs(1200), false), secs(1200));
17867        assert_eq!(retry_budget(secs(60), false), secs(60));
17868    }
17869
17870    #[test]
17871    fn the_floor_never_exceeds_the_original_budget() {
17872        // A short configured timeout must not be *raised* by the floor: the
17873        // operator asked for a bound, and a retry may not outlast the attempt
17874        // it is retrying.
17875        assert_eq!(retry_budget(secs(60), true), secs(60));
17876        assert_eq!(retry_budget(secs(480), true), secs(120));
17877        assert_eq!(retry_budget(secs(0), true), secs(0));
17878    }
17879
17880    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17881        agent::CommandEvidence {
17882            id: "item1".to_owned(),
17883            description: "cargo test".to_owned(),
17884            exit_code,
17885            result_summary: String::new(),
17886            source: "codex".to_owned(),
17887        }
17888    }
17889
17890    #[test]
17891    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17892        // No evidence is not the same fact as unconfirmed evidence: a
17893        // backend with no adapter, or a reply that ran no commands at all,
17894        // must not be misread as carrying a dangling job.
17895        assert!(!has_unconfirmed_command(&[]));
17896    }
17897
17898    #[test]
17899    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17900        // Deliberately not a check on the exit code's *value*: a fixer
17901        // legitimately runs something that fails mid-iteration before it
17902        // succeeds, and that must never by itself reopen a valid report.
17903        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17904        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17905        assert!(!has_unconfirmed_command(&[
17906            evidence(Some(0)),
17907            evidence(Some(101))
17908        ]));
17909    }
17910
17911    #[test]
17912    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17913        assert!(has_unconfirmed_command(&[
17914            evidence(Some(0)),
17915            evidence(None)
17916        ]));
17917    }
17918
17919    #[test]
17920    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17921        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17922        assert_eq!(
17923            verified_noop_claim(true, &[], text).as_deref(),
17924            Some("already fixed by b32cfc4, on main.")
17925        );
17926    }
17927
17928    #[test]
17929    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17930        // A timeout or a bad exit code reads as the ordinary loss it is,
17931        // whatever the reply's own prose claims.
17932        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17933        assert!(verified_noop_claim(false, &[], text).is_none());
17934    }
17935
17936    #[test]
17937    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17938        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17939        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17940        // A confirmed command alongside the marker is fine.
17941        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17942    }
17943
17944    #[test]
17945    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17946        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17947    }
17948
17949    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
17950    /// pair, in order, labelled A, B, C, ...
17951    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17952        runner.state.candidates = shape
17953            .iter()
17954            .enumerate()
17955            .map(|(i, &(empty, verified))| Candidate {
17956                index: i,
17957                label: (b'A' + i as u8) as char,
17958                agent: "sonnet".to_owned(),
17959                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17960                worktree: PathBuf::from(format!("/wt/{i}")),
17961                summary: String::new(),
17962                stat: String::new(),
17963                files: 0,
17964                commits: 0,
17965                empty,
17966                failed: None,
17967                verified_noop: verified.map(str::to_owned),
17968                duration_ms: 0,
17969                folded: false,
17970            })
17971            .collect();
17972    }
17973
17974    #[test]
17975    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17976        ask_test_home();
17977        let mut runner = runner_at(RunStatus::Implementing);
17978        set_candidates(
17979            &mut runner,
17980            &[
17981                (true, Some("already on main at b32cfc4")),
17982                (true, Some("same fix, see the existing test")),
17983            ],
17984        );
17985
17986        runner
17987            .after_implement()
17988            .expect("a verified no-op is not an error");
17989
17990        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17991    }
17992
17993    #[test]
17994    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17995        ask_test_home();
17996        let mut runner = runner_at(RunStatus::Implementing);
17997        // Candidate A declares a verified no-op; candidate B simply wrote
17998        // nothing and said nothing about why. One candidate's claim is not
17999        // the whole run's agreement.
18000        set_candidates(
18001            &mut runner,
18002            &[(true, Some("already on main at b32cfc4")), (true, None)],
18003        );
18004
18005        let err = runner
18006            .after_implement()
18007            .expect_err("an unverified empty candidate must still fail the run");
18008
18009        assert!(
18010            err.to_string().contains("no candidate produced a change"),
18011            "{err}"
18012        );
18013        assert_eq!(runner.state.status, RunStatus::Failed);
18014    }
18015
18016    #[test]
18017    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18018        ask_test_home();
18019        let mut runner = runner_at(RunStatus::Implementing);
18020        set_candidates(&mut runner, &[(true, None), (true, None)]);
18021
18022        let err = runner
18023            .after_implement()
18024            .expect_err("no candidate declared anything; this is an ordinary failure");
18025
18026        assert!(
18027            err.to_string().contains("no candidate produced a change"),
18028            "{err}"
18029        );
18030        assert_eq!(runner.state.status, RunStatus::Failed);
18031    }
18032}
18033
18034/// A graft rollback whose restore could not be confirmed (see
18035/// [`Runner::graft`]).
18036#[derive(Debug)]
18037struct GraftRestoreFailed(String);
18038
18039impl std::fmt::Display for GraftRestoreFailed {
18040    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18041        write!(
18042            f,
18043            "could not restore the winner to its pre-graft commit: {}",
18044            self.0
18045        )
18046    }
18047}
18048
18049impl std::error::Error for GraftRestoreFailed {}
18050
18051/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18052/// not the agent: they never draw on an agent's own timeout, and running out of
18053/// them never reads as an agent failure.
18054#[cfg(test)]
18055mod resource_wait_tests {
18056    use super::*;
18057    use std::collections::BTreeMap;
18058
18059    fn sh(id: &str, script: &str) -> AgentSpec {
18060        AgentSpec {
18061            id: id.to_owned(),
18062            kind: crate::config::AgentKind::Command,
18063            model: None,
18064            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18065            extra_args: Vec::new(),
18066            env: BTreeMap::new(),
18067            prompt_delivery: None,
18068        }
18069    }
18070
18071    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18072        SeatJob {
18073            seat: SeatState::new("impl-A", &spec.id, 1),
18074            spec,
18075            cwd: dir.to_path_buf(),
18076            prompt: "do it".to_owned(),
18077            timeout,
18078            allow_write: true,
18079            sessions: false,
18080            artifacts: dir.join("art"),
18081            stem: "impl-A".to_owned(),
18082            handover: None,
18083        }
18084    }
18085
18086    fn state(resource_wait_secs: u64) -> RunState {
18087        let mut config = Config::default();
18088        config.graph.timeout_resource_wait = resource_wait_secs;
18089        RunState::new(
18090            PathBuf::from("/repo"),
18091            "main".to_owned(),
18092            "abc1234".to_owned(),
18093            "task".to_owned(),
18094            config,
18095        )
18096    }
18097
18098    fn hold(cache: &Path) -> crate::cache::Guard {
18099        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18100        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18101            crate::cache::AcquireOutcome::Acquired(g) => g,
18102            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18103        }
18104    }
18105
18106    async fn run(
18107        jobs: Vec<SeatJob>,
18108        sem: Arc<Semaphore>,
18109        cache: Option<&Path>,
18110        st: &mut RunState,
18111    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18112        let prompts = Prompts::default();
18113        let ctx = WaveCtx {
18114            run: "r",
18115            node: "implement",
18116            prompts: &prompts,
18117            cache,
18118            round: None,
18119            carry_seats: false,
18120            pauses: None,
18121        };
18122        wave(jobs, sem, &ctx, st, 0).await
18123    }
18124
18125    #[tokio::test]
18126    async fn verification_cache_wait_preserves_the_command_budget() {
18127        crate::run::pin_test_home();
18128        let tmp = tempfile::tempdir().unwrap();
18129        let cache = tmp.path().join("target");
18130        let guard = hold(&cache);
18131        let release = tokio::spawn(async move {
18132            tokio::time::sleep(Duration::from_millis(100)).await;
18133            guard.release();
18134        });
18135        let mut st = state(10);
18136        let budget = Duration::from_millis(50);
18137        let wait = repository_wait(&st, None);
18138        let (outcomes, _) = with_cache_lease(
18139            &mut st,
18140            Some(&cache),
18141            "gate",
18142            "gate",
18143            tmp.path(),
18144            "abc1234",
18145            budget,
18146            "test gate",
18147            &wait,
18148            |_, available| async move {
18149                assert_eq!(
18150                    available, budget,
18151                    "contention must not shorten verification"
18152                );
18153                (Vec::new(), false, Vec::new())
18154            },
18155        )
18156        .await
18157        .unwrap();
18158        release.await.unwrap();
18159        assert!(outcomes.is_empty(), "{outcomes:?}");
18160        assert!(
18161            st.events
18162                .iter()
18163                .any(|event| event.message.contains("waiting for"))
18164        );
18165        assert!(st.resource_wait.is_none());
18166        // The verification body no longer owns the lease after returning.
18167        hold(&cache).release();
18168    }
18169
18170    #[tokio::test]
18171    async fn verification_cache_wait_expires_without_starting_commands() {
18172        crate::run::pin_test_home();
18173        let tmp = tempfile::tempdir().unwrap();
18174        let cache = tmp.path().join("target");
18175        let _held = hold(&cache);
18176        let mut st = state(0);
18177        let wait = repository_wait(&st, None);
18178        let (outcomes, _) = with_cache_lease(
18179            &mut st,
18180            Some(&cache),
18181            "gate",
18182            "gate",
18183            tmp.path(),
18184            "abc1234",
18185            Duration::from_secs(60),
18186            "test gate",
18187            &wait,
18188            |_, _| async { panic!("commands must not start without the cache lease") },
18189        )
18190        .await
18191        .unwrap();
18192        assert_eq!(outcomes.len(), 1);
18193        assert!(outcomes[0].resource_blocked);
18194        assert!(st.resource_wait.is_none());
18195    }
18196
18197    #[tokio::test]
18198    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18199        crate::run::pin_test_home();
18200        let tmp = tempfile::tempdir().unwrap();
18201        let cache = tmp.path().join("target-a");
18202        let guard = hold(&cache);
18203        // Held for longer than the agent's whole 1s budget.
18204        tokio::spawn(async move {
18205            tokio::time::sleep(Duration::from_millis(1500)).await;
18206            guard.release();
18207        });
18208        let mut st = state(60);
18209        let results = run(
18210            vec![job(
18211                sh("alpha", "echo ok"),
18212                tmp.path(),
18213                Duration::from_secs(1),
18214            )],
18215            Arc::new(Semaphore::new(1)),
18216            Some(&cache),
18217            &mut st,
18218        )
18219        .await;
18220        assert!(
18221            matches!(results[0].2, AgentOutcome::Ok(_)),
18222            "the wait must not be charged to the agent"
18223        );
18224        assert!(st.resource_block.is_none());
18225        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18226        assert!(st.resource_wait.is_none() && st.active.is_empty());
18227    }
18228
18229    #[tokio::test]
18230    #[cfg(unix)]
18231    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18232        use std::os::fd::AsRawFd as _;
18233        crate::run::pin_test_home();
18234        let tmp = tempfile::tempdir().unwrap();
18235        crate::git::git(tmp.path(), &["init"]).await.unwrap();
18236        let common = tmp.path().join(".git");
18237        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18238        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18239        assert_eq!(
18240            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18241            0
18242        );
18243        let mut st = state(1);
18244        st.config.coordination = crate::config::test_coordination();
18245        let mut reader = job(
18246            sh("reader", "echo must-not-run"),
18247            tmp.path(),
18248            Duration::from_secs(30),
18249        );
18250        reader.allow_write = false;
18251        let results = run(
18252            vec![reader.clone()],
18253            Arc::new(Semaphore::new(1)),
18254            None,
18255            &mut st,
18256        )
18257        .await;
18258        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18259        assert_eq!(results[0].1.turns, 0);
18260        assert!(FailClass::of(&results[0].2).is_none());
18261        assert_eq!(
18262            st.resource_block.as_ref().unwrap().kind,
18263            "repository coordination"
18264        );
18265        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18266        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18267        drop(external);
18268        let editor = crate::repo_lock::SeatLease::for_checkout(
18269            tmp.path(),
18270            true,
18271            crate::config::test_coordination().as_ref(),
18272        )
18273        .await
18274        .unwrap();
18275        let pause = Pause::new();
18276        let interrupt = Pause::new();
18277        let prompts = Prompts::default();
18278        let ctx = WaveCtx {
18279            run: "r",
18280            node: "review",
18281            prompts: &prompts,
18282            cache: None,
18283            round: None,
18284            carry_seats: false,
18285            pauses: Some((&pause, &interrupt)),
18286        };
18287        let (results, ()) = tokio::join!(
18288            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18289            async {
18290                tokio::time::sleep(Duration::from_millis(50)).await;
18291                pause.park();
18292            }
18293        );
18294        assert!(
18295            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18296        );
18297        assert_eq!(results[0].1.turns, 0);
18298        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18299        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18300        drop(editor);
18301    }
18302
18303    #[tokio::test]
18304    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18305        crate::run::pin_test_home();
18306        let tmp = tempfile::tempdir().unwrap();
18307        let cache = tmp.path().join("target-b");
18308        let _held = hold(&cache);
18309        let mut st = state(1);
18310        let results = run(
18311            vec![job(
18312                sh("alpha", "echo ok"),
18313                tmp.path(),
18314                Duration::from_secs(1),
18315            )],
18316            Arc::new(Semaphore::new(1)),
18317            Some(&cache),
18318            &mut st,
18319        )
18320        .await;
18321        let (_, seat, out) = &results[0];
18322        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18323        assert!(wave_blocked(&results));
18324        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18325        assert_eq!(seat.turns, 0, "nothing was asked");
18326        assert!(
18327            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18328            "no agent was spawned"
18329        );
18330        assert!(FailClass::of(out).is_none(), "not a failure class");
18331        let block = st.resource_block.as_ref().expect("recorded for the resume");
18332        assert_eq!(block.kind, "shared build cache");
18333        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18334        assert!(st.active.is_empty());
18335    }
18336
18337    #[tokio::test]
18338    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18339        crate::run::pin_test_home();
18340        let tmp = tempfile::tempdir().unwrap();
18341        let mut st = state(1);
18342        let results = run(
18343            vec![job(
18344                sh("alpha", "echo ok"),
18345                tmp.path(),
18346                Duration::from_secs(1),
18347            )],
18348            Arc::new(Semaphore::new(0)),
18349            None,
18350            &mut st,
18351        )
18352        .await;
18353        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18354        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18355        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18356    }
18357
18358    #[tokio::test]
18359    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18360        crate::run::pin_test_home();
18361        let tmp = tempfile::tempdir().unwrap();
18362        let mut st = state(60);
18363        let results = run(
18364            vec![job(
18365                sh("alpha", "sleep 30"),
18366                tmp.path(),
18367                Duration::from_secs(1),
18368            )],
18369            Arc::new(Semaphore::new(1)),
18370            None,
18371            &mut st,
18372        )
18373        .await;
18374        let out = &results[0].2;
18375        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18376        let class = FailClass::of(out).expect("a failure");
18377        assert_eq!(class, FailClass::Timeout);
18378        assert!(should_hand_over(None, &class), "existing handover policy");
18379        assert!(st.resource_block.is_none());
18380    }
18381}