Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54    Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58    ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62/// How much verification output is kept and fed back to the fixer.
63const OUTPUT_TAIL: usize = 8_000;
64
65/// Bytes of a failing command's output kept in an event, so the reason a run
66/// stopped is readable from the report without opening `run.json`.
67const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
70/// command's pid before releasing the build cache's lease.
71const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
74/// command's pid to actually exit before giving up and releasing anyway.
75///
76/// A timeout means the process was asked to die (`kill_on_drop`,
77/// `start_kill`), not that it already has — on Windows in particular that can
78/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
79/// the instant the command returns would let the very next acquirer (this
80/// run's own next round, another run's verification, the janitor's prune)
81/// start touching the same directory while it might still be writing to it,
82/// so this polls the actual pid — real confirmation, not a fixed guess —
83/// until it is gone or this ceiling is reached. It is still not full
84/// process-tree reaping: a grandchild the timed-out process spawned and that
85/// outlives it independently is invisible to a pid check, and continuing to
86/// observe and collect *that* stays a different piece of work with its own
87/// owner. Set generously because the common case returns early the moment
88/// the pid is confirmed gone, not because every timeout pays this in full.
89const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91/// Consecutive review rounds with no tree progress (see
92/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
93/// instead of spending the rest of the round budget.
94///
95/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
96/// legitimately finds nothing left to change (its previous round's fix already
97/// covered it, and this round's reviewers re-raised only nits) looks the same
98/// as one that is spinning, for exactly one round. Two in a row is where the
99/// two stop being distinguishable, and a review round on this workload has
100/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
101/// third attempt at a tree that has not moved twice running is pure cost.
102/// This does not touch `review_rounds` itself, which stays the operator's
103/// call.
104pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
107/// a base that moved before giving up and leaving the run `Blocked` for a
108/// person.
109///
110/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
111/// that keeps moving faster than a run can catch it is not something more
112/// rebasing fixes, it is a person's call. Not the same *number as*
113/// `land_rounds` - this budget is spent before a pull request exists, land's
114/// after - but bounded for the identical reason, so it uses the same
115/// default. Counted across both call sites in [`Runner::finish_after_tally`]
116/// (once before review, once before the gate), because either one finding
117/// the base still moving is the same signal.
118const BASE_SYNC_ROUNDS: usize = 4;
119
120/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
121/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
122/// reply held no [`FixReport`].
123///
124/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
125/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
126/// "I'll pause here until the `cargo make check` background run reports
127/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
128/// No `FixReport` was ever collected from that seat, and the run moved on to
129/// the next review round regardless.
130///
131/// Bounded independently of `review_rounds` and `graph.retries`: this
132/// recovers one seat's missing report mid-round, not a new round of review or
133/// an ordinary parse retry, and must not itself become the unbounded wait the
134/// rest of this module exists to avoid.
135const MAX_FIX_CONTINUATIONS: usize = 2;
136
137/// One queued agent invocation.
138///
139/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
140/// CLI hung up on its own stream is asked again from the same job rather than
141/// rebuilt from scratch. See [`Runner::resume_undelivered`].
142#[derive(Clone)]
143struct SeatJob {
144    spec: AgentSpec,
145    seat: SeatState,
146    cwd: PathBuf,
147    prompt: String,
148    timeout: Duration,
149    allow_write: bool,
150    sessions: bool,
151    artifacts: PathBuf,
152    stem: String,
153    /// The prompt for a seat that has been handed to another roster agent
154    /// (a fresh session): everything the original seat would have
155    /// remembered. `None` when `prompt` already carries it, as the first
156    /// ranking and the implement prompt do. Never a resume-style prompt.
157    handover: Option<String>,
158}
159
160/// How the graph reads one agent invocation.
161///
162/// Quota is split out from an ordinary failure on purpose: a rate-limited call
163/// is known to fail again if retried now, so the retry loop must not spend an
164/// attempt on it. `Dropped` is split out for the opposite reason: unlike
165/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
166/// error JSON, never the agent's answer — a caller that matched only
167/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
168/// left to read that JSON as if it were usable output. `resume_undelivered`
169/// is the only caller that acts on it; everywhere else it is reported like an
170/// ordinary failure.
171enum AgentOutcome {
172    /// A usable output.
173    Ok(AgentOutput),
174    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
175    Quota(AgentOutput),
176    /// The CLI hung up on its own stream after billed work. See
177    /// [`agent::AgentOutput::work_undelivered`].
178    Dropped(AgentOutput),
179    /// Any other failure: a timeout, a bad exit code, an empty reply.
180    Failed(String),
181    /// The agent was never spawned: a shared resource it needs (the build
182    /// cache's lease) stayed busy past `[graph] timeout_resource_wait`, or the
183    /// wait was abandoned because the run is parking. Not a failure of the
184    /// agent: it must not advance a fallback chain, enter seat history, cost
185    /// quota or spend an attempt. [`RunState::resource_block`] says why.
186    ResourceBlocked(String),
187}
188
189/// A request to park the run at its next node boundary.
190///
191/// Cloning is how the request travels: the loop keeps one handle and hands a
192/// clone to each [`Runner`], and every clone points at the same flag. There
193/// is no channel because there is nothing to send - the only message is
194/// "park", it is idempotent, and a flag cannot be missed by a receiver that
195/// was not listening yet.
196///
197/// The boundary is what makes this cheap. Every node writes the run's state
198/// before the next one starts, and every node skips what is already recorded:
199/// `prep` returns early once candidates exist, `implement` asks only the seats
200/// with nothing on disk, `judge` returns early once judgements exist. So a
201/// parked run resumes into exactly the node it stopped before, and no agent
202/// work is thrown away. Killing the process mid-node, by contrast, loses
203/// whatever the seats in flight had not yet written - which for an implement
204/// wave is an hour of paid work.
205///
206/// A [`Runner`] watches two independent handles of this type - see
207/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
208/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
209/// clone covering the whole daemon's lifetime and is never asked to un-park,
210/// which is correct exactly because nothing is dispatched after it fires.
211/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
212/// that parks for an interrupted task must go on to run other tasks
213/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
214/// reusing the daemon-wide one.
215#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219    /// A pause nobody has asked for yet.
220    #[must_use]
221    pub fn new() -> Self {
222        Self::default()
223    }
224
225    /// Ask the run to park at its next node boundary. Idempotent.
226    pub fn park(&self) {
227        self.0.store(true, Ordering::SeqCst);
228    }
229
230    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
231    /// fold into the run's own `park` event - so an operator reading the run
232    /// later knows this was a deliberate interrupt rather than a shutdown or
233    /// a binary swap. The first reason recorded wins; a park already in
234    /// flight is not relabelled by a second, unrelated request.
235    pub fn park_because(&self, reason: impl Into<String>) {
236        self.park_for(ParkCause::Interrupt {
237            reason: reason.into(),
238        });
239    }
240
241    /// [`Pause::park`], recording the structured cause. The first cause
242    /// recorded wins, like [`Pause::park_because`].
243    pub fn park_for(&self, cause: ParkCause) {
244        let mut reason_guard = self
245            .1
246            .lock()
247            .unwrap_or_else(std::sync::PoisonError::into_inner);
248        if reason_guard.is_none() {
249            *reason_guard = Some(cause);
250        }
251        drop(reason_guard);
252        self.park();
253    }
254
255    /// Has a park been asked for?
256    #[must_use]
257    pub fn parked(&self) -> bool {
258        self.0.load(Ordering::SeqCst)
259    }
260
261    /// Why the park was asked for, when the caller used [`Pause::park_because`].
262    #[must_use]
263    pub fn reason(&self) -> Option<String> {
264        self.cause().map(|c| c.label())
265    }
266
267    /// The structured cause, when one was recorded.
268    #[must_use]
269    pub fn cause(&self) -> Option<ParkCause> {
270        self.1
271            .lock()
272            .unwrap_or_else(std::sync::PoisonError::into_inner)
273            .clone()
274    }
275}
276
277/// Drives one run.
278pub struct Runner {
279    /// Run state; public so the CLI can report on it.
280    pub state: RunState,
281    roles: ResolvedRoles,
282    sem: Arc<Semaphore>,
283    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
284    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
285    /// this is never the same handle as `interrupt`.
286    pause: Pause,
287    /// Set when `magi serve`'s interrupt scheduler wants this specific run
288    /// parked at its next node boundary, to let a task marked
289    /// [`crate::queue::Task::interrupt`] run alone before this one carries
290    /// on. Unlike `pause`, a fresh, unshared handle per run - see
291    /// [`Runner::watch_interrupt`].
292    interrupt: Pause,
293}
294
295/// Where the branch's own commits start: its merge base with the base branch
296/// as the remote has it now, else with the recorded `base_commit`. A branch
297/// rebased onto a base that moved past `base_commit` would otherwise count
298/// the base's commits as its own under `base_commit..branch`.
299async fn review_base(
300    repo: &Path,
301    remote: &str,
302    base_branch: &str,
303    base_commit: &str,
304    branch: &str,
305) -> String {
306    review_base_checked(repo, remote, base_branch, base_commit, branch)
307        .await
308        .0
309}
310
311/// [`review_base`] plus whether the base was read from a freshly fetched
312/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
313/// never older than `base_commit`'s view of the base), but the answer is then
314/// not trusted to rewrite a pull request's title.
315async fn review_base_checked(
316    repo: &Path,
317    remote: &str,
318    base_branch: &str,
319    base_commit: &str,
320    branch: &str,
321) -> (String, bool) {
322    let tracking = format!("{remote}/{base_branch}");
323    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324    if git::rev_exists(repo, &tracking).await
325        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326        && !mb.is_empty()
327    {
328        return (mb, fresh);
329    }
330    let mb = git::merge_base(repo, base_commit, branch)
331        .await
332        .ok()
333        .filter(|mb| !mb.is_empty())
334        .unwrap_or_else(|| base_commit.to_owned());
335    (mb, false)
336}
337
338/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
339/// when git cannot say or finds nothing: a stale list is better than a wrong
340/// or empty one.
341pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342    if !is_review_run(state) {
343        return;
344    }
345    let base = review_base(
346        &state.repo,
347        &state.config.merge.remote,
348        &state.base_branch,
349        &state.base_commit,
350        branch,
351    )
352    .await;
353    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354        && !subjects.is_empty()
355        && state.reviewed_commits.as_ref() != Some(&subjects)
356    {
357        state.reviewed_commits = Some(subjects);
358        state.save().ok();
359    }
360}
361
362/// Subjects of the base's commits between the recorded start and the branch's
363/// merge base: what a stale `base_commit..branch` would have mistaken for the
364/// branch's own work.
365async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366    let (base, trusted) = review_base_checked(
367        &state.repo,
368        &state.config.merge.remote,
369        &state.base_branch,
370        &state.base_commit,
371        branch,
372    )
373    .await;
374    if !trusted {
375        return None;
376    }
377    git::subjects(&state.repo, &state.base_commit, &base)
378        .await
379        .ok()
380}
381
382/// May an adopted pull request's title be replaced with `computed`? Only when
383/// it is empty, magi's own shape, or a base commit's subject that leaked in;
384/// a title a person wrote stays. Never when `computed` is itself a leak.
385pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387    if is_leak(computed) {
388        return false;
389    }
390    let cur = current.trim();
391    cur.is_empty()
392        || cur.starts_with(REVIEW_PROMPT_OPENING)
393        || cur.starts_with("chore: land candidate")
394        || cur.starts_with("magi: candidate")
395        || is_leak(cur)
396}
397
398/// The commit a run branches from: the base branch as the remote has it.
399///
400/// Two failures this replaces. A run used to branch off `HEAD` and so refused
401/// to start on a dirty tree, which made `magi serve` decline every task for as
402/// long as the operator had work in progress - most of the time. Branching off
403/// the *local* base branch fixed that and introduced a worse one: `land` merges
404/// the winner on GitHub, nothing updates the local ref, and the next run
405/// branches off a base missing everything the previous runs landed. Two tasks
406/// in a row from a phone would have had the second silently re-implementing
407/// against stale code and opening a pull request that reverted the first.
408///
409/// Only refs move here - no checkout, no local branch, no merge - so it is safe
410/// with uncommitted work in the tree. A machine with no network still starts:
411/// the fetch may fail and the local tip is used with a warning, because
412/// refusing to run offline is a worse failure than running against a base the
413/// operator can see for themselves.
414///
415/// One function, called by both entry points. Two answers to "where does a run
416/// branch from" is the kind of drift nobody notices until a diff is wrong.
417/// Bring the local `branch` in line with `<remote>/<branch>` before a review
418/// checks it out.
419///
420/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
421/// anything other than plain `git push` from this checkout (a jj colocated
422/// workspace, another clone) moves only the remote-tracking ref - so the local
423/// one can be a stale placeholder. It moves only when local is behind the remote or is an
424/// empty placeholder that diverged from it; unpushed local work is kept, and a real
425/// divergence is refused rather than guessed at.
426async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427    let tracking = format!("{remote}/{branch}");
428    let fetched = git::fetch(repo, remote, branch).await;
429    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430    let local_exists = git::branch_exists(repo, branch).await?;
431    if !fresh {
432        if !local_exists {
433            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434        }
435        tracing::warn!(
436            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437        );
438        return Ok(());
439    }
440    let remote_sha = git::rev_parse(repo, &tracking).await?;
441    if !local_exists {
442        git::git(repo, &["branch", branch, &tracking]).await?;
443        return Ok(());
444    }
445    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447        return Ok(());
448    }
449    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450        // Diverged. `reconcile` settles it only when it can prove nothing is
451        // lost: a local tip that is the remote's change rebased is pushed over
452        // it (lease pinned to the tip read here), a tip whose every extra
453        // commit is empty is a placeholder the remote's work replaced, and
454        // anything else is two different changes - a question for a person.
455        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456            .await?
457        {
458            crate::reconcile::Reconciliation::Pushed => {
459                tracing::warn!(
460                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461                    short(&local_sha),
462                    short(&remote_sha)
463                );
464                return Ok(());
465            }
466            crate::reconcile::Reconciliation::Placeholder => {}
467            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468        }
469    }
470    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471    if !out.ok() {
472        bail!(
473            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474            short(&local_sha),
475            short(&remote_sha),
476            out.stderr
477        );
478    }
479    tracing::warn!(
480        "local `{branch}` was stale: fast-forwarded {} -> {}",
481        short(&local_sha),
482        short(&remote_sha)
483    );
484    Ok(())
485}
486
487async fn resolve_base(
488    repo: &Path,
489    base_branch: &str,
490    remote: &str,
491    coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493    let _lease = crate::repo_lock::SeatLease::operation_with(
494        repo,
495        coordination,
496        &crate::repo_lock::WaitPolicy::current(),
497    )
498    .await?;
499    let tracking = format!("{remote}/{base_branch}");
500    let fetched = git::fetch(repo, remote, base_branch).await;
501    if let Ok(out) = &fetched
502        && out.ok()
503        && git::rev_exists(repo, &tracking).await
504    {
505        return Ok(crate::run::FetchedBase {
506            remote: remote.to_owned(),
507            branch: base_branch.to_owned(),
508            commit: git::rev_parse(repo, &tracking).await?,
509            fetched_at: jiff::Timestamp::now(),
510        });
511    }
512    let why = match &fetched {
513        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514        Ok(_) => format!("{remote} has no {base_branch}"),
515        Err(e) => e.to_string(),
516    };
517    // No fallback to the local branch: it may be behind, and branching off an
518    // old commit is the stale-checkout bug this check exists to prevent.
519    bail!(
520        "cannot read {tracking} ({why}); refusing to branch off the local \
521         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522         base / remote in magi.toml. This is launcher preparation: check the \
523         launcher's Git administration access, network and repository coordination \
524         before retrying; an unattended seat cannot approve those permissions"
525    )
526}
527
528/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
529/// on an early return or a panic.
530///
531/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
532/// manual `magi fix` invocations against the same run are otherwise
533/// invisible to each other and would race to remove and recreate the same
534/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
535/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
536/// which is only ever reclaimed later, out of band, by
537/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
538/// `magi fix` invocation is not necessarily running under either of those, so
539/// nothing would ever sweep a lock a killed or crashed process left behind.
540/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
541/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
542/// lease use: an unreadable or unparsable pid, or a liveness query the
543/// platform cannot answer, reads as alive and the lock is left in place.
544struct FixClaim {
545    path: PathBuf,
546}
547
548impl FixClaim {
549    fn acquire(dir: &Path) -> Result<Self> {
550        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551        let path = dir.join("fix.lock");
552        match Self::create(&path) {
553            Ok(claim) => Ok(claim),
554            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555                if Self::reclaim_if_dead(&path) {
556                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
557                } else {
558                    bail!(
559                        "another repair or `magi fix` is already running for this run ({} exists)",
560                        path.display()
561                    )
562                }
563            }
564            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565        }
566    }
567
568    fn create(path: &Path) -> std::io::Result<Self> {
569        let mut f = std::fs::OpenOptions::new()
570            .write(true)
571            .create_new(true)
572            .open(path)?;
573        use std::io::Write as _;
574        // Read back by `reclaim_if_dead` on a later, stuck invocation.
575        writeln!(f, "{}", std::process::id())?;
576        Ok(Self {
577            path: path.to_owned(),
578        })
579    }
580
581    /// True if the lock named a process confirmed dead, in which case it was
582    /// also removed. Never true on an unreadable file, an unparsable pid, or
583    /// a liveness query the platform cannot answer — see this type's own doc.
584    fn reclaim_if_dead(path: &Path) -> bool {
585        let dead = std::fs::read_to_string(path)
586            .ok()
587            .and_then(|body| body.trim().parse::<u32>().ok())
588            .is_some_and(|pid| !crate::proc::pid_alive(pid));
589        dead && std::fs::remove_file(path).is_ok()
590    }
591}
592
593impl Drop for FixClaim {
594    fn drop(&mut self) {
595        let _ = std::fs::remove_file(&self.path);
596    }
597}
598
599/// Holds the mutable runner as well as its durable lease: cancellation must
600/// clear both copies of ownership, including state a caller may save later.
601struct DriverExecution<'a> {
602    runner: &'a mut Runner,
603    driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607    fn drop(&mut self) {
608        self.driver.release_local(&mut self.runner.state);
609    }
610}
611
612impl Runner {
613    /// Start a fresh run against `repo`.
614    pub async fn start(
615        repo: &Path,
616        instruction: String,
617        config: Config,
618        origin: Origin,
619    ) -> Result<Self> {
620        Self::start_naming(repo, instruction, "", config, origin).await
621    }
622
623    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
624    /// searched for branch and commit references along with the instruction,
625    /// since a task may name the work it is about only in its title.
626    pub async fn start_naming(
627        repo: &Path,
628        instruction: String,
629        also_scan: &str,
630        config: Config,
631        origin: Origin,
632    ) -> Result<Self> {
633        let repo = git::toplevel(repo).await?;
634        let missing = agent::missing_programs(&config.agents);
635        if !missing.is_empty() {
636            bail!(
637                "these agent programs are not on PATH: {}. Fix the roster in \
638                 magi.toml or install them.",
639                missing.join(", ")
640            );
641        }
642        let base_branch =
643            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644                .await?;
645        let fetched_base = resolve_base(
646            &repo,
647            &base_branch,
648            &config.merge.remote,
649            config.coordination.as_ref(),
650        )
651        .await?;
652        let base_commit = fetched_base.commit.clone();
653        let roles = config.resolve_roles()?;
654        let max_parallel = config.graph.max_parallel.max(1);
655        // A task that points at work already in the repository starts from
656        // it; what the repository says about each reference is recorded.
657        let seeds = refs::resolve(
658            &repo,
659            &base_commit,
660            &config.merge.remote,
661            &format!("{also_scan}\n{instruction}"),
662        )
663        .await;
664        refs::plan(&repo, &seeds).await?;
665        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666        // Recorded before the first save, so a crash right after minting
667        // cannot leave a run with no origin. Legibility only: nothing reads it
668        // to decide anything.
669        state.fetched_base = Some(fetched_base);
670        state.origin = Some(origin);
671        for seed in &seeds {
672            state.event(
673                "seed",
674                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675            );
676        }
677        state.seeds = seeds;
678        state.event("start", format!("run {} created", state.id));
679        state.save()?;
680        Ok(Self {
681            state,
682            roles,
683            sem: Arc::new(Semaphore::new(max_parallel)),
684            pause: Pause::new(),
685            interrupt: Pause::new(),
686        })
687    }
688
689    /// Open a review-only run against work that already exists on `branch`.
690    ///
691    /// The expensive half of the graph is the implement wave — measured at
692    /// 111 and 134 internal tool-loop turns on this repository, against a
693    /// handful for a judge or a reviewer. The cheap half is worth running on
694    /// hand-written work too, and there was no way to reach it.
695    ///
696    /// No new state and no schema change are needed: a run with **one** viable
697    /// candidate and a tally already decided degrades `execute` to exactly
698    /// review → gate → merge, because `judge` skips a single-candidate field,
699    /// `deliberate` has fewer than two first choices to reconcile, `vote`
700    /// returns early, `tally` is already present and `fold_losers` has no
701    /// losers. Resuming such a run therefore does the right thing as well.
702    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703        Self::review_taking_over(repo, branch, config, None, origin).await
704    }
705
706    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
707    /// at the same task still has `branch` checked out, its worktree is
708    /// released first if that is safe (see [`crate::handover`]), and the
709    /// review refuses with the reason if it is not. `None` is a hand-run
710    /// review: it has no earlier attempts, so only a worktree of a dead run
711    /// magi recorded itself can be released.
712    pub async fn review_taking_over(
713        repo: &Path,
714        branch: &str,
715        config: Config,
716        takeover: Option<crate::handover::Takeover>,
717        origin: Origin,
718    ) -> Result<Self> {
719        let repo = git::toplevel(repo).await?;
720        let missing = agent::missing_programs(&config.agents);
721        if !missing.is_empty() {
722            bail!(
723                "these agent programs are not on PATH: {}. Fix the roster in \
724                 magi.toml or install them.",
725                missing.join(", ")
726            );
727        }
728        let base_branch =
729            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730                .await?;
731        if base_branch == branch {
732            bail!("`{branch}` is the base branch; there is nothing to review against");
733        }
734        let fetched_base = resolve_base(
735            &repo,
736            &base_branch,
737            &config.merge.remote,
738            config.coordination.as_ref(),
739        )
740        .await?;
741        let base_commit = fetched_base.commit.clone();
742
743        let roles = config.resolve_roles()?;
744        let max_parallel = config.graph.max_parallel.max(1);
745        let mut state = RunState::new(
746            repo.clone(),
747            base_branch,
748            base_commit.clone(),
749            String::new(),
750            config,
751        );
752        state.fetched_base = Some(fetched_base);
753        state.origin = Some(origin);
754
755        // Released before anything else touches the branch: a stale local
756        // branch is moved with `git branch -f`, which git refuses while an
757        // earlier attempt's worktree still has it checked out. Everything
758        // after this point that can fail puts the old run back.
759        // A hand-run review has no task, hence no earlier attempts, but a
760        // worktree of a dead run magi made may still be released.
761        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762            earlier: Vec::new(),
763            home: crate::run::home(),
764            choice: None,
765            task_instruction: None,
766        });
767        let wait = repository_wait(&state, None);
768        let coordination_config = state.config.coordination.clone();
769        let coordination =
770            crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771                .await?;
772        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773        if let Some(released) = &released {
774            state.event(
775                "release",
776                format!(
777                    "took `{branch}` over from run {}: its worktree was released: {}",
778                    crate::run::short_of(&released.old_id),
779                    released.audit
780                ),
781            );
782        }
783        // The owner's answer to an earlier divergence question is applied
784        // here: after the release (git will not move a checked-out branch)
785        // and before the sync that would otherwise ask again.
786        if let Some(choice) = takeover.choice.as_ref()
787            && let Err(e) =
788                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789                    .await
790        {
791            if let Some(released) = &released {
792                released.restore(&repo, branch).await;
793            }
794            return Err(e.context("applying the owner's answer about the diverged branch"));
795        }
796        // An earlier attempt of this task that published the task head, and
797        // whose work is the branch under review, hands that head over: the
798        // review run updates the same pull request. Without one (a legacy
799        // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
800        // own head and its own open pull request is adopted as before.
801        for id in takeover.earlier.iter().rev() {
802            if let Ok(old) = RunState::load_under(id, &takeover.home)
803                && let Some(head) = old.pr_head.clone()
804                && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805            {
806                state.pr_head = Some(head);
807                break;
808            }
809        }
810        drop(coordination);
811        let opened = Self::open_review(
812            &repo,
813            branch,
814            state,
815            roles,
816            max_parallel,
817            base_commit,
818            &takeover,
819        )
820        .await;
821        if opened.is_err()
822            && let Some(released) = &released
823        {
824            let _coordination = crate::repo_lock::SeatLease::operation_with(
825                &repo,
826                coordination_config.as_ref(),
827                &wait,
828            )
829            .await?;
830            released.restore(&repo, branch).await;
831        }
832        opened
833    }
834
835    /// Recover exactly the operator-selected public implementation, without touching old sources.
836    pub async fn recover_pr(
837        repo: &Path,
838        task: &str,
839        selection: crate::pr_recovery::Selection,
840        mut config: Config,
841    ) -> Result<Self> {
842        selection.configure(&mut config)?;
843        selection.check_task(task, &selection.instruction)?;
844        let base_branch =
845            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
846                .await?;
847        let fetched_base = resolve_base(
848            repo,
849            &base_branch,
850            &config.merge.remote,
851            config.coordination.as_ref(),
852        )
853        .await?;
854        let base_commit = fetched_base.commit.clone();
855        selection
856            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
857            .await?;
858        selection.verify_source(repo).await?;
859        let roles = config.resolve_roles()?;
860        let max_parallel = config.graph.max_parallel.max(1);
861        let mut state = RunState::new(
862            repo.to_path_buf(),
863            base_branch,
864            base_commit.clone(),
865            selection.instruction.clone(),
866            config,
867        );
868        state.fetched_base = Some(fetched_base);
869        state.origin = Some(crate::run::Origin::queue(task));
870        state.pr_head = Some(selection.branch.clone());
871        state.pr_recovery = Some(selection.clone());
872        crate::pr_head::check_predecessors(&state, &selection.branch).await?;
873        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
874    }
875
876    async fn open_selected(
877        repo: &Path,
878        state: RunState,
879        roles: ResolvedRoles,
880        max_parallel: usize,
881        base_commit: String,
882        selection: crate::pr_recovery::Selection,
883    ) -> Result<Self> {
884        let mut state = state;
885        let branch = state.branch_for('A');
886        let wait = repository_wait(&state, None);
887        let coordination = coordinate_operation(&mut state, repo, &wait).await?;
888        git::git(repo, &["branch", &branch, selection.source()]).await?;
889        drop(coordination);
890        state.event(
891            "recovery",
892            format!(
893                "selected implementation {} for {} (public lease {})",
894                selection.source(),
895                selection.url,
896                selection.head
897            ),
898        );
899        let takeover = crate::handover::Takeover {
900            earlier: Vec::new(),
901            home: crate::run::home(),
902            choice: None,
903            task_instruction: Some(selection.instruction.clone()),
904        };
905        state.save()?;
906        let id = state.id.clone();
907        let opened = Self::open_review(
908            repo,
909            &branch,
910            state,
911            roles,
912            max_parallel,
913            base_commit,
914            &takeover,
915        )
916        .await;
917        match opened {
918            Ok(runner) => Ok(runner),
919            Err(e) => {
920                if let Ok(mut partial) = RunState::load(&id) {
921                    partial.status = RunStatus::Blocked;
922                    partial.driver_exited = true;
923                    partial.event(
924                        "recovery",
925                        format!("startup failed; retained source and candidate {branch}: {e:#}"),
926                    );
927                    partial.save()?;
928                }
929                Err(crate::pr_recovery::StartupFailed {
930                    run: id,
931                    cause: format!("{e:#}"),
932                }
933                .into())
934            }
935        }
936    }
937
938    /// What the newest earlier attempt that reviewed `branch` leaves for this
939    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
940    /// no earlier attempts. Reads the earlier record only: ownership of its
941    /// worktree and branch is [`crate::handover`]'s business.
942    async fn inherit(
943        repo: &Path,
944        branch: &str,
945        base_commit: &str,
946        takeover: &crate::handover::Takeover,
947    ) -> Option<crate::recovery::RecoveryRecord> {
948        use crate::recovery::{Relation, assess};
949        let newest = takeover.earlier.last()?;
950        let mut chosen = None;
951        for id in takeover.earlier.iter().rev() {
952            if let Ok(old) = RunState::load_under(id, &takeover.home)
953                && !old.reviews.is_empty()
954                && (old.winner().is_some_and(|w| w.branch == branch)
955                    || old.pr_head.as_deref() == Some(branch))
956            {
957                chosen = Some(old);
958                break;
959            }
960        }
961        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
962            .await
963            .unwrap_or_default();
964        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
965            Some(last) if last.head == tip => Relation::Same,
966            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
967            _ => Relation::Diverged,
968        };
969        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
970        Some(assess(from, chosen.as_ref(), base_commit, relation))
971    }
972
973    /// The half of [`Runner::review_taking_over`] that can fail after an
974    /// earlier attempt's worktree was released.
975    async fn open_review(
976        repo: &Path,
977        branch: &str,
978        mut state: RunState,
979        roles: ResolvedRoles,
980        max_parallel: usize,
981        base_commit: String,
982        takeover: &crate::handover::Takeover,
983    ) -> Result<Self> {
984        let wait = repository_wait(&state, None);
985        let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
986        if state.pr_recovery.is_none() {
987            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
988        }
989        // The commit subjects are the closest thing to a task statement that
990        // existing work carries, and the reviewers are told as much.
991        let start = review_base(
992            repo,
993            &state.config.merge.remote,
994            &state.base_branch,
995            &base_commit,
996            branch,
997        )
998        .await;
999        let log = git::log_oneline(repo, &start, branch)
1000            .await
1001            .unwrap_or_default();
1002        let log_text = if log.trim().is_empty() {
1003            "(no commit messages)"
1004        } else {
1005            log.trim()
1006        };
1007        let mut instruction = format!(
1008            "Review the work already on branch `{branch}`. There is no task \
1009             statement: what the change claims to do is whatever its commits \
1010             say.\n\n{log_text}"
1011        );
1012        // A retry of a task whose earlier attempt left reviews behind carries
1013        // that attempt's evidence, and the task's own statement with it: the
1014        // completion criteria are the task's, not whatever the commits claim.
1015        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1016            if let Some(task) = takeover
1017                .task_instruction
1018                .as_deref()
1019                .filter(|t| !t.trim().is_empty())
1020                && rec.reviewed_head.is_some()
1021            {
1022                instruction = format!(
1023                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1024                    task.trim()
1025                );
1026                rec.criteria_inherited = true;
1027            }
1028            state.event("recovery", rec.summary());
1029            state.recovery = Some(rec);
1030        }
1031        if let Some(selected) = &state.pr_recovery {
1032            instruction = selected.instruction.clone();
1033        }
1034        state.task_instruction = takeover.task_instruction.clone();
1035        state.instruction = instruction;
1036        state.reviewed_commits = Some(
1037            git::subjects(repo, &start, branch)
1038                .await
1039                .unwrap_or_default(),
1040        );
1041
1042        // An attached worktree, so the fixer's commits land on the branch under
1043        // review rather than on a detached head nobody will look at again.
1044        let worktree = state.worktree_root().join("under-review");
1045        if let Some(parent) = worktree.parent() {
1046            tokio::fs::create_dir_all(parent).await.ok();
1047        }
1048        let path = worktree.to_string_lossy().to_string();
1049        git::git(repo, &["worktree", "add", &path, branch])
1050            .await
1051            .with_context(|| {
1052                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1053            })?;
1054
1055        let setup = match ensure_setup_config(&mut state, repo).await {
1056            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1057            Err(e) => Err(e),
1058        };
1059        if let Err(e) = setup {
1060            git::worktree_remove_clean(repo, &worktree).await.ok();
1061            return Err(e);
1062        }
1063        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1064            .await
1065            .unwrap_or(0);
1066        if commits == 0 {
1067            git::worktree_remove_clean(repo, &worktree).await.ok();
1068            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1069        }
1070        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1071            .await
1072            .map(|f| f.len())
1073            .unwrap_or(0);
1074        if files == 0
1075            && let (Ok(head_tree), Ok(base_tree)) = (
1076                git::tree_of(&worktree, "HEAD").await,
1077                git::tree_of(&worktree, &base_commit).await,
1078            )
1079            && head_tree == base_tree
1080        {
1081            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1082            git::worktree_remove_clean(repo, &worktree).await.ok();
1083            bail!(
1084                "`{branch}` at {} has a tree identical to base {}; this usually means \
1085                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1086                 against `{}/{branch}`) rather than an empty change",
1087                short(&head),
1088                short(&base_commit),
1089                state.config.merge.remote
1090            );
1091        }
1092        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1093            .await
1094            .unwrap_or_default();
1095
1096        state.candidates.push(Candidate {
1097            index: 0,
1098            label: 'A',
1099            // Not an agent id on purpose: nothing in the roster wrote this, and
1100            // the stats tables must not credit anyone with a win for it.
1101            agent: EXISTING_BRANCH.to_owned(),
1102            branch: branch.to_owned(),
1103            worktree,
1104            summary: String::new(),
1105            stat,
1106            files,
1107            commits,
1108            empty: false,
1109            failed: None,
1110            verified_noop: None,
1111            duration_ms: 0,
1112            folded: false,
1113        });
1114        state.tally = Some(Tally {
1115            first_choice: BTreeMap::from([('A', 0)]),
1116            borda: BTreeMap::new(),
1117            winner: 'A',
1118            rankings: 0,
1119            unanimous_initial: false,
1120            deliberated: false,
1121            changed_votes: 0,
1122            unanimous_final: false,
1123            tie_break: None,
1124            // No panel sat, so no quorum applies. Zero judges is the correct
1125            // number for work that never competed, and must not be reported as
1126            // a collapsed panel.
1127            judges: 0,
1128            present: 0,
1129            quorum: 0,
1130            met_quorum: true,
1131            uncontested: Some("review-only run: nothing competed".to_owned()),
1132        });
1133        state.status = RunStatus::Reviewing;
1134        state.prep_complete = Some(true);
1135        state.event(
1136            "start",
1137            format!(
1138                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1139                state.id
1140            ),
1141        );
1142        state.save()?;
1143        Ok(Self {
1144            state,
1145            roles,
1146            sem: Arc::new(Semaphore::new(max_parallel)),
1147            pause: Pause::new(),
1148            interrupt: Pause::new(),
1149        })
1150    }
1151
1152    /// Reopen an existing run.
1153    pub fn resume(id: &str) -> Result<Self> {
1154        let state = RunState::load(id)?;
1155        crate::pr_head::ensure_owner(&state)?;
1156        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1157            bail!(
1158                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1159            );
1160        }
1161        if let Some(to) = &state.released_to {
1162            bail!(
1163                "run {} cannot be resumed: its worktree was released to run {}",
1164                state.short(),
1165                crate::run::short_of(to)
1166            );
1167        }
1168        let roles = state.config.resolve_roles()?;
1169        if let Some(id) = &state.config.graph.selected_implementer
1170            && state.candidates.iter().any(|c| c.agent != *id)
1171        {
1172            bail!(
1173                "replacement implementer pin does not match the saved candidates; task must stay held"
1174            );
1175        }
1176        let max_parallel = state.config.graph.max_parallel.max(1);
1177        Ok(Self {
1178            state,
1179            roles,
1180            sem: Arc::new(Semaphore::new(max_parallel)),
1181            pause: Pause::new(),
1182            interrupt: Pause::new(),
1183        })
1184    }
1185
1186    /// Walk the graph to a terminal state, skipping nodes already recorded.
1187    ///
1188    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1189    /// the phone - ends here, so this is the one place a run that ended
1190    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1191    /// notification centre. Best-effort: see [`crate::notices::raise`].
1192    pub async fn execute(&mut self) -> Result<()> {
1193        // Queue claims cover daemon work; this also serializes manual drivers
1194        // consuming a repair answer, including the new review and decision.
1195        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1196            || self.state.land_approval.is_some()
1197            || self.state.merge_decision.is_some()
1198            || !self.state.merge_repairs.is_empty()
1199            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1200        {
1201            let claim = FixClaim::acquire(&self.state.dir())?;
1202            let disk = RunState::load(&self.state.id)?;
1203            let checkpoint = |s: &RunState| {
1204                (
1205                    s.merge_repairs.len(),
1206                    s.merge_repairs
1207                        .last()
1208                        .map(|r| (r.question.clone(), r.phase)),
1209                    s.land_approval
1210                        .as_ref()
1211                        .map(|a| (a.question.clone(), a.head.clone())),
1212                )
1213            };
1214            if checkpoint(&disk) != checkpoint(&self.state) {
1215                bail!(
1216                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1217                );
1218            }
1219            Some(claim)
1220        } else {
1221            None
1222        };
1223        let mut execution = self.claim_driver()?;
1224        let runner = &mut *execution.runner;
1225        let wait = runner.repository_wait();
1226        // Keep the graph's large future off the callers' stack, including
1227        // standalone task attempts running on a default-sized test thread.
1228        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1229        let result = match result {
1230            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1231                record_repository_block(&mut runner.state, &e);
1232                runner.park_here()?;
1233                Ok(())
1234            }
1235            other => other,
1236        };
1237        execution.driver.finish(&mut runner.state)?;
1238        let ended = if result.is_err() {
1239            Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1240        } else {
1241            crate::notices::run_ended(&runner.state)
1242        };
1243        if let Some(notice) = ended {
1244            crate::notices::raise(notice);
1245        }
1246        result
1247    }
1248
1249    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1250        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1251    }
1252
1253    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1254        let wait = self.repository_wait();
1255        coordinate_operation(&mut self.state, cwd, &wait).await
1256    }
1257
1258    /// Validate ownership and persist the driver claim in one critical section.
1259    fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1260        let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1261        let disk = RunState::load(&self.state.id)?;
1262        crate::pr_head::ensure_owner(&disk)?;
1263        if disk.liveness(false) == crate::run::Liveness::Live {
1264            bail!("run already has a live driver; wait for its execution to end");
1265        }
1266        if let Some(to) = &disk.released_to {
1267            bail!(
1268                "run {} cannot continue: its worktree was released to run {}",
1269                self.state.short(),
1270                crate::run::short_of(to)
1271            );
1272        }
1273        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1274        let mut execution = DriverExecution {
1275            runner: self,
1276            driver,
1277        };
1278        // Declare after the execution guard: on any claim error this short
1279        // lock must drop before that guard reloads and releases ownership.
1280        let _owner = owner;
1281        let state = &mut execution.runner.state;
1282        // Moving again, so it is no longer parked. Set before the walk rather
1283        // than in `resume`, so every way of re-entering the graph clears it
1284        // and a card cannot claim a run is waiting to be resumed while the
1285        // agents are already working.
1286        state.parked = false;
1287        state.resource_wait = None;
1288        state.resource_block = None;
1289        // Any seat this state still lists as answering belongs to whatever
1290        // process last drove this run — this one included, if it crashed
1291        // mid-wave. Cleared and flushed immediately, before anything else
1292        // runs, so a resume can never show a seat as live when nothing is
1293        // asking it anything yet; the node that actually dispatches the next
1294        // wave repopulates it.
1295        state.clear_active();
1296        // Recorded in the same spot, and flushed together with the clear
1297        // above: this is the pid a reader checks (`RunState::liveness`) when
1298        // no daemon claim exists to answer "is a process still driving this
1299        // run" — a plain `magi run` / `magi review` typed into a terminal
1300        // claims nothing there. Always overwritten, never only-if-absent, so
1301        // a resumed run's stale pid from a previous, possibly-dead process
1302        // can never survive into this one's own report. Unlike
1303        // `clear_active`, this changes on every single `execute()` call, so
1304        // the save below is now unconditional rather than only-if-cleared.
1305        //
1306        // `driver_started_at` is recorded in the same breath, from this same
1307        // pid, so `liveness` can tell a live pid that is genuinely still us
1308        // apart from one the OS has since handed to an unrelated process —
1309        // see that field's own doc for why the pid alone is not enough.
1310        let pid = std::process::id();
1311        state.driver_pid = Some(pid);
1312        state.driver_started_at = crate::proc::process_started_at(pid);
1313        state.driver_exited = false;
1314        state.begin_pass();
1315        execution.driver.bind(state);
1316        state.save()?;
1317        execution.driver.record(state)?;
1318        Ok(execution)
1319    }
1320
1321    async fn execute_graph(&mut self) -> Result<()> {
1322        self.state.configure_recovery_resume()?;
1323        // Preservation may invalidate an old empty classification. Resolve it
1324        // before any resume shortcut can reuse a verdict or publish a branch.
1325        self.persist_candidates().await?;
1326        if self
1327            .state
1328            .merge_decision
1329            .as_ref()
1330            .is_some_and(|decision| !decision.closed)
1331            && self.state.status != RunStatus::Landing
1332        {
1333            if crate::merge_decision::resumable_hold(&self.state) {
1334                self.state.merge = Some(MergeOutcome {
1335                    mode: MergeMode::Pr,
1336                    ok: false,
1337                    detail: self
1338                        .state
1339                        .merge_decision
1340                        .as_ref()
1341                        .unwrap()
1342                        .target
1343                        .pr
1344                        .clone(),
1345                    empty: false,
1346                });
1347                self.run_land().await?;
1348                self.settle_questions();
1349                return Ok(());
1350            }
1351            bail!(
1352                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1353            );
1354        }
1355        if self.state.config.graph.selected_implementer.is_some()
1356            && self.state.status == RunStatus::Blocked
1357            && self.state.tally.is_none()
1358            && self.state.candidate_preservation.is_empty()
1359        {
1360            bail!(
1361                "selected implementation is held; explicitly release the owning task to retry it"
1362            );
1363        }
1364
1365        if self.state.merge_repairs.last().is_some_and(|r| {
1366            matches!(
1367                r.phase,
1368                crate::run::MergeRepairPhase::Requested
1369                    | crate::run::MergeRepairPhase::Fixing
1370                    | crate::run::MergeRepairPhase::Reviewing
1371            )
1372        }) {
1373            if self.repair_merge_review().await? {
1374                self.run_land().await?;
1375            }
1376            self.settle_questions();
1377            return Ok(());
1378        }
1379        // Publication already passed review and gate. Resume only that node:
1380        // syncing to a newer base here could change the verified commit.
1381        if self.state.pr_publication_pending {
1382            self.merge().await?;
1383            if self.resource_halt()? {
1384                return Ok(());
1385            }
1386            self.state.save()?;
1387            self.settle_questions();
1388            return Ok(());
1389        }
1390        // A run that already lost its quorum never resumes into the verdict
1391        // machinery: `deliberate` and `vote` would otherwise clobber the
1392        // stalled marker back to Voting and the run would keep going past a
1393        // verdict that is no longer trustworthy. Everything already recorded is
1394        // kept, so the run stays resumable (or foldable) for a human to pick up.
1395        //
1396        // On --resume the run gets one chance to repair itself: the seats a
1397        // rate limit took out are re-asked. If their quota has since reset and
1398        // the quorum is restored, the run picks up and finishes; otherwise it
1399        // stays stale and still-resumable for a later retry. If it does not
1400        // recover, the returned status stays `Stalled` and nothing was
1401        // clobbered (the recovery only mutates entries for the lost seats).
1402        if self.state.status == RunStatus::Stalled {
1403            if self.recover_stall().await? {
1404                self.finish_after_tally().await?;
1405            } else {
1406                // Still below quorum: persist the marker and stay resumable.
1407                self.state.save()?;
1408            }
1409            return Ok(());
1410        }
1411        // A run parked inside `land` - watching CI, mid fix-round, or
1412        // waiting on the owner's merge approval - resumes directly into it,
1413        // never back through `prep`. Everything before `merge` already
1414        // concluded; that is the only way `status` reaches `Landing` in the
1415        // first place. Re-walking `review_loop` first would also be actively
1416        // wrong: its own status recomputation (see its doc) treats any
1417        // clean round as reason to set `status` to `Gating`, which would
1418        // clobber this marker before `merge` ever ran, and this run would
1419        // never find its way back into `land` at all.
1420        if self.state.status == RunStatus::Landing {
1421            self.run_land().await?;
1422            // `run_land` may have settled the run right here - CI came back
1423            // green and the PR merged, say - without ever passing back
1424            // through `merge`'s own trailing call. Whatever it left `status`
1425            // as is what this has to read.
1426            self.settle_questions();
1427            return Ok(());
1428        }
1429        // A run parked at `merge` on the owner's word about a withheld PR text
1430        // resumes straight into `merge`. Re-walking the tail would sync to the
1431        // base again (possibly rebasing, possibly spending the fixer) after the
1432        // gate had passed, and then push a tree nobody reviewed or gated.
1433        if self.state.github_text.is_some()
1434            && self.state.merge.is_none()
1435            && self.state.status == RunStatus::Gating
1436        {
1437            self.merge().await?;
1438            if self.resource_halt()? {
1439                return Ok(());
1440            }
1441            self.state.save()?;
1442            return Ok(());
1443        }
1444        if let Err(e) = self.prep().await {
1445            if e.is::<crate::repo_lock::ResourceBlocked>() {
1446                return Err(e);
1447            }
1448            self.state.status = RunStatus::Blocked;
1449            self.state
1450                .prep_failure
1451                .get_or_insert_with(|| format!("{e:#}"));
1452            self.state
1453                .event("prep", format!("preparation stopped: {e:#}"));
1454            self.state.save()?;
1455            return Err(e);
1456        }
1457        if self.park_here()? {
1458            return Ok(());
1459        }
1460        self.advise().await?;
1461        if self.park_here()? {
1462            return Ok(());
1463        }
1464        self.implement().await?;
1465        if self.park_here()? {
1466            return Ok(());
1467        }
1468        // `after_implement` already saved the state and settled any open
1469        // questions when it set this; nothing later in the graph has
1470        // anything to judge.
1471        if self.state.status == RunStatus::VerifiedNoop {
1472            return Ok(());
1473        }
1474        self.judge().await?;
1475        if self.park_here()? {
1476            return Ok(());
1477        }
1478        self.deliberate().await?;
1479        if self.park_here()? {
1480            return Ok(());
1481        }
1482        self.vote().await?;
1483        if self.park_here()? {
1484            return Ok(());
1485        }
1486        self.tally()?;
1487        // A verdict that lost its quorum is not trustworthy: do not review,
1488        // gate, or merge on it. Everything already done is kept, so the run
1489        // stays resumable (or foldable); the human can replace the agent that
1490        // ran out of quota and pick it up.
1491        if self.state.status == RunStatus::Stalled {
1492            // Persist the stalled marker now — the normal end-of-execute save
1493            // below is below this early return, and without it a resumed run
1494            // would reload a pre-tally status and keep going.
1495            self.state.save()?;
1496            return Ok(());
1497        }
1498        self.finish_after_tally().await?;
1499        Ok(())
1500    }
1501
1502    /// A wave could not start its agents because a shared resource stayed busy
1503    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
1504    /// resumable. Unstarted seats have no result; completed peers retain
1505    /// their saved candidate state.
1506    fn stop_for_resource(&mut self) -> Result<()> {
1507        // A wait abandoned because the run is parking stays a park.
1508        if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1509        {
1510            self.state.status = RunStatus::Blocked;
1511        }
1512        self.state.save()
1513    }
1514
1515    /// Stop walking the graph if a node ended on a resource block.
1516    fn resource_halt(&mut self) -> Result<bool> {
1517        if self.state.resource_block.is_none() {
1518            return Ok(false);
1519        }
1520        self.stop_for_resource()?;
1521        Ok(true)
1522    }
1523
1524    /// Park here if asked to, recording it in the run's own timeline.
1525    ///
1526    /// Returns whether the caller should stop walking the graph. The state is
1527    /// saved either way by the node that just finished; this adds the event so
1528    /// the operator's card says why a run that is neither finished nor moving
1529    /// is sitting where it is.
1530    fn park_here(&mut self) -> Result<bool> {
1531        let parking = self.pause.parked() || self.interrupt.parked();
1532        if !parking && self.state.resource_block.is_some() {
1533            self.stop_for_resource()?;
1534            return Ok(true);
1535        }
1536        // Either handle asking is enough - see `Pause`'s own doc for why
1537        // they are never the same one. `interrupt` is checked second so a
1538        // reason it carries is preferred in the message below over a plain
1539        // shutdown park racing it at the same boundary.
1540        if !self.pause.parked() && !self.interrupt.parked() {
1541            return Ok(false);
1542        }
1543        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1544        let why = match cause.as_ref().map(ParkCause::label) {
1545            Some(reason) => format!(
1546                "parked after `{}` ({reason}) — resume to carry on from here",
1547                self.state.status.as_str()
1548            ),
1549            None => format!(
1550                "parked after `{}` — resume to carry on from here",
1551                self.state.status.as_str()
1552            ),
1553        };
1554        self.state.event("park", why);
1555        self.state
1556            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1557        self.state.parked = true;
1558        self.state.save()?;
1559        Ok(true)
1560    }
1561
1562    /// Only a new explicit operator approval may retry failed preparation.
1563    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1564        if self.state.prep_failure.is_none() {
1565            return Ok(());
1566        }
1567        let failed_at = self
1568            .state
1569            .events
1570            .iter()
1571            .rev()
1572            .find(|event| event.node == "prep")
1573            .map_or(self.state.updated_at, |event| event.at);
1574        anyhow::ensure!(
1575            approval_at > failed_at,
1576            "preparation retry requires a new explicit release after the failure"
1577        );
1578        self.state.prep_failure = None;
1579        self.state.prep_complete = Some(false);
1580        self.state
1581            .event("prep-retry", "operator approved retry of preparation");
1582        Ok(())
1583    }
1584
1585    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1586    pub fn on_pause(&mut self, pause: Pause) {
1587        self.pause = pause;
1588    }
1589
1590    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1591    /// scheduler asking this one run - and no other - to park so a task
1592    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1593    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1594    /// handle.
1595    pub fn watch_interrupt(&mut self, pause: Pause) {
1596        self.interrupt = pause;
1597    }
1598
1599    /// Abandon this run's own open questions, once `status` has actually
1600    /// settled rather than merely paused.
1601    ///
1602    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1603    /// either back up with the candidates, the review round and the seat
1604    /// sessions already on disk, so a question an implementer asked mid-round
1605    /// may still get a real answer read by a real resume. Only the statuses
1606    /// `resumable` excludes are actually final: the run merged, it reached
1607    /// `Ready` with nothing left to do, it failed outright with no
1608    /// established point to continue from, or every candidate agreed, with
1609    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1610    /// every one of those the seat that asked is gone for good, exactly like
1611    /// the run being deleted under `magi run rm` - so the same cleanup
1612    /// applies, worded for what actually happened instead of "the run was
1613    /// deleted".
1614    ///
1615    /// Best-effort and silent on success: called from every place `status`
1616    /// can land on one of those three, including ones a resumed run revisits,
1617    /// so it must cost nothing when there was nothing open to begin with.
1618    fn settle_questions(&mut self) {
1619        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1620            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1621        }
1622    }
1623
1624    /// The tail of the graph after a trustworthy tally: fold losers, review,
1625    /// gate, merge, and persist.
1626    async fn finish_after_tally(&mut self) -> Result<()> {
1627        // Before the losers are folded: their branches are what the graft
1628        // reads from.
1629        self.graft().await?;
1630        if self.resource_halt()? {
1631            return Ok(());
1632        }
1633        self.fold_losers().await?;
1634        // Before review starts, and again right before the gate: a run's
1635        // review rounds can themselves take long enough for the base to move
1636        // a second time, and the gate is the one node whose "green" gets
1637        // acted on.
1638        self.sync_to_base().await?;
1639        if self.state.status == RunStatus::AlreadyInBase {
1640            return Ok(());
1641        }
1642        self.review_loop().await?;
1643        if self.resource_halt()? {
1644            return Ok(());
1645        }
1646        self.sync_to_base().await?;
1647        if self.state.status == RunStatus::AlreadyInBase {
1648            return Ok(());
1649        }
1650        self.gate().await?;
1651        self.merge().await?;
1652        if self.resource_halt()? {
1653            return Ok(());
1654        }
1655        self.state.save()?;
1656        Ok(())
1657    }
1658
1659    // ---------------------------------------------------------------- prep
1660
1661    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
1662    /// run with the step and its output in the event; it is never swallowed.
1663    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
1664    /// resume starts over instead of finding candidates and missing seats.
1665    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1666        let result = match ensure_setup_config(&mut self.state, repo).await {
1667            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1668            Err(e) => Err(e),
1669        };
1670        if let Err(e) = result {
1671            git::worktree_remove_clean(repo, wt).await.ok();
1672            if in_prep {
1673                self.rollback_prep(repo).await;
1674            }
1675            self.state.status = RunStatus::Blocked;
1676            self.state.event(
1677                "setup",
1678                format!("worktree setup failed in {}: {e:#}", wt.display()),
1679            );
1680            self.state.save()?;
1681            return Err(e);
1682        }
1683        Ok(())
1684    }
1685
1686    /// Undo everything `prep` created, so a resume re-runs it whole.
1687    async fn rollback_prep(&mut self, repo: &Path) {
1688        let root = self.state.worktree_root();
1689        for c in std::mem::take(&mut self.state.candidates) {
1690            if matches!(
1691                git::worktree_remove_clean(repo, &c.worktree).await,
1692                Ok(true)
1693            ) {
1694                git::branch_delete(repo, &c.branch).await.ok();
1695            } else {
1696                self.state.candidates.push(c);
1697            }
1698        }
1699        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1700        for j in 1..=seats {
1701            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1702                .await
1703                .ok();
1704        }
1705        for k in 1..=self.state.config.graph.advisors {
1706            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1707                .await
1708                .ok();
1709        }
1710    }
1711
1712    async fn prep(&mut self) -> Result<()> {
1713        if let Some(reason) = &self.state.prep_failure {
1714            bail!(
1715                "candidate preparation previously failed: {reason}; correct the source before retrying"
1716            );
1717        }
1718        if self
1719            .state
1720            .prep_complete
1721            .unwrap_or(!self.state.candidates.is_empty())
1722        {
1723            return Ok(());
1724        }
1725        let repo = self.state.repo.clone();
1726        // Includes rollback, config, refs, checkout creation and setup. No
1727        // repository mutation may precede the real shared lock acquisition.
1728        let _coordination = self.coordinate(&repo).await?;
1729        self.rollback_prep(&repo).await;
1730        if !self.state.candidates.is_empty() {
1731            bail!(
1732                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1733            );
1734        }
1735        self.state.prep_complete = Some(false);
1736        self.state.status = RunStatus::Prep;
1737        self.state.save()?;
1738        let repo = self.state.repo.clone();
1739        let base = self.state.base_commit.clone();
1740        let plan = refs::plan(&repo, &self.state.seeds).await?;
1741        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1742        let root = self.state.worktree_root();
1743        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1744
1745        // The hook is the write-time half of the blindness contract; the
1746        // presentation filter in `blind` is the half that cannot be bypassed.
1747        let hooks_dir = self.state.dir().join("hooks");
1748        if self.state.config.blind.commit_msg_hook {
1749            std::fs::create_dir_all(&hooks_dir)
1750                .with_context(|| format!("create {}", hooks_dir.display()))?;
1751            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1752            let path = hooks_dir.join("commit-msg");
1753            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1754            make_executable(&path)?;
1755            // Ref-counted rather than a plain idempotent set: with more than
1756            // one run able to be in flight in the same repository at once
1757            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1758            // true?" check cannot tell "another run of mine still needs
1759            // this" from "nobody does", and the run that happens to finish
1760            // first would disable the hook out from under a sibling still
1761            // relying on it.
1762            git::acquire_worktree_config(&repo).await?;
1763            self.state.enabled_worktree_config = true;
1764        }
1765
1766        // `[worktree] setup` hides its products per worktree, which needs the
1767        // same per-worktree config the hook does. Held until fold, like it.
1768        ensure_setup_config(&mut self.state, &repo).await?;
1769
1770        for (index, (spec, label)) in self
1771            .roles
1772            .implementers
1773            .clone()
1774            .into_iter()
1775            .zip(labels)
1776            .enumerate()
1777        {
1778            let branch = self.state.branch_for(label);
1779            let worktree = root.join(format!("cand-{label}"));
1780            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1781            if self.state.config.blind.commit_msg_hook {
1782                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1783            }
1784            git::local_exclude(&worktree, "/.magi/").await?;
1785            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1786                // Nothing of this prep may survive: a resume must start over,
1787                // not find half the candidates and skip the rest.
1788                git::worktree_remove_clean(&repo, &worktree).await.ok();
1789                git::branch_delete(&repo, &branch).await.ok();
1790                self.rollback_prep(&repo).await;
1791                self.state.status = RunStatus::Blocked;
1792                self.state
1793                    .event("prep", format!("worktree setup failed: {e:#}"));
1794                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1795                self.state.save()?;
1796                return Err(e);
1797            }
1798            self.state.candidates.push(Candidate {
1799                index,
1800                label,
1801                agent: spec.id.clone(),
1802                branch,
1803                worktree,
1804                summary: String::new(),
1805                stat: String::new(),
1806                files: 0,
1807                commits: 0,
1808                empty: false,
1809                failed: None,
1810                verified_noop: None,
1811                duration_ms: 0,
1812                folded: false,
1813            });
1814            self.state.save()?;
1815            for pick in &plan.picks {
1816                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1817                {
1818                    self.state.status = RunStatus::Blocked;
1819                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1820                    self.state
1821                        .event("prep", format!("cannot apply referenced commit: {e}"));
1822                    self.state.save()?;
1823                    return Err(e);
1824                }
1825            }
1826        }
1827
1828        for j in 1..=self.roles.judges.len() {
1829            let wt = root.join(format!("judge-{j}"));
1830            if !wt.exists() {
1831                git::worktree_add_detached(&repo, &wt, &base).await?;
1832                self.setup_seat_worktree(&repo, &wt, true).await?;
1833            }
1834        }
1835
1836        // Disposable, detached checkouts for the design-deliberation stage's
1837        // advisor seats — the same shape as the judges' above, at the same
1838        // base commit, since advisors also only ever read. Sized off the
1839        // configured count directly rather than a resolved roster: unlike
1840        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1841        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1842        // `prep` has no `ResolvedRoles` field to read a count from here.
1843        if self.state.config.graph.advise {
1844            for k in 1..=self.state.config.graph.advisors {
1845                let wt = root.join(format!("advisor-{k}"));
1846                if !wt.exists() {
1847                    git::worktree_add_detached(&repo, &wt, &base).await?;
1848                    self.setup_seat_worktree(&repo, &wt, true).await?;
1849                }
1850            }
1851        }
1852
1853        // A judge cannot tell it is looking at its own patch — the seats keep
1854        // separate conversations — but a panel that shares agents with the
1855        // field is less independent than it looks, and that is worth saying out
1856        // loud once per run rather than leaving it in the config.
1857        let authors: Vec<&str> = self
1858            .roles
1859            .implementers
1860            .iter()
1861            .map(|a| a.id.as_str())
1862            .collect();
1863        let overlap: Vec<String> = self
1864            .roles
1865            .judges
1866            .iter()
1867            .enumerate()
1868            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1869            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1870            .collect();
1871        if !overlap.is_empty() {
1872            let note = format!(
1873                "{} also authored a candidate; blind, but the panel is less \
1874                 independent than {} distinct agents would be",
1875                overlap.join(", "),
1876                self.roles.judges.len()
1877            );
1878            self.state.event("prep", note);
1879        }
1880
1881        self.state.event(
1882            "prep",
1883            format!(
1884                "{} candidates, {} judges, base {} ({})",
1885                self.state.candidates.len(),
1886                self.roles.judges.len(),
1887                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1888                self.state.base_branch
1889            ),
1890        );
1891        self.state.prep_complete = Some(true);
1892        self.state.status = RunStatus::Implementing;
1893        self.state.save()?;
1894        Ok(())
1895    }
1896
1897    // -------------------------------------------------------------- advise
1898
1899    /// The design-deliberation stage: independent, read-only advisor seats
1900    /// each sketch a design before any implementer touches the repository,
1901    /// and (when at least one produced a usable proposal) a synthesis seat
1902    /// blends them into a brief `implement` carries in every candidate's
1903    /// prompt.
1904    ///
1905    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1906    /// advisors` is the proposal count. Everything here is best-effort and
1907    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1908    /// that cannot reach quota, or a synthesis seat that produced nothing
1909    /// usable all leave `implement` exactly as it was before this stage
1910    /// existed — the task instruction alone — rather than failing the whole
1911    /// competition over an enrichment stage. Every outcome is still recorded
1912    /// as an event, so a run that got nothing from this stage says why.
1913    ///
1914    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1915    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1916    /// resumed run whose stage failed would re-run it, and re-spend the
1917    /// agent calls, on every reentry before `implement`.
1918    ///
1919    /// Also skipped once any candidate shows implementation progress — the
1920    /// exact predicate `implement` itself uses to decide a candidate is no
1921    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1922    /// is not enough: a run created by an older binary that predates this
1923    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1924    /// an already-`Implementing`-or-later run under this build would
1925    /// otherwise walk straight back through `prep` (a no-op once candidates
1926    /// exist) into this node and spawn every advisor seat against worktrees
1927    /// `prep` never recreated — after implementation has already started,
1928    /// which is exactly the invariant this stage exists to guarantee.
1929    async fn advise(&mut self) -> Result<()> {
1930        let implement_untouched = self
1931            .state
1932            .candidates
1933            .iter()
1934            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1935        if !self.state.config.graph.advise || self.state.advise_attempted {
1936            return Ok(());
1937        }
1938        if !implement_untouched {
1939            self.state.event(
1940                "advise",
1941                "skipping the design-deliberation stage: at least one \
1942                 candidate already shows implementation progress, so this \
1943                 run is past the point the stage exists to run before"
1944                    .to_owned(),
1945            );
1946            self.state.advise_attempted = true;
1947            self.state.save()?;
1948            return Ok(());
1949        }
1950        let run_id = self.state.id.clone();
1951        let prompts = self.state.config.prompts.clone();
1952        let instruction = self.state.instruction.clone();
1953        let language = self.state.config.graph.language.clone();
1954        let root = self.state.worktree_root();
1955        let n = self.state.config.graph.advisors;
1956        let where_recorded = self.state.dir().join("run.json");
1957
1958        let seats = match self.state.config.advisors() {
1959            Ok(seats) if !seats.is_empty() => seats,
1960            Ok(_) => {
1961                self.state.event(
1962                    "advise",
1963                    format!(
1964                        "[graph] advisors is 0; skipping the design-deliberation \
1965                         stage and continuing without a synthesis brief (see {})",
1966                        where_recorded.display()
1967                    ),
1968                );
1969                self.state.advise_attempted = true;
1970                self.state.save()?;
1971                return Ok(());
1972            }
1973            Err(e) => {
1974                self.state.event(
1975                    "advise",
1976                    format!(
1977                        "could not resolve advisor seats ({e:#}); continuing \
1978                         without a design-deliberation brief (see {})",
1979                        where_recorded.display()
1980                    ),
1981                );
1982                self.state.advise_attempted = true;
1983                self.state.save()?;
1984                return Ok(());
1985            }
1986        };
1987
1988        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1989        let artifacts = agent::artifacts_dir(&self.state.dir());
1990        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1991
1992        let mut jobs = Vec::new();
1993        for (i, spec) in seats.iter().cloned().enumerate() {
1994            let seat_key = format!("advisor-{}", i + 1);
1995            let seat = self.seat(&seat_key, &spec.id);
1996            jobs.push(SeatJob {
1997                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1998                spec,
1999                seat,
2000                cwd: worktrees[i % worktrees.len()].clone(),
2001                timeout,
2002                allow_write: false,
2003                sessions: false,
2004                artifacts: artifacts.clone(),
2005                stem: seat_key,
2006                handover: None,
2007            });
2008        }
2009
2010        self.state.event(
2011            "advise",
2012            format!(
2013                "{} advisor seat(s) sketching a design in parallel",
2014                jobs.len()
2015            ),
2016        );
2017        let mut quota_losses = Vec::new();
2018        let cache = self.state.config.cache_dir();
2019        let ctx = WaveCtx {
2020            carry_seats: false,
2021            pauses: Some((&self.pause, &self.interrupt)),
2022            run: &run_id,
2023            node: "advise",
2024            prompts: &prompts,
2025            cache: cache.as_deref(),
2026            round: None,
2027        };
2028        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2029        let results = ask_json_wave::<Proposal>(
2030            jobs,
2031            Arc::clone(&self.sem),
2032            self.state.config.graph.retries,
2033            &advisor_roster,
2034            &ctx,
2035            &mut quota_losses,
2036            &mut self.state,
2037            &|p: &Proposal| p.validate(),
2038        )
2039        .await;
2040        if self.state.resource_block.is_some() {
2041            return self.stop_for_resource();
2042        }
2043        self.state.quota.extend(quota_losses);
2044
2045        let mut records = Vec::with_capacity(results.len());
2046        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2047            let agent_id = seat.agent.clone();
2048            self.state.seats.insert(seat.key.clone(), seat);
2049            match res {
2050                Ok((proposal, out)) => {
2051                    self.state
2052                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2053                    records.push(advise::AdvisorRecord::proposed(
2054                        i + 1,
2055                        agent_id,
2056                        proposal,
2057                        out.duration_ms,
2058                    ));
2059                }
2060                Err(e) => {
2061                    self.state.event(
2062                        "advise",
2063                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2064                    );
2065                    records.push(advise::AdvisorRecord::failed(
2066                        i + 1,
2067                        agent_id,
2068                        e.to_string(),
2069                    ));
2070                }
2071            }
2072        }
2073
2074        let mut advice = advise::Advice {
2075            records,
2076            synthesis: None,
2077        };
2078        if advice.proposals().is_empty() {
2079            self.state.event(
2080                "advise",
2081                "no advisor produced a usable proposal; continuing without a \
2082                 synthesis brief"
2083                    .to_owned(),
2084            );
2085        } else {
2086            match self
2087                .synthesize_brief(
2088                    &advice,
2089                    &instruction,
2090                    &language,
2091                    &worktrees[0],
2092                    &artifacts,
2093                    &run_id,
2094                    &prompts,
2095                    cache.as_deref(),
2096                )
2097                .await
2098            {
2099                Ok(Some(text)) => {
2100                    self.state.event(
2101                        "advise",
2102                        "synthesized a design brief for the implementer".to_owned(),
2103                    );
2104                    advice.synthesis = Some(text);
2105                }
2106                Ok(None) => {
2107                    self.state.event(
2108                        "advise",
2109                        "the synthesis seat produced nothing usable; continuing \
2110                         without a design brief"
2111                            .to_owned(),
2112                    );
2113                }
2114                Err(e) => {
2115                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2116                        return Err(e);
2117                    }
2118                    self.state.event(
2119                        "advise",
2120                        format!("could not synthesize a design brief: {e:#}"),
2121                    );
2122                }
2123            }
2124        }
2125        advise::apply_reflection(&mut advice);
2126
2127        self.state.advice = Some(advice);
2128        self.state.advise_attempted = true;
2129        self.state.save()?;
2130        Ok(())
2131    }
2132
2133    /// The synthesis seat: reads every advisor's proposal and blends them
2134    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2135    /// out of [`Runner::advise`] only for readability — it is not called
2136    /// anywhere else.
2137    ///
2138    /// Picked the same way [`crate::talk`]'s standing conversation and
2139    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2140    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2141    /// order (a claude seat, else the first runnable agent in roster order)
2142    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2143    /// in [`crate::config`] for why a dedicated field exists here at all.
2144    #[allow(clippy::too_many_arguments)]
2145    async fn synthesize_brief(
2146        &mut self,
2147        advice: &advise::Advice,
2148        instruction: &str,
2149        language: &str,
2150        cwd: &Path,
2151        artifacts: &Path,
2152        run_id: &str,
2153        prompts: &Prompts,
2154        cache: Option<&Path>,
2155    ) -> Result<Option<String>> {
2156        let chain = agent::pick_chain(
2157            &self.state.config.agents,
2158            self.state.config.roles.synthesizer.as_ref(),
2159            &agent::installed,
2160            "synthesizer",
2161        )?;
2162        let proposals = advice.proposals();
2163        let mut prompt = prompt::with_overlay(
2164            prompt::synthesize_brief(instruction, &proposals, language),
2165            prompts.overlay("advise"),
2166        );
2167        if cache.is_some() {
2168            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2169            // below — see `prompt::build_cache_note`'s doc for why telling a
2170            // read-only seat to build through the shared cache is exactly how
2171            // a sandbox's write refusal gets misread as a defect.
2172            prompt.push('\n');
2173            prompt.push_str(&prompt::build_cache_note("advise", false));
2174        }
2175        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2176        // Each id is tried once, in order; a quota hit, error or unusable
2177        // answer moves to the next. The seat is single-turn (`sessions:
2178        // false`) and the prompt is the whole context, so a fallback agent
2179        // needs nothing carried over.
2180        let mut last = None;
2181        for (n, spec) in chain.iter().enumerate() {
2182            if n > 0 {
2183                self.state
2184                    .event("advise", format!("synthesis falling back to {}", spec.id));
2185            }
2186            let mut seat = self.seat("advise-synthesis", &spec.id);
2187            let wait = self.repository_wait();
2188            let outcome = wait
2189                .scope(agent::invoke(
2190                    spec,
2191                    &mut seat,
2192                    &Invocation {
2193                        scope: crate::agent::InvocationScope::Repository,
2194                        coordination: self.state.config.coordination.as_ref(),
2195                        fetched_base: None,
2196                        cwd,
2197                        prompt: &prompt,
2198                        timeout,
2199                        allow_write: false,
2200                        unsandboxed: false,
2201                        sessions: false,
2202                        artifacts,
2203                        stem: &if n == 0 {
2204                            "advise-synthesis".to_owned()
2205                        } else {
2206                            format!("advise-synthesis-{}", spec.id)
2207                        },
2208                        run: run_id,
2209                        node: "advise",
2210                        cache_dir: None,
2211                        attachments: &[],
2212                        writable: &[],
2213                    },
2214                ))
2215                .await;
2216            if outcome.is_ok() {
2217                self.state.seats.insert(seat.key.clone(), seat);
2218            }
2219            let advance = agent::chain_advances(&outcome);
2220            last = Some(outcome);
2221            if !advance {
2222                break;
2223            }
2224        }
2225        // Exhausted: the last attempt's result is what a single failed seat
2226        // would have produced.
2227        let out = last.expect("a chain holds at least one agent")?;
2228        if !out.usable() {
2229            return Ok(None);
2230        }
2231        let text =
2232            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2233        Ok((!text.trim().is_empty()).then_some(text))
2234    }
2235
2236    // ----------------------------------------------------------- implement
2237
2238    async fn implement(&mut self) -> Result<()> {
2239        self.persist_candidates().await?;
2240        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2241        // agent files with `magi task add` name the run that paid for it. The
2242        // prompt overlay is cloned alongside it because the waves borrow it
2243        // while `self` is mutably borrowed by the node's own bookkeeping.
2244        let run_id = self.state.id.clone();
2245        let prompts = self.state.config.prompts.clone();
2246        let todo: Vec<usize> = self
2247            .state
2248            .candidates
2249            .iter()
2250            .enumerate()
2251            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2252            .map(|(i, _)| i)
2253            .collect();
2254        if todo.is_empty() {
2255            return self.after_implement();
2256        }
2257        self.state.status = RunStatus::Implementing;
2258
2259        let language = self.state.config.graph.language.clone();
2260        let github_language = self.state.config.graph.github_language.clone();
2261        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2262        let sessions = self.state.config.graph.sessions;
2263        let artifacts = agent::artifacts_dir(&self.state.dir());
2264        // The design-deliberation stage's blended brief, when `advise` found
2265        // one — carried into every implementer's prompt the same way
2266        // regardless of which candidate it is.
2267        let brief = self
2268            .state
2269            .advice
2270            .as_ref()
2271            .and_then(|a| a.synthesis.as_deref())
2272            .map(str::to_owned);
2273        let attachments = self.state.attachments.clone();
2274
2275        let mut jobs = Vec::new();
2276        for &i in &todo {
2277            let (index, label, worktree) = {
2278                let c = &self.state.candidates[i];
2279                (c.index, c.label, c.worktree.clone())
2280            };
2281            let spec = self.roles.implementers[index].clone();
2282            let seat_key = format!("impl-{label}");
2283            let seat = self.seat(&seat_key, &spec.id);
2284            let instruction = seeded_instruction(&self.state);
2285            jobs.push(SeatJob {
2286                spec,
2287                seat,
2288                prompt: prompt::implement_in(
2289                    &instruction,
2290                    &worktree.to_string_lossy(),
2291                    &language,
2292                    &github_language,
2293                    brief.as_deref(),
2294                    &attachments,
2295                ),
2296                cwd: worktree,
2297                timeout,
2298                allow_write: true,
2299                sessions,
2300                artifacts: artifacts.clone(),
2301                stem: format!("impl-{label}"),
2302                handover: None,
2303            });
2304        }
2305
2306        self.state.event(
2307            "implement",
2308            format!("{} candidates in parallel", jobs.len()),
2309        );
2310        // Kept so a seat whose CLI hung up can be asked again from the same
2311        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2312        // can update a seat's own entry once a fallback agent takes it over —
2313        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2314        // whichever agent actually answered, not the one that quota'd out.
2315        let mut sent = jobs.clone();
2316        let cache = self.state.config.cache_dir();
2317        let ctx = WaveCtx {
2318            carry_seats: false,
2319            pauses: Some((&self.pause, &self.interrupt)),
2320            run: &run_id,
2321            node: "implement",
2322            prompts: &prompts,
2323            cache: cache.as_deref(),
2324            round: None,
2325        };
2326        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2327        if !wave_blocked(&results) {
2328            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2329                .await;
2330            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2331                .await?;
2332            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2333                .await;
2334        }
2335        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2336
2337        let selected_failed = self.state.config.graph.selected_implementer.is_some()
2338            && results
2339                .iter()
2340                .any(|(_, _, out)| FailClass::of(out).is_some());
2341        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2342            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2343                continue;
2344            }
2345            let seat_key = seat.key.clone();
2346            // A quota fallback (`resume_seat_handovers`) may have handed this
2347            // seat to a different agent than the one `prep` recorded on the
2348            // candidate; the stats tables and any later fixer-defaults-to-
2349            // winner's-author lookup must credit whoever actually answered —
2350            // unless every fallback also quota'd out, in which case nobody
2351            // actually answered and crediting the last agent tried would
2352            // erase every earlier agent's own quota loss from the stats
2353            // tables instead of just this one seat's.
2354            let agent = seat.agent.clone();
2355            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2356            self.state.seats.insert(seat.key.clone(), seat);
2357            let label = self.state.candidates[i].label;
2358
2359            let (summary, duration, failed, verified_claim) = match out {
2360                AgentOutcome::Ok(o) => {
2361                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2362                    let failed = (!o.usable()).then(|| {
2363                        if o.timed_out {
2364                            "agent timed out".to_owned()
2365                        } else {
2366                            format!(
2367                                "agent exited with {}",
2368                                crate::notices::exit_status(o.exit_code)
2369                            )
2370                        }
2371                    });
2372                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2373                    (text, o.duration_ms, failed, verified_claim)
2374                }
2375                // Left un-resumed by `resume_undelivered` (a dirty tree
2376                // already rescues the work, or there was no session left to
2377                // resume into) — reported like the ordinary failure it is,
2378                // never as if `o.text` (the CLI's raw error JSON) were an
2379                // answer.
2380                AgentOutcome::Dropped(o) => {
2381                    let why = o
2382                        .dropped
2383                        .as_ref()
2384                        .map(|d| d.why.as_str())
2385                        .unwrap_or("the CLI ended the stream without delivering its answer");
2386                    (
2387                        String::new(),
2388                        o.duration_ms,
2389                        Some(format!("the CLI dropped the stream ({why})")),
2390                        None,
2391                    )
2392                }
2393                AgentOutcome::Quota(o) => {
2394                    self.state.quota.push(QuotaLoss {
2395                        seat: seat_key,
2396                        node: "implement".to_owned(),
2397                        at: Timestamp::now(),
2398                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2399                    });
2400                    (
2401                        String::new(),
2402                        o.duration_ms,
2403                        Some("rate limited (quota); produced no change".to_owned()),
2404                        None,
2405                    )
2406                }
2407                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2408                    (String::new(), 0, Some(e), None)
2409                }
2410            };
2411
2412            // Save every completed turn before any fallible Git operation. A failed
2413            // rescue must not erase the reply or prevent collection of other seats.
2414            let c = &mut self.state.candidates[i];
2415            if !exhausted_the_fallback_chain {
2416                c.agent = agent;
2417            }
2418            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2419            c.duration_ms = duration;
2420            c.failed = failed;
2421            c.verified_noop = verified_claim;
2422            self.state
2423                .candidate_preservation
2424                .insert(label, "awaiting persistence".to_owned());
2425        }
2426        self.state.save()?;
2427        self.persist_candidates().await?;
2428
2429        if resource_blocked {
2430            return self.stop_for_resource();
2431        }
2432        if selected_failed {
2433            self.state.status = RunStatus::Blocked;
2434            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2435            self.state.save()?;
2436            bail!(
2437                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2438            );
2439        }
2440        self.after_implement()
2441    }
2442
2443    /// Retry only persistence after a completed implementation turn.
2444    async fn persist_candidates(&mut self) -> Result<()> {
2445        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2446        for label in labels {
2447            let i = self
2448                .state
2449                .candidates
2450                .iter()
2451                .position(|c| c.label == label)
2452                .context("pending candidate is missing")?;
2453            let c = self.state.candidates[i].clone();
2454            let result = async {
2455                let wait = crate::repo_lock::WaitPolicy::new(
2456                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2457                    || false,
2458                );
2459                let _coordination =
2460                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2461                // Best effort evidence requires no index writes. A successful snapshot
2462                // never authorizes removal of a tree whose rescue failed.
2463                if let Err(e) = git::snapshot_uncommitted(
2464                    &c.worktree,
2465                    &self
2466                        .state
2467                        .dir()
2468                        .join("artifacts")
2469                        .join(format!("candidate-{label}-recovery")),
2470                )
2471                .await
2472                {
2473                    self.state.event(
2474                        "preserve",
2475                        format!("candidate {label} snapshot incomplete: {e:#}"),
2476                    );
2477                }
2478                let rescued = git::rescue_commit(
2479                    &c.worktree,
2480                    &format!("magi: candidate {label} (uncommitted work)"),
2481                )
2482                .await?;
2483                self.state.note_withheld("implement", &rescued.withheld);
2484                let base = &self.state.base_commit;
2485                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2486                let patch = git::diff(&c.worktree, base, "HEAD").await?;
2487                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2488                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2489                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2490                if c.empty && commits > 0 && !patch.trim().is_empty()
2491                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
2492                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2493                        || self.state.tally.is_some())
2494                {
2495                    // Keep the old classification and pending marker so every
2496                    // retry stops too. Retain decisions as recovery evidence;
2497                    // only a supported recovery operation may replace them.
2498                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2499                }
2500                let c = &mut self.state.candidates[i];
2501                c.commits = commits;
2502                c.stat = stat;
2503                c.files = files;
2504                c.empty = commits == 0 || patch.trim().is_empty();
2505                if !c.empty {
2506                    c.failed = None;
2507                    c.verified_noop = None;
2508                }
2509                self.state.event(
2510                    "preserve",
2511                    format!("candidate {label}: {files} files, {commits} commits"),
2512                );
2513                Ok::<_, anyhow::Error>(())
2514            }
2515            .await;
2516            match result {
2517                Ok(()) => {
2518                    self.state.candidate_preservation.remove(&label);
2519                }
2520                Err(e) => {
2521                    let reason = format!("candidate {label} preservation failed: {e:#}");
2522                    self.state.event("preserve", &reason);
2523                    self.state.candidate_preservation.insert(label, reason);
2524                }
2525            }
2526            if self.state.candidate_preservation.is_empty()
2527                && self.state.status == RunStatus::Blocked
2528                && self.state.candidates.iter().all(|c| c.failed.is_none())
2529            {
2530                // Persist the recovered phase with the cleared pending marker;
2531                // a crash between saves must not turn this into a selected-seat
2532                // failure hold on the next resume.
2533                self.state.status = RunStatus::Implementing;
2534                self.state.event(
2535                    "preserve",
2536                    "candidate persistence recovered; implementation can continue",
2537                );
2538            }
2539            self.state.save()?;
2540        }
2541        if !self.state.candidate_preservation.is_empty() {
2542            self.state.status = RunStatus::Blocked;
2543            self.state.save()?;
2544            bail!(
2545                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2546            );
2547        }
2548        Ok(())
2549    }
2550
2551    /// Ask again, once, for work a CLI did and then failed to hand over.
2552    ///
2553    /// [`agent::dropped_stream`] recognises the one shape observed: an error
2554    /// status with an empty response and a usage report showing output tokens,
2555    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
2556    /// seven minutes and 14,267 output tokens that arrived as an empty
2557    /// candidate, because `agy`'s own subscriber fell behind and hung up.
2558    ///
2559    /// Two conditions, and both matter:
2560    ///
2561    /// - **Only when the tree is untouched.** Often the agent has already
2562    ///   written its files and only the closing message was lost; the rescue
2563    ///   commit below picks that up and there is nothing to ask for. Re-asking
2564    ///   then would pay for a second implementation of work already on disk.
2565    /// - **Once.** A CLI that drops one stream can drop the next, and this
2566    ///   node is the most expensive in the graph.
2567    ///
2568    /// The re-ask is a resume, not a re-run: `has_context` is true because the
2569    /// dropped reply still carried its `conversation_id`, so the seat is asked
2570    /// to finish what it was doing rather than sent the whole task again. It
2571    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
2572    /// node's - for the same reason a re-ranked judge does: restating finished
2573    /// work is not the work.
2574    ///
2575    /// Unlike a quota this is worth retrying at all: a rate limit fails the
2576    /// same way until it resets, while an abandoned conversation is still
2577    /// there to be picked up.
2578    async fn resume_undelivered(
2579        &mut self,
2580        results: &mut [(usize, SeatState, AgentOutcome)],
2581        sent: &[SeatJob],
2582        prompts: &Prompts,
2583        run_id: &str,
2584    ) {
2585        for (wi, seat, out) in results.iter_mut() {
2586            let Some(dropped) = (match &*out {
2587                AgentOutcome::Dropped(o) => o.dropped.clone(),
2588                _ => None,
2589            }) else {
2590                continue;
2591            };
2592            let Some(job) = sent.get(*wi) else { continue };
2593            // Already on disk? Then only the closing message was lost.
2594            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2595                self.state.event(
2596                    "implement",
2597                    format!(
2598                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2599                         work is in the tree",
2600                        seat.key, dropped.output_tokens, dropped.why
2601                    ),
2602                );
2603                continue;
2604            }
2605            // The re-ask only makes sense as a resume: `resume_after_drop`
2606            // says nothing about the task, trusting the seat to still hold it.
2607            // Without a session to resume — sessions disabled, or this CLI's
2608            // drop shape happened not to carry a session id — that prompt
2609            // would open a brand-new conversation with no context at all,
2610            // which is worse than leaving this as the ordinary failure it
2611            // already is.
2612            if !has_context(&job.spec, seat, job.sessions) {
2613                self.state.event(
2614                    "implement",
2615                    format!(
2616                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2617                         is no session left to resume",
2618                        seat.key, dropped.output_tokens, dropped.why
2619                    ),
2620                );
2621                continue;
2622            }
2623            self.state.event(
2624                "implement",
2625                format!(
2626                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2627                     conversation",
2628                    seat.key, dropped.output_tokens, dropped.why
2629                ),
2630            );
2631            let mut retry = job.clone();
2632            retry.seat = seat.clone();
2633            retry.prompt = prompt::resume_after_drop(&dropped.why);
2634            retry.timeout = retry_budget(job.timeout, true);
2635            retry.stem = format!("{}-resume", job.stem);
2636            let cache = self.state.config.cache_dir();
2637            let ctx = WaveCtx {
2638                carry_seats: false,
2639                pauses: Some((&self.pause, &self.interrupt)),
2640                run: run_id,
2641                node: "implement",
2642                prompts,
2643                cache: cache.as_deref(),
2644                round: None,
2645            };
2646            let (resumed_seat, resumed) =
2647                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2648            *seat = resumed_seat;
2649            *out = resumed;
2650        }
2651    }
2652
2653    /// Fall an implement seat through to the next untried agent in the
2654    /// implementer roster when it lost to quota — or, since the handover was
2655    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
2656    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
2657    /// quota path itself being unchanged — instead of leaving the
2658    /// seat's loss final the moment one agent's account runs dry.
2659    ///
2660    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
2661    /// are the motivating case: `Config::resolve_roles`'s `implementers`
2662    /// truncates to the single slot rotation picked, so a solo task whose one
2663    /// implementer hits quota mid-run used to have nothing else to try. This
2664    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
2665    /// unrotated roster — which is the only place the *other* candidates in
2666    /// the machine's roster still exist once `implementers` has been cut down
2667    /// to size.
2668    ///
2669    /// Walks forward from just past the seat's own original position in the
2670    /// roster, never wrapping back to the front: a later candidate slot (say
2671    /// `beta`, the roster's second entry) must fall through to the *next*
2672    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
2673    /// almost certainly a different candidate's own agent already — and once
2674    /// the roster's tail is exhausted there is nothing left to fall through
2675    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
2676    /// whole [`AgentSpec`]: a roster with the same id named twice must not
2677    /// let this retry that id forever. The loop keeps falling through until
2678    /// an attempt lands something other than `Quota` or the roster's tail
2679    /// runs out of untried ids, at which point the seat is left exactly as
2680    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
2681    /// `QuotaLoss` recorded, the candidate failed/empty.
2682    ///
2683    /// `sent` is taken mutably and updated with the fallback agent's spec:
2684    /// `resume_unconfirmed_commands`, which runs after this and also reads
2685    /// `sent`, must see whichever agent actually ended up answering the seat
2686    /// — reading the stale, original spec there would check session
2687    /// eligibility against the wrong CLI and could hand a fallback agent's
2688    /// session id to the agent that just lost the seat to quota.
2689    ///
2690    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
2691    /// — `self.seat` only reuses state when the agent id is unchanged, so
2692    /// handing it a different id already gets this for free. Reusing the old
2693    /// seat would resume a different CLI's session as if it were a
2694    /// continuation of this one.
2695    ///
2696    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
2697    /// a quota loss cuts an agent off mid-turn, so anything already in the
2698    /// tree is unfinished work, not a completed candidate a re-ask would pay
2699    /// for twice. A dirty tree is rescued into a commit first (the same
2700    /// neutral-identity rescue `implement`'s own outcome loop gives every
2701    /// candidate) so the next agent starts clean.
2702    ///
2703    /// The new agent gets the implementer's full prompt and full
2704    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
2705    /// it has no session and no context, and is implementing the task from
2706    /// nothing, unlike a resumed drop which is only restating work already
2707    /// done.
2708    ///
2709    /// Every intermediate `Quota` this loop absorbs is folded into a plain
2710    /// `implement` event, never into `self.state.quota` — that is what
2711    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
2712    /// go unspent, and a seat that ultimately recovered on its second or
2713    /// third agent is not the stalled panel that check exists to catch. Only
2714    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
2715    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
2716    /// outcome loop already has — this helper never pushes to it itself.
2717    async fn resume_seat_handovers(
2718        &mut self,
2719        results: &mut [(usize, SeatState, AgentOutcome)],
2720        sent: &mut [SeatJob],
2721        prompts: &Prompts,
2722        run_id: &str,
2723    ) -> Result<()> {
2724        let instruction = seeded_instruction(&self.state);
2725        let language = self.state.config.graph.language.clone();
2726        let github_language = self.state.config.graph.github_language.clone();
2727        let brief = self
2728            .state
2729            .advice
2730            .as_ref()
2731            .and_then(|a| a.synthesis.as_deref())
2732            .map(str::to_owned);
2733        let attachments = self.state.attachments.clone();
2734        for (wi, seat, out) in results.iter_mut() {
2735            // Who holds the other candidate seats of this wave right now
2736            // (earlier handovers already written back to `sent`).
2737            let others: BTreeSet<String> = sent
2738                .iter()
2739                .enumerate()
2740                .filter(|(j, _)| j != wi)
2741                .map(|(_, j)| j.spec.id.clone())
2742                .collect();
2743            let Some(job) = sent.get_mut(*wi) else {
2744                continue;
2745            };
2746            // Where the seat's own original agent sits in the roster — the
2747            // fallback walk starts just past here, never at the front, so a
2748            // later candidate slot's quota loss does not fall back onto an
2749            // earlier slot's own agent.
2750            let start = self
2751                .roles
2752                .implementer_roster
2753                .iter()
2754                .position(|s| s.id == job.spec.id)
2755                .unwrap_or(0);
2756            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2757            let mut fallback_attempt = 0usize;
2758            let mut prev: Option<FailClass> = None;
2759            while let Some(cur) = FailClass::of(&*out) {
2760                if !should_hand_over(prev.as_ref(), &cur) {
2761                    break;
2762                }
2763                let Some(next) =
2764                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2765                        .cloned()
2766                else {
2767                    break;
2768                };
2769                tried.insert(next.id.clone());
2770                fallback_attempt += 1;
2771
2772                let coordination = self.coordinate(&job.cwd).await?;
2773                if let Ok(r) = git::rescue_commit(
2774                    &job.cwd,
2775                    &format!(
2776                        "magi: candidate {} (uncommitted work before {} fallback)",
2777                        seat.key,
2778                        if cur == FailClass::Quota {
2779                            "quota"
2780                        } else {
2781                            "handover"
2782                        }
2783                    ),
2784                )
2785                .await
2786                {
2787                    self.state.note_withheld("implement", &r.withheld);
2788                }
2789                drop(coordination);
2790
2791                record_handover(
2792                    &mut self.state,
2793                    "implement",
2794                    &seat.key,
2795                    &seat.agent,
2796                    &next.id,
2797                    &cur,
2798                    &fail_reason(&*out),
2799                );
2800                prev = Some(cur.clone());
2801
2802                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2803                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2804                // Kept in sync on `sent` itself, not just the local retry: a
2805                // later helper (`resume_unconfirmed_commands`) reads `sent`
2806                // after this one returns and must see whichever agent is now
2807                // occupying the seat, not the one that just quota'd out —
2808                // otherwise it would judge session/continuation eligibility
2809                // by the wrong CLI and could resend a fallback's session id
2810                // to the agent that lost it the seat in the first place.
2811                job.spec = next.clone();
2812                let mut retry = job.clone();
2813                retry.seat = new_seat;
2814                retry.prompt = prompt::implement_in(
2815                    &instruction,
2816                    &job.cwd.to_string_lossy(),
2817                    &language,
2818                    &github_language,
2819                    brief.as_deref(),
2820                    &attachments,
2821                );
2822                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2823                let cache = self.state.config.cache_dir();
2824                let ctx = WaveCtx {
2825                    carry_seats: false,
2826                    pauses: Some((&self.pause, &self.interrupt)),
2827                    run: run_id,
2828                    node: "implement",
2829                    prompts,
2830                    cache: cache.as_deref(),
2831                    round: None,
2832                };
2833                let (fallback_seat, fallback_out) = run_one(
2834                    retry,
2835                    Arc::clone(&self.sem),
2836                    &ctx,
2837                    &mut self.state,
2838                    fallback_attempt,
2839                )
2840                .await;
2841                *seat = fallback_seat;
2842                *out = fallback_out;
2843            }
2844        }
2845        Ok(())
2846    }
2847
2848    /// Ask an implement seat's own CLI to confirm what it started, once, when
2849    /// its reply reported a command whose completion status it never
2850    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2851    /// that does and does not mean.
2852    ///
2853    /// The completion contract this task asks for, extended to `implement`
2854    /// with the same signal `continue_fix_report` reads for the fixer,
2855    /// rather than a keyword search over the reply or a hard requirement on
2856    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2857    /// a candidate's CLI turn ended cleanly while a test run it had started
2858    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2859    /// named in it at all is untouched by this: `commands` is empty, so
2860    /// there is nothing to be unconfirmed.
2861    ///
2862    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2863    /// this is not about recovering edits that might already be on disk, it
2864    /// is about a result the seat itself never vouched for, which resuming
2865    /// asks for regardless of what the tree already holds. Bounded to one
2866    /// attempt for the same reason `resume_undelivered` is — this is the
2867    /// most expensive node in the graph — and a seat that still cannot
2868    /// confirm on that attempt is left as whatever its (possibly still
2869    /// unconfirmed) reply says; this does not invent a new "failed" reason
2870    /// for a candidate that otherwise produced a real, committed change.
2871    async fn resume_unconfirmed_commands(
2872        &mut self,
2873        results: &mut [(usize, SeatState, AgentOutcome)],
2874        sent: &[SeatJob],
2875        prompts: &Prompts,
2876        run_id: &str,
2877    ) {
2878        for (wi, seat, out) in results.iter_mut() {
2879            let AgentOutcome::Ok(o) = &*out else {
2880                continue;
2881            };
2882            if !has_unconfirmed_command(&o.commands) {
2883                continue;
2884            }
2885            let Some(job) = sent.get(*wi) else { continue };
2886            if !has_context(&job.spec, seat, job.sessions) {
2887                self.state.event(
2888                    "implement",
2889                    format!(
2890                        "{}: the reply named a command whose own CLI never confirmed the exit \
2891                         status of, but there is no session left to resume",
2892                        seat.key
2893                    ),
2894                );
2895                continue;
2896            }
2897            self.state.event(
2898                "implement",
2899                format!(
2900                    "{}: the reply named a command whose own CLI never confirmed the exit \
2901                     status of; resuming the conversation",
2902                    seat.key
2903                ),
2904            );
2905            let mut retry = job.clone();
2906            retry.seat = seat.clone();
2907            retry.prompt = prompt::resume_incomplete(
2908                "a command in your last reply had no confirmed exit status",
2909            );
2910            retry.timeout = retry_budget(job.timeout, true);
2911            retry.stem = format!("{}-confirm", job.stem);
2912            let cache = self.state.config.cache_dir();
2913            let ctx = WaveCtx {
2914                carry_seats: false,
2915                pauses: Some((&self.pause, &self.interrupt)),
2916                run: run_id,
2917                node: "implement",
2918                prompts,
2919                cache: cache.as_deref(),
2920                round: None,
2921            };
2922            let (resumed_seat, resumed) =
2923                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2924            *seat = resumed_seat;
2925            *out = resumed;
2926        }
2927    }
2928
2929    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
2930    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
2931    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
2932    /// that motivated this.
2933    ///
2934    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
2935    /// own nudge loop already covers for judge/review/vote seats, and not a
2936    /// dropped stream, which [`Runner::resume_undelivered`] covers for
2937    /// implement seats: here the CLI turn genuinely finished while the node's
2938    /// own work — the fixer's account of what it did — had not. Gated purely
2939    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
2940    /// reply, never on any wording in it, so a fixer whose valid, first-try
2941    /// `FixReport` happens to mention having waited on a background test is
2942    /// never resumed — the `Ok(report)` branch at the call site returns
2943    /// before this is ever invoked.
2944    ///
2945    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
2946    /// attempt ([`retry_budget`]), nothing attempted once the session is
2947    /// gone, and a quota hit ends the loop immediately rather than retrying a
2948    /// rate limit that fails the same way again.
2949    async fn continue_fix_report(
2950        &mut self,
2951        mut seat: SeatState,
2952        parse_err: String,
2953        job: &SeatJob,
2954        prompts: &Prompts,
2955        run_id: &str,
2956        round: usize,
2957    ) -> (
2958        SeatState,
2959        Option<FixReport>,
2960        Option<String>,
2961        ContinuationRecord,
2962    ) {
2963        let mut last_err = parse_err;
2964        let mut cumulative_wait_ms = 0u64;
2965        let mut attempts = 0usize;
2966        loop {
2967            if !has_context(&job.spec, &seat, job.sessions) {
2968                self.state.event(
2969                    "fix",
2970                    format!(
2971                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
2972                         session left to resume into"
2973                    ),
2974                );
2975                let outcome = if attempts == 0 {
2976                    ContinuationOutcome::NoSession
2977                } else {
2978                    ContinuationOutcome::Exhausted
2979                };
2980                return (
2981                    seat,
2982                    None,
2983                    Some(format!("unparsable fix report: {last_err}")),
2984                    ContinuationRecord {
2985                        attempts,
2986                        cumulative_wait_ms,
2987                        outcome,
2988                    },
2989                );
2990            }
2991            if attempts >= MAX_FIX_CONTINUATIONS {
2992                self.state.event(
2993                    "fix",
2994                    format!(
2995                        "round {round}: fixer's reply still had no adoption report after \
2996                         {attempts} continuation(s) ({last_err}); giving up"
2997                    ),
2998                );
2999                return (
3000                    seat,
3001                    None,
3002                    Some(format!(
3003                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3004                    )),
3005                    ContinuationRecord {
3006                        attempts,
3007                        cumulative_wait_ms,
3008                        outcome: ContinuationOutcome::Exhausted,
3009                    },
3010                );
3011            }
3012            attempts += 1;
3013            self.state.event(
3014                "fix",
3015                format!(
3016                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3017                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3018                ),
3019            );
3020            let mut retry = job.clone();
3021            retry.seat = seat.clone();
3022            retry.prompt = prompt::resume_incomplete(&last_err);
3023            retry.timeout = retry_budget(job.timeout, true);
3024            retry.stem = format!("{}-continue{attempts}", job.stem);
3025            let cache = self.state.config.cache_dir();
3026            let ctx = WaveCtx {
3027                carry_seats: false,
3028                pauses: Some((&self.pause, &self.interrupt)),
3029                run: run_id,
3030                node: "fix",
3031                prompts,
3032                cache: cache.as_deref(),
3033                round: Some(round),
3034            };
3035            let (resumed_seat, resumed_out) = run_one(
3036                retry,
3037                Arc::clone(&self.sem),
3038                &ctx,
3039                &mut self.state,
3040                attempts,
3041            )
3042            .await;
3043            seat = resumed_seat;
3044            match resumed_out {
3045                AgentOutcome::Ok(o) => {
3046                    cumulative_wait_ms += o.duration_ms;
3047                    match verdict::extract_json::<FixReport>(&o.text) {
3048                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3049                            self.state.event(
3050                                "fix",
3051                                format!(
3052                                    "round {round}: fixer's adoption report recovered after \
3053                                     {attempts} continuation(s)"
3054                                ),
3055                            );
3056                            return (
3057                                seat,
3058                                Some(report),
3059                                None,
3060                                ContinuationRecord {
3061                                    attempts,
3062                                    cumulative_wait_ms,
3063                                    outcome: ContinuationOutcome::Resumed,
3064                                },
3065                            );
3066                        }
3067                        // The report parsed, but this same reply's own
3068                        // CommandEvidence — the identical record `state.jobs`
3069                        // renders — names a command whose CLI never
3070                        // confirmed an exit status. Read together, that is
3071                        // not a resolved answer: keep nudging rather than
3072                        // accept a report standing next to a command the
3073                        // seat's own CLI cannot vouch for.
3074                        Ok(_) => {
3075                            last_err = "the reply parsed, but it reported a command whose own CLI \
3076                                 never confirmed an exit status"
3077                                .to_owned();
3078                        }
3079                        Err(e) => last_err = e.to_string(),
3080                    }
3081                }
3082                AgentOutcome::Quota(o) => {
3083                    cumulative_wait_ms += o.duration_ms;
3084                    self.state.quota.push(QuotaLoss {
3085                        seat: seat.key.clone(),
3086                        node: "fix".to_owned(),
3087                        at: Timestamp::now(),
3088                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3089                    });
3090                    self.state.event(
3091                        "fix",
3092                        format!(
3093                            "round {round}: continuation rate limited (quota); not retrying now"
3094                        ),
3095                    );
3096                    return (
3097                        seat,
3098                        None,
3099                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3100                        ContinuationRecord {
3101                            attempts,
3102                            cumulative_wait_ms,
3103                            outcome: ContinuationOutcome::QuotaLost,
3104                        },
3105                    );
3106                }
3107                AgentOutcome::Dropped(o) => {
3108                    cumulative_wait_ms += o.duration_ms;
3109                    let why = o
3110                        .dropped
3111                        .as_ref()
3112                        .map(|d| d.why.as_str())
3113                        .unwrap_or("the CLI ended the stream without delivering its answer");
3114                    last_err = format!("the CLI dropped the stream ({why})");
3115                }
3116                AgentOutcome::Failed(e) => last_err = e,
3117                // The continuation was never started; the report stays lost
3118                // and the caller records that, without any handover.
3119                AgentOutcome::ResourceBlocked(e) => {
3120                    return (
3121                        seat,
3122                        None,
3123                        Some(format!("the continuation was not started: {e}")),
3124                        ContinuationRecord {
3125                            attempts: attempts.saturating_sub(1),
3126                            cumulative_wait_ms,
3127                            outcome: ContinuationOutcome::NoSession,
3128                        },
3129                    );
3130                }
3131            }
3132        }
3133    }
3134
3135    fn after_implement(&mut self) -> Result<()> {
3136        if !self.state.candidate_preservation.is_empty() {
3137            bail!("candidate preservation incomplete; judging is unsafe");
3138        }
3139        // Scan every candidate patch once the set is complete.
3140        if self.state.leaks.is_empty() {
3141            let cfg = self.state.config.blind.clone();
3142            let mut leaks = Vec::new();
3143            for c in &self.state.candidates {
3144                let Some(patch) =
3145                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3146                else {
3147                    continue;
3148                };
3149                leaks.extend(blind::scan(
3150                    &format!("candidate {} patch", c.label),
3151                    &patch,
3152                    &cfg.vendor_tokens,
3153                ));
3154            }
3155            if !leaks.is_empty() {
3156                let summary = leaks
3157                    .iter()
3158                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3159                    .collect::<Vec<_>>()
3160                    .join(", ");
3161                match cfg.on_leak {
3162                    LeakPolicy::Fail => {
3163                        self.state.status = RunStatus::Failed;
3164                        self.state
3165                            .event("blind", format!("vendor text in a patch: {summary}"));
3166                        self.state.leaks = leaks;
3167                        self.state.save()?;
3168                        self.settle_questions();
3169                        bail!(
3170                            "blind.on_leak = \"fail\" and vendor text reached a \
3171                             judged patch: {summary}"
3172                        );
3173                    }
3174                    LeakPolicy::Redact => self.state.event(
3175                        "blind",
3176                        format!("redacting vendor text for judging: {summary}"),
3177                    ),
3178                    LeakPolicy::Warn => self.state.event(
3179                        "blind",
3180                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3181                    ),
3182                }
3183                self.state.leaks = leaks;
3184            }
3185        }
3186
3187        if self.state.viable().is_empty() {
3188            if self.state.all_candidates_verified_noop() {
3189                // Every candidate agreed, with evidence the adoption guard
3190                // accepted, that nothing belongs in this worktree. That is
3191                // not the same fact as a candidate that simply failed to
3192                // write anything, and settling it as an ordinary `Failed`
3193                // (see `SCHEMA`'s doc for schema 10) is what let two of
3194                // task 391f's attempts burn a retry each re-discovering the
3195                // same already-landed fix. Terminal either way, so `judge`
3196                // must never run over an empty candidate set — unlike the
3197                // `Failed` branch below this returns `Ok`, not an error:
3198                // nothing here failed.
3199                self.state.status = RunStatus::VerifiedNoop;
3200                self.state.save()?;
3201                self.settle_questions();
3202                return Ok(());
3203            }
3204            self.state.status = RunStatus::Failed;
3205            self.state.save()?;
3206            self.settle_questions();
3207            bail!("no candidate produced a change; nothing to judge");
3208        }
3209        self.state.status = RunStatus::Judging;
3210        self.state.save()?;
3211        Ok(())
3212    }
3213
3214    // --------------------------------------------------------------- judge
3215
3216    async fn judge(&mut self) -> Result<()> {
3217        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3218        // agent files with `magi task add` name the run that paid for it. The
3219        // prompt overlay is cloned alongside it because the waves borrow it
3220        // while `self` is mutably borrowed by the node's own bookkeeping.
3221        let run_id = self.state.id.clone();
3222        let prompts = self.state.config.prompts.clone();
3223        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3224            return Ok(());
3225        }
3226        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3227        if viable.len() == 1 {
3228            // Recorded so this is a one-time event: `judgements` stays empty
3229            // either way, which without this flag is indistinguishable from
3230            // "not yet judged" on the next reentry — and status is left
3231            // untouched, so a later node's conclusion (e.g. `Blocked` after
3232            // the review budget ran out) survives a resume instead of being
3233            // clobbered back to `Judging` by this node running again.
3234            self.state.judge_skipped = true;
3235            self.state.event(
3236                "judge",
3237                format!(
3238                    "only candidate {} produced a change; judging skipped",
3239                    viable[0].label
3240                ),
3241            );
3242            self.state.save()?;
3243            return Ok(());
3244        }
3245        self.state.status = RunStatus::Judging;
3246
3247        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3248        let language = self.state.config.graph.language.clone();
3249        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3250        let sessions = self.state.config.graph.sessions;
3251        let artifacts = agent::artifacts_dir(&self.state.dir());
3252        let root = self.state.worktree_root();
3253        let base_short = short(&self.state.base_commit);
3254
3255        let mut jobs = Vec::new();
3256        let mut orders = Vec::new();
3257        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3258            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3259            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3260            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3261            let seat_key = format!("judge-{}", j + 1);
3262            let seat = self.seat(&seat_key, &spec.id);
3263            jobs.push(SeatJob {
3264                prompt: prompt::judge(
3265                    &self.state.instruction,
3266                    &views,
3267                    self.roles.judges.len(),
3268                    &base_short,
3269                    &language,
3270                ),
3271                spec,
3272                seat,
3273                cwd: root.join(format!("judge-{}", j + 1)),
3274                timeout,
3275                allow_write: false,
3276                sessions,
3277                artifacts: artifacts.clone(),
3278                stem: format!("judge-{}", j + 1),
3279                handover: None,
3280            });
3281        }
3282
3283        self.state.event(
3284            "judge",
3285            format!(
3286                "{} judges ranking {} candidates blind",
3287                jobs.len(),
3288                viable.len()
3289            ),
3290        );
3291        let labels_for_check = labels.clone();
3292        let mut quota_losses = Vec::new();
3293        let cache = self.state.config.cache_dir();
3294        let ctx = WaveCtx {
3295            carry_seats: false,
3296            pauses: Some((&self.pause, &self.interrupt)),
3297            run: &run_id,
3298            node: "judge",
3299            prompts: &prompts,
3300            cache: cache.as_deref(),
3301            round: None,
3302        };
3303        let results = ask_json_wave::<Ranking>(
3304            jobs,
3305            Arc::clone(&self.sem),
3306            self.state.config.graph.retries,
3307            &self.roles.judge_roster,
3308            &ctx,
3309            &mut quota_losses,
3310            &mut self.state,
3311            &move |r: &Ranking| r.validate(&labels_for_check),
3312        )
3313        .await;
3314        if self.state.resource_block.is_some() {
3315            return self.stop_for_resource();
3316        }
3317        self.state.quota.extend(quota_losses);
3318
3319        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3320            let agent_id = seat.agent.clone();
3321            self.state.seats.insert(seat.key.clone(), seat);
3322            let mut record = Judgement {
3323                judge: j + 1,
3324                seat: format!("judge-{}", j + 1),
3325                agent: agent_id,
3326                ranking: Vec::new(),
3327                reasons: BTreeMap::new(),
3328                confidence: None,
3329                strengths: BTreeMap::new(),
3330                order: orders[j].clone(),
3331                failed: None,
3332                duration_ms: 0,
3333            };
3334            match res {
3335                Ok((ranking, out)) => {
3336                    record.ranking = ranking.normalized();
3337                    record.reasons = ranking.reasons;
3338                    record.strengths = ranking.strengths;
3339                    record.confidence = ranking.confidence;
3340                    record.duration_ms = out.duration_ms;
3341                    self.state.event(
3342                        "judge",
3343                        format!(
3344                            "judge {} ranked {}",
3345                            j + 1,
3346                            record.ranking.iter().collect::<String>()
3347                        ),
3348                    );
3349                }
3350                Err(e) => {
3351                    record.failed = Some(e.to_string());
3352                    self.state
3353                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3354                }
3355            }
3356            self.state.judgements.push(record);
3357            self.state.save()?;
3358        }
3359        Ok(())
3360    }
3361
3362    // ---------------------------------------------------------- deliberate
3363
3364    async fn deliberate(&mut self) -> Result<()> {
3365        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3366        // agent files with `magi task add` name the run that paid for it. The
3367        // prompt overlay is cloned alongside it because the waves borrow it
3368        // while `self` is mutably borrowed by the node's own bookkeeping.
3369        let run_id = self.state.id.clone();
3370        let prompts = self.state.config.prompts.clone();
3371        if !self.state.deliberation.is_empty() {
3372            return Ok(());
3373        }
3374        let tops: Vec<char> = self
3375            .state
3376            .judgements
3377            .iter()
3378            .filter_map(|j| j.ranking.first().copied())
3379            .collect();
3380        let rounds = self.state.config.graph.deliberate_rounds;
3381        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3382            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3383                self.state.event(
3384                    "deliberate",
3385                    format!("judges agreed on {} outright; no deliberation", tops[0]),
3386                );
3387            }
3388            self.state.status = RunStatus::Voting;
3389            self.state.save()?;
3390            return Ok(());
3391        }
3392
3393        self.state.status = RunStatus::Deliberating;
3394        self.state.event(
3395            "deliberate",
3396            format!(
3397                "split: first choices were {} — opening {rounds} round(s)",
3398                tops.iter().collect::<String>()
3399            ),
3400        );
3401
3402        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3403        let language = self.state.config.graph.language.clone();
3404        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3405        let sessions = self.state.config.graph.sessions;
3406        let artifacts = agent::artifacts_dir(&self.state.dir());
3407        let root = self.state.worktree_root();
3408        let base_short = short(&self.state.base_commit);
3409
3410        // Judges argue in sequence so that a turn can answer the one before it;
3411        // that is the difference between deliberation and three parallel
3412        // monologues.
3413        for round in 1..=rounds {
3414            let mut turns: Vec<DeliberationTurn> = Vec::new();
3415            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3416                if self.state.judgements[j].failed.is_some() {
3417                    continue;
3418                }
3419                let seat_key = format!("judge-{}", j + 1);
3420                let spec = self.occupant(&seat_key, spec);
3421                let mut seat = self.seat(&seat_key, &spec.id);
3422                let transcript = self.transcript(&turns, j);
3423                let build = |context: Option<&str>| {
3424                    prompt::deliberate(
3425                        &self.state.instruction,
3426                        context,
3427                        &transcript,
3428                        round,
3429                        rounds,
3430                        &language,
3431                    )
3432                };
3433                let block = self.candidate_block(&viable, &base_short);
3434                let full = build(Some(&block));
3435                let text = if has_context(&spec, &seat, sessions) {
3436                    build(None)
3437                } else {
3438                    full.clone()
3439                };
3440                let job = SeatJob {
3441                    spec,
3442                    seat: seat.clone(),
3443                    prompt: text,
3444                    cwd: root.join(format!("judge-{}", j + 1)),
3445                    timeout,
3446                    allow_write: false,
3447                    sessions,
3448                    artifacts: artifacts.clone(),
3449                    stem: format!("delib-{round}-judge-{}", j + 1),
3450                    handover: Some(full),
3451                };
3452                let cache = self.state.config.cache_dir();
3453                let ctx = WaveCtx {
3454                    carry_seats: false,
3455                    pauses: Some((&self.pause, &self.interrupt)),
3456                    run: &run_id,
3457                    node: "deliberate",
3458                    prompts: &prompts,
3459                    cache: cache.as_deref(),
3460                    round: None,
3461                };
3462                // A turn is never nudged (`retries` 0): a failed seat is
3463                // handed to the next roster agent, which gets the full
3464                // context. An empty answer is a turn, not a failure.
3465                let mut losses = Vec::new();
3466                let mut results = ask_wave_with::<String>(
3467                    vec![job],
3468                    Arc::clone(&self.sem),
3469                    0,
3470                    &self.roles.judge_roster,
3471                    &ctx,
3472                    &mut losses,
3473                    &mut self.state,
3474                    &|text: &str| {
3475                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3476                    },
3477                )
3478                .await;
3479                self.state.quota.extend(losses);
3480                let (updated, res, _) = results.pop().expect("one job in, one result out");
3481                seat = updated;
3482                let agent_id = seat.agent.clone();
3483                self.state.seats.insert(seat.key.clone(), seat);
3484                let body = match res {
3485                    Ok((body, _)) => body,
3486                    // Skip the seat; a CLI's raw error JSON is never read as
3487                    // this judge's position.
3488                    Err(e) => {
3489                        self.state
3490                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3491                        continue;
3492                    }
3493                };
3494                let tentative = verdict::extract_json::<Position>(&body)
3495                    .ok()
3496                    .and_then(|p| p.tentative)
3497                    .and_then(|s| s.trim().chars().next())
3498                    .map(|c| c.to_ascii_uppercase());
3499                self.state.event(
3500                    "deliberate",
3501                    format!(
3502                        "round {round}: judge {} now favours {}",
3503                        j + 1,
3504                        tentative.map_or("—".to_owned(), |c| c.to_string())
3505                    ),
3506                );
3507                turns.push(DeliberationTurn {
3508                    judge: j + 1,
3509                    agent: agent_id,
3510                    body: blind::sanitize_prose(&body, &self.state.config.blind),
3511                    tentative,
3512                });
3513            }
3514            self.state
3515                .deliberation
3516                .push(DeliberationRound { round, turns });
3517            self.state.save()?;
3518        }
3519
3520        self.state.status = RunStatus::Voting;
3521        self.state.save()?;
3522        Ok(())
3523    }
3524
3525    // ---------------------------------------------------------------- vote
3526
3527    async fn vote(&mut self) -> Result<()> {
3528        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3529        // agent files with `magi task add` name the run that paid for it. The
3530        // prompt overlay is cloned alongside it because the waves borrow it
3531        // while `self` is mutably borrowed by the node's own bookkeeping.
3532        let run_id = self.state.id.clone();
3533        let prompts = self.state.config.prompts.clone();
3534        if !self.state.votes.is_empty() {
3535            return Ok(());
3536        }
3537        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3538        if viable.len() == 1 {
3539            return Ok(());
3540        }
3541        self.state.status = RunStatus::Voting;
3542
3543        let language = self.state.config.graph.language.clone();
3544        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3545        let sessions = self.state.config.graph.sessions;
3546        let artifacts = agent::artifacts_dir(&self.state.dir());
3547        let root = self.state.worktree_root();
3548        let base_short = short(&self.state.base_commit);
3549        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3550
3551        let mut jobs = Vec::new();
3552        let mut seats_at = Vec::new();
3553        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3554            if self
3555                .state
3556                .judgements
3557                .get(j)
3558                .is_some_and(|r| r.failed.is_some())
3559            {
3560                continue;
3561            }
3562            let seat_key = format!("judge-{}", j + 1);
3563            let spec = self.occupant(&seat_key, spec);
3564            let seat = self.seat(&seat_key, &spec.id);
3565            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3566            let text = if has_context(&spec, &seat, sessions) {
3567                prompt::final_vote(&viable, &language)
3568            } else {
3569                full.clone()
3570            };
3571            jobs.push(SeatJob {
3572                spec,
3573                seat,
3574                prompt: text,
3575                cwd: root.join(format!("judge-{}", j + 1)),
3576                timeout,
3577                allow_write: false,
3578                sessions,
3579                artifacts: artifacts.clone(),
3580                stem: format!("vote-judge-{}", j + 1),
3581                handover: Some(full),
3582            });
3583            seats_at.push(j);
3584        }
3585
3586        self.state.event(
3587            "vote",
3588            format!(
3589                "collecting {} final votes one by one, privately",
3590                jobs.len()
3591            ),
3592        );
3593        let allowed = viable.clone();
3594        let mut quota_losses = Vec::new();
3595        let cache = self.state.config.cache_dir();
3596        let ctx = WaveCtx {
3597            carry_seats: false,
3598            pauses: Some((&self.pause, &self.interrupt)),
3599            run: &run_id,
3600            node: "vote",
3601            prompts: &prompts,
3602            cache: cache.as_deref(),
3603            round: None,
3604        };
3605        let results = ask_json_wave::<FinalVote>(
3606            jobs,
3607            Arc::clone(&self.sem),
3608            self.state.config.graph.retries,
3609            &self.roles.judge_roster,
3610            &ctx,
3611            &mut quota_losses,
3612            &mut self.state,
3613            &move |v: &FinalVote| match v.label() {
3614                Some(c) if allowed.contains(&c) => Ok(()),
3615                other => bail!("vote {other:?} is not one of {allowed:?}"),
3616            },
3617        )
3618        .await;
3619        if self.state.resource_block.is_some() {
3620            return self.stop_for_resource();
3621        }
3622        self.state.quota.extend(quota_losses);
3623
3624        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3625            let agent_id = seat.agent.clone();
3626            self.state.seats.insert(seat.key.clone(), seat);
3627            let initial = self
3628                .state
3629                .judgements
3630                .get(j)
3631                .and_then(|r| r.ranking.first().copied());
3632            let mut record = VoteRecord {
3633                judge: j + 1,
3634                agent: agent_id,
3635                vote: None,
3636                reason: String::new(),
3637                changed: false,
3638            };
3639            match res {
3640                Ok((v, _)) => {
3641                    record.vote = v.label();
3642                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3643                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3644                    self.state.event(
3645                        "vote",
3646                        format!(
3647                            "judge {} voted {}{}",
3648                            j + 1,
3649                            record.vote.unwrap_or('?'),
3650                            if record.changed { " (changed)" } else { "" }
3651                        ),
3652                    );
3653                }
3654                Err(e) => {
3655                    self.state
3656                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3657                }
3658            }
3659            self.state.votes.push(record);
3660            self.state.save()?;
3661        }
3662        Ok(())
3663    }
3664
3665    // --------------------------------------------------------------- tally
3666
3667    fn tally(&mut self) -> Result<()> {
3668        if self.state.tally.is_some() {
3669            return Ok(());
3670        }
3671        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3672        let tops: Vec<char> = self
3673            .state
3674            .judgements
3675            .iter()
3676            .filter_map(|j| j.ranking.first().copied())
3677            .collect();
3678        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3679
3680        // A judge whose private vote failed still counted once, in the initial
3681        // ranking; using it beats discarding a whole seat.
3682        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3683        let mut cast: Vec<char> = Vec::new();
3684        for (i, j) in self.state.judgements.iter().enumerate() {
3685            let vote = self
3686                .state
3687                .votes
3688                .iter()
3689                .find(|v| v.judge == i + 1)
3690                .and_then(|v| v.vote)
3691                .or_else(|| j.ranking.first().copied());
3692            if let Some(v) = vote {
3693                *first_choice.entry(v).or_insert(0) += 1;
3694                cast.push(v);
3695            }
3696        }
3697
3698        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3699        for j in &self.state.judgements {
3700            let n = j.ranking.len();
3701            for (pos, label) in j.ranking.iter().enumerate() {
3702                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3703            }
3704        }
3705
3706        let best = first_choice.values().copied().max().unwrap_or(0);
3707        let mut leaders: Vec<char> = first_choice
3708            .iter()
3709            .filter(|(_, v)| **v == best)
3710            .map(|(k, _)| *k)
3711            .collect();
3712        let mut tie_break = None;
3713        if leaders.len() > 1 {
3714            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3715            let borda_leaders: Vec<char> = leaders
3716                .iter()
3717                .copied()
3718                .filter(|l| borda[l] == top_borda)
3719                .collect();
3720            tie_break = Some(if borda_leaders.len() == 1 {
3721                format!(
3722                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3723                    leaders.len()
3724                )
3725            } else {
3726                format!(
3727                    "{} way tie on both first-choice votes and Borda points, broken by label order",
3728                    leaders.len()
3729                )
3730            });
3731            leaders = borda_leaders;
3732            leaders.sort_unstable();
3733        }
3734        let winner = *leaders
3735            .first()
3736            .or(viable.first())
3737            .context("no candidate to declare a winner from")?;
3738
3739        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3740        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3741        let deliberated = !self.state.deliberation.is_empty();
3742
3743        // Whose verdict is this? A rate-limited seat is absent even if it
3744        // ranked before the limit hit, so presence is measured against the
3745        // recorded losses, not just "did a ranking ever appear".
3746        let quota_seats: std::collections::BTreeSet<&str> =
3747            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3748        let mut present = 0usize;
3749        for (i, j) in self.state.judgements.iter().enumerate() {
3750            if quota_seats.contains(j.seat.as_str()) {
3751                continue;
3752            }
3753            let ranked = !j.ranking.is_empty() && j.failed.is_none();
3754            let voted = self
3755                .state
3756                .votes
3757                .iter()
3758                .any(|v| v.judge == i + 1 && v.vote.is_some());
3759            if ranked || voted {
3760                present += 1;
3761            }
3762        }
3763        // Strict majority of the configured panel. A bare majority is real
3764        // signal we can act on, while a minority verdict must never stand in
3765        // for a healthy one. A one-candidate run needs no panel at all, and
3766        // `judges` stays `0` rather than the roster size a panel that never
3767        // sat would otherwise be credited with.
3768        let needs_quorum = viable.len() > 1;
3769        let judges_total = if needs_quorum {
3770            self.roles.judges.len()
3771        } else {
3772            0
3773        };
3774        let quorum = if needs_quorum {
3775            judges_total / 2 + 1
3776        } else {
3777            0
3778        };
3779        let met_quorum = !needs_quorum || present >= quorum;
3780        let uncontested = (!needs_quorum).then(|| {
3781            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3782        });
3783
3784        self.state.event(
3785            "tally",
3786            match &uncontested {
3787                Some(reason) => format!("winner {winner} — {reason}"),
3788                None => format!(
3789                    "winner {winner} — votes {} | initial {} | {} changed | \
3790                     {present}/{judges_total} judges{}",
3791                    first_choice
3792                        .iter()
3793                        .map(|(k, v)| format!("{k}:{v}"))
3794                        .collect::<Vec<_>>()
3795                        .join(" "),
3796                    if unanimous_initial {
3797                        "unanimous"
3798                    } else {
3799                        "split"
3800                    },
3801                    changed_votes,
3802                    if met_quorum {
3803                        String::new()
3804                    } else {
3805                        format!(" — below quorum ({quorum} required)")
3806                    },
3807                ),
3808            },
3809        );
3810        if !met_quorum {
3811            self.state.event(
3812                "stall",
3813                format!(
3814                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3815                     the run stops here, resumable"
3816                ),
3817            );
3818        }
3819        self.state.tally = Some(Tally {
3820            first_choice,
3821            borda,
3822            winner,
3823            rankings: tops.len(),
3824            unanimous_initial,
3825            deliberated,
3826            changed_votes,
3827            unanimous_final,
3828            tie_break,
3829            judges: judges_total,
3830            present,
3831            quorum,
3832            met_quorum,
3833            uncontested,
3834        });
3835        self.state.status = if met_quorum {
3836            RunStatus::Reviewing
3837        } else {
3838            RunStatus::Stalled
3839        };
3840        self.state.save()?;
3841        Ok(())
3842    }
3843
3844    // ------------------------------------------------------------- recover
3845
3846    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3847    /// resumed toward completion once the transient cause clears.
3848    ///
3849    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3850    /// it toward the quorum, which is exactly the set of seats whose absence
3851    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3852    /// not depend on which node happened to hit the limit), or an ordinary
3853    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3854    /// seat is never disturbed.
3855    ///
3856    /// A seat that now answers with a usable ranking is "recovered": its
3857    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3858    /// `tally` counts it present again), and its vote re-collected. A seat that
3859    /// still fails keeps its loss and stays absent.
3860    ///
3861    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3862    /// to review/gate/merge), `false` when it is still below quorum (the run
3863    /// stays `Stalled`, still resumable for a later retry).
3864    #[allow(clippy::too_many_lines)]
3865    async fn recover_stall(&mut self) -> Result<bool> {
3866        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3867        // agent files with `magi task add` name the run that paid for it. The
3868        // prompt overlay is cloned alongside it because the waves borrow it
3869        // while `self` is mutably borrowed by the node's own bookkeeping.
3870        let run_id = self.state.id.clone();
3871        let prompts = self.state.config.prompts.clone();
3872        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3873        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3874        // is what keeps a non-quota collapse — or a quota loss recorded at the
3875        // deliberate node — from being a permanent dead-end on `--resume`.
3876        let quota_seats: BTreeSet<&str> =
3877            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3878        let absent: Vec<String> = self
3879            .state
3880            .judgements
3881            .iter()
3882            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3883            .map(|j| j.seat.clone())
3884            .collect();
3885        if absent.is_empty() {
3886            return Ok(false);
3887        }
3888        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3889        if viable.len() <= 1 {
3890            return Ok(false);
3891        }
3892        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3893        let language = self.state.config.graph.language.clone();
3894        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3895        let sessions = self.state.config.graph.sessions;
3896        let artifacts = agent::artifacts_dir(&self.state.dir());
3897        let root = self.state.worktree_root();
3898        let base_short = short(&self.state.base_commit);
3899        let candidates: Vec<Candidate> = viable.clone();
3900
3901        // Map each absent seat key to its 0-based position in `roles.judges`.
3902        let mut positions: Vec<usize> = absent
3903            .iter()
3904            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3905            .collect();
3906        if positions.is_empty() {
3907            return Ok(false);
3908        }
3909        positions.sort_unstable();
3910        positions.dedup();
3911
3912        // Re-rank the lost seats, one blind prompt each.
3913        let mut judge_jobs = Vec::new();
3914        for &j in &positions {
3915            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3916            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3917            let seat_key = format!("judge-{}", j + 1);
3918            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3919            let seat = self.seat(&seat_key, &spec.id);
3920            judge_jobs.push(SeatJob {
3921                spec,
3922                seat,
3923                prompt: prompt::judge(
3924                    &self.state.instruction,
3925                    &views,
3926                    self.roles.judges.len(),
3927                    &base_short,
3928                    &language,
3929                ),
3930                cwd: root.join(seat_key),
3931                timeout,
3932                allow_write: false,
3933                sessions,
3934                artifacts: artifacts.clone(),
3935                stem: format!("judge-{}-recover", j + 1),
3936                handover: None,
3937            });
3938        }
3939
3940        let labels_for_check = labels.clone();
3941        let mut judge_losses = Vec::new();
3942        let retries = self.state.config.graph.retries;
3943        let cache = self.state.config.cache_dir();
3944        let ctx = WaveCtx {
3945            carry_seats: false,
3946            pauses: Some((&self.pause, &self.interrupt)),
3947            run: &run_id,
3948            node: "judge",
3949            prompts: &prompts,
3950            cache: cache.as_deref(),
3951            round: None,
3952        };
3953        let results = ask_json_wave::<Ranking>(
3954            judge_jobs,
3955            Arc::clone(&self.sem),
3956            retries,
3957            &self.roles.judge_roster,
3958            &ctx,
3959            &mut judge_losses,
3960            &mut self.state,
3961            &move |r: &Ranking| r.validate(&labels_for_check),
3962        )
3963        .await;
3964        if self.state.resource_block.is_some() {
3965            self.stop_for_resource()?;
3966            return Ok(false);
3967        }
3968
3969        // Refresh the judgement of every seat that ranked again.
3970        let mut recovered: BTreeSet<usize> = BTreeSet::new();
3971        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3972            let agent_id = seat.agent.clone();
3973            self.state.seats.insert(seat.key.clone(), seat);
3974            let record = &mut self.state.judgements[j];
3975            match res {
3976                Ok((ranking, out)) => {
3977                    record.agent = agent_id;
3978                    record.ranking = ranking.normalized();
3979                    record.reasons = ranking.reasons;
3980                    record.strengths = ranking.strengths;
3981                    record.confidence = ranking.confidence;
3982                    record.failed = None;
3983                    record.duration_ms = out.duration_ms;
3984                    recovered.insert(j);
3985                    self.state.event(
3986                        "recover",
3987                        format!("judge {} ranked again after the limit", j + 1),
3988                    );
3989                }
3990                Err(e) => {
3991                    self.state
3992                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3993                }
3994            }
3995        }
3996
3997        // Re-ask the votes of the seats that recovered a ranking.
3998        let mut vote_jobs = Vec::new();
3999        let mut vote_pos: Vec<usize> = Vec::new();
4000        for &j in &recovered {
4001            let seat_key = format!("judge-{}", j + 1);
4002            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4003            let seat = self.seat(&seat_key, &spec.id);
4004            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4005            let text = if has_context(&spec, &seat, sessions) {
4006                prompt::final_vote(&labels, &language)
4007            } else {
4008                full.clone()
4009            };
4010            vote_jobs.push(SeatJob {
4011                spec,
4012                seat,
4013                prompt: text,
4014                cwd: root.join(seat_key),
4015                timeout,
4016                allow_write: false,
4017                sessions,
4018                artifacts: artifacts.clone(),
4019                stem: format!("vote-judge-{}-recover", j + 1),
4020                handover: Some(full),
4021            });
4022            vote_pos.push(j);
4023        }
4024        let allowed = labels.clone();
4025        let mut vote_losses = Vec::new();
4026        let vote_retries = self.state.config.graph.retries;
4027        let vote_cache = self.state.config.cache_dir();
4028        let ctx = WaveCtx {
4029            carry_seats: false,
4030            pauses: Some((&self.pause, &self.interrupt)),
4031            run: &run_id,
4032            node: "vote",
4033            prompts: &prompts,
4034            cache: vote_cache.as_deref(),
4035            round: None,
4036        };
4037        let votes = ask_json_wave::<FinalVote>(
4038            vote_jobs,
4039            Arc::clone(&self.sem),
4040            vote_retries,
4041            &self.roles.judge_roster,
4042            &ctx,
4043            &mut vote_losses,
4044            &mut self.state,
4045            &move |v: &FinalVote| match v.label() {
4046                Some(c) if allowed.contains(&c) => Ok(()),
4047                other => bail!("vote {other:?} is not one of {allowed:?}"),
4048            },
4049        )
4050        .await;
4051        if self.state.resource_block.is_some() {
4052            self.stop_for_resource()?;
4053            return Ok(false);
4054        }
4055        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4056            let agent_id = seat.agent.clone();
4057            self.state.seats.insert(seat.key.clone(), seat);
4058            match res {
4059                Ok((v, _)) => {
4060                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4061                        rec.vote = v.label();
4062                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4063                    } else {
4064                        self.state.votes.push(VoteRecord {
4065                            judge: j + 1,
4066                            agent: agent_id,
4067                            vote: v.label(),
4068                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4069                            changed: false,
4070                        });
4071                    }
4072                    self.state.event(
4073                        "recover",
4074                        format!("judge {} voted again after the limit", j + 1),
4075                    );
4076                }
4077                Err(e) => {
4078                    self.state
4079                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4080                }
4081            }
4082        }
4083
4084        // A seat that ranked again is present even if its re-vote failed —
4085        // `tally` falls back to the initial ranking's first choice — so clear
4086        // its quota loss. Seats that still fail keep theirs and stay absent.
4087        let recovered_keys: BTreeSet<String> = recovered
4088            .iter()
4089            .map(|&j| format!("judge-{}", j + 1))
4090            .collect();
4091        self.state
4092            .quota
4093            .retain(|q| !recovered_keys.contains(&q.seat));
4094        // A seat that hit the limit again is a fresh loss, not the old one:
4095        // replace the stale entry so the history stays one-per-seat and the
4096        // daemon can tell this attempt's loss from a previous session's.
4097        for loss in judge_losses.into_iter().chain(vote_losses) {
4098            if recovered_keys.contains(&loss.seat) {
4099                continue;
4100            }
4101            self.state.quota.retain(|q| q.seat != loss.seat);
4102            self.state.quota.push(loss);
4103        }
4104
4105        // Recompute the verdict from the refreshed panel.
4106        self.state.tally = None;
4107        self.tally()?;
4108        Ok(self
4109            .state
4110            .tally
4111            .as_ref()
4112            .map(|t| t.met_quorum)
4113            .unwrap_or(false))
4114    }
4115
4116    // ---------------------------------------------------------------- graft
4117
4118    /// Port the strengths the judges attributed to the losing candidates onto
4119    /// the winner, once, between the tally and the review loop.
4120    ///
4121    /// Best-effort by construction: it returns nothing and fails nothing. A
4122    /// skip, a fixer that failed or hit quota, a round that changed nothing
4123    /// and a result over the size limit ([`graft::limit`]) all leave the
4124    /// winner exactly at the commit it had before (`GraftRecord::pre`) and the
4125    /// run carries on as if the phase were off. See [`graft`] for the
4126    /// decisions and `AGENTS.md` for the invariants.
4127    ///
4128    /// The one exception is a rollback that could not be confirmed: the
4129    /// rejected commit may still be the winner's tip, so the run is blocked
4130    /// (resumable; the record stays `Running`, which makes a resume retry the
4131    /// restore) instead of reviewing, gating and merging it.
4132    async fn graft(&mut self) -> Result<()> {
4133        let Err(e) = self.graft_inner().await else {
4134            return Ok(());
4135        };
4136        if e.is::<crate::repo_lock::ResourceBlocked>() {
4137            return Err(e);
4138        }
4139        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4140            self.state.status = RunStatus::Blocked;
4141            self.state.event("graft", format!("blocked: {e:#}"));
4142            self.state.save()?;
4143            return Err(e);
4144        }
4145        // An error after the `Running` record was saved may have left the
4146        // fixer's work in the tree: the record must not stay unsettled
4147        // (a later resume would reset past the review fixes), so confirm the
4148        // restore now, or block.
4149        if let Some(rec) = self.state.graft.clone()
4150            && !rec.outcome.settled()
4151        {
4152            if let Some(winner) = self.state.winner().cloned() {
4153                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4154                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4155                        return Err(re);
4156                    }
4157                    self.state.status = RunStatus::Blocked;
4158                    self.state.event("graft", format!("blocked: {re:#}"));
4159                    self.state.save()?;
4160                    return Err(re);
4161                }
4162            }
4163            if let Some(g) = self.state.graft.as_mut() {
4164                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4165            }
4166        }
4167        self.state.event("graft", format!("skipped: {e:#}"));
4168        if let Err(e) = self.state.save() {
4169            tracing::warn!("save after graft: {e:#}");
4170        }
4171        Ok(())
4172    }
4173
4174    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4175    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4176    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4177        let _coordination = self.coordinate(wt).await?;
4178        crate::recovery::restore_repair(wt, pre)
4179            .await
4180            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4181    }
4182
4183    /// A skip is a run event, recorded once however often the node is re-entered.
4184    fn graft_skipped(&mut self, why: &str) {
4185        let message = format!("skipped: {why}");
4186        if !self
4187            .state
4188            .events
4189            .iter()
4190            .any(|e| e.node == "graft" && e.message == message)
4191        {
4192            self.state.event("graft", message);
4193        }
4194    }
4195
4196    async fn graft_inner(&mut self) -> Result<()> {
4197        // A round that was started and never finished (crash, kill): the
4198        // fixer may have left anything in the tree, so put the winner back.
4199        if let Some(rec) = self.state.graft.clone()
4200            && !rec.outcome.settled()
4201        {
4202            // Reviews exist only if the graft ended without its record being
4203            // settled (a swallowed error): the tree has moved on since, and
4204            // a reset to `pre` would discard the review fixes.
4205            if self.state.reviews.is_empty()
4206                && let Some(winner) = self.state.winner().cloned()
4207            {
4208                self.graft_restore(&winner.worktree, &rec.pre).await?;
4209            }
4210            if rec.outcome == GraftOutcome::WaitingForRestore {
4211                self.state.graft = None;
4212                self.state.save()?;
4213            } else {
4214                if let Some(g) = self.state.graft.as_mut() {
4215                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4216                }
4217                self.state.event(
4218                    "graft",
4219                    "rolled back: the previous round was interrupted; the winner is untouched",
4220                );
4221                self.state.save()?;
4222                return Ok(());
4223            }
4224        }
4225        let plan = match graft::plan(&self.state) {
4226            Ok(plan) => plan,
4227            Err(skip) => {
4228                if skip != graft::Skip::AlreadyDone {
4229                    self.graft_skipped(skip.reason());
4230                    self.state.save()?;
4231                }
4232                return Ok(());
4233            }
4234        };
4235        let Some(winner) = self.state.winner().cloned() else {
4236            return Ok(());
4237        };
4238        let wt = winner.worktree.clone();
4239        if !git::is_clean(&wt).await? {
4240            self.graft_skipped("the winner's worktree has uncommitted changes");
4241            self.state.save()?;
4242            return Ok(());
4243        }
4244        let pre = git::rev_parse(&wt, "HEAD").await?;
4245        let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4246        else {
4247            self.graft_skipped("the winner's diff could not be measured");
4248            self.state.save()?;
4249            return Ok(());
4250        };
4251        let limit = graft::limit(winner_lines);
4252        // Saved before the fixer is called: a stop from here on is repaired
4253        // by the branch above on the next entry.
4254        self.state.graft = Some(GraftRecord {
4255            pre: pre.clone(),
4256            post: None,
4257            lines: None,
4258            limit: Some(limit),
4259            agent: None,
4260            outcome: GraftOutcome::Running,
4261            requested: graft::requested(&plan),
4262            adoption: None,
4263            adoption_error: None,
4264        });
4265        self.state.event(
4266            "graft",
4267            format!(
4268                "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4269                plan.strengths
4270                    .iter()
4271                    .map(|(l, _, _)| l.to_string())
4272                    .collect::<Vec<_>>()
4273                    .join(", "),
4274                plan.winner
4275            ),
4276        );
4277        self.state.save()?;
4278
4279        let fix_prompt = prompt::graft(
4280            &self.state.instruction,
4281            plan.winner,
4282            &plan.strengths,
4283            limit,
4284            &self.state.config.graph.language,
4285        );
4286        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4287        let sessions = self.state.config.graph.sessions;
4288        let artifacts = agent::artifacts_dir(&self.state.dir());
4289        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4290        let (_, seat, out) = self
4291            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4292                SeatJob {
4293                    prompt: fix_prompt.clone(),
4294                    spec,
4295                    seat,
4296                    cwd: wt.clone(),
4297                    timeout,
4298                    allow_write: true,
4299                    sessions,
4300                    artifacts: artifacts.clone(),
4301                    stem: "graft".to_owned(),
4302                    handover: None,
4303                }
4304            })
4305            .await;
4306        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4307            // Preserve the need to restore earlier fallback edits without
4308            // confusing an unstarted seat with an interrupted invocation.
4309            self.state.graft.as_mut().expect("graft record").outcome =
4310                GraftOutcome::WaitingForRestore;
4311            return self.stop_for_resource();
4312        }
4313        let mut why: Option<String> = None;
4314        match &out {
4315            AgentOutcome::Ok(o) if agent::output_advances(o) => {
4316                why = Some("the fixer produced nothing usable".to_owned());
4317            }
4318            AgentOutcome::Ok(_) => {}
4319            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4320            AgentOutcome::Quota(o) => {
4321                self.state.quota.push(QuotaLoss {
4322                    seat: seat.key.clone(),
4323                    node: "graft".to_owned(),
4324                    at: Timestamp::now(),
4325                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4326                });
4327                why = Some("rate limited (quota)".to_owned());
4328            }
4329            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4330                why = Some(format!(
4331                    "the fixer failed: {}",
4332                    e.lines()
4333                        .next()
4334                        .unwrap_or("")
4335                        .chars()
4336                        .take(160)
4337                        .collect::<String>()
4338                ));
4339            }
4340        }
4341        let agent_id = seat.agent.clone();
4342        self.state.seats.insert(seat.key.clone(), seat);
4343
4344        let mut post = None;
4345        let mut lines = None;
4346        let _coordination = self.coordinate(&wt).await?;
4347        if why.is_none() {
4348            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4349            if let Ok(r) = &rescued {
4350                self.state.note_withheld("graft", &r.withheld);
4351            }
4352            let head = git::rev_parse(&wt, "HEAD").await?;
4353            // What is measured must be what gets reviewed and gated: a failed
4354            // rescue, or anything still uncommitted (withheld lockfiles
4355            // aside, which are never part of the tree), rejects the graft.
4356            let dirty = match &rescued {
4357                Ok(r) => git::status_porcelain(&wt)
4358                    .await
4359                    .map(|st| {
4360                        st.lines().any(|l| {
4361                            !r.withheld
4362                                .iter()
4363                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
4364                        })
4365                    })
4366                    .unwrap_or(true),
4367                Err(_) => true,
4368            };
4369            if dirty {
4370                why = Some("the fixer left changes that could not be committed".to_owned());
4371            } else if head == pre {
4372                why = Some("the fixer changed nothing".to_owned());
4373            } else {
4374                lines = git::changed_lines(&wt, &pre, &head).await;
4375                if graft::within_budget(lines, limit) {
4376                    post = Some(head);
4377                } else {
4378                    why = Some(format!(
4379                        "{} changed lines against a limit of {limit}",
4380                        lines.map_or_else(
4381                            || "an unmeasurable number of".to_owned(),
4382                            |l| l.to_string()
4383                        )
4384                    ));
4385                }
4386            }
4387        }
4388        let (adoption, adoption_error) = match &out {
4389            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4390            _ => (None, None),
4391        };
4392        let requested = graft::requested(&plan);
4393        let outcome = match why {
4394            None => {
4395                self.state.event(
4396                    "graft",
4397                    format!(
4398                        "applied: {} changed lines (limit {limit})",
4399                        lines.unwrap_or(0)
4400                    ),
4401                );
4402                GraftOutcome::Applied
4403            }
4404            Some(why) => {
4405                // Not settled until the restore is confirmed: the record
4406                // stays `Running`, so a resume retries it.
4407                crate::recovery::restore_repair(&wt, &pre)
4408                    .await
4409                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4410                self.state.event(
4411                    "graft",
4412                    format!("rolled back: {why}; the winner is untouched"),
4413                );
4414                GraftOutcome::RolledBack(why)
4415            }
4416        };
4417        self.state.graft = Some(GraftRecord {
4418            pre,
4419            post,
4420            lines,
4421            limit: Some(limit),
4422            agent: Some(agent_id),
4423            outcome,
4424            requested,
4425            adoption,
4426            adoption_error,
4427        });
4428        self.state.save()?;
4429        Ok(())
4430    }
4431
4432    // ----------------------------------------------------------------- fold
4433
4434    async fn fold_losers(&mut self) -> Result<()> {
4435        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4436            return Ok(());
4437        };
4438        if !self.state.candidate_preservation.is_empty() {
4439            bail!("candidate preservation incomplete; refusing cleanup");
4440        }
4441        let repo = self.state.repo.clone();
4442        let _coordination = self.coordinate(&repo).await?;
4443        let mut folded = Vec::new();
4444        for i in 0..self.state.candidates.len() {
4445            let c = &self.state.candidates[i];
4446            if c.label == winner || c.folded {
4447                continue;
4448            }
4449            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4450            remove_preserved_worktree(&repo, &wt).await?;
4451            if git::branch_exists(&repo, &branch).await?
4452                && !git::branch_delete(&repo, &branch).await?
4453            {
4454                bail!("candidate branch removal refused");
4455            }
4456            self.state.candidates[i].folded = true;
4457            self.state.save()?;
4458            folded.push(label.to_string());
4459        }
4460        // The judges are finished; their checkouts are pure cost from here.
4461        let root = self.state.worktree_root();
4462        for j in 1..=self.roles.judges.len() {
4463            let wt = root.join(format!("judge-{j}"));
4464            if wt.exists() {
4465                remove_preserved_worktree(&repo, &wt).await?;
4466            }
4467        }
4468        // The design-deliberation stage is finished by the time a tally
4469        // exists — same reasoning as the judges above.
4470        if self.state.config.graph.advise {
4471            for k in 1..=self.state.config.graph.advisors {
4472                let wt = root.join(format!("advisor-{k}"));
4473                if wt.exists() {
4474                    remove_preserved_worktree(&repo, &wt).await?;
4475                }
4476            }
4477        }
4478        if !folded.is_empty() {
4479            self.state
4480                .event("fold", format!("folded candidates {}", folded.join(", ")));
4481            self.state.save()?;
4482        }
4483        Ok(())
4484    }
4485
4486    // ------------------------------------------------------------ base sync
4487
4488    /// Land the winner's tree on the current tip of `<remote>/<base>` before
4489    /// anything verifies it.
4490    ///
4491    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
4492    /// read whatever is checked out in the winner's worktree. Left alone that
4493    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
4494    /// it when the run *branched* - and a run takes long enough that the base
4495    /// has usually moved by the time it gets here. A gate that ran there
4496    /// answers "green on the commit this run started from", not "green on
4497    /// what is about to land", and the difference showed up three times in
4498    /// one day as a green run whose merge would have reverted a file another
4499    /// pull request had already landed.
4500    ///
4501    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
4502    /// `land::Step::Rebase` calls, rather than a second implementation of the
4503    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
4504    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
4505    /// doc warns about ("two answers to a question nobody notices until a
4506    /// diff is wrong").
4507    ///
4508    /// A conflict is not the end of the road: the standing rebase is handed
4509    /// to the fixer seat, at most `graph.review_rounds` times, counted in
4510    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
4511    /// land). Once it finishes, review and the gate run as usual on the
4512    /// rebased tree, which is where a breakage the new base caused is caught
4513    /// by the ordinary gate-fix round. magi resolves nothing itself.
4514    ///
4515    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
4516    /// `state.base_sync.attempts`, is how many times the base is *rebased
4517    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
4518    /// *conflict* was given to a fixer. When the fixer cannot finish the
4519    /// rebase the branch is restored, `state.base_sync.conflict` is set with
4520    /// what was tried (rounds spent, paths still conflicted) and the branch
4521    /// and worktree stay exactly as they were - untouched, for a person to
4522    /// look at - which is also what makes re-entering this function
4523    /// afterwards a no-op instead of a second attempt at the same wall. A
4524    /// push failure ends the same way.
4525    async fn sync_to_base(&mut self) -> Result<()> {
4526        if self.state.status == RunStatus::AlreadyInBase {
4527            return Ok(());
4528        }
4529        let conflicted = self
4530            .state
4531            .base_sync
4532            .as_ref()
4533            .is_some_and(|s| s.conflict.is_some());
4534        let Some(winner) = self.state.winner().cloned() else {
4535            return Ok(());
4536        };
4537
4538        let repo = self.state.repo.clone();
4539        let remote = self.state.config.merge.remote.clone();
4540        let base_branch = self.state.base_branch.clone();
4541        let tracking = format!("{remote}/{base_branch}");
4542
4543        git::fetch(&repo, &remote, &base_branch).await.ok();
4544        // No network, or the remote never had this branch: the run already
4545        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
4546        // otherwise), and one that got this far is not blocked by a fetch
4547        // that fails now. It just has no newer tip to compare against.
4548        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4549            return Ok(());
4550        };
4551
4552        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4553        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4554        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4555
4556        // Before any rebase, and before a recorded conflict is honoured: a
4557        // branch whose change reached the base under other commit ids has
4558        // nothing to rebase and nothing to conflict with, and a run that
4559        // already stopped on that phantom conflict recovers here on resume.
4560        // `behind == 0` with head == tip is a branch the base has since taken
4561        // in whole, whether or not a conflict was ever recorded: the ancestry
4562        // proof must still run (`classify` ignores a head still on the start
4563        // commit).
4564        if (behind > 0 || conflicted || head == tip)
4565            && self
4566                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4567                .await?
4568        {
4569            return Ok(());
4570        }
4571        if conflicted {
4572            return Ok(());
4573        }
4574
4575        if behind == 0 {
4576            // A fixer-finished rebase moves the branch ref before the
4577            // winner's worktree is told (`sync_to_head` below). A run that
4578            // died in between resumes here with `behind == 0` and a tree still
4579            // holding the pre-rebase files, which review and the gate would
4580            // then read. That state is exactly: HEAD moved off the tip the
4581            // rebase started from, yet the tree is still identical to that
4582            // tip. A tree with edits of its own differs from it, so nothing
4583            // is thrown away.
4584            if let Some(from) = self
4585                .state
4586                .rebase_fixes
4587                .iter()
4588                .rev()
4589                .find_map(|r| r.from.clone())
4590                && from != head
4591                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4592                    .await
4593                    .is_ok_and(|o| o.ok())
4594            {
4595                git::sync_to_head(&winner.worktree).await?;
4596            }
4597            // An earlier attempt may have rebased the branch locally and died
4598            // before pushing it (only the fresh-rebase arm below pushes).
4599            // Publish it now, so the plain push at PR time is not refused as
4600            // a non-fast-forward. A run already holding a recorded conflict
4601            // never reaches here; that case is out of scope.
4602            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4603            if let Some(why) = &conflict {
4604                self.state.status = RunStatus::Blocked;
4605                self.state.event("land", why.clone());
4606            }
4607            self.state.base_sync = Some(BaseSync {
4608                tip,
4609                behind: 0,
4610                attempts,
4611                conflict,
4612                already_in: None,
4613            });
4614            self.state.save()?;
4615            return Ok(());
4616        }
4617
4618        if attempts >= BASE_SYNC_ROUNDS {
4619            let why = format!(
4620                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4621                 rebase(s); rebasing again would only race it",
4622                winner.branch
4623            );
4624            self.state.status = RunStatus::Blocked;
4625            self.state.base_sync = Some(BaseSync {
4626                tip,
4627                behind,
4628                attempts,
4629                conflict: Some(why.clone()),
4630                already_in: None,
4631            });
4632            self.state.event("land", why);
4633            self.state.save()?;
4634            return Ok(());
4635        }
4636
4637        self.state.event(
4638            "land",
4639            format!(
4640                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4641                winner.branch
4642            ),
4643        );
4644        self.state.save()?;
4645
4646        // The remote's copy of the branch, read now and only if the fetch
4647        // really succeeded (a stale tracking ref must never pin a lease). It is
4648        // pushed over after a rebase only when it is a commit this branch
4649        // already contains, by ancestry or by patch (an earlier rebase of ours
4650        // that never reached the remote): anything else is somebody else's work.
4651        let branch_tracking = format!("{remote}/{}", winner.branch);
4652        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4653        let remote_tip = if self.state.task_pr_branch().is_none()
4654            && matches!(&fetched_branch, Ok(o) if o.ok())
4655        {
4656            git::rev_parse(&repo, &branch_tracking).await.ok()
4657        } else {
4658            None
4659        };
4660        // A remote tip this branch does not contain is somebody else's work:
4661        // rebasing would leave a local tip that can never be pushed. Stop
4662        // before touching anything and say so.
4663        if let Some(theirs) = &remote_tip
4664            && !git::is_ancestor(&repo, theirs, &head).await
4665            && !crate::reconcile::origin_missing(&repo, &head, theirs)
4666                .await
4667                .is_ok_and(|missing| missing.is_empty())
4668        {
4669            let why = format!(
4670                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4671                 them",
4672                short(theirs),
4673                winner.branch
4674            );
4675            self.state.status = RunStatus::Blocked;
4676            self.state.base_sync = Some(BaseSync {
4677                tip,
4678                behind,
4679                attempts,
4680                conflict: Some(why.clone()),
4681                already_in: None,
4682            });
4683            self.state.event("land", why);
4684            self.state.save()?;
4685            return Ok(());
4686        }
4687
4688        let scratch = self.state.dir().join("base-sync");
4689        let rebased = match crate::rebase::rebase_with_fixer(
4690            &mut self.state,
4691            &scratch,
4692            &winner.branch,
4693            &tracking,
4694        )
4695        .await
4696        {
4697            Ok(crate::rebase::Rebased::Applied) => Ok(None),
4698            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4699            Err(e) => Err(e),
4700        };
4701        let attempts = attempts + 1;
4702        match rebased {
4703            Ok(None) => {
4704                // The branch ref moved, but a worktree that already had it
4705                // checked out (the winner's) was not told; sync its index and
4706                // files before anything reads them.
4707                git::sync_to_head(&winner.worktree).await?;
4708                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4709                let mut conflict = None;
4710                if let Some(pinned) = &remote_tip {
4711                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4712                    match pushed {
4713                        Ok(o) if o.ok() => self.state.event(
4714                            "land",
4715                            format!("pushed rebased {} to {remote}", winner.branch),
4716                        ),
4717                        Ok(o) => {
4718                            conflict = Some(format!(
4719                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
4720                                winner.branch,
4721                                short(pinned),
4722                                o.stderr.chars().take(600).collect::<String>()
4723                            ));
4724                        }
4725                        Err(e) => {
4726                            conflict = Some(format!(
4727                                "rebased {} locally but could not push it: {e:#}",
4728                                winner.branch
4729                            ));
4730                        }
4731                    }
4732                }
4733                if let Some(why) = &conflict {
4734                    self.state.status = RunStatus::Blocked;
4735                    self.state.event("land", why.clone());
4736                }
4737                self.state.base_sync = Some(BaseSync {
4738                    tip: tip.clone(),
4739                    behind: 0,
4740                    attempts,
4741                    conflict,
4742                    already_in: None,
4743                });
4744                self.state
4745                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
4746            }
4747            Ok(Some(conflict)) => {
4748                let why = format!(
4749                    "{} conflicts with {tracking} and did not rebase: {}",
4750                    winner.branch,
4751                    conflict.chars().take(600).collect::<String>()
4752                );
4753                self.state.status = RunStatus::Blocked;
4754                self.state.base_sync = Some(BaseSync {
4755                    tip,
4756                    behind,
4757                    attempts,
4758                    conflict: Some(why.clone()),
4759                    already_in: None,
4760                });
4761                self.state.event("land", why);
4762            }
4763            Err(e) => {
4764                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4765                self.state.status = RunStatus::Blocked;
4766                self.state.base_sync = Some(BaseSync {
4767                    tip,
4768                    behind,
4769                    attempts,
4770                    conflict: Some(why.clone()),
4771                    already_in: None,
4772                });
4773                self.state.event("land", why);
4774            }
4775        }
4776        self.state.save()?;
4777        Ok(())
4778    }
4779
4780    /// Push a branch an earlier attempt rebased locally but never published,
4781    /// pinned to the remote tip read right after a successful fetch. Returns
4782    /// the reason when the run must stop; `None` when there was nothing to do
4783    /// (no remote copy, the same tip, or a remote copy this branch already
4784    /// contains, which the PR-time push fast-forwards) or the push succeeded.
4785    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4786        // Task competitions publish only after the PR text gate, to their
4787        // public head; never resurrect a candidate branch on the remote.
4788        if self.state.task_pr_branch().is_some() {
4789            return None;
4790        }
4791        let repo = self.state.repo.clone();
4792        let remote = self.state.config.merge.remote.clone();
4793        let fetched = git::fetch(&repo, &remote, branch).await;
4794        if !matches!(&fetched, Ok(o) if o.ok()) {
4795            return None;
4796        }
4797        let branch_tracking = format!("{remote}/{branch}");
4798        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4799        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4800            return None;
4801        }
4802        if !crate::reconcile::origin_missing(&repo, head, &theirs)
4803            .await
4804            .is_ok_and(|missing| missing.is_empty())
4805        {
4806            return Some(format!(
4807                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4808                 them",
4809                short(&theirs)
4810            ));
4811        }
4812        match git::push_pinned(&repo, &remote, branch, &theirs).await {
4813            Ok(o) if o.ok() => {
4814                self.state
4815                    .event("land", format!("pushed rebased {branch} to {remote}"));
4816                None
4817            }
4818            Ok(o) => Some(format!(
4819                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4820                 someone may have pushed): {}",
4821                short(&theirs),
4822                o.stderr.chars().take(600).collect::<String>()
4823            )),
4824            Err(e) => Some(format!(
4825                "{branch} is rebased locally but could not be pushed: {e:#}"
4826            )),
4827        }
4828    }
4829
4830    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
4831    /// is already on `tip` under other commit ids ([`crate::already`]); returns
4832    /// whether it did.
4833    ///
4834    /// Checked only when the base is ahead of the branch. A failing check is
4835    /// "not proven" - the ordinary rebase path then decides - never a reason to
4836    /// stop the run.
4837    ///
4838    /// The remote copy of the branch is held to the same standard as the local
4839    /// one: if it carries a tip this worktree does not, that tip must itself be
4840    /// proven in the base, or nothing is settled (a pull request would
4841    /// otherwise be closed over commits nobody checked). The pull request is
4842    /// closed *before* the terminal status is saved; if that fails for a
4843    /// reason other than a refusal (no network, a `gh` error) the run is left
4844    /// `Blocked` with the reason as its conflict, which a resume retries -
4845    /// the same recovery a phantom conflict gets.
4846    async fn settle_already_in(
4847        &mut self,
4848        branch: &str,
4849        tip: &str,
4850        head: &str,
4851        attempts: usize,
4852        behind: usize,
4853    ) -> Result<bool> {
4854        let repo = self.state.repo.clone();
4855        let remote = self.state.config.merge.remote.clone();
4856        let start = self.state.base_commit.clone();
4857        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4858            Ok(Some(e)) => e,
4859            Ok(None) => return Ok(false),
4860            Err(e) => {
4861                tracing::warn!("already-in-base check for {branch}: {e:#}");
4862                return Ok(false);
4863            }
4864        };
4865        let mut verified = vec![head.to_owned()];
4866        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4867        let fetched = git::fetch(&repo, &remote, &public_branch).await;
4868        if matches!(&fetched, Ok(o) if o.ok())
4869            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4870            && theirs != head
4871        {
4872            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4873                Ok(Some(_)) => verified.push(theirs),
4874                _ => return Ok(false),
4875            }
4876        }
4877        let base_branch = self.state.base_branch.clone();
4878        let message = format!(
4879            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4880             land",
4881            evidence.names(),
4882            evidence.proof.as_str()
4883        );
4884        let closed =
4885            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4886                .await;
4887        match closed {
4888            Ok(Ok(url)) => self
4889                .state
4890                .event("land", format!("closed {url}: superseded on {base_branch}")),
4891            Ok(Err(why)) => self
4892                .state
4893                .event("land", format!("did not close a pull request: {why}")),
4894            Err(e) => {
4895                let why = format!(
4896                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
4897                     not be closed ({e:#}); resume to retry"
4898                );
4899                self.state.status = RunStatus::Blocked;
4900                self.state.base_sync = Some(BaseSync {
4901                    tip: tip.to_owned(),
4902                    behind,
4903                    attempts,
4904                    conflict: Some(why.clone()),
4905                    already_in: None,
4906                });
4907                self.state.event("land", why);
4908                self.state.save()?;
4909                return Ok(true);
4910            }
4911        }
4912        self.state.status = RunStatus::AlreadyInBase;
4913        self.state.base_sync = Some(BaseSync {
4914            tip: tip.to_owned(),
4915            behind,
4916            attempts,
4917            conflict: None,
4918            already_in: Some(evidence),
4919        });
4920        self.state.event("land", message);
4921        self.state.save()?;
4922        self.settle_questions();
4923        Ok(true)
4924    }
4925
4926    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
4927    /// last landed the winner on, once it has run, else the commit the run
4928    /// branched from.
4929    ///
4930    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
4931    /// and `vote` all happen before there is a winner to rebase, so they
4932    /// compare every candidate against the branch point on purpose, and a
4933    /// base that moves after they are already done cannot change an answer
4934    /// they already gave.
4935    fn landing_base(&self) -> String {
4936        self.state
4937            .base_sync
4938            .as_ref()
4939            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4940    }
4941
4942    // ------------------------------------------------------- operator fix
4943
4944    /// Route specific, already-recorded review findings to a fixer for a
4945    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
4946    /// entry point.
4947    ///
4948    /// Distinct from `review_loop`'s own fix step in three ways: it never
4949    /// runs a reviewer wave, it never spends review-round budget, and what
4950    /// happened is recorded as an [`OperatorFixRequest`] appended to
4951    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
4952    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
4953    /// and vote must never be rewritten to look like a manufactured blocking
4954    /// verdict.
4955    ///
4956    /// Only meaningful once review has actually concluded: `Ready` (handed
4957    /// off with findings still open, or simply concluded clean while minor
4958    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
4959    /// gate failed). Everything else is refused: a run still in progress
4960    /// should simply be resumed, and a `Merged` run's branch has already
4961    /// landed — reopening *this* run's own record cannot change that, so the
4962    /// answer there is a fresh `magi review <branch>`.
4963    ///
4964    /// A real commit here re-verifies through a fresh, ordinary review-only
4965    /// run on the same branch ([`Self::review`]) rather than reopening this
4966    /// run's own `review_loop`: once any round in this run's history went
4967    /// clean, `review_conclusion` treats that as permanent by design (the
4968    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
4969    /// way to force one more genuine reviewer wave out of *this* run without
4970    /// either rewriting history or weakening that guarantee for every other
4971    /// caller. A review-only run costs nothing extra — no implementation, no
4972    /// judging, no vote — and exercises the exact same review → verify →
4973    /// gate → (human) merge path, unmodified.
4974    pub async fn fix_selected(
4975        &mut self,
4976        ids: &[String],
4977        reason: &str,
4978        allow_stale: bool,
4979    ) -> Result<()> {
4980        let reason = reason.trim();
4981        if reason.is_empty() {
4982            bail!("a fix request needs a reason — that is the operator's own record of why");
4983        }
4984        if ids.is_empty() {
4985            bail!("no finding id given");
4986        }
4987        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4988            bail!(
4989                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4990                 has already concluded — can be given a targeted fix. A run still \
4991                 in progress should simply be resumed; a `merged` run's branch has \
4992                 already landed, so its answer is a fresh `magi review <branch>`, \
4993                 not reopening this run's own record",
4994                self.state.id,
4995                self.state.status.as_str()
4996            );
4997        }
4998        let Some(winner) = self.state.winner().cloned() else {
4999            bail!("run {} has no winning candidate to fix", self.state.id);
5000        };
5001        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5002            bail!(
5003                "branch `{}` no longer exists; this run cannot be extended",
5004                winner.branch
5005            );
5006        }
5007        let home = crate::run::home();
5008        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5009            bail!(
5010                "run {} is currently being worked on by another magi process",
5011                self.state.id
5012            );
5013        }
5014        // Held for the rest of this call, including the follow-up review
5015        // below: two `magi fix` invocations against the same run must not
5016        // both reach the worktree manipulation further down, which would
5017        // otherwise race to remove and recreate the same directory — see
5018        // [`FixClaim`]'s own doc.
5019        let _claim = FixClaim::acquire(&self.state.dir())?;
5020
5021        // Resolve every id before spending anything — an unknown id refuses
5022        // the whole request rather than silently dropping it — and dedup
5023        // while keeping the operator's own order.
5024        let mut seen = BTreeSet::new();
5025        let mut findings = Vec::new();
5026        let mut missing = Vec::new();
5027        for id in ids {
5028            if !seen.insert(id.clone()) {
5029                continue;
5030            }
5031            match self.state.finding(id) {
5032                Some((round, rec, f)) => findings.push(OperatorFixFinding {
5033                    id: f.id.clone(),
5034                    severity: f.severity,
5035                    reviewer_vote: rec.vote,
5036                    round: round.round,
5037                    round_head: round.head.clone(),
5038                    reviewer: rec.reviewer,
5039                    agent: rec.agent.clone(),
5040                    file: f.file.clone(),
5041                    line: f.line,
5042                    title: f.title.clone(),
5043                    detail: f.detail.clone(),
5044                    outcome: OperatorFixOutcome::Pending,
5045                }),
5046                None => missing.push(id.clone()),
5047            }
5048        }
5049        if !missing.is_empty() {
5050            bail!(
5051                "unknown finding id(s): {}; nothing was changed",
5052                missing.join(", ")
5053            );
5054        }
5055
5056        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5057        let stale_details: Vec<(String, String)> = findings
5058            .iter()
5059            .filter(|f| f.round_head != head_at_request)
5060            .map(|f| (f.id.clone(), f.round_head.clone()))
5061            .collect();
5062        let stale = !stale_details.is_empty();
5063        if stale && !allow_stale {
5064            bail!(
5065                "the branch has moved since some finding(s) were raised — {} — now \
5066                 at {}; pass --allow-stale to fix anyway, or re-run review first",
5067                stale_details
5068                    .iter()
5069                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5070                    .collect::<Vec<_>>()
5071                    .join(", "),
5072                short(&head_at_request)
5073            );
5074        }
5075
5076        let request = OperatorFixRequest {
5077            requested_at: Timestamp::now(),
5078            reason: reason.to_owned(),
5079            findings,
5080            head_at_request: head_at_request.clone(),
5081            allow_stale,
5082            stale,
5083            fix: None,
5084            result_head: None,
5085            follow_up_review_run: None,
5086        };
5087        self.state.event(
5088            "fix",
5089            format!(
5090                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5091                request.findings.len(),
5092                request
5093                    .findings
5094                    .iter()
5095                    .map(|f| f.id.as_str())
5096                    .collect::<Vec<_>>()
5097                    .join(", "),
5098            ),
5099        );
5100        // Recorded now, before any worktree work or the fixer call itself —
5101        // and re-saved at each checkpoint below: a crash at any point after
5102        // this (mid fixer call, mid follow-up review) must not lose the fact
5103        // that this was requested, for which findings, and why. Everything
5104        // past this point reads and writes through `request_index` rather
5105        // than a local variable, since `request` itself is moved here.
5106        self.state.operator_fixes.push(request);
5107        self.state.save()?;
5108        let request_index = self.state.operator_fixes.len() - 1;
5109
5110        // A fresh, dedicated worktree for this one call, never the winner's
5111        // own worktree in place: that one may already be gone (folded away),
5112        // and reusing it in place would leave the branch checked out there
5113        // when the follow-up review below tries to check it out again. Freed
5114        // immediately after, either way — but only once confirmed clean:
5115        // `worktree_remove` is a `git worktree remove --force`, which would
5116        // otherwise discard uncommitted work left there by the operator or
5117        // another process before this had a chance to even look at it.
5118        let repo = self.state.repo.clone();
5119        let coordination = self.coordinate(&repo).await?;
5120        if winner.worktree.exists() {
5121            // Lockfiles a rescue commit withheld stay untracked on purpose and
5122            // are already recorded; they are not the operator's work to protect.
5123            let dirty = git::git(
5124                &winner.worktree,
5125                &["status", "--porcelain", "--untracked-files=all"],
5126            )
5127            .await?;
5128            let only_withheld = dirty.lines().all(|l| {
5129                l.strip_prefix("?? ")
5130                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5131            });
5132            if !only_withheld {
5133                bail!(
5134                    "`{}` has uncommitted changes; refusing to touch it — commit or \
5135                     discard them first",
5136                    winner.worktree.display()
5137                );
5138            }
5139            git::worktree_remove(&self.state.repo, &winner.worktree)
5140                .await
5141                .ok();
5142        }
5143        let fix_worktree = self.state.worktree_root().join("operator-fix");
5144        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5145        git::git(
5146            &self.state.repo,
5147            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5148        )
5149        .await
5150        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5151        if !git::is_clean(&fix_worktree).await? {
5152            git::worktree_remove(&self.state.repo, &fix_worktree)
5153                .await
5154                .ok();
5155            bail!(
5156                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5157                winner.branch
5158            );
5159        }
5160        let repo_for_setup = self.state.repo.clone();
5161        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5162            Ok(()) => {
5163                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5164            }
5165            Err(e) => Err(e),
5166        };
5167        if let Err(e) = setup {
5168            git::worktree_remove(&self.state.repo, &fix_worktree)
5169                .await
5170                .ok();
5171            return Err(e);
5172        }
5173
5174        let run_id = self.state.id.clone();
5175        let prompts = self.state.config.prompts.clone();
5176        let language = self.state.config.graph.language.clone();
5177        let sessions = self.state.config.graph.sessions;
5178        let artifacts = agent::artifacts_dir(&self.state.dir());
5179        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5180            .findings
5181            .iter()
5182            .map(|f| Finding {
5183                id: f.id.clone(),
5184                severity: f.severity,
5185                file: f.file.clone(),
5186                line: f.line,
5187                title: f.title.clone(),
5188                detail: f.detail.clone(),
5189            })
5190            .collect();
5191        let fix_prompt = prompt::operator_fix_in(
5192            &self.state.instruction,
5193            &finding_list,
5194            reason,
5195            &stale_details,
5196            &head_at_request,
5197            &language,
5198            &self.state.config.graph.github_language,
5199        );
5200        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5201        drop(coordination);
5202        let (job, seat, out) = self
5203            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5204                prompt: fix_prompt.clone(),
5205                spec,
5206                seat,
5207                cwd: fix_worktree.clone(),
5208                timeout,
5209                allow_write: true,
5210                sessions,
5211                artifacts: artifacts.clone(),
5212                stem: "operator-fix".to_owned(),
5213                handover: None,
5214            })
5215            .await;
5216        let agent_id = seat.agent.clone();
5217
5218        let mut fix = FixRecord {
5219            agent: agent_id,
5220            addressed: Vec::new(),
5221            rejected: Vec::new(),
5222            notes: String::new(),
5223            committed: false,
5224            failed: None,
5225            duration_ms: 0,
5226            continuation: None,
5227        };
5228        let mut final_seat = seat.clone();
5229        match out {
5230            AgentOutcome::Ok(o) => {
5231                fix.duration_ms = o.duration_ms;
5232                let parsed = verdict::extract_json::<FixReport>(&o.text);
5233                let incomplete_reason = match &parsed {
5234                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5235                        "the reply parsed, but it reported a command whose own CLI \
5236                         never confirmed an exit status"
5237                            .to_owned(),
5238                    ),
5239                    Ok(_) => None,
5240                    Err(e) => Some(e.to_string()),
5241                };
5242                match incomplete_reason {
5243                    None => {
5244                        let report = parsed.expect("checked Ok above");
5245                        fix.addressed = report.addressed;
5246                        fix.rejected = report.rejected;
5247                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5248                    }
5249                    Some(reason) => {
5250                        let (resumed_seat, resolved, failure, cont) = self
5251                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5252                            .await;
5253                        fix.duration_ms += cont.cumulative_wait_ms;
5254                        fix.continuation = Some(cont);
5255                        final_seat = resumed_seat;
5256                        match resolved {
5257                            Some(report) => {
5258                                fix.addressed = report.addressed;
5259                                fix.rejected = report.rejected;
5260                                fix.notes =
5261                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
5262                            }
5263                            None => fix.failed = failure,
5264                        }
5265                    }
5266                }
5267            }
5268            AgentOutcome::Dropped(o) => {
5269                fix.duration_ms = o.duration_ms;
5270                let why = o
5271                    .dropped
5272                    .as_ref()
5273                    .map(|d| d.why.as_str())
5274                    .unwrap_or("the CLI ended the stream without delivering its answer");
5275                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5276            }
5277            AgentOutcome::Quota(o) => {
5278                self.state.quota.push(QuotaLoss {
5279                    seat: final_seat.key.clone(),
5280                    node: "fix".to_owned(),
5281                    at: Timestamp::now(),
5282                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5283                });
5284                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5285            }
5286            AgentOutcome::Failed(e) => fix.failed = Some(e),
5287            AgentOutcome::ResourceBlocked(e) => {
5288                fix.failed = Some(format!("the fixer was not started: {e}"));
5289            }
5290        }
5291        if fix.continuation.is_none() {
5292            fix.continuation = Some(ContinuationRecord::not_needed());
5293        }
5294        self.state.seats.insert(final_seat.key.clone(), final_seat);
5295
5296        let rescue_message = format!(
5297            "magi: operator-selected fix ({}) (uncommitted work)",
5298            self.state.operator_fixes[request_index]
5299                .findings
5300                .iter()
5301                .map(|f| f.id.as_str())
5302                .collect::<Vec<_>>()
5303                .join(", ")
5304        );
5305        let repo = self.state.repo.clone();
5306        let coordination = self.coordinate(&repo).await?;
5307        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5308            self.state.note_withheld("fix", &r.withheld);
5309        }
5310        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5311        fix.committed = after != head_at_request;
5312        git::worktree_remove(&self.state.repo, &fix_worktree)
5313            .await
5314            .ok();
5315        drop(coordination);
5316
5317        self.state.event(
5318            "fix",
5319            match &fix.failed {
5320                Some(reason) => format!(
5321                    "operator fix: adoption report was lost ({reason}); {}",
5322                    if fix.committed {
5323                        "committed"
5324                    } else {
5325                        "NO new commit"
5326                    }
5327                ),
5328                None => format!(
5329                    "operator fix: {} addressed, {} rejected, {}",
5330                    fix.addressed.len(),
5331                    fix.rejected.len(),
5332                    if fix.committed {
5333                        "committed"
5334                    } else {
5335                        "NO new commit"
5336                    }
5337                ),
5338            },
5339        );
5340
5341        // Every selected finding gets an outcome — never left `Pending` once
5342        // the fixer's own turn is over. A report that never came back at all
5343        // marks every one of them `Unreported`, not silently "not addressed":
5344        // quota, a dropped stream, or an exhausted continuation are gaps in
5345        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
5346        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
5347        for f in &mut self.state.operator_fixes[request_index].findings {
5348            f.outcome = if fix.failed.is_some() {
5349                OperatorFixOutcome::Unreported
5350            } else if fix.addressed.contains(&f.id) {
5351                OperatorFixOutcome::Addressed
5352            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5353                OperatorFixOutcome::Rejected { why: r.why.clone() }
5354            } else {
5355                OperatorFixOutcome::Unreported
5356            };
5357        }
5358
5359        let committed = fix.committed;
5360        if committed {
5361            self.state.operator_fixes[request_index].result_head = Some(after.clone());
5362        }
5363        self.state.operator_fixes[request_index].fix = Some(fix);
5364        // Saved again now that the fixer's own outcome is final, on top of
5365        // the save right after the request was first pushed above.
5366        self.state.save()?;
5367
5368        if committed {
5369            self.state.event(
5370                "fix",
5371                format!(
5372                    "operator fix committed {}; opening a follow-up review-only run",
5373                    short(&after)
5374                ),
5375            );
5376            // The operator asked for the fix, and the follow-up serves whatever
5377            // task the run it follows served.
5378            let origin =
5379                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5380            match Self::review(
5381                &self.state.repo,
5382                &winner.branch,
5383                self.state.config.clone(),
5384                origin,
5385            )
5386            .await
5387            {
5388                Ok(mut follow_up) => {
5389                    follow_up.state.event(
5390                        "start",
5391                        format!(
5392                            "requested by an operator fix on run {} for finding(s) {}",
5393                            self.state.id,
5394                            self.state.operator_fixes[request_index]
5395                                .findings
5396                                .iter()
5397                                .map(|f| f.id.as_str())
5398                                .collect::<Vec<_>>()
5399                                .join(", "),
5400                        ),
5401                    );
5402                    follow_up.state.save()?;
5403                    let follow_up_id = follow_up.state.id.clone();
5404                    // The follow-up is a run like any other: it belongs to the
5405                    // task of the run it follows, or to one filed for it.
5406                    let adopted = match crate::direct::adopt(
5407                        &follow_up.state,
5408                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5409                    ) {
5410                        Ok(a) => a,
5411                        Err(e) => {
5412                            // An ownerless run must not spend agent calls; it
5413                            // stays saved, and `magi run --resume` adopts it.
5414                            self.state.event(
5415                                "fix",
5416                                format!(
5417                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5418                                ),
5419                            );
5420                            self.state.save()?;
5421                            return Ok(());
5422                        }
5423                    };
5424                    let executed = follow_up.execute().await;
5425                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5426                    if let Some(a) = adopted {
5427                        a.finish(&follow_up.state, executed);
5428                    }
5429                    if let Some(e) = failure {
5430                        self.state.event(
5431                            "fix",
5432                            format!(
5433                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5434                            ),
5435                        );
5436                    }
5437                    self.state.operator_fixes[request_index].follow_up_review_run =
5438                        Some(follow_up_id);
5439                }
5440                Err(e) => {
5441                    self.state.event(
5442                        "fix",
5443                        format!("committed the fix but could not open a follow-up review: {e:#}"),
5444                    );
5445                }
5446            }
5447            self.state.save()?;
5448        }
5449
5450        Ok(())
5451    }
5452
5453    // --------------------------------------------------------------- review
5454
5455    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
5456    /// names, in order, each at most once, moving on only when the call
5457    /// advances (an error, a quota hit or nothing usable - the same decision
5458    /// point `agent::chain_advances` is for the other chained roles).
5459    ///
5460    /// Each agent gets its own seat from `Runner::seat`: the same agent
5461    /// continues its conversation (the winner's own implementer seat when it
5462    /// is the winner's author, now that the competition is over), another
5463    /// takes a fresh one so the full prompt is sent again. A handover is
5464    /// recorded under `node`, which is what makes the fallback stick for the
5465    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
5466    /// is returned, so an exhausted chain reads exactly like a single failed
5467    /// fixer: one quota loss, the same wording, the same refund. An earlier
5468    /// attempt's edits are left in the tree and judged, with the final
5469    /// attempt's, by what git says afterwards.
5470    ///
5471    /// Returns the job that produced the outcome, for `continue_fix_report`.
5472    async fn ask_fixer(
5473        &mut self,
5474        winner: &Candidate,
5475        node: &'static str,
5476        round: Option<usize>,
5477        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5478    ) -> (SeatJob, SeatState, AgentOutcome) {
5479        let attempts = fixer::attempts(&self.state, &self.roles, winner);
5480        self.ask_fixer_among(attempts, node, round, None, build)
5481            .await
5482    }
5483
5484    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
5485    /// worktree and a commit: before the next agent is asked after one that
5486    /// did not satisfy the call, the worktree is put back on that commit, so
5487    /// a failed agent's partial edits are not carried into the next one.
5488    async fn ask_fixer_among(
5489        &mut self,
5490        attempts: Vec<(AgentSpec, String)>,
5491        node: &'static str,
5492        round: Option<usize>,
5493        restore: Option<(&Path, &str)>,
5494        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5495    ) -> (SeatJob, SeatState, AgentOutcome) {
5496        let run_id = self.state.id.clone();
5497        let prompts = self.state.config.prompts.clone();
5498        let cache = self.state.config.cache_dir();
5499        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5500        let mut last = None;
5501        for (i, (spec, key)) in attempts.into_iter().enumerate() {
5502            let seat = self.seat(&key, &spec.id);
5503            let mut job = build(spec.clone(), seat);
5504            if i > 0 {
5505                job.stem = format!("{}-{}", job.stem, spec.id);
5506            }
5507            let ctx = WaveCtx {
5508                carry_seats: false,
5509                pauses: Some((&self.pause, &self.interrupt)),
5510                run: &run_id,
5511                node,
5512                prompts: &prompts,
5513                cache: cache.as_deref(),
5514                round,
5515            };
5516            let (seat, out) =
5517                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5518            // A resource block never advances the chain: the chosen agent
5519            // was not asked, so it must be the one asked again on resume.
5520            let advances = match &out {
5521                AgentOutcome::Ok(o) => agent::output_advances(o),
5522                AgentOutcome::ResourceBlocked(_) => false,
5523                _ => true,
5524            };
5525            if let Some(next) = ids.get(i + 1)
5526                && advances
5527            {
5528                self.state.seats.insert(seat.key.clone(), seat.clone());
5529                let class =
5530                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5531                let reason = match &out {
5532                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5533                    other => fail_reason(other),
5534                };
5535                // A failed agent's edits must not reach the next one, and a
5536                // restore that did not work ends the chain: the call fails
5537                // and the caller's own (confirmed) restore decides.
5538                if let Some((wt, rev)) = restore
5539                    && let Err(e) = self.graft_restore(wt, rev).await
5540                {
5541                    let why =
5542                        format!("could not restore the worktree before the next fixer: {e:#}");
5543                    tracing::warn!("{why}");
5544                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5545                        AgentOutcome::ResourceBlocked(why)
5546                    } else {
5547                        AgentOutcome::Failed(why)
5548                    };
5549                    last = Some((job, seat, out));
5550                    break;
5551                }
5552                record_handover(
5553                    &mut self.state,
5554                    node,
5555                    &seat.key,
5556                    &spec.id,
5557                    next,
5558                    &class,
5559                    &reason,
5560                );
5561                continue;
5562            }
5563            last = Some((job, seat, out));
5564            break;
5565        }
5566        last.expect("the fixer chain always has an entry")
5567    }
5568
5569    /// The one repair a verification-only failure earns (see
5570    /// [`crate::recovery`]), before the first review round of a run that took
5571    /// a branch over.
5572    ///
5573    /// Saved as started *before* the fixer is called, so a resume never
5574    /// repairs twice; a start found on re-entry is rolled back to the commit
5575    /// it began from. A failed or unusable fixer leaves the tree as it was and
5576    /// the review goes on over it, so the first round's own verification
5577    /// reports the failure again. The failure is never carried into a round as
5578    /// a result: verification is re-run on whatever commit this leaves.
5579    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5580        use crate::recovery::{Mode, Repair};
5581        if !self.state.reviews.is_empty() {
5582            return Ok(());
5583        }
5584        let Some(mut rec) = self.state.recovery.clone() else {
5585            return Ok(());
5586        };
5587        if rec.mode != Mode::VerifyRepair {
5588            return Ok(());
5589        }
5590        let wt = winner.worktree.clone();
5591        let mut head = git::rev_parse(&wt, "HEAD").await?;
5592        match rec.repair.clone() {
5593            Repair::Done { .. } => return Ok(()),
5594            Repair::WaitingForRestore { before } => {
5595                self.graft_restore(&wt, &before).await?;
5596                head = before;
5597                rec.repair = Repair::NotStarted;
5598                self.state.recovery = Some(rec.clone());
5599                self.state.save()?;
5600            }
5601            Repair::Started { before } => {
5602                // Interrupted: the unfinished edits are not trusted.
5603                // An unconfirmed rollback blocks the run with `Started` kept,
5604                // so a resume retries it; the tree is never reviewed as is.
5605                if let Err(e) = self.graft_restore(&wt, &before).await {
5606                    if e.is::<crate::repo_lock::ResourceBlocked>() {
5607                        return Err(e);
5608                    }
5609                    return self
5610                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5611                        .await;
5612                }
5613                rec.repair = Repair::Done {
5614                    before: before.clone(),
5615                    after: before,
5616                    committed: false,
5617                    failed: Some("interrupted; rolled back".to_owned()),
5618                };
5619                self.state
5620                    .event("recovery", "the repair was interrupted and rolled back");
5621                self.state.recovery = Some(rec);
5622                self.state.save()?;
5623                return Ok(());
5624            }
5625            Repair::NotStarted => {}
5626        }
5627        if crate::recovery::recheck(&mut rec, &head) {
5628            self.state.event("recovery", rec.summary());
5629            self.state.recovery = Some(rec);
5630            self.state.save()?;
5631            return Ok(());
5632        }
5633        rec.repair = Repair::Started {
5634            before: head.clone(),
5635        };
5636        self.state.recovery = Some(rec.clone());
5637        self.state.save()?;
5638
5639        let language = self.state.config.graph.language.clone();
5640        let sessions = self.state.config.graph.sessions;
5641        let artifacts = agent::artifacts_dir(&self.state.dir());
5642        let fix_prompt = prompt::recovery_fix(
5643            &self.state.instruction,
5644            &rec,
5645            self.state.config.graph.review_rounds,
5646            &language,
5647        );
5648        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5649        self.state.event(
5650            "recovery",
5651            format!(
5652                "repair: the fixer starts from the failed verification on {}",
5653                short(&head)
5654            ),
5655        );
5656        let (_job, seat, out) = self
5657            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5658                prompt: fix_prompt.clone(),
5659                spec,
5660                seat,
5661                cwd: wt.clone(),
5662                timeout,
5663                allow_write: true,
5664                sessions,
5665                artifacts: artifacts.clone(),
5666                stem: "fix-recovery".to_owned(),
5667                handover: None,
5668            })
5669            .await;
5670        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5671            rec.repair = Repair::WaitingForRestore { before: head };
5672            self.state.recovery = Some(rec);
5673            return self.stop_for_resource();
5674        }
5675        let failed = match &out {
5676            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5677            AgentOutcome::Quota(o) => {
5678                self.state.quota.push(QuotaLoss {
5679                    seat: seat.key.clone(),
5680                    node: "fix".to_owned(),
5681                    at: Timestamp::now(),
5682                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5683                });
5684                Some("rate limited (quota); the fixer could not run".to_owned())
5685            }
5686            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5687            AgentOutcome::Failed(e) => Some(e.clone()),
5688        };
5689        self.state.seats.insert(seat.key.clone(), seat);
5690        let coordination = self.coordinate(&wt).await?;
5691        let failed = match failed {
5692            // The agent's partial edits are not kept.
5693            Some(why) => {
5694                if let Err(e) = git::restore_to(&wt, &head).await {
5695                    return self
5696                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5697                        .await;
5698                }
5699                Some(why)
5700            }
5701            None => {
5702                if let Ok(r) = git::rescue_commit(
5703                    &wt,
5704                    "magi: recovery repair of the failed verification (uncommitted work)",
5705                )
5706                .await
5707                {
5708                    self.state.note_withheld("fix", &r.withheld);
5709                }
5710                None
5711            }
5712        };
5713        let after = git::rev_parse(&wt, "HEAD").await?;
5714        drop(coordination);
5715        self.state.event(
5716            "recovery",
5717            match &failed {
5718                Some(why) => format!("repair: nothing kept ({why})"),
5719                None if after != head => {
5720                    format!("repair: committed {}", short(&after))
5721                }
5722                None => "repair: the fixer made no commit".to_owned(),
5723            },
5724        );
5725        rec.repair = Repair::Done {
5726            before: head.clone(),
5727            committed: after != head,
5728            after,
5729            failed,
5730        };
5731        self.state.recovery = Some(rec);
5732        self.state.save()?;
5733        Ok(())
5734    }
5735
5736    /// A repair whose rollback could not be confirmed: the run is blocked and
5737    /// resumable, and the unconfirmed state is never reviewed.
5738    async fn stop_recovery(
5739        &mut self,
5740        mut rec: crate::recovery::RecoveryRecord,
5741        head: &str,
5742        why: String,
5743    ) -> Result<()> {
5744        // `Started` stays recorded: a resume retries the restore.
5745        rec.repair = crate::recovery::Repair::Started {
5746            before: head.to_owned(),
5747        };
5748        self.state.recovery = Some(rec);
5749        self.state
5750            .event("recovery", format!("repair could not be undone: {why}"));
5751        self.state.status = RunStatus::Blocked;
5752        self.state.save()?;
5753        bail!("recovery repair could not be rolled back: {why}")
5754    }
5755
5756    async fn review_loop(&mut self) -> Result<()> {
5757        // A base that would not rebase is a person's decision, not a review
5758        // round: nothing here would change the answer, and reviewers and a
5759        // fixer would be spending real budget on a tree that cannot land
5760        // regardless of what they find.
5761        if self
5762            .state
5763            .base_sync
5764            .as_ref()
5765            .is_some_and(|s| s.conflict.is_some())
5766        {
5767            return Ok(());
5768        }
5769        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
5770        // agent files with `magi task add` name the run that paid for it. The
5771        // prompt overlay is cloned alongside it because the waves borrow it
5772        // while `self` is mutably borrowed by the node's own bookkeeping.
5773        let run_id = self.state.id.clone();
5774        let prompts = self.state.config.prompts.clone();
5775        let Some(winner) = self.state.winner().cloned() else {
5776            return Ok(());
5777        };
5778        let max_rounds = self.state.config.graph.review_rounds;
5779        // A clean round, an exhausted round budget, or a stalled tree (see
5780        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
5781        // they are recorded — recomputed here, not read off `status`, so a
5782        // reentry into a run that already stopped restates the identical
5783        // verdict instead of silently handing back whatever an earlier node
5784        // in this same walk clobbered `status` to (a solo-candidate
5785        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
5786        // below runs an empty range once the budget is spent, and would
5787        // otherwise fall through without touching `status` at all.
5788        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5789            // A reentry after a crash between the last round's save and
5790            // `stop_reviewing` reaches the hand-off here, not there.
5791            if status == RunStatus::Gating {
5792                self.record_contested_handoff();
5793            }
5794            self.state.status = status;
5795            self.state.save()?;
5796            return Ok(());
5797        }
5798        self.state.status = RunStatus::Reviewing;
5799        // A last recorded round whose own verification never resolved
5800        // (`ResourceBlocked` — the shared build cache, not the patch) is
5801        // never a concluded round, whatever the round budget says: starting
5802        // a fresh round on top of it would spend a whole new reviewer wave
5803        // re-reading an unchanged patch instead of just retrying the one
5804        // check that actually needs it, and once the budget is spent the
5805        // loop below has nothing left to do at all (its range is empty).
5806        // Retry that check directly instead, exactly the same retry
5807        // `stop_reviewing` already does for its own catch-up case.
5808        if self
5809            .state
5810            .reviews
5811            .last()
5812            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5813        {
5814            let shell = self.state.config.shell();
5815            return self
5816                .stop_reviewing(
5817                    "the last round's own verification never resolved",
5818                    &shell,
5819                    &winner.worktree,
5820                )
5821                .await;
5822        }
5823
5824        self.recovery_repair(&winner).await?;
5825        if self.resource_halt()? {
5826            return Ok(());
5827        }
5828
5829        let repo = self.state.repo.clone();
5830        let root = self.state.worktree_root();
5831        let language = self.state.config.graph.language.clone();
5832        let sessions = self.state.config.graph.sessions;
5833        let artifacts = self.merge_repair_artifacts();
5834        let base = self.landing_base();
5835        let base_short = short(&base);
5836        let reviewers = self.roles.reviewers.clone();
5837        let shell = self.state.config.shell();
5838
5839        for round in (self.state.reviews.len() + 1)..=max_rounds {
5840            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5841            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5842            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5843            // The prior round's own record, already persisted — never a
5844            // hand-carried variable of just its failing output: that is
5845            // exactly what let a round's e2e result drift out of sync with
5846            // which commit it was actually about (see `SCHEMA`'s doc for
5847            // schema 8). Judged against `head`, the commit reviewers are
5848            // about to look at now, so the summary always reads as "an
5849            // earlier head" here — this round's own patch has not been
5850            // checked yet.
5851            let prev_verification = self
5852                .state
5853                .reviews
5854                .last()
5855                .and_then(|r| r.verification_summary(&head));
5856            // Only the first round of a run that took a branch over: later
5857            // rounds have this run's own history.
5858            let prior_section = match (&self.state.recovery, round) {
5859                (Some(rec), 1) if self.state.reviews.is_empty() => {
5860                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5861                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5862                            Some(crate::recovery::head_of(
5863                                &git::diff(&winner.worktree, reviewed, "HEAD")
5864                                    .await
5865                                    .unwrap_or_default(),
5866                                crate::recovery::SINCE_DIFF_MAX,
5867                            ))
5868                        }
5869                        _ => None,
5870                    };
5871                    Some(prompt::recovery_review(rec, since.as_deref()))
5872                }
5873                _ => None,
5874            };
5875
5876            // Each reviewer gets its own detached checkout of exactly this
5877            // commit: nobody can perturb the winner's tree, and the fixer can
5878            // keep working without racing a reviewer.
5879            let mut jobs = Vec::new();
5880            let coordination = self.coordinate(&repo).await?;
5881            for (r, spec) in reviewers.iter().cloned().enumerate() {
5882                let wt = root.join(format!("review-{}", r + 1));
5883                // `clean -fdx` in a reset wipes setup's products, so every
5884                // round sets the seat up again.
5885                if wt.exists() {
5886                    git::reset_detached(&wt, &head).await?;
5887                } else {
5888                    git::worktree_add_detached(&repo, &wt, &head).await?;
5889                }
5890                self.setup_seat_worktree(&repo, &wt, false).await?;
5891                let seat_key = format!("review-{}", r + 1);
5892                // The seat starts the round on whoever answered it last, not
5893                // on the agent the spec names, so a failure is not re-paid.
5894                let spec = pick_start_spec(
5895                    &self.roles.reviewer_roster,
5896                    spec,
5897                    self.state.seat_history.get(&seat_key),
5898                );
5899                let seat = self.seat(&seat_key, &spec.id);
5900                jobs.push(SeatJob {
5901                    prompt: prompt::review(&prompt::ReviewCtx {
5902                        instruction: &self.state.instruction,
5903                        branch: &winner.branch,
5904                        base_short: &base_short,
5905                        stat: &stat,
5906                        patch: &patch,
5907                        verification: prev_verification.as_ref(),
5908                        prior: prior_section.as_deref(),
5909                        reviewers: reviewers.len(),
5910                        round,
5911                        rounds: max_rounds,
5912                        // A review-only run has no rankings, so nothing
5913                        // competed for this patch and the reviewer is told so.
5914                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5915                        lens: Lens::for_seat(r),
5916                        language: &language,
5917                        github_language: &self.state.config.graph.github_language,
5918                    }),
5919                    spec,
5920                    seat,
5921                    cwd: wt,
5922                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5923                    allow_write: false,
5924                    sessions,
5925                    artifacts: artifacts.clone(),
5926                    stem: format!("review-{round}-{}", r + 1),
5927                    handover: None,
5928                });
5929            }
5930
5931            drop(coordination);
5932            self.state.event(
5933                "review",
5934                format!(
5935                    "round {round}: {} reviewers on {}",
5936                    jobs.len(),
5937                    short(&head)
5938                ),
5939            );
5940            let mut quota_losses = Vec::new();
5941            let review_retries = self.state.config.graph.retries;
5942            let review_cache = self.state.config.cache_dir();
5943            let ctx = WaveCtx {
5944                carry_seats: true,
5945                pauses: Some((&self.pause, &self.interrupt)),
5946                run: &run_id,
5947                node: "review",
5948                prompts: &prompts,
5949                cache: review_cache.as_deref(),
5950                round: Some(round),
5951            };
5952            let results = ask_json_wave::<Review>(
5953                jobs,
5954                Arc::clone(&self.sem),
5955                review_retries,
5956                &self.roles.reviewer_roster,
5957                &ctx,
5958                &mut quota_losses,
5959                &mut self.state,
5960                &|_: &Review| Ok(()),
5961            )
5962            .await;
5963            if self.state.resource_block.is_some() {
5964                return self.stop_for_resource();
5965            }
5966            // Counted before the move below: how many of *this* round's
5967            // reviewer seats were lost to their own rate limit, as opposed to
5968            // a crash, a timeout, or unparsable output — see `round_is_clean`.
5969            let round_quota_missing = quota_losses.len();
5970            self.state.quota.extend(quota_losses);
5971
5972            let mut records = Vec::new();
5973            let mut all_findings = Vec::new();
5974            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5975                let agent_id = seat.agent.clone();
5976                self.state.seats.insert(seat.key.clone(), seat);
5977                let mut record = ReviewRecord {
5978                    reviewer: r + 1,
5979                    agent: agent_id,
5980                    summary: String::new(),
5981                    findings: Vec::new(),
5982                    vote: None,
5983                    failed: None,
5984                    duration_ms: 0,
5985                    // Set for both outcomes: `failed: Some(_)` with
5986                    // `attempts > 0` is a seat every retry still lost, not a
5987                    // recovered one — only `failed: None` with `attempts > 0`
5988                    // reads as "answered after a nudge" (see this field's own
5989                    // doc).
5990                    attempts,
5991                };
5992                match res {
5993                    Ok((review, out)) => {
5994                        // Sanitized here, at the point every other piece of
5995                        // agent prose in this file is (candidate summaries,
5996                        // deliberation turns, vote reasons): a reviewer's own
5997                        // words are the one thing about it that could name
5998                        // it, and reconsideration below broadcasts this same
5999                        // summary and these same findings to every other
6000                        // seat on the panel.
6001                        record.summary =
6002                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
6003                        record.vote = Some(review.vote);
6004                        record.duration_ms = out.duration_ms;
6005                        for (n, mut f) in review.findings.into_iter().enumerate() {
6006                            // ids are magi's, never the agent's: the fixer's
6007                            // adoption report is keyed by them.
6008                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6009                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6010                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6011                            // `file` is agent-supplied prose too, never
6012                            // checked against the real tree — the same
6013                            // exposure `title`/`detail` above have, just in
6014                            // a field easy to forget because it looks like a
6015                            // path rather than free text.
6016                            f.file = f
6017                                .file
6018                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6019                            all_findings.push(f.clone());
6020                            record.findings.push(f);
6021                        }
6022                        self.state.event(
6023                            "review",
6024                            format!(
6025                                "round {round}: reviewer {} voted {} with {} finding(s)",
6026                                r + 1,
6027                                review.vote.label(),
6028                                record.findings.len()
6029                            ),
6030                        );
6031                    }
6032                    Err(e) => {
6033                        record.failed = Some(e.to_string());
6034                        self.state.event(
6035                            "review",
6036                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6037                        );
6038                    }
6039                }
6040                records.push(record);
6041            }
6042
6043            // Tally the round's votes and, if they split, spend the one
6044            // round of reconsideration the split -> deliberate -> revote
6045            // shape `judge`/`vote` use for the panel, sized down to what a
6046            // read-only review round can afford: one round, and a revote
6047            // rather than an argument, because the panel already wrote its
6048            // reasoning down as findings the first time around.
6049            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6050            let vote_split =
6051                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6052            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6053            if vote_split {
6054                self.state.event(
6055                    "review",
6056                    format!(
6057                        "round {round}: votes split ({}) — one round of reconsideration",
6058                        initial_votes
6059                            .iter()
6060                            .map(|v| v.label())
6061                            .collect::<Vec<_>>()
6062                            .join(", ")
6063                    ),
6064                );
6065                // Seats read every seat's findings and votes, still numbered
6066                // and never named — the same anonymity `review` itself keeps.
6067                let panel: Vec<ReviewSeatReport<'_>> = records
6068                    .iter()
6069                    .filter_map(|r| {
6070                        r.vote.map(|vote| ReviewSeatReport {
6071                            reviewer: r.reviewer,
6072                            vote,
6073                            summary: &r.summary,
6074                            findings: &r.findings,
6075                        })
6076                    })
6077                    .collect();
6078
6079                let mut jobs = Vec::new();
6080                let mut seats_at = Vec::new();
6081                for (r, spec) in reviewers.iter().cloned().enumerate() {
6082                    // A seat with no initial vote has nothing to reconsider
6083                    // from and stays absent, the same as it stayed absent
6084                    // from `panel` above.
6085                    if records[r].vote.is_none() {
6086                        continue;
6087                    }
6088                    let wt = root.join(format!("review-{}", r + 1));
6089                    let seat_key = format!("review-{}", r + 1);
6090                    let spec = self.occupant(&seat_key, spec);
6091                    let seat = self.seat(&seat_key, &spec.id);
6092                    // A seat with no live session has already forgotten the
6093                    // initial review's prompt — restate the patch it is
6094                    // voting on, the same as `deliberate`/`vote` do for a
6095                    // judge in the same position.
6096                    // The panel already carries this seat's own review and
6097                    // vote, so restating the patch makes the prompt whole for
6098                    // a seat handed to another agent.
6099                    let build = |with_patch: bool| {
6100                        prompt::review_reconsider(&ReviewReconsiderCtx {
6101                            instruction: &self.state.instruction,
6102                            reviewer: r + 1,
6103                            lens: Lens::for_seat(r),
6104                            panel: &panel,
6105                            patch: with_patch.then_some(ReviewPatch {
6106                                branch: &winner.branch,
6107                                base_short: &base_short,
6108                                stat: &stat,
6109                                patch: &patch,
6110                            }),
6111                            round,
6112                            rounds: max_rounds,
6113                            language: &language,
6114                        })
6115                    };
6116                    let full = build(true);
6117                    let prompt = if has_context(&spec, &seat, sessions) {
6118                        build(false)
6119                    } else {
6120                        full.clone()
6121                    };
6122                    jobs.push(SeatJob {
6123                        prompt,
6124                        spec,
6125                        seat,
6126                        cwd: wt,
6127                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6128                        allow_write: false,
6129                        sessions,
6130                        artifacts: artifacts.clone(),
6131                        stem: format!("review-{round}-reconsider-{}", r + 1),
6132                        handover: Some(full),
6133                    });
6134                    seats_at.push(r);
6135                }
6136
6137                let mut recon_quota_losses = Vec::new();
6138                let recon_cache = self.state.config.cache_dir();
6139                let recon_ctx = WaveCtx {
6140                    carry_seats: true,
6141                    pauses: Some((&self.pause, &self.interrupt)),
6142                    run: &run_id,
6143                    node: "review",
6144                    prompts: &prompts,
6145                    cache: recon_cache.as_deref(),
6146                    round: Some(round),
6147                };
6148                let recon_results = ask_json_wave::<ReviewRevote>(
6149                    jobs,
6150                    Arc::clone(&self.sem),
6151                    review_retries,
6152                    &self.roles.reviewer_roster,
6153                    &recon_ctx,
6154                    &mut recon_quota_losses,
6155                    &mut self.state,
6156                    &|_: &ReviewRevote| Ok(()),
6157                )
6158                .await;
6159                if self.state.resource_block.is_some() {
6160                    return self.stop_for_resource();
6161                }
6162                self.state.quota.extend(recon_quota_losses);
6163
6164                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6165                    let agent_id = seat.agent.clone();
6166                    self.state.seats.insert(seat.key.clone(), seat);
6167                    let mut rec = ReviewRevoteRecord {
6168                        reviewer: r + 1,
6169                        agent: agent_id,
6170                        vote: None,
6171                        reason: String::new(),
6172                        failed: None,
6173                    };
6174                    match res {
6175                        Ok((rv, _)) => {
6176                            rec.vote = Some(rv.vote);
6177                            rec.reason =
6178                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6179                            self.state.event(
6180                                "review",
6181                                format!(
6182                                    "round {round}: reviewer {} revoted {}",
6183                                    r + 1,
6184                                    rv.vote.label()
6185                                ),
6186                            );
6187                        }
6188                        Err(e) => {
6189                            rec.failed = Some(e.to_string());
6190                            self.state.event(
6191                                "review",
6192                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6193                            );
6194                        }
6195                    }
6196                    reconsideration.push(rec);
6197                }
6198            } else if initial_votes.len() > 1 {
6199                self.state.event(
6200                    "review",
6201                    format!(
6202                        "round {round}: votes agreed ({}) — no reconsideration",
6203                        initial_votes[0].label()
6204                    ),
6205                );
6206            }
6207
6208            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6209            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6210            // A round that already has a blocking finding and a round left to
6211            // try is going back to the fixer no matter what `verify.e2e`
6212            // says, so running it first only spends the loop's slowest step
6213            // (minutes, for a Rust repo's full test suite) on a head about
6214            // to be rewritten. Deferred, never skipped: `verify.e2e` still
6215            // runs once a round has no blocking findings left (see
6216            // `round_is_clean`, which a deferred — empty — `e2e` can never
6217            // satisfy since `blocking` is nonzero whenever this branch is
6218            // taken), and `stop_reviewing` forces a real run before it will
6219            // ever read a deferred round as green.
6220            let defer_e2e =
6221                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6222            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6223                let reason =
6224                    format!("{blocking} blocking finding(s) already required a fix this round");
6225                self.state.event(
6226                    "verify",
6227                    format!(
6228                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6229                         {}); it will run once a round has none left",
6230                        short(&head)
6231                    ),
6232                );
6233                (Vec::new(), false, true, Some(reason))
6234            } else {
6235                let e2e_commands = self.state.config.verify.e2e.clone();
6236                let cache_dir = self.state.config.cache_dir();
6237                let context = format!("round {round}");
6238                let wait = self.repository_wait();
6239                let (e2e, verify_retried) = with_cache_lease(
6240                    &mut self.state,
6241                    cache_dir.as_deref(),
6242                    "e2e",
6243                    "e2e",
6244                    &winner.worktree,
6245                    &head,
6246                    verify_timeout,
6247                    &context,
6248                    &wait,
6249                    |state, budget| {
6250                        let shell = shell.clone();
6251                        let e2e_commands = e2e_commands.clone();
6252                        let worktree = winner.worktree.clone();
6253                        let context = context.clone();
6254                        async move {
6255                            run_e2e_with_retry(
6256                                state,
6257                                &shell,
6258                                &e2e_commands,
6259                                &worktree,
6260                                budget,
6261                                &context,
6262                            )
6263                            .await
6264                        }
6265                    },
6266                )
6267                .await?;
6268                (e2e, verify_retried, false, None)
6269            };
6270
6271            let expected = records.len();
6272            let answered = records.iter().filter(|r| r.failed.is_none()).count();
6273            let incomplete = answered < expected;
6274            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6275            let policy = self.state.config.graph.incomplete_review;
6276            let clean = round_is_clean(
6277                blocking,
6278                e2e_ok,
6279                answered,
6280                expected,
6281                round_quota_missing,
6282                policy,
6283            );
6284
6285            let mut round_record = ReviewRound {
6286                round,
6287                head: head.clone(),
6288                verified_head: None,
6289                verified_at: None,
6290                reviews: records,
6291                e2e,
6292                verify_retried,
6293                e2e_deferred,
6294                e2e_defer_reason,
6295                fix: None,
6296                blocking,
6297                answered,
6298                expected,
6299                clean,
6300                progressed: false,
6301                vote_split,
6302                reconsideration,
6303                verdict: None,
6304            };
6305            // The final vote per seat is its revote where reconsideration
6306            // ran and answered, its initial vote otherwise — the same
6307            // fallback `tally` uses for a judge whose private vote failed.
6308            round_record.verdict = ReviewVote::worst(
6309                round_record
6310                    .final_votes()
6311                    .into_iter()
6312                    .map(|(_, _, vote)| vote),
6313            );
6314            // Which commit and when magi actually attempted to check —
6315            // known the moment a command was dispatched against `head`,
6316            // whether or not it finished: a resource-blocked attempt still
6317            // targeted a specific commit at a specific time, and leaving
6318            // that unrecorded is exactly what made `verification_summary`
6319            // report a fresh attempt as "commit unknown ... recorded before
6320            // this was tracked", indistinguishable from a genuinely old,
6321            // untracked record. Only a deferred or unconfigured round never
6322            // ran at all and has nothing to record — see
6323            // `ReviewRound::verified_head`'s own doc.
6324            if !matches!(
6325                round_record.e2e_status(),
6326                E2eStatus::Deferred | E2eStatus::NotConfigured
6327            ) {
6328                round_record.verified_head = Some(head.clone());
6329                round_record.verified_at = Some(Timestamp::now());
6330            }
6331            let this_round_verification = round_record.verification_summary(&head);
6332
6333            if incomplete {
6334                let missing: Vec<String> = round_record
6335                    .reviews
6336                    .iter()
6337                    .filter(|r| r.failed.is_some())
6338                    .map(|r| format!("review-{}", r.reviewer))
6339                    .collect();
6340                self.state.event(
6341                    "review",
6342                    format!(
6343                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6344                        missing.join(", ")
6345                    ),
6346                );
6347            }
6348
6349            if clean {
6350                self.state.event(
6351                    "review",
6352                    if incomplete && policy == IncompleteReviewPolicy::Warn {
6353                        format!(
6354                            "round {round}: clean (warn policy, incomplete panel) — no \
6355                             blocking findings from the seats that answered, verification green"
6356                        )
6357                    } else if incomplete {
6358                        format!(
6359                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6360                             quorum) — no blocking findings from the seats that answered, \
6361                             verification green",
6362                            expected - answered
6363                        )
6364                    } else {
6365                        format!("round {round}: clean — no blocking findings, verification green")
6366                    },
6367                );
6368                self.state.reviews.push(round_record);
6369                self.state.status = RunStatus::Gating;
6370                self.state.save()?;
6371                return Ok(());
6372            }
6373
6374            // Nothing was raised and verification passed, but not every seat
6375            // answered and `round_is_clean` still refused to call it clean —
6376            // either a seat is missing for a reason other than its own quota
6377            // (a crash, a timeout, unparsable output — worth another try), or
6378            // every seat that could have answered lost its quota and nobody
6379            // is left to decide on: re-review rather than send the fixer
6380            // after a round with nothing to fix.
6381            if incomplete && blocking == 0 && e2e_ok {
6382                self.state.reviews.push(round_record);
6383                self.state.save()?;
6384                if round == max_rounds {
6385                    self.state.status = RunStatus::Blocked;
6386                    self.state.event(
6387                        "review",
6388                        format!(
6389                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6390                             refusing to call it clean",
6391                            expected - answered
6392                        ),
6393                    );
6394                    return Ok(());
6395                }
6396                continue;
6397            }
6398
6399            // Nothing for the fixer to act on (`blocking == 0`) and the only
6400            // reason this round is not clean is that magi itself never got
6401            // a command to run — the shared build cache, not the patch (see
6402            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
6403            // the fixer would invite a change to appease contention that has
6404            // nothing to do with the diff, and would leave this attempt
6405            // sitting in the next round's prompt as if it were about an
6406            // earlier, superseded commit rather than what it actually is:
6407            // the same head, still waiting to be checked. Wait for it the
6408            // same way the final round's own contention is already handled,
6409            // whatever round this happens to be.
6410            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6411                self.state.reviews.push(round_record);
6412                return self
6413                    .stop_reviewing(
6414                        "the round's own verification could not run",
6415                        &shell,
6416                        &winner.worktree,
6417                    )
6418                    .await;
6419            }
6420
6421            if round == max_rounds {
6422                self.state.reviews.push(round_record);
6423                return self
6424                    .stop_reviewing(
6425                        &format!(
6426                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6427                        ),
6428                        &shell,
6429                        &winner.worktree,
6430                    )
6431                    .await;
6432            }
6433
6434            // Fix. The winner's own implementer seat continues its conversation:
6435            // the competition is over, so context is pure benefit now.
6436            let blocking_findings: Vec<_> = all_findings
6437                .iter()
6438                .filter(|f| f.severity.blocks())
6439                .cloned()
6440                .collect();
6441            let fix_prompt = prompt::fix_in(
6442                &self.state.instruction,
6443                &blocking_findings,
6444                this_round_verification.as_ref(),
6445                round,
6446                max_rounds,
6447                &language,
6448                &self.state.config.graph.github_language,
6449            );
6450            let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6451            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6452            let (job, seat, out) = self
6453                .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6454                    prompt: fix_prompt.clone(),
6455                    spec,
6456                    seat,
6457                    cwd: winner.worktree.clone(),
6458                    timeout,
6459                    allow_write: true,
6460                    sessions,
6461                    artifacts: artifacts.clone(),
6462                    stem: format!("fix-{round}"),
6463                    handover: None,
6464                })
6465                .await;
6466            // Not asked: the round is not recorded (so a resume repeats it),
6467            // no fixer was handed over to, and the seat is left as it was.
6468            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6469                return self.stop_for_resource();
6470            }
6471            let agent_id = seat.agent.clone();
6472
6473            let mut fix = FixRecord {
6474                agent: agent_id,
6475                addressed: Vec::new(),
6476                rejected: Vec::new(),
6477                notes: String::new(),
6478                committed: false,
6479                failed: None,
6480                duration_ms: 0,
6481                continuation: None,
6482            };
6483            let mut continuation = ContinuationRecord::not_needed();
6484            let mut final_seat = seat.clone();
6485            match out {
6486                AgentOutcome::Ok(o) => {
6487                    fix.duration_ms = o.duration_ms;
6488                    let parsed = verdict::extract_json::<FixReport>(&o.text);
6489                    // A parsed report standing next to a command this same
6490                    // reply's own CLI never confirmed the exit status of is
6491                    // not a resolved answer — the identical `CommandEvidence`
6492                    // `state.jobs` renders, read here instead of only on
6493                    // display, per the completion judgment and the shown
6494                    // record needing to agree.
6495                    let incomplete_reason = match &parsed {
6496                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6497                            "the reply parsed, but it reported a command whose own CLI never \
6498                             confirmed an exit status"
6499                                .to_owned(),
6500                        ),
6501                        Ok(_) => None,
6502                        Err(e) => Some(e.to_string()),
6503                    };
6504                    match incomplete_reason {
6505                        None => {
6506                            let report = parsed.expect("checked Ok above");
6507                            fix.addressed = report.addressed;
6508                            fix.rejected = report.rejected;
6509                            fix.notes =
6510                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
6511                        }
6512                        Some(reason) => {
6513                            let (resumed_seat, resolved, failure, cont) = self
6514                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6515                                .await;
6516                            fix.duration_ms += cont.cumulative_wait_ms;
6517                            continuation = cont;
6518                            final_seat = resumed_seat;
6519                            match resolved {
6520                                Some(report) => {
6521                                    fix.addressed = report.addressed;
6522                                    fix.rejected = report.rejected;
6523                                    fix.notes = blind::sanitize_prose(
6524                                        &report.notes,
6525                                        &self.state.config.blind,
6526                                    );
6527                                }
6528                                None => fix.failed = failure,
6529                            }
6530                        }
6531                    }
6532                }
6533                // The CLI's raw error JSON is not a fix report to parse.
6534                AgentOutcome::Dropped(o) => {
6535                    fix.duration_ms = o.duration_ms;
6536                    let why = o
6537                        .dropped
6538                        .as_ref()
6539                        .map(|d| d.why.as_str())
6540                        .unwrap_or("the CLI ended the stream without delivering its answer");
6541                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6542                }
6543                AgentOutcome::Quota(o) => {
6544                    self.state.quota.push(QuotaLoss {
6545                        seat: final_seat.key.clone(),
6546                        node: "fix".to_owned(),
6547                        at: Timestamp::now(),
6548                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6549                    });
6550                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6551                }
6552                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6553            }
6554            fix.continuation = Some(continuation);
6555            self.state.seats.insert(final_seat.key.clone(), final_seat);
6556            let coordination = self.coordinate(&winner.worktree).await?;
6557            if let Ok(r) = git::rescue_commit(
6558                &winner.worktree,
6559                &format!("magi: review round {round} fixes (uncommitted work)"),
6560            )
6561            .await
6562            {
6563                self.state.note_withheld("fix", &r.withheld);
6564            }
6565            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6566            fix.committed = after != before;
6567            // Judged by what `git` says moved against base, never by the
6568            // fixer's own `addressed`/`rejected` count — see
6569            // `ReviewRound::progressed`. Propagated with `?`, the same as the
6570            // `patch` snapshot above: swallowing this error would default
6571            // `diff_after` to empty, which almost always differs from a
6572            // non-empty `patch` and reads as "progressed" — exactly backwards
6573            // for a `git` failure the stagnation check cannot see through.
6574            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6575            drop(coordination);
6576            let progressed = diff_after != patch;
6577            let commit_note = if fix.committed {
6578                "committed"
6579            } else {
6580                "NO new commit"
6581            };
6582            let tree_note = if progressed {
6583                "changed vs base"
6584            } else {
6585                "unchanged vs base"
6586            };
6587            self.state.event(
6588                "fix",
6589                match &fix.failed {
6590                    // Distinct on purpose from "0 addressed, 0 rejected": the
6591                    // fixer's own diff still landed (blocking counts do keep
6592                    // falling round over round), only its adoption report did
6593                    // not come back, so this must never read like every
6594                    // finding was reviewed and declined.
6595                    Some(reason) => {
6596                        format!(
6597                            "round {round}: fixer's adoption report was lost ({reason}); \
6598                             {commit_note}, tree {tree_note}"
6599                        )
6600                    }
6601                    None => format!(
6602                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6603                         {tree_note}{}",
6604                        fix.addressed.len(),
6605                        fix.rejected.len(),
6606                        if continuation.outcome == ContinuationOutcome::Resumed {
6607                            format!(
6608                                " (adoption report recovered after {} continuation(s))",
6609                                continuation.attempts
6610                            )
6611                        } else {
6612                            String::new()
6613                        },
6614                    ),
6615                },
6616            );
6617            round_record.fix = Some(fix);
6618            round_record.progressed = progressed;
6619            self.state.reviews.push(round_record);
6620            self.state.save()?;
6621
6622            // The fixer's own report never came back this round, even after
6623            // `continue_fix_report`'s own budget was spent on it — not an
6624            // ordinary "no report" (dropped stream, quota, plain failure),
6625            // which already reads that way and is left to the existing round
6626            // budget. Stopping here, rather than opening another round, is
6627            // what keeps a next reviewer/fixer wave from ever being
6628            // dispatched onto `winner.worktree` while whatever the seat's
6629            // last call may still have running there is unaccounted for: no
6630            // process liveness check exists (and none is being added — see
6631            // AGENTS.md/this task's own scope), so the only way to honour
6632            // "nothing starts before a valid report returns" is to not start
6633            // anything further on this worktree from this run at all.
6634            if matches!(
6635                continuation.outcome,
6636                ContinuationOutcome::Exhausted
6637                    | ContinuationOutcome::QuotaLost
6638                    | ContinuationOutcome::NoSession
6639            ) {
6640                return self
6641                    .stop_reviewing(
6642                        "the fixer's adoption report never came back, even after resuming its \
6643                         own seat; refusing to start another round against the same worktree \
6644                         while that is unresolved",
6645                        &shell,
6646                        &winner.worktree,
6647                    )
6648                    .await;
6649            }
6650
6651            let streak = self
6652                .state
6653                .reviews
6654                .iter()
6655                .rev()
6656                .take_while(|r| !r.progressed)
6657                .count();
6658            if streak >= STAGNANT_LIMIT {
6659                return self
6660                    .stop_reviewing(
6661                        &format!(
6662                            "the tree has not moved against base for {streak} round(s) in a row"
6663                        ),
6664                        &shell,
6665                        &winner.worktree,
6666                    )
6667                    .await;
6668            }
6669        }
6670        Ok(())
6671    }
6672
6673    /// Decide, from the last recorded round's own verification, whether
6674    /// stopping the review loop is a hand-off or a genuine block.
6675    ///
6676    /// Called once the loop has given up trying — the round budget is spent,
6677    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
6678    /// findings still open, never while a round is still clean or the
6679    /// incomplete-panel case handled inline above. Gate and e2e are facts
6680    /// about the tree; a lingering review finding is an opinion, and this
6681    /// workload's own `magi stats` puts reviewer precision low enough
6682    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
6683    /// must not by itself stand between a green, verified change and the
6684    /// human who decides what to do with it. A red e2e is not an opinion, so
6685    /// that case still blocks, with the failing command and a tail of its
6686    /// output recorded here rather than left in `run.json` for someone to go
6687    /// find.
6688    ///
6689    /// A round that deferred its own e2e (see [`Config::graph`]'s
6690    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
6691    /// only because nothing ran, and treating an empty list as a passing one
6692    /// here is exactly the "deferred painted green" bug this function exists
6693    /// to not have. When the last round's own verification never resolved —
6694    /// deferred on purpose, or a real attempt the shared build cache blocked
6695    /// — this makes (or retries) the real run, on the actual worktree this
6696    /// loop is about to stop touching, before deciding anything. A
6697    /// resource-blocked attempt is likewise never read as either green or
6698    /// red: it is evidence about the machine, not the patch (see
6699    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
6700    /// blocked cache leaves this call without deciding rather than guessing
6701    /// — the caller retries on a later reentry.
6702    /// Record, once, that the review loop handed off over a blocking finding
6703    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
6704    /// `land` asks the owner even with `land_approval` off. Called from every
6705    /// path that concludes `Gating`; a reentry keeps the first record.
6706    fn record_contested_handoff(&mut self) {
6707        if self.state.contested_handoff.is_some() {
6708            return;
6709        }
6710        let Some(contested) = self
6711            .state
6712            .reviews
6713            .last()
6714            .and_then(ReviewRound::contested_handoff)
6715        else {
6716            return;
6717        };
6718        self.state.event(
6719            "review",
6720            format!(
6721                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6722                 wait for the owner's approval",
6723                contested.findings.len(),
6724                contested.rejecters.len()
6725            ),
6726        );
6727        self.state.contested_handoff = Some(contested);
6728    }
6729
6730    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6731        let round_idx = self.state.reviews.len() - 1;
6732        // A deferred round and a resource-blocked one are the same shape
6733        // here: neither has a real result yet, and both get one more
6734        // attempt. Read off `e2e_status` — the single source for this —
6735        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
6736        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
6737        // still retries instead of being read as a settled result the
6738        // instant it stops being empty.
6739        let needs_catchup_run = matches!(
6740            self.state.reviews[round_idx].e2e_status(),
6741            E2eStatus::Deferred | E2eStatus::ResourceBlocked
6742        );
6743        if needs_catchup_run {
6744            let round = self.state.reviews[round_idx].round;
6745            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6746            let commands = self.state.config.verify.e2e.clone();
6747            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6748            let cache_dir = self.state.config.cache_dir();
6749            let context = format!(
6750                "round {round}: verification unresolved, catching up before the final decision"
6751            );
6752            let wait = self.repository_wait();
6753            let (outcomes, verify_retried) = with_cache_lease(
6754                &mut self.state,
6755                cache_dir.as_deref(),
6756                "e2e",
6757                "e2e",
6758                worktree,
6759                &attempted_head,
6760                timeout,
6761                &context,
6762                &wait,
6763                |state, budget| {
6764                    let shell = shell.to_vec();
6765                    let commands = commands.clone();
6766                    let context = context.clone();
6767                    async move {
6768                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6769                            .await
6770                    }
6771                },
6772            )
6773            .await?;
6774            let last = &mut self.state.reviews[round_idx];
6775            last.e2e = outcomes;
6776            last.verify_retried = verify_retried;
6777            // Always the commit and time this attempt actually targeted,
6778            // whether or not it happens to equal the reviewed `head` and
6779            // whether or not a command finished — see
6780            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
6781            // attempt is recorded too, so a later reader sees "attempted
6782            // again at T2" rather than silence.
6783            last.verified_head = Some(attempted_head);
6784            last.verified_at = Some(Timestamp::now());
6785            if verify_inconclusive(&last.e2e) {
6786                // Still not a real result: `e2e_deferred` is left exactly
6787                // as it was, so `needs_catchup_run` above reads
6788                // `ResourceBlocked` (via `e2e_status`, which checks
6789                // `resource_blocked` before `e2e_deferred`) and retries
6790                // again on the next reentry, rather than recording
6791                // contention as a red e2e and blocking the run on it.
6792                self.state.save()?;
6793                return Ok(());
6794            }
6795            last.e2e_deferred = false;
6796        }
6797        let last = &self.state.reviews[round_idx];
6798        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6799
6800        match last.e2e_status() {
6801            E2eStatus::Failed => {
6802                let red: Vec<String> = last
6803                    .e2e
6804                    .iter()
6805                    .filter(|o| !o.ok())
6806                    .map(|o| {
6807                        format!(
6808                            "`{}` -> {:?}\n{}",
6809                            o.command,
6810                            o.code,
6811                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6812                        )
6813                    })
6814                    .collect();
6815                self.state
6816                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6817                self.state.status = RunStatus::Blocked;
6818            }
6819            // `needs_catchup_run` above already retried once this call; if
6820            // it is still blocked, this is magi's own admission it could
6821            // not get a command to run, never a verdict on the patch — the
6822            // run is left exactly where a later reentry can retry again.
6823            E2eStatus::ResourceBlocked => {
6824                self.state.event(
6825                    "review",
6826                    format!(
6827                        "{why}; e2e could not run (shared build cache unavailable); not \
6828                         deciding yet"
6829                    ),
6830                );
6831            }
6832            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6833                self.state.event(
6834                    "review",
6835                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6836                );
6837                self.record_contested_handoff();
6838                self.state.status = RunStatus::Gating;
6839            }
6840        }
6841        self.state.save()?;
6842        Ok(())
6843    }
6844
6845    // ----------------------------------------------------------------- gate
6846
6847    async fn gate(&mut self) -> Result<()> {
6848        // Judged by the review record itself, not by `status`: a solo
6849        // candidate's `judge`/`deliberate` skip rewrites `status` on every
6850        // reentry (see `judge`), and trusting it here is exactly how a run
6851        // that exhausted its review budget got gated and merged a second
6852        // time around. `review_conclusion` recomputes the review loop's own
6853        // verdict from the round records themselves — `Gating` for a clean
6854        // round or a hand-off (see `stop_reviewing`), anything else means the
6855        // loop is still going or genuinely blocked.
6856        // A base the winner could not be replayed onto is a decision, not a
6857        // round: there is no landing tree to gate. Read as its own record for
6858        // the same reason the review verdict is.
6859        if self.state.status == RunStatus::Failed
6860            || self
6861                .state
6862                .base_sync
6863                .as_ref()
6864                .is_some_and(|s| s.conflict.is_some())
6865            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6866                != Some(RunStatus::Gating)
6867        {
6868            return Ok(());
6869        }
6870        if self.state.gate_ran {
6871            // `review_loop` derives its conclusion from the clean review
6872            // record on every reentry and therefore puts a completed run back
6873            // in `Gating`. A recorded gate is a stronger, terminal fact:
6874            // retain its original command output (or lack of any, for a repo
6875            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
6876            // doc) and restore `Blocked` on a real failure rather than
6877            // pretending the command is still running or running it a second
6878            // time. `gate_ran == false` remains the only shape — unattempted,
6879            // or a resource-blocked retry — that may still need to execute a
6880            // command.
6881            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6882                self.state.status = RunStatus::Blocked;
6883                self.state.save()?;
6884            }
6885            return Ok(());
6886        }
6887        let Some(winner) = self.state.winner().cloned() else {
6888            return Ok(());
6889        };
6890        self.state.status = RunStatus::Gating;
6891        let mut outcomes = self.run_gate(&winner).await?;
6892        loop {
6893            // A resource-blocked outcome means the gate command never actually
6894            // ran - the shared build cache could not be acquired or confirmed
6895            // fresh in time - which is evidence about the machine, not about
6896            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
6897            // Recording it as a red gate would mark a run `Blocked` on nothing
6898            // but contention magi has already logged; leaving `self.state.gate`
6899            // empty and `self.state.gate_ran` false instead keeps the shape
6900            // this function already treats as "still needs to run" (see the
6901            // early-return above), so the next call retries the command
6902            // rather than concluding anything.
6903            if verify_inconclusive(&outcomes) {
6904                self.state.save()?;
6905                return Ok(());
6906            }
6907            if outcomes.iter().all(CommandOutcome::ok) {
6908                break;
6909            }
6910            match self.gate_fix_round(&winner, &outcomes).await? {
6911                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6912                GateFix::Stop => break,
6913                GateFix::Defer => {
6914                    self.state.save()?;
6915                    return Ok(());
6916                }
6917            }
6918        }
6919        let passed = outcomes.iter().all(CommandOutcome::ok);
6920        self.state.gate = outcomes;
6921        self.state.gate_ran = true;
6922        self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6923        if !passed {
6924            self.state.status = RunStatus::Blocked;
6925            let spent = self.state.gate_fixes.len();
6926            self.state.event(
6927                "gate",
6928                if spent == 0 {
6929                    "gate failed; not merging".to_owned()
6930                } else {
6931                    format!("gate failed after {spent} gate-fix round(s); not merging")
6932                },
6933            );
6934        }
6935        self.state.save()?;
6936        Ok(())
6937    }
6938
6939    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
6940    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
6941    /// after review is clean and never on a candidate awaiting judging.
6942    ///
6943    /// Never fails the run: a non-zero exit or timeout is a warning and a
6944    /// recorded outcome, and the gate remains the single arbiter. Nothing
6945    /// configured means nothing happens - no event, no commit. `commit_all`
6946    /// commits any leftover change under the neutral identity and returns
6947    /// `false` when the tree is clean, so no empty commit is ever made.
6948    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6949        let commands = state.config.verify.pre_gate.clone();
6950        if commands.is_empty() {
6951            return;
6952        }
6953        let shell = state.config.shell();
6954        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6955        let (outcomes, _) = run_commands(
6956            state,
6957            "pre_gate",
6958            "pre_gate",
6959            0,
6960            &shell,
6961            &commands,
6962            &winner.worktree,
6963            timeout,
6964        )
6965        .await;
6966        for o in &outcomes {
6967            if !o.ok() {
6968                tracing::warn!(
6969                    "pre_gate `{}` failed ({:?}); the gate decides",
6970                    o.command,
6971                    o.code
6972                );
6973            }
6974            state.event(
6975                "pre_gate",
6976                format!(
6977                    "`{}` -> {}",
6978                    o.command,
6979                    if o.ok() {
6980                        "pass".to_owned()
6981                    } else {
6982                        format!(
6983                            "FAIL ({:?})\n{}",
6984                            o.code,
6985                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6986                        )
6987                    }
6988                ),
6989            );
6990        }
6991        state.pre_gate = outcomes;
6992        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6993            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6994                Ok(head) => {
6995                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6996                    state.pre_gate_commit = Some(head);
6997                }
6998                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6999            },
7000            Ok(false) => {}
7001            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7002        }
7003        if let Err(e) = state.save() {
7004            tracing::warn!("could not persist the pre_gate record: {e:#}");
7005        }
7006    }
7007
7008    /// Run `verify.gate` once against the winner's current tree, logging one
7009    /// event per command. Empty when nothing is configured.
7010    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7011        let shell = self.state.config.shell();
7012        let gate_commands = self.state.config.verify.gate.clone();
7013        let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7014        let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7015            Vec::new()
7016        } else {
7017            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7018            let cache_dir = self.state.config.cache_dir();
7019            let wait = self.repository_wait();
7020            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7021            let (outcomes, _) = with_cache_lease(
7022                &mut self.state,
7023                cache_dir.as_deref(),
7024                "gate",
7025                "gate",
7026                &winner.worktree,
7027                &head,
7028                timeout,
7029                "final gate",
7030                &wait,
7031                |state, budget| {
7032                    let shell = shell.clone();
7033                    let gate_commands = gate_commands.clone();
7034                    let winner = winner.clone();
7035                    async move {
7036                        Self::run_pre_gate(state, &winner).await;
7037                        let (outcomes, timed_out_pids) = run_commands(
7038                            state,
7039                            "gate",
7040                            "gate",
7041                            0,
7042                            &shell,
7043                            &gate_commands,
7044                            &winner.worktree,
7045                            budget,
7046                        )
7047                        .await;
7048                        (outcomes, false, timed_out_pids)
7049                    }
7050                },
7051            )
7052            .await?;
7053            outcomes
7054        };
7055        if outcomes.is_empty() {
7056            // Nothing configured to check — distinct from every other
7057            // silence in this run's event log, since an empty `gate` alone
7058            // no longer says whether the gate ran at all (see
7059            // `RunState::gate_ran`'s own doc).
7060            self.state
7061                .event("gate", "no gate commands configured; not tested");
7062        }
7063        for o in &outcomes {
7064            self.state.event(
7065                "gate",
7066                format!(
7067                    "`{}` -> {}",
7068                    o.command,
7069                    if o.ok() {
7070                        "pass".to_owned()
7071                    } else {
7072                        format!(
7073                            "FAIL ({:?})\n{}",
7074                            o.code,
7075                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7076                        )
7077                    }
7078                ),
7079            );
7080        }
7081        Ok(outcomes)
7082    }
7083
7084    /// One bounded fix round for a failing gate.
7085    ///
7086    /// The fixer is told the failure came from the gate itself, not from a
7087    /// reviewer, and is shown the failed commands, their exit codes and a tail
7088    /// of their output - whatever `[verify].gate` holds, nothing here knows
7089    /// what those commands run. Only a normal non-zero exit that printed
7090    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
7091    /// command or a full disk says nothing about the code, and a fixer sent
7092    /// after it can only appease the machine. The round is judged by what git
7093    /// says moved, never by the fixer's own report, and `verify.e2e` runs
7094    /// again before the gate does, so a fix cannot trade a green gate for a
7095    /// red e2e unnoticed.
7096    async fn gate_fix_round(
7097        &mut self,
7098        winner: &Candidate,
7099        outcomes: &[CommandOutcome],
7100    ) -> Result<GateFix> {
7101        let cap = self.state.config.graph.gate_fix_rounds;
7102        let spent = self.state.gate_fixes.len();
7103        if spent >= cap {
7104            if cap > 0 {
7105                self.state.event(
7106                    "gate",
7107                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
7108                );
7109            }
7110            return Ok(GateFix::Stop);
7111        }
7112        if !gate_fixable(outcomes) {
7113            self.state.event(
7114                "gate",
7115                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7116                 command or similar); not spending a fix round on it",
7117            );
7118            return Ok(GateFix::Stop);
7119        }
7120        let min_free = self.state.config.disk.min_free_bytes;
7121        if min_free > 0 {
7122            match crate::disk::free_bytes(&winner.worktree) {
7123                Ok(free) if crate::disk::enough_space(free, min_free) => {}
7124                Ok(free) => {
7125                    self.state.event(
7126                        "gate",
7127                        format!(
7128                            "only {free} bytes free ({min_free} required by `[disk] \
7129                             min_free_bytes`); not spending a fix round on a failure the disk \
7130                             may explain"
7131                        ),
7132                    );
7133                    return Ok(GateFix::Stop);
7134                }
7135                Err(e) => {
7136                    self.state.event(
7137                        "gate",
7138                        format!("free disk space could not be measured ({e:#}); no fix round"),
7139                    );
7140                    return Ok(GateFix::Stop);
7141                }
7142            }
7143        }
7144
7145        let attempt = spent + 1;
7146        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7147        let base = self.landing_base();
7148        let fix_prompt = prompt::gate_fix_in(
7149            &self.state.instruction,
7150            &failed,
7151            attempt,
7152            cap,
7153            &self.state.config.graph.language,
7154            &self.state.config.graph.github_language,
7155        );
7156        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7157        let sessions = self.state.config.graph.sessions;
7158        let artifacts = self.merge_repair_artifacts();
7159        self.state.event(
7160            "gate",
7161            format!("gate failed; gate-fix round {attempt} of {cap}"),
7162        );
7163        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7164        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7165        let (_, seat, out) = self
7166            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7167                prompt: fix_prompt.clone(),
7168                spec,
7169                seat,
7170                cwd: winner.worktree.clone(),
7171                timeout,
7172                allow_write: true,
7173                sessions,
7174                artifacts: artifacts.clone(),
7175                stem: format!("gate-fix-{attempt}"),
7176                handover: None,
7177            })
7178            .await;
7179        // The fixer was never asked: no round is spent (nothing is recorded)
7180        // and a later reentry asks the same agent again.
7181        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7182            return Ok(GateFix::Defer);
7183        }
7184        let mut record = GateFixRecord {
7185            agent: seat.agent.clone(),
7186            failed,
7187            notes: String::new(),
7188            committed: false,
7189            error: None,
7190        };
7191        match out {
7192            AgentOutcome::Ok(o) => {
7193                // A missing report is not a failed fix: the round is judged
7194                // by the tree below, and the report only carries prose.
7195                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7196                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7197                }
7198            }
7199            AgentOutcome::Dropped(_) => {
7200                record.error = Some("the CLI dropped the stream".to_owned());
7201            }
7202            AgentOutcome::Quota(o) => {
7203                self.state.quota.push(QuotaLoss {
7204                    seat: seat.key.clone(),
7205                    node: "gate-fix".to_owned(),
7206                    at: Timestamp::now(),
7207                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7208                });
7209                record.error = Some("rate limited (quota); fixer could not run".to_owned());
7210            }
7211            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7212        }
7213        self.state.seats.insert(seat.key.clone(), seat);
7214        let coordination = self.coordinate(&winner.worktree).await?;
7215        if let Ok(r) = git::rescue_commit(
7216            &winner.worktree,
7217            &format!("magi: gate fix {attempt} (uncommitted work)"),
7218        )
7219        .await
7220        {
7221            self.state.note_withheld("gate-fix", &r.withheld);
7222        }
7223        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7224        record.committed = after != before;
7225        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7226        let note = record.error.clone();
7227        self.state.gate_fixes.push(record);
7228        self.state.save()?;
7229        if !changed {
7230            self.state.event(
7231                "gate",
7232                match note {
7233                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7234                    None => format!("gate-fix round {attempt}: the tree did not change"),
7235                },
7236            );
7237            return Ok(GateFix::Stop);
7238        }
7239        self.state.event(
7240            "gate",
7241            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7242        );
7243
7244        drop(coordination);
7245        let commands = self.state.config.verify.e2e.clone();
7246        if !commands.is_empty() {
7247            let shell = self.state.config.shell();
7248            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7249            let cache_dir = self.state.config.cache_dir();
7250            let context = format!("gate-fix round {attempt}");
7251            let wait = self.repository_wait();
7252            let (e2e, _) = with_cache_lease(
7253                &mut self.state,
7254                cache_dir.as_deref(),
7255                "e2e",
7256                "e2e",
7257                &winner.worktree,
7258                &after,
7259                timeout,
7260                &context,
7261                &wait,
7262                |state, budget| {
7263                    let shell = shell.clone();
7264                    let commands = commands.clone();
7265                    let context = context.clone();
7266                    let worktree = winner.worktree.clone();
7267                    async move {
7268                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7269                            .await
7270                    }
7271                },
7272            )
7273            .await?;
7274            if verify_inconclusive(&e2e) {
7275                return Ok(GateFix::Defer);
7276            }
7277            if e2e.iter().any(|o| !o.ok()) {
7278                self.state.event(
7279                    "gate",
7280                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7281                );
7282                return Ok(GateFix::Stop);
7283            }
7284        }
7285        Ok(GateFix::Retry)
7286    }
7287
7288    // ---------------------------------------------------------------- merge
7289
7290    /// One read-only rewrite by the summary's author; a text the gate still
7291    /// withholds goes to the owner (see `github_text`) and the fixed fallback
7292    /// applies on their word or on silence. `None` means the run is parked on
7293    /// that question and nothing may be pushed or opened yet.
7294    async fn guarded_pr_message(
7295        &mut self,
7296        winner: &Candidate,
7297        facts: Option<&BranchFacts>,
7298        posting: bool,
7299    ) -> Result<Option<PrMessage>> {
7300        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7301        let head = match final_head.as_deref() {
7302            Some(h) => crate::pr_outcome::FinalHead::At(h),
7303            None => crate::pr_outcome::FinalHead::Unknown,
7304        };
7305        // Only a pull request that is about to be posted pays for a summary.
7306        let summarized = if posting {
7307            Some(self.summarize_change(winner, final_head.as_deref()).await)
7308        } else {
7309            None
7310        };
7311        // A busy shared resource is not a permanent cause: publishing an
7312        // "unavailable" description would stick (a later head move never
7313        // edits the body). Stop before anything is pushed or opened; the
7314        // run is `Blocked` and a resume asks the summary again.
7315        if self.state.resource_block.is_some() {
7316            self.state.event(
7317                "pr-summary",
7318                "a shared resource was busy; nothing was published, resume to summarize again",
7319            );
7320            return Ok(None);
7321        }
7322        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7323            anyhow::bail!(
7324                "the final commit moved during public description generation; retry before publishing"
7325            );
7326        }
7327        let note = match (&summarized, &self.state.change_summary) {
7328            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7329            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7330            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7331            (None, _) => ChangeNote::NotAsked,
7332        };
7333        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7334        if !posting {
7335            let identity = crate::scrub::Identity::current();
7336            return Ok(Some(PrMessage {
7337                title: crate::scrub::scrub(&pr.title, &identity),
7338                body: crate::scrub::scrub(&pr.body, &identity),
7339            }));
7340        }
7341        // This very text was already rejected and asked about: no second
7342        // judge call, rewrite or question, just carry the record on.
7343        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7344        if let Some(known) = self
7345            .state
7346            .github_text
7347            .clone()
7348            .filter(|g| g.fingerprint == raw_fp)
7349        {
7350            return self.settle_withheld(winner, &pr, known).await;
7351        }
7352        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7353        let mut violations =
7354            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7355        let identity = crate::scrub::Identity::current();
7356        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7357            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7358            && !violations.contains(&crate::github_text::Violation::SensitiveData)
7359        {
7360            violations.push(crate::github_text::Violation::SensitiveData);
7361        }
7362        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
7363        // only a language violation starts one; sensitive data alone goes
7364        // straight to the owner, with the redacted text as the fallback.
7365        let has_language = violations
7366            .iter()
7367            .any(|v| *v != crate::github_text::Violation::SensitiveData);
7368        if self.state.config.graph.github_text_guard && !violations.is_empty() {
7369            self.state.event(
7370                "github-text",
7371                if has_language {
7372                    format!("description rejected: {violations:?}; requesting one rewrite")
7373                } else {
7374                    format!("description rejected: {violations:?}; asking the owner")
7375                },
7376            );
7377            let mut rewritten = false;
7378            let mut rejected_rewrite = None;
7379            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7380            {
7381                let output = text;
7382                if let Some(title) = summary_title(&output) {
7383                    // Judge the redacted candidate that will actually be posted.
7384                    // Sensitive data alone must not discard a useful rewrite.
7385                    let raw_body = summary_without_title(&output);
7386                    let clean_title = crate::scrub::scrub(&title, &identity);
7387                    let mut body = crate::scrub::scrub(&raw_body, &identity);
7388                    let redacted = clean_title != title || body != raw_body;
7389                    let title = clean_title;
7390                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7391                    if !complete {
7392                        self.state.event(
7393                            "github-text",
7394                            "description rewrite omitted required findings",
7395                        );
7396                    }
7397                    if !body.trim().is_empty() {
7398                        body.push_str("\n\n");
7399                        body.push_str(&public_pr_evidence(&self.state, head));
7400                    }
7401                    let marker = format!("magi:run/{}", self.state.id);
7402                    if !body.trim().is_empty() && !body.contains(&marker) {
7403                        body.push_str(&format!("\n\n{marker}"));
7404                    }
7405                    let rewrite = PrMessage {
7406                        title: title.clone(),
7407                        body: body.clone(),
7408                    };
7409                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7410                    let mut failed = crate::github_text::check_for(
7411                        &self.state.config.graph,
7412                        &title,
7413                        &body,
7414                        redecision,
7415                    );
7416                    if (!crate::github_text::shareable(&title)
7417                        || !crate::github_text::shareable(&body))
7418                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
7419                    {
7420                        failed.push(crate::github_text::Violation::SensitiveData);
7421                    }
7422                    if complete
7423                        && !title.trim().is_empty()
7424                        && !body.trim().is_empty()
7425                        && !failed.is_empty()
7426                    {
7427                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
7428                            title: title.clone(),
7429                            body: body.clone(),
7430                            failed: failed
7431                                .iter()
7432                                .map(|v| crate::github_text::category(*v).to_owned())
7433                                .collect(),
7434                        });
7435                    }
7436                    if complete
7437                        && !title.trim().is_empty()
7438                        && !raw_body.trim().is_empty()
7439                        && failed.is_empty()
7440                    {
7441                        decision = redecision;
7442                        pr = PrMessage { title, body };
7443                        rewritten = true;
7444                        self.state
7445                            .event("github-text", "description rewrite passed");
7446                        if redacted {
7447                            self.state.event(
7448                                "github-text",
7449                                "sensitive data removed from description rewrite",
7450                            );
7451                        }
7452                    }
7453                }
7454            }
7455            if !rewritten {
7456                self.state.event(
7457                    "github-text",
7458                    "description rewrite unavailable or rejected; asking the owner",
7459                );
7460                if let Some(w) =
7461                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7462                {
7463                    // A different text than the one asked about before: that
7464                    // question no longer describes anything.
7465                    self.retire_github_text_question("the withheld text changed");
7466                    let record = crate::run::GithubTextAsk {
7467                        rewrite: rejected_rewrite,
7468                        fingerprint: raw_fp,
7469                        title: w.title,
7470                        body: w.body,
7471                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7472                        question: None,
7473                        asks: 0,
7474                        resolved: false,
7475                        chosen_title: None,
7476                        choice: None,
7477                    };
7478                    return self.settle_withheld(winner, &pr, record).await;
7479                }
7480            }
7481        }
7482        let (title, body) =
7483            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7484        Ok(Some(PrMessage { title, body }))
7485    }
7486
7487    /// Describe the whole change from its final diff in either run mode.
7488    ///
7489    /// The summary is tied to the final commit and to the merge base of the
7490    /// resolved remote base; a record for any other pair is dropped and the
7491    /// agent asked again, so a description never outlives the diff it was
7492    /// written from. The seat is read-only and the prompt tells it to read the
7493    /// diff and the code around it. `[roles] synthesizer` selects the agents
7494    /// (unset: the default pick), each asked at most once, advancing on an
7495    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
7496    /// carries the reason it is unavailable; nothing is invented then.
7497    async fn summarize_change(
7498        &mut self,
7499        winner: &Candidate,
7500        head: Option<&str>,
7501    ) -> std::result::Result<(), String> {
7502        let Some(head) = head else {
7503            return Err("the final commit could not be read".to_owned());
7504        };
7505        let remote = self.state.config.merge.remote.clone();
7506        let tracking = format!("{remote}/{}", self.state.base_branch);
7507        if !git::rev_exists(&winner.worktree, &tracking).await {
7508            return Err(format!("the remote base `{tracking}` could not be read"));
7509        }
7510        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7511            Ok(b) if !b.is_empty() => b,
7512            _ => return Err("the merge base with the remote base could not be read".to_owned()),
7513        };
7514        let evidence = crate::pr_outcome::summary_evidence(
7515            &self.state,
7516            crate::pr_outcome::FinalHead::At(head),
7517        );
7518        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7519        if self
7520            .state
7521            .change_summary
7522            .as_ref()
7523            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7524        {
7525            return Ok(());
7526        }
7527        if self.state.change_summary.take().is_some() {
7528            self.state.event(
7529                "pr-summary",
7530                "the final diff or evidence changed; the earlier change summary was discarded",
7531            );
7532        }
7533        // Save the source snapshot even if no generation agent is available.
7534        if let Err(e) = write_artifact(
7535            &self.state,
7536            &format!("public-description-{fingerprint}.md"),
7537            &format!("Final diff: {base}..{head}\n\n{evidence}"),
7538        ) {
7539            tracing::warn!("could not save public description evidence: {e:#}");
7540        }
7541        let chain = match agent::pick_chain(
7542            &self.state.config.agents,
7543            self.state.config.roles.synthesizer.as_ref(),
7544            &agent::installed,
7545            "synthesizer",
7546        ) {
7547            Ok(chain) => chain,
7548            Err(e) => {
7549                tracing::warn!("no agent for the change summary: {e:#}");
7550                self.state.event("pr-summary", "no summary agent available");
7551                return Err("no agent was available to summarize it".to_owned());
7552            }
7553        };
7554        let subjects = git::subjects(&winner.worktree, &base, head)
7555            .await
7556            .unwrap_or_default();
7557        let stat = git::diff_stat(&winner.worktree, &base, head)
7558            .await
7559            .unwrap_or_default();
7560        let stat: String = stat.chars().take(6_000).collect();
7561        let mut prompt = crate::prompt::change_summary_in(
7562            &base,
7563            head,
7564            &subjects,
7565            stat.trim(),
7566            &self.state.config.graph.github_language,
7567        );
7568        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7569        let sessions = self.state.config.graph.sessions;
7570        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7571        let prompts = self.state.config.prompts.clone();
7572        let cache = self.state.config.cache_dir();
7573        let run = self.state.id.clone();
7574        for (i, spec) in chain.into_iter().enumerate() {
7575            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7576            let job = SeatJob {
7577                spec: spec.clone(),
7578                seat,
7579                cwd: winner.worktree.clone(),
7580                prompt: prompt.clone(),
7581                timeout,
7582                allow_write: false,
7583                sessions,
7584                artifacts: agent::artifacts_dir(&self.state.dir()),
7585                stem: if i == 0 {
7586                    "change-summary".to_owned()
7587                } else {
7588                    format!("change-summary-{}", spec.id)
7589                },
7590                handover: None,
7591            };
7592            let ctx = WaveCtx {
7593                run: &run,
7594                node: "pr-summary",
7595                prompts: &prompts,
7596                cache: cache.as_deref(),
7597                round: None,
7598                carry_seats: false,
7599                pauses: Some((&self.pause, &self.interrupt)),
7600            };
7601            let (seat, outcome) =
7602                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7603            self.state.seats.insert(seat.key.clone(), seat);
7604            let text = match outcome {
7605                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7606                AgentOutcome::ResourceBlocked(why) => {
7607                    // No agent was asked: the run stops resumably before
7608                    // anything is published (see `guarded_pr_message`).
7609                    if self.state.resource_block.is_none() {
7610                        self.state.resource_block = Some(crate::run::ResourceBlock {
7611                            node: "pr-summary".to_owned(),
7612                            kind: "shared resource".to_owned(),
7613                            detail: why,
7614                            at: Timestamp::now(),
7615                        });
7616                    }
7617                    return Err("a shared resource was busy".to_owned());
7618                }
7619                _ => continue,
7620            };
7621            let blind = &self.state.config.blind;
7622            let Some(title) = summary_title(&text) else {
7623                continue;
7624            };
7625            let body: String = summary_without_title(&text)
7626                .lines()
7627                .filter(|l| !l.contains("magi:run/"))
7628                .collect::<Vec<_>>()
7629                .join("\n");
7630            let body = blind::sanitize_prose(body.trim(), blind);
7631            let title = blind::sanitize_prose(&title, blind);
7632            if body.trim().is_empty() || title.trim().is_empty() {
7633                continue;
7634            }
7635            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7636                continue;
7637            }
7638            // The branch must still be the one the agent was shown.
7639            if git::rev_parse(&winner.worktree, "HEAD")
7640                .await
7641                .ok()
7642                .as_deref()
7643                != Some(head)
7644            {
7645                return Err("the branch moved while the summary was written".to_owned());
7646            }
7647            if git::merge_base(&winner.worktree, &tracking, head)
7648                .await
7649                .ok()
7650                .as_deref()
7651                != Some(base.as_str())
7652                || crate::pr_outcome::summary_evidence(
7653                    &self.state,
7654                    crate::pr_outcome::FinalHead::At(head),
7655                ) != evidence
7656            {
7657                return Err("the summary evidence changed while it was written".to_owned());
7658            }
7659            self.state.change_summary = Some(crate::run::ChangeSummary {
7660                head: head.to_owned(),
7661                base,
7662                evidence: fingerprint,
7663                title,
7664                body,
7665            });
7666            self.state.event(
7667                "pr-summary",
7668                format!("change summary written by {} for the final diff", spec.id),
7669            );
7670            if let Err(e) = self.state.save() {
7671                tracing::warn!("could not save the change summary: {e:#}");
7672            }
7673            return Ok(());
7674        }
7675        self.state
7676            .event("pr-summary", "no agent produced a change summary");
7677        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7678    }
7679
7680    /// One read-only rewrite request for a withheld description. The
7681    /// implementer's own conversation is reused only when it exists and can be
7682    /// resumed; otherwise the `[roles] language_judge` chain (or the default
7683    /// pick) writes it in a separate conversation. The prompt carries the whole
7684    /// text, so no agent needs any memory. Each agent is asked at most once and
7685    /// the chain advances on an error, quota, unusable answer or a reply with
7686    /// no `TITLE:` line. Nothing is recorded as a handover.
7687    async fn rewrite_pr_text(
7688        &mut self,
7689        winner: &Candidate,
7690        pr: &PrMessage,
7691        violations: &[crate::github_text::Violation],
7692    ) -> Option<String> {
7693        let prompt = format!(
7694            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7695            pr.title, pr.body
7696        );
7697        let sessions = self.state.config.graph.sessions;
7698        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7699        // (spec, seat, resumed)
7700        let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7701        let key = format!("impl-{}", winner.label);
7702        if let Some(spec) = self
7703            .state
7704            .config
7705            .agents
7706            .iter()
7707            .find(|a| a.id == winner.agent)
7708            .cloned()
7709            && let Some(existing) = self.state.seats.get(&key)
7710            && existing.agent == spec.id
7711            && agent::has_session(spec.kind, existing, sessions)
7712        {
7713            attempts.push((spec, existing.clone(), true));
7714        } else {
7715            let chain = agent::pick_chain(
7716                &self.state.config.agents,
7717                self.state.config.roles.language_judge.as_ref(),
7718                &agent::installed,
7719                "language_judge",
7720            );
7721            match chain {
7722                Ok(chain) => {
7723                    for spec in chain {
7724                        let seat = crate::fixer::seat_for(
7725                            &mut self.state,
7726                            "github-text-rewrite",
7727                            &spec.id,
7728                        );
7729                        attempts.push((spec, seat, false));
7730                    }
7731                }
7732                Err(e) => {
7733                    tracing::warn!("no agent for the description rewrite: {e:#}");
7734                    self.state
7735                        .event("github-text", "no rewrite agent available");
7736                    return None;
7737                }
7738            }
7739        }
7740        let prompts = self.state.config.prompts.clone();
7741        let cache = self.state.config.cache_dir();
7742        let run = self.state.id.clone();
7743        for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7744            let job = SeatJob {
7745                spec: spec.clone(),
7746                seat,
7747                cwd: winner.worktree.clone(),
7748                prompt: prompt.clone(),
7749                timeout: retry_budget(timeout_full, resumed),
7750                allow_write: false,
7751                sessions,
7752                artifacts: agent::artifacts_dir(&self.state.dir()),
7753                stem: if i == 0 {
7754                    "github-text-rewrite".to_owned()
7755                } else {
7756                    format!("github-text-rewrite-{}", spec.id)
7757                },
7758                handover: None,
7759            };
7760            let ctx = WaveCtx {
7761                run: &run,
7762                node: "github-text",
7763                prompts: &prompts,
7764                cache: cache.as_deref(),
7765                round: None,
7766                carry_seats: false,
7767                pauses: Some((&self.pause, &self.interrupt)),
7768            };
7769            let (seat, outcome) =
7770                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7771            self.state.seats.insert(seat.key.clone(), seat);
7772            match outcome {
7773                AgentOutcome::Ok(o)
7774                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7775                {
7776                    return Some(o.text);
7777                }
7778                AgentOutcome::ResourceBlocked(_) => return None,
7779                _ => {}
7780            }
7781        }
7782        None
7783    }
7784
7785    /// Abandon the standing posting-gate question, if any.
7786    fn retire_github_text_question(&mut self, why: &str) {
7787        let Some(id) = self
7788            .state
7789            .github_text
7790            .as_ref()
7791            .and_then(|g| g.question.clone())
7792        else {
7793            return;
7794        };
7795        let _ = ask::Questions::open().update(&id, |q| {
7796            q.abandon(why);
7797            Ok(())
7798        });
7799    }
7800
7801    /// Carry a withheld text to its end: file the question, park while it is
7802    /// open, and on an answer, silence or timeout pick the text to post. The
7803    /// decision is saved before it is used, so it applies once.
7804    async fn settle_withheld(
7805        &mut self,
7806        winner: &Candidate,
7807        pr: &PrMessage,
7808        mut g: crate::run::GithubTextAsk,
7809    ) -> Result<Option<PrMessage>> {
7810        use crate::github_text as gt;
7811        let store = ask::Questions::open();
7812        let timeout = self.state.config.graph.answer_timeout;
7813        if !g.resolved {
7814            let mut standing = g
7815                .question
7816                .as_ref()
7817                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7818            if let Some(q) = standing.as_ref()
7819                && q.status.open()
7820                && gt::expired(q, timeout)
7821            {
7822                let why = format!("no answer within {}s of asking", timeout.max(1));
7823                standing = store
7824                    .update(&q.id, |q| {
7825                        q.abandon(&why);
7826                        Ok(())
7827                    })
7828                    .ok()
7829                    .map(|(q, ())| q);
7830            }
7831            let mut ask_again: Option<Option<Vec<String>>> = None;
7832            match standing {
7833                // Recorded but never written (a stop between the two saves),
7834                // or removed: ask again without spending the retry.
7835                None if g.asks <= 1 => {
7836                    g.asks = 0;
7837                    ask_again = Some(None);
7838                }
7839                None => {
7840                    g.resolved = true;
7841                    g.chosen_title = None;
7842                    g.choice = Some(GithubTextChoice::Fallback);
7843                }
7844                Some(q) if q.status.open() => {
7845                    self.park_on_github_text(&g)?;
7846                    return Ok(None);
7847                }
7848                Some(q) => match gt::read_reply(&q) {
7849                    gt::Reply::Fallback => {
7850                        g.resolved = true;
7851                        g.chosen_title = None;
7852                        g.choice = Some(GithubTextChoice::Fallback);
7853                    }
7854                    gt::Reply::Original => {
7855                        g.resolved = true;
7856                        g.chosen_title = None;
7857                        g.choice = Some(GithubTextChoice::Original);
7858                    }
7859                    gt::Reply::Rewrite => {
7860                        g.resolved = true;
7861                        g.chosen_title = None;
7862                        g.choice = Some(if g.rewrite.is_some() {
7863                            GithubTextChoice::Rewrite
7864                        } else {
7865                            GithubTextChoice::Fallback
7866                        });
7867                    }
7868                    gt::Reply::Title(text) => {
7869                        let first = text
7870                            .lines()
7871                            .map(str::trim)
7872                            .find(|l| !l.is_empty())
7873                            .unwrap_or("");
7874                        let decision = if first.is_empty() {
7875                            None
7876                        } else {
7877                            let candidate = PrMessage {
7878                                title: first.to_owned(),
7879                                body: String::new(),
7880                            };
7881                            self.judge_pr_language(&winner.worktree, &candidate).await
7882                        };
7883                        match gt::vet_title(
7884                            &text,
7885                            decision,
7886                            self.state.config.graph.github_english(),
7887                        ) {
7888                            Ok(title) => {
7889                                g.resolved = true;
7890                                g.chosen_title = Some(title);
7891                                g.choice = Some(GithubTextChoice::MyTitle);
7892                            }
7893                            Err(cats) if g.asks < 2 => {
7894                                ask_again = Some(Some(
7895                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7896                                ))
7897                            }
7898                            Err(_) => {
7899                                g.resolved = true;
7900                                g.chosen_title = None;
7901                                g.choice = Some(GithubTextChoice::Fallback);
7902                            }
7903                        }
7904                    }
7905                },
7906            }
7907            if let Some(rejected) = ask_again {
7908                // A retry (after a rejected replacement title) is still about
7909                // the original text: the same ORIGINAL / FALLBACK comparison.
7910                let w = gt::Withheld {
7911                    title: g.title,
7912                    body: g.body,
7913                    categories: g.categories.clone(),
7914                };
7915                // Named by the original text's fingerprint alone, so a resume
7916                // rewrites the same file and the detail stays identical.
7917                let artifact = match crate::run::write_artifact(
7918                    &self.state,
7919                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
7920                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7921                ) {
7922                    Ok(p) => Some(p),
7923                    Err(e) => {
7924                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
7925                        None
7926                    }
7927                };
7928                let (detail, comparison) = gt::question_content(
7929                    &w,
7930                    &pr.title,
7931                    &pr.body,
7932                    &self.state.id,
7933                    rejected.as_deref(),
7934                    artifact.as_deref(),
7935                    g.rewrite.as_ref(),
7936                );
7937                let mut q = ask::Question::new(
7938                    self.state.id.clone(),
7939                    gt::ASK_NODE.to_owned(),
7940                    gt::ASK_SEAT.to_owned(),
7941                    gt::question_summary(&self.state.config.graph.language, &w),
7942                    detail,
7943                    gt::question_choices(&w, g.rewrite.as_ref()),
7944                );
7945                q.posting_comparison = Some(comparison);
7946                q.answer_timeout = timeout;
7947                // Recorded before it is written: a stop in between leaves a
7948                // record naming a question the resume then files again.
7949                g.question = Some(q.id.clone());
7950                g.asks += 1;
7951                self.state.github_text = Some(g.clone());
7952                self.state.event(
7953                    "github-text",
7954                    format!("asking the owner about the withheld text ({})", q.short()),
7955                );
7956                self.state.save()?;
7957                store
7958                    .put_executable(&mut q)
7959                    .context("file the posting-gate question")?;
7960                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7961                    tracing::warn!(
7962                        "could not notify about posting-gate question {}: {e:#}",
7963                        q.short()
7964                    );
7965                }
7966                self.park_on_github_text(&g)?;
7967                return Ok(None);
7968            }
7969        }
7970        let resolved = g.chosen_title.clone();
7971        let original = g.choice == Some(GithubTextChoice::Original);
7972        // Field names only: the event log never records the text.
7973        self.state.event(
7974            "github-text",
7975            if original {
7976                format!(
7977                    "posting the original text for: {}",
7978                    match (g.title, g.body) {
7979                        (true, true) => "title, description",
7980                        (true, false) => "title",
7981                        _ => "description",
7982                    }
7983                )
7984            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7985                "posting the saved rewrite title and description".to_owned()
7986            } else if resolved.is_some() {
7987                "posting the owner's replacement title".to_owned()
7988            } else {
7989                "posting the neutral text for the withheld fields".to_owned()
7990            },
7991        );
7992        self.state.github_text = Some(g.clone());
7993        self.state.save()?;
7994        if g.choice == Some(GithubTextChoice::Rewrite)
7995            && let Some(r) = g.rewrite.as_ref()
7996        {
7997            return Ok(Some(PrMessage {
7998                title: r.title.clone(),
7999                body: r.body.clone(),
8000            }));
8001        }
8002        let w = gt::Withheld {
8003            title: g.title,
8004            body: g.body,
8005            categories: g.categories.clone(),
8006        };
8007        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8008        if let Some(t) = resolved {
8009            title = t;
8010        }
8011        // Every withheld field is now neutral or vetted English.
8012        let english = gt::LanguageDecision {
8013            title_english: true,
8014            body_english: true,
8015            body_complete: true,
8016        };
8017        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8018        if original {
8019            // The owner's explicit choice overrides the gate for the fields it
8020            // was made for; the others keep what the gate produced.
8021            if g.title {
8022                title = pr.title.clone();
8023            }
8024            if g.body {
8025                body = pr.body.clone();
8026            }
8027        }
8028        Ok(Some(PrMessage { title, body }))
8029    }
8030
8031    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8032        self.state.github_text = Some(g.clone());
8033        self.state.end_pass_parked(ParkCause::Question);
8034        self.state.parked = true;
8035        self.state.event(
8036            "github-text",
8037            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8038        );
8039        self.state.save()
8040    }
8041
8042    /// Ask `[roles] language_judge` about this text and record which source
8043    /// decided; the text itself is never recorded.
8044    async fn judge_pr_language(
8045        &mut self,
8046        cwd: &Path,
8047        pr: &PrMessage,
8048    ) -> Option<crate::github_text::LanguageDecision> {
8049        self.state.config.roles.language_judge.as_ref()?;
8050        if !self.state.config.graph.language_gate() {
8051            return None;
8052        }
8053        let decision =
8054            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8055        self.state.event(
8056            "github-text",
8057            if decision.is_some() {
8058                "language decided by the language judge"
8059            } else {
8060                "language judge unavailable; using built-in heuristics"
8061            },
8062        );
8063        decision
8064    }
8065
8066    async fn merge(&mut self) -> Result<()> {
8067        // Same reasoning as `gate`: ask the review and gate records directly
8068        // rather than `status`, which a solo-candidate `judge`/`deliberate`
8069        // skip can rewrite on reentry to something that no longer says
8070        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
8071        // so a hand-off (open findings, green verification) reaches merge
8072        // exactly like a genuinely clean round does.
8073        //
8074        // A run resumed mid-`land` never reaches here at all: `execute`
8075        // recognises `RunStatus::Landing` before it even calls `prep`, and
8076        // routes straight to `run_land` instead. That has to happen a level
8077        // up from this function, not with a check in here, because
8078        // `review_loop`'s own status recomputation (see its doc) runs
8079        // *before* `merge` on every reentry and would otherwise overwrite
8080        // the `Landing` marker with `Gating` before this node ever saw it.
8081        if self
8082            .state
8083            .base_sync
8084            .as_ref()
8085            .is_some_and(|s| s.conflict.is_some())
8086            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8087                != Some(RunStatus::Gating)
8088            // `gate_ran == false` is not "passed" - `gate` leaves it false
8089            // both before it has ever run and when its last attempt was
8090            // resource-blocked (see `Runner::gate`'s own doc), and neither is
8091            // permission to merge on nothing but the review record. Only a
8092            // gate that actually ran may proceed; an intentional local
8093            // review can also finish with verification explicitly not configured.
8094            // `RunState::gate_status` computes that distinction.
8095            || !self.state.gate_allows_progress()
8096        {
8097            return Ok(());
8098        }
8099        // This node's own record, not `status`: `status == Ready` is not
8100        // unique to the harmless `MergeMode::None` path this line was
8101        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
8102        // run's PR was closed without merging — and on that run `mode` is
8103        // still `Pr`, so a reentry that fell through here would push and
8104        // open a second pull request. `self.state.merge` is set exactly once
8105        // this node (or `land`) has already produced a verdict, under every
8106        // mode, which is what "already done" actually means here.
8107        if self.state.merge.is_some() && !self.state.pr_publication_pending {
8108            return Ok(());
8109        }
8110        let Some(winner) = self.state.winner().cloned() else {
8111            return Ok(());
8112        };
8113        let repo = self.state.repo.clone();
8114        let base = self.state.base_branch.clone();
8115        let mode = self.state.config.merge.mode;
8116        let style = self.state.config.merge.style;
8117        let facts = if is_review_run(&self.state) {
8118            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8119            let start = review_base(
8120                &repo,
8121                &self.state.config.merge.remote,
8122                &base,
8123                &self.state.base_commit,
8124                &winner.branch,
8125            )
8126            .await;
8127            branch_facts(&repo, &start, &winner.branch).await
8128        } else {
8129            None
8130        };
8131        // `None` when the base could not be freshly read: then an adopted
8132        // pull request's title is left alone.
8133        let leaked = if is_review_run(&self.state) {
8134            leaked_subjects(&self.state, &winner.branch).await
8135        } else {
8136            Some(Vec::new())
8137        };
8138        // `None`: parked on the owner's word about a withheld title or body.
8139        // Nothing has been pushed or opened, and `merge` stays unrecorded so
8140        // the resume arrives here again.
8141        let Some(pr) = self
8142            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8143            .await?
8144        else {
8145            return Ok(());
8146        };
8147        let message = pr.commit_message();
8148
8149        let outcome = match mode {
8150            MergeMode::None => MergeOutcome {
8151                mode,
8152                ok: true,
8153                detail: manual_merge_command(style, &repo, &winner.branch, &message),
8154                empty: false,
8155            },
8156            MergeMode::Pr | MergeMode::Local
8157                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8158            {
8159                MergeOutcome {
8160                    mode,
8161                    ok: false,
8162                    detail: empty_candidate_detail(&self.state, &base),
8163                    empty: true,
8164                }
8165            }
8166            MergeMode::Local => {
8167                let on = git::current_branch(&repo).await?;
8168                if on.as_deref() != Some(base.as_str()) {
8169                    MergeOutcome {
8170                        mode,
8171                        ok: false,
8172                        detail: format!(
8173                            "{} has {} checked out, not the base branch {base}",
8174                            repo.display(),
8175                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
8176                        ),
8177                        empty: false,
8178                    }
8179                } else if !git::is_clean(&repo).await? {
8180                    MergeOutcome {
8181                        mode,
8182                        ok: false,
8183                        detail: format!("{} is dirty; refusing to merge", repo.display()),
8184                        empty: false,
8185                    }
8186                } else {
8187                    let out = match style {
8188                        MergeStyle::Merge => {
8189                            git::merge_no_ff(&repo, &winner.branch, &message).await?
8190                        }
8191                        MergeStyle::Squash => {
8192                            git::merge_squash(&repo, &winner.branch, &message).await?
8193                        }
8194                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8195                    };
8196                    MergeOutcome {
8197                        mode,
8198                        ok: out.ok(),
8199                        detail: if out.ok() { out.stdout } else { out.stderr },
8200                        empty: false,
8201                    }
8202                }
8203            }
8204            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8205                self.state.pr_publication_pending = true;
8206                self.state.save()?;
8207                let out =
8208                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8209                self.state.pr_publication_pending = out.is_err();
8210                MergeOutcome {
8211                    mode,
8212                    ok: out.is_ok(),
8213                    detail: match out {
8214                        Ok(url) => url,
8215                        Err(e) => format!("{e:#}"),
8216                    },
8217                    empty: false,
8218                }
8219            }
8220            MergeMode::Pr => {
8221                let remote = self.state.config.merge.remote.clone();
8222                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8223                if !pushed.ok() {
8224                    MergeOutcome {
8225                        mode,
8226                        ok: false,
8227                        detail: pushed.stderr,
8228                        empty: false,
8229                    }
8230                } else {
8231                    // A retry or resume of a run whose branch already has an
8232                    // open pull request adopts it rather than failing on a
8233                    // duplicate. Only this winner branch into this base:
8234                    // `branch_for` derives the name from the run id, so a
8235                    // different run's pull request never matches.
8236                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8237                    let out = match pr_merge_plan(found) {
8238                        PrPlan::Create => {
8239                            gh_pr_create(
8240                                &winner.worktree,
8241                                &base,
8242                                &winner.branch,
8243                                &pr.title,
8244                                &pr.body,
8245                            )
8246                            .await
8247                        }
8248                        PrPlan::Adopt { url, title } => {
8249                            self.state
8250                                .event("merge", format!("Pr: adopted open pull request {url}"));
8251                            if self.state.github_text.as_ref().is_some_and(|g| {
8252                                g.resolved
8253                                    && g.choice == Some(GithubTextChoice::Rewrite)
8254                                    && g.rewrite.is_some()
8255                            }) {
8256                                // The owner chose the entire pair, even if the
8257                                // existing title already matches or was edited.
8258                                land::set_pr_text_verbatim(
8259                                    &winner.worktree,
8260                                    &url,
8261                                    &pr.title,
8262                                    &pr.body,
8263                                )
8264                                .await
8265                                .map(|()| url)
8266                            } else {
8267                                if title != pr.title
8268                                    && (!is_review_run(&self.state)
8269                                        || leaked
8270                                            .as_deref()
8271                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8272                                    && let Err(e) =
8273                                        if self.state.github_text.as_ref().is_some_and(|g| {
8274                                            g.resolved
8275                                                && g.title
8276                                                && g.choice == Some(GithubTextChoice::Original)
8277                                        }) {
8278                                            // The owner chose to post this title unmodified.
8279                                            land::set_pr_title_verbatim(
8280                                                &winner.worktree,
8281                                                &url,
8282                                                &pr.title,
8283                                            )
8284                                            .await
8285                                        } else {
8286                                            land::set_pr_title(
8287                                                &mut self.state,
8288                                                &winner.worktree,
8289                                                &url,
8290                                                &pr.title,
8291                                            )
8292                                            .await
8293                                        }
8294                                {
8295                                    tracing::warn!("could not refresh title of {url}: {e:#}");
8296                                    self.state
8297                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
8298                                }
8299                                Ok(url)
8300                            }
8301                        }
8302                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8303                    };
8304                    match out {
8305                        Ok(url) => MergeOutcome {
8306                            mode,
8307                            ok: true,
8308                            detail: url,
8309                            empty: false,
8310                        },
8311                        Err(e) => MergeOutcome {
8312                            mode,
8313                            ok: false,
8314                            detail: e.to_string(),
8315                            empty: false,
8316                        },
8317                    }
8318                }
8319            }
8320        };
8321
8322        self.state.status = match (mode, outcome.ok) {
8323            (MergeMode::None, _) => RunStatus::Ready,
8324            (_, true) if outcome.empty => RunStatus::Ready,
8325            (MergeMode::Pr, true) => RunStatus::Ready,
8326            (_, true) => RunStatus::Merged,
8327            (_, false) => RunStatus::Blocked,
8328        };
8329        self.state.event(
8330            "merge",
8331            format!(
8332                "{:?}: {}",
8333                mode,
8334                outcome.detail.lines().next().unwrap_or("")
8335            ),
8336        );
8337        self.state.merge = Some(outcome);
8338        self.state.save()?;
8339
8340        // The PR is open and the run would historically stop here, leaving the
8341        // operator to watch checks, feed review comments back to a fixer, and
8342        // merge. That was done by hand six times in one session before this
8343        // existed. Opt-in, because merging is the one irreversible thing magi
8344        // can do to a repository.
8345        if self.state.config.graph.land
8346            && mode == MergeMode::Pr
8347            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8348        {
8349            self.run_land().await?;
8350        }
8351        // `run_land` may have left `status` at `Landing` - still waiting on
8352        // CI or the owner's approval, not actually settled - so this has to
8353        // read whatever `status` ended up as here, not the `Merged` this
8354        // function set a few lines up.
8355        self.settle_questions();
8356        Ok(())
8357    }
8358
8359    /// Enter `land`.
8360    ///
8361    /// Shared between a fresh run's first pass through [`Runner::merge`] and
8362    /// a resumed run's re-entry. `land::land` itself is what serialises the
8363    /// two git-mutating moments inside the loop — the rebase push and
8364    /// `gh pr merge` — per repository (see its own doc); nothing here needs
8365    /// to hold a lock across the whole call, and doing so would serialise
8366    /// this run's CI wait against a *different* run's land-approval resume
8367    /// in the same repository, which is exactly the "must not wait on
8368    /// another task" property the daemon's slot-freeing exists to give.
8369    async fn run_land(&mut self) -> Result<()> {
8370        crate::pr_head::ensure_owner(&self.state)?;
8371        let url = self
8372            .state
8373            .merge
8374            .as_ref()
8375            .map(|m| m.detail.clone())
8376            .unwrap_or_default();
8377        let url = url.lines().next().unwrap_or("").trim().to_owned();
8378        if !url.starts_with("http") {
8379            return Ok(());
8380        }
8381        // A land failure is not a lost run: the work is on a branch and the
8382        // pull request is open, which is exactly where a human takes over.
8383        loop {
8384            match land::land(&mut self.state, &url).await {
8385                Ok(_)
8386                    if self
8387                        .state
8388                        .merge_repairs
8389                        .last()
8390                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8391                {
8392                    if self.repair_merge_review().await? {
8393                        continue;
8394                    }
8395                }
8396                Ok(pr) if self.state.parked => {
8397                    // `land` already saved the parked marker; nothing here
8398                    // overrides `status` back to a terminal value while an
8399                    // approval is still outstanding.
8400                    let _ = pr;
8401                }
8402                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8403                    self.state.status = RunStatus::Superseded;
8404                    self.state.save()?;
8405                }
8406                Ok(pr) => {
8407                    self.state.status = match pr.state {
8408                        land::PrLifecycle::Merged => RunStatus::Merged,
8409                        _ => RunStatus::Blocked,
8410                    };
8411                    // Downstream of a confirmed merge only - see
8412                    // `bump::should_release_bump`'s own doc for why this one
8413                    // check covers all three of `land`'s success paths.
8414                    // Best-effort: the run already landed, so a failure here
8415                    // (the decision call, `gh`, `cargo`) is recorded and never
8416                    // turns a landed run into a failed one.
8417                    if bump::should_release_bump(self.state.status)
8418                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8419                    {
8420                        // The event is the run's own record. Not-eligible cases
8421                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
8422                        // `Err` is a bump that was tried and failed:
8423                        // `after_merge` itself raises the operator notice for
8424                        // that, whether or not a release PR exists yet.
8425                        self.state
8426                            .event("bump", format!("release bump skipped: {e:#}"));
8427                    }
8428                    // Independent of the bump, and best-effort in the same way:
8429                    // findings the merge left open become follow-up tasks.
8430                    if self.state.status == RunStatus::Merged {
8431                        crate::followup::after_merge(&mut self.state, &pr.url).await;
8432                    }
8433                    self.state.save()?;
8434                }
8435                Err(e) => {
8436                    if e.is::<crate::repo_lock::ResourceBlocked>() {
8437                        return Err(e);
8438                    }
8439                    self.state.status = RunStatus::Blocked;
8440                    self.state.event("land", format!("gave up: {e}"));
8441                    self.state.save()?;
8442                }
8443            }
8444            break;
8445        }
8446        Ok(())
8447    }
8448
8449    /// One repair cycle stays under the existing driver/task claim. An
8450    /// interrupted fixer is rolled back and held rather than run twice.
8451    async fn repair_merge_review(&mut self) -> Result<bool> {
8452        use crate::run::MergeRepairPhase as Phase;
8453        crate::pr_head::ensure_owner(&self.state)?;
8454        let i = self.state.merge_repairs.len() - 1;
8455        let request = self.state.merge_repairs[i].clone();
8456        if request.phase == Phase::Fixing {
8457            let restored = match self.state.winner().cloned() {
8458                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8459                None => Err(anyhow::anyhow!("repair has no winning branch")),
8460            };
8461            let why = match restored {
8462                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8463                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8464                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8465            };
8466            self.hold_merge_repair(i, why)?;
8467            return Ok(false);
8468        }
8469        let failure = if i >= self.state.config.graph.land_rounds
8470            || self.state.config.graph.review_rounds == 0
8471        {
8472            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8473        } else {
8474            None
8475        };
8476        if let Some(why) = failure {
8477            self.hold_merge_repair(i, why)?;
8478            return Ok(false);
8479        }
8480        if request.phase == Phase::Requested {
8481            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8482                Ok(pr) => pr,
8483                Err(e) => {
8484                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8485                    return Ok(false);
8486                }
8487            };
8488            self.state.merge_repairs[i].phase = Phase::Fixing;
8489            self.state.land_approval = None;
8490            self.state.land_armed_head = None;
8491            self.state.save()?;
8492            // Serialize the exact records, including finding ids/details,
8493            // reconsideration reasons, final votes and verification output.
8494            let evidence = serde_json::to_string_pretty(&request)?;
8495            let reason = format!(
8496                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8497            );
8498            let budget = self.state.config.graph.land_rounds;
8499            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8500            match result {
8501                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8502                Ok(land::Fixed::Failed(why)) => {
8503                    self.hold_merge_repair(
8504                        i,
8505                        format!(
8506                            "repair failed: {why}; inspect the branch and run an explicit review"
8507                        ),
8508                    )?;
8509                    return Ok(false);
8510                }
8511                Err(e) => {
8512                    self.hold_merge_repair(
8513                        i,
8514                        format!(
8515                            "repair failed: {e:#}; inspect the branch and run an explicit review"
8516                        ),
8517                    )?;
8518                    return Ok(false);
8519                }
8520            }
8521            // Even no change gets a real new reviewer wave. The previous
8522            // evidence remains in the request; a former clean/concluded round
8523            // cannot short-circuit the new review or gate.
8524            self.begin_merge_repair_review(i)?;
8525        }
8526        self.review_loop().await?;
8527        self.gate().await?;
8528        if self.resource_halt()? || self.state.parked {
8529            return Ok(false);
8530        }
8531        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8532            != Some(RunStatus::Gating)
8533            || !self.state.gate_allows_progress()
8534        {
8535            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8536            return Ok(false);
8537        }
8538        // Review and gate may have added commits after the initial repair.
8539        // Publish exactly their final head to the existing PR, with a lease.
8540        let winner = self
8541            .state
8542            .winner()
8543            .cloned()
8544            .context("repair has no winning branch")?;
8545        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8546        if self.state.gate_head.as_deref() != Some(after.as_str())
8547            || !git::is_clean(&winner.worktree).await?
8548        {
8549            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8550            return Ok(false);
8551        }
8552        let expected = self
8553            .state
8554            .pr_pushed_head
8555            .as_deref()
8556            .unwrap_or(&request.head)
8557            .to_owned();
8558        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8559        // Push the checked commit itself: a local ref moving after the check
8560        // must not change which commit the forge receives.
8561        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8562        let refspec = format!("{after}:refs/heads/{public}");
8563        let push = git::git_raw(
8564            &winner.worktree,
8565            &["push", &lease, &self.state.config.merge.remote, &refspec],
8566        )
8567        .await;
8568        match push {
8569            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8570            Ok(out) => {
8571                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8572                return Ok(false);
8573            }
8574            Err(e) => {
8575                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8576                return Ok(false);
8577            }
8578        }
8579        self.state.merge_repairs[i].phase = Phase::Complete;
8580        self.state.status = RunStatus::Landing;
8581        self.state.event(
8582            "repair",
8583            "new review and gate completed; fresh owner merge decision required",
8584        );
8585        self.state.save()?;
8586        Ok(true)
8587    }
8588
8589    fn merge_repair_artifacts(&self) -> PathBuf {
8590        let artifacts = agent::artifacts_dir(&self.state.dir());
8591        match self
8592            .state
8593            .merge_repairs
8594            .last()
8595            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8596        {
8597            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8598            None => artifacts,
8599        }
8600    }
8601
8602    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8603        self.state.reviews.clear();
8604        self.state.gate.clear();
8605        self.state.gate_ran = false;
8606        self.state.gate_head = None;
8607        self.state.gate_fixes.clear();
8608        self.state.pre_gate.clear();
8609        self.state.pre_gate_commit = None;
8610        self.state.contested_handoff = None;
8611        self.state.land_reruns = None;
8612        self.state.land_approval = None;
8613        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8614        self.state.status = RunStatus::Reviewing;
8615        self.state.save()
8616    }
8617
8618    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8619        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8620        self.state.merge_repairs[index].failure = Some(why.clone());
8621        self.state.status = RunStatus::Blocked;
8622        self.state.event("repair", why);
8623        self.state.save()
8624    }
8625
8626    // -------------------------------------------------------------- helpers
8627
8628    /// Fetch or create a seat, keeping its conversation across nodes.
8629    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8630        if let Some(existing) = self.state.seats.get(key)
8631            && existing.agent == agent
8632        {
8633            return existing.clone();
8634        }
8635        // A seat that changes agent mints its session id from the agent too,
8636        // like a handover: the old agent's uuid is already taken by the CLI.
8637        let fresh = if self.state.seats.contains_key(key) {
8638            handover_seat(key, agent, self.state.next_seat_seed())
8639        } else {
8640            SeatState::new(key, agent, self.state.seed)
8641        };
8642        self.state.seats.insert(key.to_owned(), fresh.clone());
8643        fresh
8644    }
8645
8646    /// The agent now holding seat `key`: `spec`, unless a handover moved the
8647    /// seat to another roster agent, in which case that agent. Nodes that
8648    /// continue a seat's conversation (deliberation, the votes, a reviewer's
8649    /// reconsideration) must keep talking to whoever answered it, not slip
8650    /// back to the agent that failed it.
8651    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8652        match self.state.seats.get(key) {
8653            Some(s) if s.agent != spec.id => {
8654                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8655            }
8656            _ => spec,
8657        }
8658    }
8659
8660    /// A candidate rendered for judging, with the leak policy applied.
8661    fn view(&self, c: &Candidate) -> CandidateView {
8662        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8663            .unwrap_or_default();
8664        let (patch, _) = blind::sanitize_patch(
8665            &format!("candidate {} patch", c.label),
8666            &raw,
8667            &self.state.config.blind,
8668        );
8669        CandidateView {
8670            label: c.label,
8671            branch: c.branch.clone(),
8672            summary: c.summary.clone(),
8673            stat: c.stat.clone(),
8674            patch,
8675        }
8676    }
8677
8678    /// The full candidate set as prompt text, for seats with no live session.
8679    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8680        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8681        prompt::judge(
8682            "(see above)",
8683            &views,
8684            self.roles.judges.len(),
8685            base_short,
8686            "en",
8687        )
8688    }
8689
8690    /// The final-vote prompt with everything a seat that has no session of its
8691    /// own needs: the candidates, the seat's own ranking and reasons, and the
8692    /// anonymised deliberation it took part in (only when there was one, so a
8693    /// handed-over seat never sees more than the seat it replaces did). The
8694    /// `Final vote` heading stays first.
8695    fn vote_prompt_full(
8696        &self,
8697        j: usize,
8698        labels: &[char],
8699        language: &str,
8700        candidates: &[Candidate],
8701        base_short: &str,
8702    ) -> String {
8703        let mut text = format!(
8704            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8705            prompt::final_vote(labels, language),
8706            self.state.instruction,
8707            self.candidate_block(candidates, base_short)
8708        );
8709        if let Some(own) = self
8710            .state
8711            .judgements
8712            .get(j)
8713            .filter(|r| !r.ranking.is_empty())
8714        {
8715            let reasons = own
8716                .reasons
8717                .iter()
8718                .map(|(k, v)| format!("- {k}: {v}"))
8719                .collect::<Vec<_>>()
8720                .join("\n");
8721            text.push_str(&format!(
8722                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8723                own.ranking.iter().collect::<String>(),
8724                if reasons.is_empty() {
8725                    ""
8726                } else {
8727                    ", because:\n"
8728                }
8729            ));
8730        }
8731        if !self.state.deliberation.is_empty() {
8732            text.push_str("\n# What was argued before this vote\n");
8733            for t in self.transcript(&[], j) {
8734                text.push_str(&format!(
8735                    "\n## {}{}\n\n{}\n",
8736                    t.who,
8737                    if t.is_self { " (you)" } else { "" },
8738                    t.body.trim()
8739                ));
8740            }
8741        }
8742        text
8743    }
8744
8745    /// Anonymised transcript for judge `self_idx`.
8746    ///
8747    /// The initial rankings are always the opening statements. Seeding them
8748    /// only when no turn had been taken yet meant every judge after the first
8749    /// argued against a single voice instead of against the actual split — the
8750    /// disagreement is the information, so it is always on the table.
8751    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8752        let mut turns = Vec::new();
8753        for j in &self.state.judgements {
8754            if j.ranking.is_empty() {
8755                continue;
8756            }
8757            let reasons = j
8758                .reasons
8759                .iter()
8760                .map(|(k, v)| format!("- {k}: {v}"))
8761                .collect::<Vec<_>>()
8762                .join("\n");
8763            turns.push(Turn {
8764                who: format!("Judge {} (opening ranking)", j.judge),
8765                is_self: j.judge == self_idx + 1,
8766                body: format!(
8767                    "Ranked {}{}{reasons}",
8768                    j.ranking.iter().collect::<String>(),
8769                    if reasons.is_empty() {
8770                        ""
8771                    } else {
8772                        ", because:\n"
8773                    }
8774                ),
8775            });
8776        }
8777        for t in self
8778            .state
8779            .deliberation
8780            .iter()
8781            .flat_map(|r| r.turns.iter())
8782            .chain(current)
8783        {
8784            turns.push(Turn {
8785                who: format!("Judge {}", t.judge),
8786                is_self: t.judge == self_idx + 1,
8787                body: t.body.clone(),
8788            });
8789        }
8790        turns
8791    }
8792}
8793
8794/// Does this seat still hold the context a follow-up prompt would rely on?
8795fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8796    agent::has_session(spec.kind, seat, sessions)
8797}
8798
8799/// The next entry in `roster` after `start`, never wrapping back to the
8800/// front, whose id is not in `tried` yet.
8801///
8802/// Starts one past `start` rather than at the front of `roster`: `start` is
8803/// the seat's own original position, and a seat whose candidate slot already
8804/// sits on the roster's second entry must fall through to the third next, not
8805/// restart at the first — which is very likely a different candidate's own
8806/// agent already. Never wraps back past `start`, for the same reason: an
8807/// entry earlier in the roster than the seat's own position is almost
8808/// certainly some *other* candidate slot's own agent, and once the tail of
8809/// the roster is exhausted there are no more untried agents for *this* seat
8810/// to fall through to — the caller's fallback chain ends there, exactly as
8811/// "no further untried agents remain in the list for that seat" asks for.
8812///
8813/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
8814/// the same id twice (an operator's `roles.implementers` typo, or a
8815/// `[[agents]]` list reused across roles) must not let
8816/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
8817/// over `roster` either finds an untried id or runs out, so this always
8818/// terminates regardless of duplicates.
8819fn next_untried_in_roster<'a>(
8820    roster: &'a [AgentSpec],
8821    start: usize,
8822    tried: &BTreeSet<String>,
8823) -> Option<&'a AgentSpec> {
8824    roster
8825        .get(start + 1..)?
8826        .iter()
8827        .find(|s| !tried.contains(&s.id))
8828}
8829
8830/// The successor for a seat, shared by all four seat kinds (implement, judge,
8831/// review, advise). `others` holds the ids that *currently* occupy the other
8832/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
8833/// sees them), so roster entries beyond the seat count act as spares: a failed
8834/// seat goes to an agent no other seat holds whenever the roster permits.
8835///
8836/// `carried` is `Some` only for the review loop's carried failure history.
8837/// Order: (1) forward from `start`, never wrapping, untried, not a carried
8838/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
8839/// whole roster: untried and not another seat's occupant; (3) the plain walk
8840/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
8841/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
8842/// a better outcome than an empty seat, and it keeps the answer to "is there
8843/// a successor at all" exactly what it was before occupants were considered,
8844/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
8845/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
8846/// carried failure is a cheaper bet than doubling an agent on the panel, and
8847/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
8848/// the same round are handled one at a time, so a seat later in the batch
8849/// sees an earlier one's new occupant but not yet its own freed agent.
8850fn pick_successor<'a>(
8851    roster: &'a [AgentSpec],
8852    start: usize,
8853    tried: &BTreeSet<String>,
8854    carried: Option<&BTreeSet<String>>,
8855    others: &BTreeSet<String>,
8856) -> Option<&'a AgentSpec> {
8857    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8858    roster
8859        .get(start + 1..)
8860        .and_then(|tail| {
8861            tail.iter()
8862                .filter(free)
8863                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8864        })
8865        .or_else(|| {
8866            carried
8867                .is_some()
8868                .then(|| roster.iter().find(free))
8869                .flatten()
8870        })
8871        .or_else(|| match carried {
8872            Some(c) => next_for_seat(roster, start, tried, c),
8873            None => next_untried_in_roster(roster, start, tried),
8874        })
8875}
8876
8877/// The next agent for a seat that carries its failure history across rounds
8878/// (the review loop). `round_tried` is this round's own bound and starts
8879/// empty every round; `carried_failed` only decides priority.
8880///
8881/// First: an id walking forward from `start`, never wrapping, that is neither
8882/// tried this round nor failed in an earlier one. Only when that is exhausted
8883/// does it rescue: the first roster id (in roster order, so this one *does*
8884/// look before `start`) not yet tried this round, which is by then a carried
8885/// failure. Each id is rescued at most once per round, so it cannot loop.
8886fn next_for_seat<'a>(
8887    roster: &'a [AgentSpec],
8888    start: usize,
8889    round_tried: &BTreeSet<String>,
8890    carried_failed: &BTreeSet<String>,
8891) -> Option<&'a AgentSpec> {
8892    roster
8893        .get(start + 1..)?
8894        .iter()
8895        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8896        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8897}
8898
8899/// Where a reviewer seat starts a round: the agent that last answered it when
8900/// it is still on the roster and not marked failed, else the spec's own agent
8901/// unless it failed, else the next roster agent that has not failed, else the
8902/// spec's own agent again (the whole roster failed). Ids no longer on the
8903/// roster are ignored. An empty roster has no handover, so the spec stands.
8904fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8905    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8906        return spec;
8907    };
8908    let ok = |id: &str| !h.failed.contains(id);
8909    if let Some(last) = h.last_ok.as_deref()
8910        && ok(last)
8911        && let Some(s) = roster.iter().find(|s| s.id == last)
8912    {
8913        return s.clone();
8914    }
8915    if ok(&spec.id) {
8916        return spec;
8917    }
8918    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8919    roster
8920        .get(from + 1..)
8921        .into_iter()
8922        .flatten()
8923        .chain(roster.iter())
8924        .find(|s| ok(&s.id))
8925        .cloned()
8926        .unwrap_or(spec)
8927}
8928
8929/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
8930/// seed so a CLI that mints its session id up front (`--session-id`) never
8931/// reuses the uuid the previous agent already opened under the same seat key.
8932pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8933    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8934}
8935
8936/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
8937/// so the classifier below and the code that builds the message cannot drift.
8938const TIMED_OUT: &str = "timed out";
8939
8940impl FailClass {
8941    /// `None` for an answer; otherwise how the turn failed.
8942    fn of(out: &AgentOutcome) -> Option<Self> {
8943        match out {
8944            AgentOutcome::Ok(_) => None,
8945            AgentOutcome::Quota(_) => Some(Self::Quota),
8946            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8947            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8948            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8949            // Not a failure of the agent. Callers check for it first; `None`
8950            // here means "no class to hand over on".
8951            AgentOutcome::ResourceBlocked(_) => None,
8952        }
8953    }
8954
8955    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
8956    fn stem_word(&self) -> &'static str {
8957        match self {
8958            Self::Quota => "quota",
8959            _ => "handover",
8960        }
8961    }
8962}
8963
8964/// The message's first line with its variable parts removed — digit runs and
8965/// path-like tokens — so "exit code 2" and "exit code 7" read
8966/// as one kind of failure.
8967fn failure_signature(msg: &str) -> String {
8968    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8969    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
8970    // the historical machine key while the process diagnosis uses readable prose.
8971    if let Some(code) = line
8972        .strip_prefix("process exited with no usable output (exit code ")
8973        .and_then(|s| s.strip_suffix(')'))
8974        .and_then(|s| s.parse::<i32>().ok())
8975    {
8976        line = format!("exited with some({code}) and no usable output");
8977    } else if matches!(
8978        line.as_str(),
8979        "process exited with no usable output (no exit code reported)"
8980            | "process exited with no usable output (exit code unavailable)"
8981    ) {
8982        line = "exited with none and no usable output".to_owned();
8983    }
8984    let mut out = Vec::new();
8985    for word in line.split_whitespace() {
8986        if word.contains('/') || word.contains('\\') {
8987            out.push("<path>".to_owned());
8988            continue;
8989        }
8990        let mut w = String::new();
8991        let mut in_digits = false;
8992        for c in word.chars() {
8993            if c.is_ascii_digit() {
8994                if !in_digits {
8995                    w.push('#');
8996                }
8997                in_digits = true;
8998            } else {
8999                in_digits = false;
9000                w.push(c);
9001            }
9002        }
9003        out.push(w);
9004    }
9005    out.join(" ").chars().take(120).collect()
9006}
9007
9008/// Whether a seat that just failed with `cur` may go to the next roster agent.
9009/// A quota or a timeout always may. Any other failure may not when the agent
9010/// before it failed the same way: an error the prompt causes would otherwise
9011/// walk the whole roster. `prev` is the class of the immediately preceding
9012/// agent's failure, so a quota or timeout in between breaks the run of
9013/// identical failures by itself.
9014fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9015    match cur {
9016        FailClass::Quota | FailClass::Timeout => true,
9017        FailClass::Other(_) => prev != Some(cur),
9018    }
9019}
9020
9021/// A short human reason for a failed outcome, for the handover record.
9022fn fail_reason(out: &AgentOutcome) -> String {
9023    match out {
9024        AgentOutcome::Ok(_) => String::new(),
9025        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9026        AgentOutcome::Dropped(o) => format!(
9027            "the CLI dropped the stream ({})",
9028            o.dropped
9029                .as_ref()
9030                .map(|d| d.why.as_str())
9031                .unwrap_or("it ended without delivering its answer")
9032        ),
9033        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9034            e.lines().next().unwrap_or("").chars().take(160).collect()
9035        }
9036    }
9037}
9038
9039/// Note one handover in the run: the structured record and, in the timeline,
9040/// the sentence a person reads. A quota keeps the wording it always had.
9041pub(crate) fn record_handover(
9042    state: &mut RunState,
9043    node: &str,
9044    seat: &str,
9045    from: &str,
9046    to: &str,
9047    class: &FailClass,
9048    reason: &str,
9049) {
9050    let message = if *class == FailClass::Quota {
9051        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9052    } else {
9053        format!("{seat}: handed over {from} -> {to} ({reason})")
9054    };
9055    state.event(node, message);
9056    state.handovers.push(Handover {
9057        at: Timestamp::now(),
9058        node: node.to_owned(),
9059        seat: seat.to_owned(),
9060        from: from.to_owned(),
9061        to: to.to_owned(),
9062        reason: reason.to_owned(),
9063    });
9064}
9065
9066/// Did this reply report running a command whose own CLI never confirmed an
9067/// exit status?
9068///
9069/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
9070/// command *finished* (see that type's own doc), so this can only be `true`
9071/// for a command whose completion event carried no readable exit code — not
9072/// for one that simply is not mentioned at all. That is the one signal this
9073/// crate can read, from the same record `state.jobs` renders, about a reply
9074/// standing next to work its own CLI cannot vouch for finishing; it is
9075/// deliberately not a check on the exit code's *value* (a fixer legitimately
9076/// runs a command that fails mid-iteration before it succeeds) and not a
9077/// guess at a command still running in the background (which emits no event
9078/// at all, and so leaves no evidence here to find).
9079fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9080    commands.iter().any(|c| c.exit_code.is_none())
9081}
9082
9083/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
9084/// trusted as a verified no-op — the adoption guard's own text-level half.
9085///
9086/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
9087/// timed out): a marker only earns the benefit of the doubt from a turn the
9088/// CLI itself vouches for finishing properly, the same house style
9089/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
9090/// *fix* report to for `commands`. A candidate that timed out, exited
9091/// non-zero, or left a command unconfirmed is read as the ordinary loss it
9092/// is, whatever prose it wrote — this returns `None` before it ever looks at
9093/// `text`. The remaining guards (the tree really is empty, the evidence is
9094/// non-empty) are the caller's: this only reads what the reply *claimed*.
9095fn verified_noop_claim(
9096    usable: bool,
9097    commands: &[agent::CommandEvidence],
9098    text: &str,
9099) -> Option<String> {
9100    (usable && !has_unconfirmed_command(commands))
9101        .then(|| verdict::verified_noop(text))
9102        .flatten()
9103}
9104
9105fn short(commit: &str) -> String {
9106    commit.chars().take(7).collect()
9107}
9108
9109fn make_executable(path: &Path) -> Result<()> {
9110    #[cfg(unix)]
9111    {
9112        use std::os::unix::fs::PermissionsExt as _;
9113        let mut perms = std::fs::metadata(path)?.permissions();
9114        perms.set_mode(0o755);
9115        std::fs::set_permissions(path, perms)?;
9116    }
9117    #[cfg(not(unix))]
9118    {
9119        let _ = path;
9120    }
9121    Ok(())
9122}
9123
9124/// What every seat in one batch shares: where the answers are attributed, the
9125/// prompt overlay they inherit, and the build cache they are told to use.
9126///
9127/// A struct rather than four more parameters: `wave` also needs the run's
9128/// state (to record who is answering right now) and the attempt number, and
9129/// eight positional arguments is both unreadable and a clippy error.
9130struct WaveCtx<'a> {
9131    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
9132    /// paid for it.
9133    run: &'a str,
9134    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
9135    node: &'a str,
9136    prompts: &'a Prompts,
9137    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
9138    cache: Option<&'a Path>,
9139    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
9140    /// `JobRecord::round`. `None` for every other node.
9141    round: Option<usize>,
9142    /// Carry each seat's failed-agent history across waves (the review loop
9143    /// only): start-of-round priority and handover choice read
9144    /// [`RunState::seat_history`], and every answer or failure writes it.
9145    carry_seats: bool,
9146    /// The run's park / interrupt requests, so a wait for a shared resource
9147    /// can be abandoned the moment the operator asks to park.
9148    pauses: Option<(&'a Pause, &'a Pause)>,
9149}
9150
9151/// Shared scheduling policy for operations and spawned seat invocations.
9152fn repository_wait(
9153    state: &RunState,
9154    pauses: Option<(&Pause, &Pause)>,
9155) -> crate::repo_lock::WaitPolicy {
9156    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9157    let inherited = crate::repo_lock::WaitPolicy::current();
9158    crate::repo_lock::WaitPolicy::new(
9159        Duration::from_secs(state.config.graph.timeout_resource_wait),
9160        move || {
9161            inherited.cancelled()
9162                || pauses
9163                    .as_ref()
9164                    .is_some_and(|(a, b)| a.parked() || b.parked())
9165        },
9166    )
9167}
9168
9169fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9170    if error.is::<crate::repo_lock::ResourceBlocked>() {
9171        state.resource_block = Some(crate::run::ResourceBlock {
9172            node: "repository".into(),
9173            kind: "repository coordination".into(),
9174            detail: error.to_string(),
9175            at: Timestamp::now(),
9176        });
9177        state.event("resource", error.to_string());
9178    }
9179}
9180
9181async fn coordinate_operation(
9182    state: &mut RunState,
9183    cwd: &Path,
9184    wait: &crate::repo_lock::WaitPolicy,
9185) -> Result<Option<crate::repo_lock::SeatLease>> {
9186    let result =
9187        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9188            .await;
9189    if let Err(e) = &result {
9190        record_repository_block(state, e);
9191    }
9192    result
9193}
9194
9195/// Run one job, honouring the parallelism budget.
9196async fn run_one(
9197    job: SeatJob,
9198    sem: Arc<Semaphore>,
9199    ctx: &WaveCtx<'_>,
9200    state: &mut RunState,
9201    attempt: usize,
9202) -> (SeatState, AgentOutcome) {
9203    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9204        .await
9205        .pop()
9206        .expect("one job in, one result out");
9207    (seat, out)
9208}
9209
9210/// Run every job concurrently, capped by the semaphore, preserving order.
9211///
9212/// Every seat in the batch is recorded into [`RunState::active`] before the
9213/// wave starts and cleared as each answer lands, so the run's own record says
9214/// who is still being waited on rather than only who finished.
9215async fn wave(
9216    jobs: Vec<SeatJob>,
9217    sem: Arc<Semaphore>,
9218    ctx: &WaveCtx<'_>,
9219    state: &mut RunState,
9220    attempt: usize,
9221) -> Vec<(usize, SeatState, AgentOutcome)> {
9222    let WaveCtx {
9223        run,
9224        node,
9225        prompts,
9226        cache,
9227        round,
9228        carry_seats: _,
9229        pauses,
9230    } = *ctx;
9231    for job in &jobs {
9232        state.seat_started(node, &job.seat.key, job.timeout, attempt);
9233    }
9234    if let Err(e) = state.save() {
9235        // A failed persist of "who is answering right now" must not abort the
9236        // wave: the seats are already being asked, and the alternative is
9237        // losing the answers to save a status line nobody may even be
9238        // watching.
9239        tracing::warn!("could not persist in-progress seats: {e:#}");
9240    }
9241    // Hold the shared build cache's lease for the whole batch, not per job:
9242    // several candidates (an implement wave) or a fixer legitimately share
9243    // one cache concurrently within this run, and that stays untouched — a
9244    // single lease taken once for the whole wave and released once it is
9245    // done is what stops a *different* borrower (another run's own wave, its
9246    // e2e/gate, a human's `magi review`) from interleaving a build into the
9247    // same directory while this one is in flight. Best-effort, not
9248    // all-or-nothing: a wave that cannot get the lease within its own
9249    // longest job's budget still runs — an hour of paid implementer calls is
9250    // not thrown away over cache contention — but every write-allowed seat
9251    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
9252    // below), the same fallback a read-only seat always gets, rather than
9253    // building into a directory this run was never granted. The identity
9254    // record is still invalidated below either way, so the next tracked
9255    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
9256    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9257    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
9258    // never borrowed from a job's agent budget (and never charged to it either,
9259    // see below). The agent's clock starts in `agent::invoke`, at spawn.
9260    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9261    let cache_guard = if let Some(cache_dir) = cache {
9262        if jobs_had_a_writer {
9263            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9264            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9265            match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9266            {
9267                Ok(Some(g)) => Some(g),
9268                // Parking: leave without spawning anything. The run parks at
9269                // the next boundary and the seats are asked afresh on resume.
9270                Ok(None) => {
9271                    state.resource_block = Some(crate::run::ResourceBlock {
9272                        node: node.to_owned(),
9273                        kind: "shared build cache".to_owned(),
9274                        detail: "the run is parking".to_owned(),
9275                        at: Timestamp::now(),
9276                    });
9277                    return block_wave(
9278                        jobs,
9279                        state,
9280                        node,
9281                        attempt,
9282                        "parked while waiting for the shared build cache",
9283                    );
9284                }
9285                // An unreadable lease is not contention: keep the old
9286                // fallback (no shared cache for this wave's seats).
9287                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9288                Err(e) => {
9289                    // Writing seats may not build into a cache this run does
9290                    // not hold, and an agent that cannot be given one is not
9291                    // asked at all: the run stops resumably instead.
9292                    let why = format!("{e:#}");
9293                    state.resource_block = Some(crate::run::ResourceBlock {
9294                        node: node.to_owned(),
9295                        kind: "shared build cache".to_owned(),
9296                        detail: why.clone(),
9297                        at: Timestamp::now(),
9298                    });
9299                    state.event(
9300                        "resource",
9301                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9302                    );
9303                    return block_wave(jobs, state, node, attempt, &why);
9304                }
9305            }
9306        } else {
9307            None
9308        }
9309    } else {
9310        None
9311    };
9312    // Every job keeps its whole budget: the wait above was not the agent's.
9313    // Restart the seats' clocks so "started" means "being spawned now".
9314    state.seats_spawning(node, attempt);
9315    if let Err(e) = state.save() {
9316        tracing::warn!("could not persist the start of a wave: {e:#}");
9317    }
9318    // Queueing behind this wave's own seats for a parallelism slot is normal
9319    // (each of them is bounded by its own agent timeout); only a wait beyond
9320    // all of them plus the resource limit means the slots never came back.
9321    let slot_limit = resource_limit
9322        + jobs
9323            .iter()
9324            .map(|j| j.timeout)
9325            .max()
9326            .unwrap_or_default()
9327            .saturating_mul(jobs.len() as u32);
9328    let mut set = tokio::task::JoinSet::new();
9329    // Seats announce themselves here the moment they hold a parallelism
9330    // permit, i.e. just before their CLI is spawned.
9331    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9332    let overlay = prompts.overlay(node);
9333    let repository_wait = repository_wait(state, pauses);
9334    for (i, mut job) in jobs.into_iter().enumerate() {
9335        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9336        if cache.is_some() {
9337            job.prompt.push('\n');
9338            job.prompt
9339                .push_str(&prompt::build_cache_note(node, job.allow_write));
9340        }
9341        let sem = Arc::clone(&sem);
9342        let run = run.to_owned();
9343        let node = node.to_owned();
9344        // Only implementers were told about the task's attachments, so only
9345        // their seats get the directory widened for reading.
9346        let attachments = if node == "implement" {
9347            state.attachments.clone()
9348        } else {
9349            Vec::new()
9350        };
9351        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
9352        // `prompt::build_cache_note`'s doc for why setting it anyway is
9353        // exactly how a sandboxed reviewer's write refusal got reported as a
9354        // defect in the patch, not a property of its own seat. And a
9355        // write-allowed one is handed it only when the lease above was
9356        // actually acquired: a wave that could not get it (`cache_guard` is
9357        // `None`, see its own comment) must not send seats to build into a
9358        // directory this run does not hold - that is the exact concurrent,
9359        // unmanaged-write race this module exists to prevent, not something
9360        // "proceeding anyway" is allowed to reintroduce.
9361        let cache = cache
9362            .filter(|_| job.allow_write && cache_guard.is_some())
9363            .map(Path::to_path_buf);
9364        let spawned_tx = spawned_tx.clone();
9365        let repository_wait = repository_wait.clone();
9366        let coordination = state.config.coordination.clone();
9367        let fetched_base = state.fetched_base.clone();
9368        set.spawn(async move {
9369            // A wait for a parallelism slot is scheduling, not the agent: it
9370            // is bounded by its own limit and ends as a resource block, never
9371            // as a timeout of the agent that was not started.
9372            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9373                Ok(p) => p,
9374                Err(_) => {
9375                    return (
9376                        i,
9377                        job.seat,
9378                        AgentOutcome::ResourceBlocked(format!(
9379                            "no parallel slot became free within {}s",
9380                            slot_limit.as_secs()
9381                        )),
9382                    );
9383                }
9384            };
9385            spawned_tx.send(job.seat.key.clone()).ok();
9386            let mut seat = job.seat;
9387            let out = repository_wait
9388                .scope(agent::invoke(
9389                    &job.spec,
9390                    &mut seat,
9391                    &Invocation {
9392                        scope: crate::agent::InvocationScope::Repository,
9393                        coordination: coordination.as_ref(),
9394                        fetched_base: fetched_base.as_ref(),
9395                        cwd: &job.cwd,
9396                        prompt: &job.prompt,
9397                        timeout: job.timeout,
9398                        allow_write: job.allow_write,
9399                        unsandboxed: false,
9400                        sessions: job.sessions,
9401                        artifacts: &job.artifacts,
9402                        stem: &job.stem,
9403                        run: &run,
9404                        node: &node,
9405                        cache_dir: cache.as_deref(),
9406                        attachments: &attachments,
9407                        writable: &[],
9408                    },
9409                ))
9410                .await;
9411            let out = match out {
9412                Ok(o) if o.usable() => AgentOutcome::Ok(o),
9413                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9414                // Billed work the CLI failed to hand over is not an ordinary
9415                // failure, but its text is the CLI's raw error JSON, not an
9416                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
9417                // read it as one by forgetting to check. `usable()` is always
9418                // false here (dropped implies an empty response), so this has
9419                // to be checked before the catch-all `Failed` below or the
9420                // one shape this exists for is lost with the rest.
9421                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9422                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9423                Ok(o) => AgentOutcome::Failed(format!(
9424                    "process exited with no usable output ({})",
9425                    crate::notices::exit_status(o.exit_code)
9426                )),
9427                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9428                    AgentOutcome::ResourceBlocked(e.to_string())
9429                }
9430                Err(e) => AgentOutcome::Failed(e.to_string()),
9431            };
9432            (i, seat, out)
9433        });
9434    }
9435    drop(spawned_tx);
9436    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9437    loop {
9438        let joined = tokio::select! {
9439            Some(key) = spawned_rx.recv() => {
9440                // The clock of this seat starts now, not when it queued.
9441                state.seat_spawned(&key);
9442                if let Err(e) = state.save() {
9443                    tracing::warn!("could not persist a seat's start: {e:#}");
9444                }
9445                continue;
9446            }
9447            joined = set.join_next() => joined,
9448        };
9449        let Some(joined) = joined else {
9450            break;
9451        };
9452        let (i, seat, out) = match joined {
9453            Ok(v) => v,
9454            // No seat to clear: a panicked task never reported which one it
9455            // was. The defensive sweep below this loop is what stops that
9456            // seat's `active` entry from surviving forever.
9457            Err(e) => {
9458                tracing::error!("agent task panicked: {e}");
9459                continue;
9460            }
9461        };
9462        state.seat_finished(&seat.key);
9463        if let AgentOutcome::ResourceBlocked(why) = &out
9464            && state.resource_block.is_none()
9465        {
9466            state.resource_block = Some(crate::run::ResourceBlock {
9467                node: node.to_owned(),
9468                kind: if why.contains("repository coordination") {
9469                    "repository coordination"
9470                } else {
9471                    "parallel slot"
9472                }
9473                .to_owned(),
9474                detail: why.clone(),
9475                at: Timestamp::now(),
9476            });
9477            state.event("resource", format!("{node}: no agent was started - {why}"));
9478        }
9479        record_jobs(state, node, round, &seat.key, &out);
9480        if let Err(e) = state.save() {
9481            tracing::warn!("could not persist a seat's completion: {e:#}");
9482        }
9483        if collected.len() <= i {
9484            collected.resize_with(i + 1, || None);
9485        }
9486        collected[i] = Some((i, seat, out));
9487    }
9488    // Belt-and-braces for the panic branch above: every seat this exact batch
9489    // started shares this `(node, attempt)` pair, and every seat that finished
9490    // normally already cleared itself, so anything left tagged with it here
9491    // can only be a panicked task's leftover. Cleared unconditionally rather
9492    // than left to read as still answering forever.
9493    if state
9494        .active
9495        .values()
9496        .any(|a| a.node == node && a.attempt == attempt)
9497    {
9498        state
9499            .active
9500            .retain(|_, a| !(a.node == node && a.attempt == attempt));
9501        if let Err(e) = state.save() {
9502            tracing::warn!("could not persist the end of a wave: {e:#}");
9503        }
9504    }
9505    // Whether or not the lease above was actually held, several worktrees
9506    // may just have built into the cache with nothing here able to name one
9507    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
9508    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
9509    // clean it might not have strictly needed; trusting a stale match would
9510    // cost it a wrong answer.
9511    if let Some(cache_dir) = cache
9512        && jobs_had_a_writer
9513    {
9514        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9515    }
9516    if let Some(guard) = cache_guard {
9517        guard.release();
9518    }
9519    collected.into_iter().flatten().collect()
9520}
9521
9522/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
9523/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
9524/// this is data collection, not the fix-specific completion contract in
9525/// [`Runner::continue_fix_report`], and applies regardless of which node
9526/// asked.
9527///
9528/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
9529/// evidence from; `Failed` does not, and correctly contributes nothing — a
9530/// timeout or crash is not itself evidence about a command the seat may have
9531/// started.
9532fn record_jobs(
9533    state: &mut RunState,
9534    node: &str,
9535    round: Option<usize>,
9536    seat: &str,
9537    out: &AgentOutcome,
9538) {
9539    let commands: &[agent::CommandEvidence] = match out {
9540        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9541        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9542    };
9543    let checked_at = Timestamp::now();
9544    for c in commands {
9545        state.jobs.push(JobRecord {
9546            node: node.to_owned(),
9547            round,
9548            seat: seat.to_owned(),
9549            id: c.id.clone(),
9550            description: c.description.clone(),
9551            checked_at,
9552            status: match c.exit_code {
9553                Some(0) => JobStatus::Completed,
9554                Some(_) => JobStatus::Failed,
9555                None => JobStatus::Unknown,
9556            },
9557            exit_code: c.exit_code,
9558            result_summary: c.result_summary.clone(),
9559            source: c.source.clone(),
9560        });
9561    }
9562}
9563
9564/// Is a review round clean, given how many reviewer seats answered against
9565/// how many the round expected?
9566///
9567/// A seat that never answered (timeout, crash, unparsable output) is not a
9568/// seat that read the patch and found nothing — treating it as such is
9569/// exactly the bug this function exists to close. Under the default `block`
9570/// policy a missing seat can never be clean; `warn` still requires the seats
9571/// that *did* answer to have found nothing blocking and verification to be
9572/// green.
9573///
9574/// `quota_missing` narrows that `block` default for exactly one cause of
9575/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
9576/// a session limit lifts by the very next round buys nothing — the seat is
9577/// asked again with the same quota — so once every missing seat is accounted
9578/// for by a quota loss (and at least one seat *did* answer, so a decision has
9579/// something to rest on) the round is decided on the panel that could answer,
9580/// same as `warn` would. A panel that lost every seat to quota is not
9581/// decided here: `answered == 0` falls through to the existing `block`
9582/// fallback so a fully collapsed panel still waits rather than landing on no
9583/// review at all.
9584fn round_is_clean(
9585    blocking: usize,
9586    e2e_ok: bool,
9587    answered: usize,
9588    expected: usize,
9589    quota_missing: usize,
9590    policy: IncompleteReviewPolicy,
9591) -> bool {
9592    if blocking != 0 || !e2e_ok {
9593        return false;
9594    }
9595    if answered == expected || policy == IncompleteReviewPolicy::Warn {
9596        return true;
9597    }
9598    answered > 0 && expected - answered <= quota_missing
9599}
9600
9601/// The review loop's own conclusion, derived entirely from its persisted
9602/// round records and the round budget that produced them — never from
9603/// `status`, so a reentry (or `gate`/`merge` reading it independently)
9604/// recomputes the identical answer regardless of what an earlier node in the
9605/// same walk, or a previous walk, did to `status`.
9606///
9607/// `None` while more rounds remain to try, including when review never ran
9608/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
9609/// gone clean, or the budget is spent, or the tree has stopped moving (see
9610/// [`STAGNANT_LIMIT`]), the answer is one of two things:
9611///
9612/// - An incomplete panel that raised nothing is missing input, not a
9613///   verified tree — never a hand-off candidate, whatever verification said
9614///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
9615/// - Otherwise, green e2e on the last round hands off (see
9616///   [`Runner::stop_reviewing`]); red e2e blocks.
9617///
9618/// A last round whose own verification is still `ResourceBlocked` — magi
9619/// itself never got a command to run, not evidence the patch is broken —
9620/// is neither: this returns `None` for it too, the same as "more rounds
9621/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
9622/// handling of that shape) instead of this cheap recomputation guessing a
9623/// verdict a real attempt never produced.
9624fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9625    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9626        return Some(RunStatus::Gating);
9627    }
9628    let last = reviews.last()?;
9629    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9630    if reviews.len() < max_rounds && !stagnant {
9631        return None;
9632    }
9633    if last.incomplete() && last.blocking == 0 {
9634        return Some(RunStatus::Blocked);
9635    }
9636    if last.e2e_status() == E2eStatus::ResourceBlocked {
9637        return None;
9638    }
9639    Some(if last.e2e.iter().all(CommandOutcome::ok) {
9640        RunStatus::Gating
9641    } else {
9642        RunStatus::Blocked
9643    })
9644}
9645
9646/// How long a re-ask may take, given the budget the first attempt had.
9647///
9648/// A `nudged` retry is a request to restate an answer the seat has already
9649/// worked out: it carries no new work, so it does not deserve the original
9650/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
9651/// 133 seconds while a third sat for over ten minutes on a resumed session
9652/// holding 410 KB of prior output - and because the retry had inherited the
9653/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
9654/// judging round whose other seats were long finished.
9655///
9656/// A quarter of the budget, with a floor so that a deliberately short timeout
9657/// does not collapse to nothing. A retry that re-sends the whole prompt
9658/// (because the seat kept no context) is the original job again, and keeps the
9659/// original budget.
9660pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9661    if nudged {
9662        (full / 4).max(Duration::from_secs(120)).min(full)
9663    } else {
9664        full
9665    }
9666}
9667
9668/// Run a wave and parse each reply, re-asking the seats whose reply was
9669/// unusable.
9670///
9671/// The re-ask is a nudge rather than the whole prompt again when the seat still
9672/// holds its conversation, which is the difference between a cheap retry and
9673/// paying for the entire candidate set twice.
9674///
9675/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
9676/// successor in `roster` is handed to it instead of being re-asked: the
9677/// handover is the retry. A seat with no successor left (a single-agent
9678/// roster, the roster's tail) is nudged as before, up to `retries` times. So
9679/// the asks to one seat in one node number at most
9680/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
9681/// is asked once (a dropped stream is nudged first), and only the last agent
9682/// of the chain gets the `retries` same-agent nudges. Each roster agent is
9683/// tried at most once per seat, walking forward from the seat's own position and never wrapping
9684/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
9685/// other failure stops the chain when the previous agent failed the same way
9686/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
9687/// [`has_context`] is false and the job's own full prompt and full budget are
9688/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
9689/// intermediate ones are not losses) and is returned as a failure like any
9690/// other absent seat — the caller decides whether the panel still has a
9691/// quorum. An empty `roster` disables handover: failures are nudged as they
9692/// always were, and a quota is simply lost. A reply that fails to parse or
9693/// validate is the prompt's doing and is only ever nudged, never handed over.
9694///
9695/// The returned [`SeatState`] names the agent that answered (or tried last).
9696#[allow(clippy::too_many_arguments)]
9697async fn ask_json_wave<T>(
9698    jobs: Vec<SeatJob>,
9699    sem: Arc<Semaphore>,
9700    retries: usize,
9701    roster: &[AgentSpec],
9702    ctx: &WaveCtx<'_>,
9703    losses: &mut Vec<QuotaLoss>,
9704    state: &mut RunState,
9705    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9706) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9707where
9708    T: serde::de::DeserializeOwned + Send + 'static,
9709{
9710    ask_wave_with(
9711        jobs,
9712        sem,
9713        retries,
9714        roster,
9715        ctx,
9716        losses,
9717        state,
9718        &|text: &str| {
9719            let v = verdict::extract_json::<T>(text)?;
9720            validate(&v)?;
9721            Ok(v)
9722        },
9723    )
9724    .await
9725}
9726
9727/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
9728/// a node whose answer is prose (deliberation) shares the same handover,
9729/// failure classification, quota bookkeeping and bounds instead of a copy.
9730///
9731/// A seat handed to another roster agent is sent the job's `handover` prompt
9732/// (when it has one) rather than `prompt`: the new agent has no session, so a
9733/// resume-style prompt would be incomplete. That holds for the handover ask
9734/// and for every nudge to that agent whose `has_context` is false.
9735#[allow(clippy::too_many_arguments)]
9736async fn ask_wave_with<T>(
9737    jobs: Vec<SeatJob>,
9738    sem: Arc<Semaphore>,
9739    retries: usize,
9740    roster: &[AgentSpec],
9741    ctx: &WaveCtx<'_>,
9742    losses: &mut Vec<QuotaLoss>,
9743    state: &mut RunState,
9744    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9745) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9746where
9747    T: Send + 'static,
9748{
9749    let n = jobs.len();
9750    let originals: Vec<SeatJob> = jobs;
9751    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9752    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9753    // Nudges each seat's *current* agent has taken — 0 for a first-ask
9754    // answer, N once it has gone through N nudges. Read back once this
9755    // returns, so a caller building a history record (`ReviewRecord`) can
9756    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
9757    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
9758    // field's own doc.
9759    let mut nudges: Vec<usize> = vec![0; n];
9760    // The agent now occupying each seat, the ids it has already been through,
9761    // where in the roster the walk began, the class of the last failure, and
9762    // the stem word of a handover not yet asked (full prompt, full budget).
9763    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9764    let mut tried: Vec<BTreeSet<String>> = specs
9765        .iter()
9766        .map(|s| BTreeSet::from([s.id.clone()]))
9767        .collect();
9768    let starts: Vec<usize> = specs
9769        .iter()
9770        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9771        .collect();
9772    let carry = ctx.carry_seats && !roster.is_empty();
9773    // Carried across rounds: ids that failed the seat earlier, and how the
9774    // last failure went (so a repeat of it is not handed over again).
9775    let carried: Vec<BTreeSet<String>> = originals
9776        .iter()
9777        .map(|j| {
9778            state
9779                .seat_history
9780                .get(&j.seat.key)
9781                .filter(|_| carry)
9782                .map(|h| h.failed.clone())
9783                .unwrap_or_default()
9784        })
9785        .collect();
9786    let mut prev: Vec<Option<FailClass>> = originals
9787        .iter()
9788        .map(|j| {
9789            state
9790                .seat_history
9791                .get(&j.seat.key)
9792                .filter(|_| carry)
9793                .and_then(|h| h.last_fail.clone())
9794        })
9795        .collect();
9796    let next_agent =
9797        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9798            let others: BTreeSet<String> = specs
9799                .iter()
9800                .enumerate()
9801                .filter(|(j, _)| *j != i)
9802                .map(|(_, s)| s.id.clone())
9803                .collect();
9804            pick_successor(
9805                roster,
9806                starts[i],
9807                tried,
9808                carry.then(|| &carried[i]),
9809                &others,
9810            )
9811            .cloned()
9812        };
9813    let mut fresh: Vec<Option<String>> = vec![None; n];
9814    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9815    let mut pending: Vec<usize> = (0..n).collect();
9816
9817    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
9818    // asks: an agent with a successor is asked once and handed over, and only
9819    // a seat with no successor spends `retries` nudges on the same agent. This
9820    // only guarantees the loop's own termination whatever those say.
9821    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9822    for round in 0..max_rounds {
9823        if pending.is_empty() {
9824            break;
9825        }
9826        let mut batch = Vec::with_capacity(pending.len());
9827        let mut renudged: Vec<&str> = Vec::new();
9828        for &i in &pending {
9829            let src = &originals[i];
9830            // A seat now held by another agent than the job named has no
9831            // session of its own: it gets the full-context prompt whenever
9832            // it is asked in full (the handover ask, a nudge it cannot
9833            // resume).
9834            let full: &str = match &src.handover {
9835                Some(h) if specs[i].id != src.spec.id => h,
9836                _ => &src.prompt,
9837            };
9838            // The prompt and the budget are one decision: a nudge restates
9839            // finished work, a re-sent prompt redoes it.
9840            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9841                (
9842                    full.to_owned(),
9843                    src.timeout,
9844                    format!("{}-{word}-{}", src.stem, specs[i].id),
9845                )
9846            } else if nudges[i] == 0 {
9847                (full.to_owned(), src.timeout, src.stem.clone())
9848            } else {
9849                renudged.push(src.seat.key.as_str());
9850                let why = done[i]
9851                    .as_ref()
9852                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
9853                    .unwrap_or_else(|| "no parsable answer".to_owned());
9854                let nudge = prompt::nudge(&why);
9855                let nudged = has_context(&specs[i], &seats[i], src.sessions);
9856                let prompt = if nudged {
9857                    nudge
9858                } else {
9859                    format!("{full}\n\n---\n\n{nudge}")
9860                };
9861                (
9862                    prompt,
9863                    retry_budget(src.timeout, nudged),
9864                    format!("{}-retry{}", src.stem, nudges[i]),
9865                )
9866            };
9867            batch.push(SeatJob {
9868                spec: specs[i].clone(),
9869                seat: seats[i].clone(),
9870                cwd: src.cwd.clone(),
9871                prompt,
9872                timeout,
9873                allow_write: src.allow_write,
9874                sessions: src.sessions,
9875                artifacts: src.artifacts.clone(),
9876                stem,
9877                handover: None,
9878            });
9879        }
9880
9881        if !renudged.is_empty() {
9882            state.event(
9883                ctx.node,
9884                format!("retry {round}: re-asking {}", renudged.join(", ")),
9885            );
9886        }
9887        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9888        let mut still = Vec::new();
9889        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9890            seats[i] = seat;
9891            // No agent was asked: nothing to hand over, nudge, remember or
9892            // charge. The caller sees `resource_block` and stops the node.
9893            if let AgentOutcome::ResourceBlocked(why) = &out {
9894                done[i] = Some(Err(anyhow::anyhow!("{why}")));
9895                continue;
9896            }
9897            let class = FailClass::of(&out);
9898            // A dropped stream is nudged first (the conversation is still
9899            // there to pick up); only a seat whose nudges are spent hands over.
9900            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9901                && nudges[i] < retries
9902                && !roster.is_empty();
9903            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9904                let next = should_hand_over(prev[i].as_ref(), &cur)
9905                    .then(|| next_agent(i, &tried[i], &specs))
9906                    .flatten();
9907                if carry {
9908                    let h = state
9909                        .seat_history
9910                        .entry(originals[i].seat.key.clone())
9911                        .or_default();
9912                    h.failed.insert(specs[i].id.clone());
9913                    h.last_fail = Some(cur.clone());
9914                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9915                        h.last_ok = None;
9916                    }
9917                    // Saved before the next agent is asked, so a restart in
9918                    // between does not forget who failed.
9919                    if let Err(e) = state.save() {
9920                        tracing::warn!("could not persist a seat's failure history: {e:#}");
9921                    }
9922                }
9923                if let Some(next) = next {
9924                    record_handover(
9925                        state,
9926                        ctx.node,
9927                        &originals[i].seat.key,
9928                        &specs[i].id,
9929                        &next.id,
9930                        &cur,
9931                        &fail_reason(&out),
9932                    );
9933                    tried[i].insert(next.id.clone());
9934                    prev[i] = Some(cur.clone());
9935                    seats[i] =
9936                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9937                    specs[i] = next;
9938                    fresh[i] = Some(cur.stem_word().to_owned());
9939                    nudges[i] = 0;
9940                    done[i] = Some(Err(anyhow::anyhow!(
9941                        "handed over after: {}",
9942                        fail_reason(&out)
9943                    )));
9944                    still.push(i);
9945                    continue;
9946                }
9947            }
9948            if carry
9949                && !nudge_first
9950                && let Some(cur) = class.clone()
9951            {
9952                // The chain ended here (no successor, or a repeated failure).
9953                let h = state
9954                    .seat_history
9955                    .entry(originals[i].seat.key.clone())
9956                    .or_default();
9957                h.failed.insert(specs[i].id.clone());
9958                h.last_fail = Some(cur);
9959            }
9960            let parsed = match out {
9961                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9962                AgentOutcome::Quota(o) => {
9963                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9964                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9965                }
9966                // Not a parseable answer: the nudge loop re-asks it, which is
9967                // exactly what a dropped stream needs. Just don't hand its raw
9968                // error JSON to `extract_json`.
9969                AgentOutcome::Dropped(o) => {
9970                    let why = o
9971                        .dropped
9972                        .as_ref()
9973                        .map(|d| d.why.as_str())
9974                        .unwrap_or("the CLI ended the stream without delivering its answer");
9975                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9976                }
9977                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9978                    Err(anyhow::anyhow!(e))
9979                }
9980            };
9981            let quota = class == Some(FailClass::Quota);
9982            let failed = parsed.is_err();
9983            done[i] = Some(parsed);
9984            if carry && !failed {
9985                let h = state
9986                    .seat_history
9987                    .entry(originals[i].seat.key.clone())
9988                    .or_default();
9989                h.failed.remove(&specs[i].id);
9990                h.last_ok = Some(specs[i].id.clone());
9991                h.last_fail = None;
9992            }
9993            // Do not re-ask a rate-limited seat (quota) — a retry is known to
9994            // fail the same way; and never re-ask a seat that already parsed.
9995            // A failed agent that still has a successor is not re-asked
9996            // either: the handover was its remedy and has just been refused
9997            // (the chain stops on a repeated failure class). A seat with no
9998            // successor left (a single-agent roster, the roster's tail, or an
9999            // empty roster) keeps the same-agent nudge, bounded by `retries`.
10000            let agent_failure = class.is_some()
10001                && !nudge_first
10002                && !roster.is_empty()
10003                && next_agent(i, &tried[i], &specs).is_some();
10004            if failed && !quota && !agent_failure && nudges[i] < retries {
10005                nudges[i] += 1;
10006                still.push(i);
10007            }
10008        }
10009        pending = still;
10010    }
10011
10012    // One loss per seat whose chain ended on a quota: the intermediate ones
10013    // were absorbed by a handover and are not losses.
10014    for (i, q) in last_quota.into_iter().enumerate() {
10015        if let Some(reset) = q {
10016            losses.push(QuotaLoss {
10017                seat: originals[i].seat.key.clone(),
10018                node: ctx.node.to_owned(),
10019                at: Timestamp::now(),
10020                reset,
10021            });
10022        }
10023    }
10024
10025    seats
10026        .into_iter()
10027        .zip(done)
10028        .zip(nudges)
10029        .map(|((seat, res), attempts)| {
10030            (
10031                seat,
10032                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10033                attempts,
10034            )
10035        })
10036        .collect()
10037}
10038
10039/// Did the wave end without asking any agent (see [`block_wave`])?
10040fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10041    results
10042        .iter()
10043        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10044}
10045
10046/// Settle a wave that never spawned anything: every job answers
10047/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
10048/// else about the seat (session, history, quota) is touched.
10049fn block_wave(
10050    jobs: Vec<SeatJob>,
10051    state: &mut RunState,
10052    node: &str,
10053    attempt: usize,
10054    why: &str,
10055) -> Vec<(usize, SeatState, AgentOutcome)> {
10056    state.resource_wait = None;
10057    state
10058        .active
10059        .retain(|_, a| !(a.node == node && a.attempt == attempt));
10060    if let Err(e) = state.save() {
10061        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10062    }
10063    jobs.into_iter()
10064        .enumerate()
10065        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10066        .collect()
10067}
10068
10069/// Acquire the shared build cache's lease, waiting out contention within
10070/// `budget` (never past it — see AGENTS.md's build-cache section on why an
10071/// unbounded wait is never acceptable).
10072///
10073/// A first, non-blocking check happens before ever waiting; if it finds the
10074/// lease busy, that fact is logged as a `verify` event *and* flushed with
10075/// [`RunState::save`] immediately — not only once the wait finally succeeds
10076/// or gives up — so a `magi show` run by a different process while this one
10077/// is still waiting reads a `run.json` that says so, rather than whatever it
10078/// looked like before the wait started. The same applies to the terminal
10079/// failure: logged and saved before this returns `Err`, so a caller that
10080/// could not get the lease at all still leaves a legible record of why.
10081async fn acquire_cache_lease(
10082    state: &mut RunState,
10083    cache_dir: &Path,
10084    owner: &crate::cache::Owner,
10085    budget: Duration,
10086    context: &str,
10087    cancel: &(dyn Fn() -> bool + Sync),
10088) -> Result<Option<crate::cache::Guard>> {
10089    let home = crate::run::home();
10090    let started = Instant::now();
10091    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10092        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10093        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10094        Err(e) => {
10095            state.event(
10096                "verify",
10097                format!("{context}: could not check the shared build cache: {e:#}"),
10098            );
10099            if let Err(e2) = state.save() {
10100                tracing::warn!("could not persist a cache-check failure: {e2:#}");
10101            }
10102            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10103        }
10104    };
10105    state.event(
10106        "verify",
10107        format!(
10108            "{context}: waiting for the shared build cache at {} ({})",
10109            cache_dir.display(),
10110            busy.describe()
10111        ),
10112    );
10113    // Seats of this node are waiting, not running: say so, so the countdown a
10114    // reader sees only ever measures an agent that has been spawned.
10115    state.resource_wait = Some(crate::run::ResourceWait {
10116        node: context.to_owned(),
10117        kind: "shared build cache".to_owned(),
10118        holder: busy.describe(),
10119        since: Timestamp::now(),
10120        limit_secs: budget.as_secs(),
10121    });
10122    if let Err(e) = state.save() {
10123        tracing::warn!("could not persist a cache wait: {e:#}");
10124    }
10125    let remaining = budget.saturating_sub(started.elapsed());
10126    let waited = crate::cache::wait_for_unless(
10127        &home,
10128        cache_dir,
10129        owner,
10130        remaining,
10131        Duration::from_secs(5),
10132        cancel,
10133    )
10134    .await;
10135    state.resource_wait = None;
10136    match waited {
10137        Ok(g) => Ok(g),
10138        Err(e) => {
10139            state.event("verify", format!("{context}: {e:#}"));
10140            if let Err(e2) = state.save() {
10141                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10142            }
10143            Err(e)
10144        }
10145    }
10146}
10147
10148/// The lease state could not be read at all (unreadable lease file, a
10149/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
10150/// known to be contended, so a wave degrades to running without a shared cache
10151/// instead of stopping on a hold that [`crate::triage`] could never release.
10152#[derive(Debug)]
10153struct CacheCheckError(String);
10154
10155impl std::fmt::Display for CacheCheckError {
10156    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10157        f.write_str(&self.0)
10158    }
10159}
10160
10161impl std::error::Error for CacheCheckError {}
10162
10163/// Run `body` — a verify command batch — while holding the shared build
10164/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
10165/// never interleave with another borrower's build against the same
10166/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
10167/// timeout, or a human's own `magi review`. See the `cache` module doc for
10168/// why this matters more than Cargo's own per-target locking covers — two
10169/// *different* worktrees building the same package name/version into one
10170/// cache directory is a staleness bug, not a lock contention one.
10171///
10172/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
10173/// agent waves do. Repository scheduling also has its own bounded,
10174/// cancellable resource-wait policy. Contention must not consume the
10175/// verification budget: `body` receives the full `budget` after both leases
10176/// are acquired and the cache is fresh. Without a shared cache, only
10177/// repository coordination is acquired before handing over the full budget.
10178///
10179/// A lease that cannot be acquired within the resource wait is reported as a single
10180/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
10181/// verification — the same shape a spawn failure already takes in
10182/// [`run_commands`], so a caller need not special-case it.
10183#[allow(clippy::too_many_arguments)]
10184async fn with_cache_lease<'s, F, Fut>(
10185    state: &'s mut RunState,
10186    cache_dir: Option<&Path>,
10187    node: &str,
10188    seat: &str,
10189    worktree: &Path,
10190    head: &str,
10191    budget: Duration,
10192    context: &str,
10193    wait: &crate::repo_lock::WaitPolicy,
10194    body: F,
10195) -> Result<(Vec<CommandOutcome>, bool)>
10196where
10197    F: FnOnce(&'s mut RunState, Duration) -> Fut,
10198    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10199{
10200    // Match editing waves: cache first, repository second. In particular,
10201    // pre_gate must never hold the repository while waiting for a wave's cache.
10202    let started = Instant::now();
10203    let home = crate::run::home();
10204    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10205    let guard = if let Some(cache_dir) = cache_dir {
10206        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10207        match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10208            wait.cancelled()
10209        })
10210        .await
10211        .and_then(|g| {
10212            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10213        }) {
10214            Ok(g) => Some(g),
10215            Err(e) => {
10216                return Ok((
10217                    vec![CommandOutcome {
10218                        command: "(waiting for the shared build cache)".into(),
10219                        code: None,
10220                        output_tail: e.to_string(),
10221                        duration_ms: started.elapsed().as_millis() as u64,
10222                        resource_blocked: true,
10223                    }],
10224                    false,
10225                ));
10226            }
10227        }
10228    } else {
10229        None
10230    };
10231    let _coordination = coordinate_operation(state, worktree, wait).await?;
10232    if let Some(cache_dir) = cache_dir {
10233        let identity = crate::cache::Identity::new(worktree, head);
10234        if let Err(e) = crate::cache::ensure_fresh(
10235            &home,
10236            cache_dir,
10237            &identity,
10238            &state.config.verify.cargo_manifests(),
10239        ) {
10240            state.event(
10241                "verify",
10242                format!(
10243                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10244                    worktree.display(),
10245                    short(head)
10246                ),
10247            );
10248            if let Some(guard) = guard {
10249                guard.release();
10250            }
10251            return Ok((
10252                vec![CommandOutcome {
10253                    command: "(confirming the shared build cache is fresh)".into(),
10254                    code: None,
10255                    output_tail: e.to_string(),
10256                    duration_ms: started.elapsed().as_millis() as u64,
10257                    resource_blocked: true,
10258                }],
10259                false,
10260            ));
10261        }
10262    }
10263    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10264    if !timed_out_pids.is_empty() {
10265        wait_for_timed_out_children_to_die(&timed_out_pids).await;
10266    }
10267    // Drop repository coordination before releasing the cache, reversing the
10268    // acquisition order. A cancelled/error future also drops both guards.
10269    drop(_coordination);
10270    if let Some(guard) = guard {
10271        guard.release();
10272    }
10273    Ok((outcomes, retried))
10274}
10275
10276/// Poll `pids` — commands [`run_commands`] reports as still running when its
10277/// own timeout elapsed — until every one is confirmed gone, or
10278/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
10279///
10280/// Real confirmation where confirmation is possible, not a substitute for
10281/// full process-tree observation: a grandchild the timed-out process spawned
10282/// and that survives independently of it is invisible to a pid check the
10283/// same way it always was, and continuing to observe and collect *that*
10284/// stays a different piece of work with its own owner. This only narrows a
10285/// fixed blind wait into an actual check of the pids this process does know
10286/// about.
10287async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10288    wait_for_pids_with(
10289        pids,
10290        crate::proc::pid_alive,
10291        LEASE_RELEASE_POLL,
10292        LEASE_RELEASE_MAX_WAIT,
10293    )
10294    .await;
10295}
10296
10297/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
10298/// interval and ceiling supplied by the caller, so the polling *logic* -
10299/// returns as soon as every pid reports dead, gives up at the ceiling
10300/// otherwise - is testable on millisecond durations without asking the real
10301/// OS about a pid at all.
10302async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10303    pids: &[u32],
10304    alive: F,
10305    poll: Duration,
10306    max_wait: Duration,
10307) {
10308    let deadline = Instant::now() + max_wait;
10309    loop {
10310        if pids.iter().all(|&pid| !alive(pid)) {
10311            return;
10312        }
10313        if Instant::now() >= deadline {
10314            return;
10315        }
10316        tokio::time::sleep(poll).await;
10317    }
10318}
10319
10320/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
10321/// admission that it could not even get a verify command to run, as opposed
10322/// to evidence the command actually produced? A caller that would otherwise
10323/// read a resource-blocked outcome as a red command must check this first:
10324/// see [`Runner::gate`], which retries rather than records `Blocked` when
10325/// this is true.
10326fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10327    outcomes.iter().any(|o| o.resource_blocked)
10328}
10329
10330/// What [`Runner::gate_fix_round`] decided.
10331enum GateFix {
10332    /// The tree changed and `verify.e2e` is still green: run the gate again.
10333    Retry,
10334    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
10335    /// last failure stands and the run ends blocked.
10336    Stop,
10337    /// `verify.e2e` could not run after the fix (magi's own contention):
10338    /// decide nothing now, a later reentry retries.
10339    Defer,
10340}
10341
10342/// Is every red command in `outcomes` an ordinary failure the code could
10343/// explain: it ran, exited non-zero, and said something?
10344///
10345/// A timeout, a spawn failure and a killed process all leave `code` `None`;
10346/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
10347/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
10348/// what the command is stays the gate's business.
10349fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10350    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10351    red.peek().is_some()
10352        && red.all(|o| {
10353            !o.resource_blocked
10354                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10355                && !o.output_tail.trim().is_empty()
10356        })
10357}
10358
10359/// Describe one verify command's outcome for the event log, distinguishing a
10360/// build/link failure — the toolchain never produced a binary to run — from
10361/// an actual test failure, since only the latter is a verdict on the patch.
10362fn e2e_outcome_label(o: &CommandOutcome) -> String {
10363    if o.ok() {
10364        return "pass".to_owned();
10365    }
10366    let reason = if o.build_failed() {
10367        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10368    } else {
10369        format!("FAIL ({:?})", o.code)
10370    };
10371    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10372}
10373
10374/// Run `verify.e2e`, retrying once if the first attempt could not build or
10375/// link — a build/link failure is frequently a race against a shared
10376/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
10377/// `verify` event per command, tagged with `context` (normally `"round N"`)
10378/// so the two call sites that need this — the ordinary per-round leg in
10379/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
10380/// it will ever call a round green — read identically in the event log.
10381async fn run_e2e_with_retry(
10382    state: &mut RunState,
10383    shell: &[String],
10384    commands: &[String],
10385    worktree: &Path,
10386    timeout: Duration,
10387    context: &str,
10388) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10389    let (mut e2e, mut timed_out_pids) = run_commands(
10390        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10391    )
10392    .await;
10393    for o in &e2e {
10394        state.event(
10395            "verify",
10396            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10397        );
10398    }
10399    // A build/link failure is not a verdict on the patch — it is frequently a
10400    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
10401    // one retry before letting a red like that decide the round.
10402    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10403    if verify_retried {
10404        state.event(
10405            "verify",
10406            format!(
10407                "{context}: verify could not build/link, not a test result — retrying once \
10408                 before concluding"
10409            ),
10410        );
10411        let retried = run_commands(
10412            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10413        )
10414        .await;
10415        e2e = retried.0;
10416        // Both attempts' timeouts matter, not just the last one: the first
10417        // attempt's descendants may still be alive alongside the retry's.
10418        timed_out_pids.extend(retried.1);
10419        for o in &e2e {
10420            state.event(
10421                "verify",
10422                format!(
10423                    "{context}: retry `{}` -> {}",
10424                    o.command,
10425                    e2e_outcome_label(o)
10426                ),
10427            );
10428        }
10429    }
10430    (e2e, verify_retried, timed_out_pids)
10431}
10432
10433/// Run configured shell commands in `cwd`, in order. The second element is
10434/// the pid of every command that hit `timeout` and was still running when
10435/// this stopped waiting on it (best-effort: `None` when the platform did not
10436/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
10437/// that releases a shared resource afterward needs to know.
10438///
10439/// Records `task` into [`RunState::active`] at every command boundary
10440/// (`RunState::task_command`) and clears it once the whole list has run
10441/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
10442/// for minutes with no seat and no output of its own to show for it (see
10443/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
10444/// apart from "not yet run" without this), and this is the only place that
10445/// knows which command is running right now and how many are left. Three
10446/// saves per command — start, not per second — matching the same "only at a
10447/// boundary" rule [`wave`] already follows for seats.
10448#[allow(clippy::too_many_arguments)]
10449async fn run_commands(
10450    state: &mut RunState,
10451    node: &str,
10452    task: &str,
10453    attempt: usize,
10454    shell: &[String],
10455    commands: &[String],
10456    cwd: &Path,
10457    timeout: Duration,
10458) -> (Vec<CommandOutcome>, Vec<u32>) {
10459    if commands.is_empty() {
10460        // Nothing to mark as running and nothing to clear — an empty list
10461        // means "not configured", and touching `active` (or the disk) over
10462        // that would be a write for every round of a repo with no
10463        // `verify.e2e` / `verify.gate` commands at all.
10464        return (Vec::new(), Vec::new());
10465    }
10466    let mut out = Vec::new();
10467    let mut timed_out_pids = Vec::new();
10468    let total = commands.len();
10469    for (idx, command) in commands.iter().enumerate() {
10470        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10471        if let Err(e) = state.save() {
10472            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10473        }
10474        let started = Instant::now();
10475        let mut cmd = tokio::process::Command::new(&shell[0]);
10476        cmd.quiet();
10477        cmd.args(&shell[1..])
10478            .arg(command)
10479            .current_dir(cwd)
10480            .stdin(std::process::Stdio::null())
10481            .stdout(std::process::Stdio::piped())
10482            .stderr(std::process::Stdio::piped())
10483            .kill_on_drop(true);
10484        let spawned = cmd.spawn();
10485        let (code, body) = match spawned {
10486            Ok(child) => {
10487                // Captured before the child is consumed below: `kill_on_drop`
10488                // only *asks* the process to die when the timeout branch
10489                // drops it, and the pid is the only way anyone downstream can
10490                // later check whether that request actually took.
10491                let pid = child.id();
10492                match tokio::time::timeout(timeout, child.wait_with_output()).await {
10493                    Ok(Ok(o)) => {
10494                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10495                        body.push_str(&String::from_utf8_lossy(&o.stderr));
10496                        (o.status.code(), body)
10497                    }
10498                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
10499                    Err(_) => {
10500                        if let Some(pid) = pid {
10501                            timed_out_pids.push(pid);
10502                        }
10503                        (None, format!("timed out after {}s", timeout.as_secs()))
10504                    }
10505                }
10506            }
10507            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10508        };
10509        out.push(CommandOutcome {
10510            command: command.clone(),
10511            code,
10512            output_tail: tail(&body, OUTPUT_TAIL),
10513            duration_ms: started.elapsed().as_millis() as u64,
10514            resource_blocked: false,
10515        });
10516    }
10517    state.task_finished(task);
10518    if let Err(e) = state.save() {
10519        tracing::warn!("could not persist the end of {task}: {e:#}");
10520    }
10521    (out, timed_out_pids)
10522}
10523
10524/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
10525/// section (`report::run`) and in the `merge` event this node records — for
10526/// the operator to run by hand.
10527///
10528/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
10529/// branch whose ruleset forbids merge commits (GitHub's "must not contain
10530/// merge commits", or "require linear history") rejects the push a `--no-ff`
10531/// merge would produce, which is exactly the guidance this function replaces.
10532/// `message`'s first line becomes the squash commit's subject, matching the
10533/// note `report::run` prints alongside this command — see that function for
10534/// why an explicit subject is not optional there.
10535fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10536    let repo = repo.display();
10537    match style {
10538        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10539        MergeStyle::Squash => {
10540            // The subject sits inside double quotes, and a title an agent
10541            // wrote may carry the characters that break out of them.
10542            let subject = message
10543                .lines()
10544                .next()
10545                .unwrap_or(branch)
10546                .replace(['\\', '"', '$', '`'], "");
10547            format!(
10548                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10549            )
10550        }
10551        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10552    }
10553}
10554
10555/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
10556/// under the hood, rejects a `title` over 256 characters and the whole
10557/// command fails — no PR at all, for a run whose body was otherwise fine
10558/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
10559/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
10560/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
10561/// this bound. It is a margin, not a guarantee — a title packed
10562/// with multi-unit characters could still in principle land close to the
10563/// edge, but a real task title's occasional emoji or accented letter fits
10564/// comfortably inside it.
10565const PR_TITLE_MAX: usize = 240;
10566
10567/// A pull request title from the opening line of `text`, or `None` when that
10568/// line is not English (GitHub text is) or has no letters.
10569///
10570/// A line within `max` is kept as is. A longer one is cut at the end of its
10571/// first sentence when that falls inside `max`, else at a word boundary with a
10572/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
10573/// make every merely-truncated title look non-English). The language check
10574/// runs on the kept text before any mark is added, so only what GitHub will
10575/// show is judged: an English opening followed by non-ASCII far past the cut
10576/// still passes.
10577fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10578    let line = queue::first_line(text)?;
10579    let chars: Vec<char> = line.chars().collect();
10580    let (kept, mark) = if chars.len() <= max {
10581        (line.to_owned(), "")
10582    } else if let Some(end) = sentence_end(&chars, max) {
10583        (chars[..end].iter().collect::<String>(), "")
10584    } else {
10585        let room = max.saturating_sub(3);
10586        // Cut at the last space inside the room; when the char just past the
10587        // room is a space the room already ends on a word.
10588        let cut = if chars[room].is_whitespace() {
10589            room
10590        } else {
10591            chars[..room]
10592                .iter()
10593                .rposition(|c| c.is_whitespace())
10594                .unwrap_or(room)
10595        };
10596        let head: String = chars[..cut].iter().collect();
10597        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10598        (head.to_owned(), "...")
10599    };
10600    if kept.is_empty() {
10601        return None;
10602    }
10603    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10604        return None;
10605    }
10606    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10607        return None;
10608    }
10609    Some(format!("{kept}{mark}"))
10610}
10611
10612/// The char length of the first sentence of `chars` when it ends within `max`
10613/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
10614/// abbreviations so `e.g. foo` does not end a title early.
10615fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10616    const MIN: usize = 20;
10617    for i in MIN..max.min(chars.len()) {
10618        if !matches!(chars[i], '.' | '!' | '?') {
10619            continue;
10620        }
10621        let Some(&next) = chars.get(i + 1) else {
10622            continue;
10623        };
10624        if !next.is_whitespace() {
10625            continue;
10626        }
10627        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10628        if after.is_some_and(|c| c.is_ascii_lowercase()) {
10629            continue;
10630        }
10631        let word: String = chars[..i]
10632            .iter()
10633            .rev()
10634            .take_while(|c| !c.is_whitespace())
10635            .collect::<Vec<_>>()
10636            .into_iter()
10637            .rev()
10638            .collect();
10639        let word = word.to_ascii_lowercase();
10640        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10641            continue;
10642        }
10643        let end = chars[..i]
10644            .iter()
10645            .rposition(|c| !c.is_whitespace())
10646            .map_or(i, |p| p + 1);
10647        return Some(end);
10648    }
10649    None
10650}
10651
10652/// What `merge = "pr"` (and the merge commit of the other modes) says about a
10653/// change: a title and a body describing what was *implemented*, not the task
10654/// that asked for it. A task reads as a request; a reader of the merged
10655/// history wants the change.
10656struct PrMessage {
10657    title: String,
10658    body: String,
10659}
10660
10661impl PrMessage {
10662    /// Title, blank line, body. The first line is the squash/merge commit
10663    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
10664    /// has to stay one sensible line.
10665    fn commit_message(&self) -> String {
10666        format!("{}\n\n{}", self.title, self.body)
10667    }
10668}
10669
10670/// The text after a leading `TITLE:` (any case) on `line`.
10671fn title_marker(line: &str) -> Option<&str> {
10672    let line = line.trim();
10673    let head = line.get(..6)?;
10674    head.eq_ignore_ascii_case("title:")
10675        .then(|| line[6..].trim())
10676}
10677
10678/// The implementer's own one-line title: the `TITLE:` line the implement
10679/// prompt asks for at the top of its SUMMARY. Candidate commits are all
10680/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
10681/// source, and a title that says as much is refused here too.
10682fn summary_title(summary: &str) -> Option<String> {
10683    let first = summary.lines().find(|l| !l.trim().is_empty())?;
10684    let raw = title_marker(first)?;
10685    if raw.is_empty() {
10686        return None;
10687    }
10688    let title = queue::title_from(raw, PR_TITLE_MAX);
10689    let lower = title.to_ascii_lowercase();
10690    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10691        return None;
10692    }
10693    Some(title)
10694}
10695
10696/// How `open_review`'s instruction begins; see [`landing_title`].
10697const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10698
10699/// Marker `open_review` gives a candidate that nothing in the roster wrote.
10700pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10701
10702/// Does this run review work that already existed, rather than implement a
10703/// task? Runs recorded before `reviewed_commits` existed carry only the
10704/// candidate marker.
10705fn is_review_run(state: &RunState) -> bool {
10706    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10707}
10708
10709/// The title of a review-only run: the subject of the oldest commit under
10710/// review. Later commits are usually fixups, and `instruction` is the review
10711/// prompt, which says nothing about the change. GitHub text is English, so a
10712/// non-ASCII or blank subject yields `None` and the caller's neutral title.
10713fn review_title(state: &RunState) -> Option<String> {
10714    english_subject(
10715        state.reviewed_commits.as_ref()?.first()?,
10716        state.config.graph.github_english(),
10717    )
10718}
10719
10720/// `raw` as a pull request title, or `None` when it is blank, not English
10721/// (GitHub text is), or one of magi's own candidate commit subjects.
10722fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10723    let raw = raw.trim();
10724    if raw.is_empty() {
10725        return None;
10726    }
10727    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10728        return None;
10729    }
10730    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10731        return None;
10732    }
10733    let title = queue::title_from(raw, PR_TITLE_MAX);
10734    let lower = title.to_ascii_lowercase();
10735    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10736        return None;
10737    }
10738    Some(title)
10739}
10740
10741/// What a review-only run's branch says about itself, read at the moment the
10742/// pull request is opened.
10743#[derive(Debug, Clone, PartialEq, Eq)]
10744struct BranchFacts {
10745    /// `(subject, body)` of each commit, oldest first.
10746    commits: Vec<(String, String)>,
10747    /// Trimmed `git diff --stat`.
10748    stat: String,
10749}
10750
10751/// Longest diff stat shown: this many file lines plus the summary line.
10752const STAT_FILE_LINES: usize = 25;
10753/// Cap on the commit list, well inside GitHub's 65536-character body limit.
10754const COMMITS_MAX_CHARS: usize = 20_000;
10755
10756/// Read the commits and diff stat of `base..branch`. `None` when git cannot
10757/// say or finds nothing, so the caller falls back to what the run recorded.
10758async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10759    let commits = git::commit_log(repo, base, branch).await.ok()?;
10760    if commits.is_empty() {
10761        return None;
10762    }
10763    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10764    let lines: Vec<&str> = stat.lines().collect();
10765    let stat = if lines.len() > STAT_FILE_LINES + 1 {
10766        let omitted = lines.len() - 1 - STAT_FILE_LINES;
10767        let more = format!(" ... {omitted} more file(s)");
10768        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10769        kept.push(&more);
10770        kept.push(lines[lines.len() - 1]);
10771        kept.join("\n")
10772    } else {
10773        lines.join("\n")
10774    };
10775    Some(BranchFacts { commits, stat })
10776}
10777
10778/// Defang what would break the surrounding markdown: a closing `</details>`
10779/// and a code fence.
10780fn markdown_safe(text: &str) -> String {
10781    text.replace("</details>", "&lt;/details&gt;")
10782        .replace("\x60\x60\x60", "~~~")
10783}
10784
10785fn neutral_title(state: &RunState, winner: char) -> String {
10786    format!(
10787        "chore: land candidate {} of run {}",
10788        winner.to_ascii_uppercase(),
10789        state.id
10790    )
10791}
10792
10793/// The pull request title to hand to `land::merge_subject`. A review-only run
10794/// opened by an earlier build titled its pull request with the review prompt;
10795/// that title is dropped (empty, so the fallback applies) rather than landed.
10796/// Any other title, including an operator's rename, passes through untouched,
10797/// and so does every title of a run that implements a task.
10798pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10799    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10800        ""
10801    } else {
10802        pr_title
10803    }
10804}
10805
10806/// What the squash subject falls back to when the pull request title is empty
10807/// or candidate-shaped: for a review-only run the derived title, never the
10808/// review prompt held in `instruction`.
10809pub fn landing_subject_source(state: &RunState) -> String {
10810    if is_review_run(state) {
10811        let winner = state.candidates.first().map_or('A', |c| c.label);
10812        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10813    }
10814    state.instruction.clone()
10815}
10816
10817/// `summary` without its `TITLE:` line, which the pull request title already
10818/// carries.
10819fn summary_without_title(summary: &str) -> String {
10820    let mut lines = summary.trim().lines().peekable();
10821    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10822        lines.next();
10823    }
10824    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10825}
10826
10827/// The pull request title and body for the winning candidate.
10828///
10829/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
10830/// the task's own opening line via [`queue::title_from`] when there is none.
10831/// `state.instruction` can open with blank lines (`task_text` only rejects a
10832/// body that is blank *entirely*), which `title_from` skips.
10833///
10834/// Publishing uses an independent description generated from the final diff
10835/// and structured evidence. Internal summaries and reports remain local.
10836/// Non-publishing previews may display the original task and candidate summary.
10837/// The footer retains the run and candidate provenance.
10838#[cfg(test)]
10839fn pr_message(state: &RunState, winner: char) -> PrMessage {
10840    pr_message_with(state, winner, None)
10841}
10842
10843/// [`pr_message`] with what the branch of a review-only run says about itself.
10844/// `facts` is ignored for a run that implements a task.
10845#[cfg(test)]
10846fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10847    let raw = pr_message_raw(
10848        state,
10849        winner,
10850        facts,
10851        crate::pr_outcome::FinalHead::Unreported,
10852        ChangeNote::NotAsked,
10853    );
10854    let id = crate::scrub::Identity::current();
10855    PrMessage {
10856        title: crate::scrub::scrub(&raw.title, &id),
10857        body: crate::scrub::scrub(&raw.body, &id),
10858    }
10859}
10860
10861/// What the public description may say about the change itself.
10862#[derive(Debug, Clone, Copy)]
10863enum ChangeNote<'a> {
10864    /// A local preview or non-publishing merge mode: no public generation needed.
10865    NotAsked,
10866    /// A summary written from the final diff.
10867    Summary(&'a crate::run::ChangeSummary),
10868    /// A summary was asked for and could not be produced; the text says why.
10869    Unavailable(&'a str),
10870}
10871
10872fn pr_message_raw(
10873    state: &RunState,
10874    winner: char,
10875    facts: Option<&BranchFacts>,
10876    head: crate::pr_outcome::FinalHead<'_>,
10877    change: ChangeNote<'_>,
10878) -> PrMessage {
10879    if !matches!(change, ChangeNote::NotAsked) {
10880        let (title, mut body) = match change {
10881            ChangeNote::Summary(c) => (
10882                c.title.clone(),
10883                format!("## Summary\n\n{}\n\n", c.body.trim()),
10884            ),
10885            ChangeNote::Unavailable(why) => (
10886                neutral_title(state, winner),
10887                format!(
10888                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10889                ),
10890            ),
10891            ChangeNote::NotAsked => unreachable!(),
10892        };
10893        body.push_str(&public_pr_evidence(state, head));
10894        body.push_str(&format!(
10895            "\n---\nmagi:run/{} magi:candidate-{}\n",
10896            state.id,
10897            winner.to_ascii_lowercase()
10898        ));
10899        return PrMessage { title, body };
10900    }
10901    pr_preview(state, winner, facts, head)
10902}
10903
10904/// Append public facts from the run records after every prose generation,
10905/// including posting-gate rewrites. The agent cannot drop this provenance.
10906fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10907    let mut body = crate::pr_outcome::review_rounds(state);
10908    body.push_str(&crate::pr_outcome::final_verification(state, head));
10909    let open = state.last_round_findings();
10910    if !open.is_empty() {
10911        body.push_str("## Open review findings\n\n");
10912        for f in open {
10913            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10914        }
10915        body.push('\n');
10916    }
10917    if let Some(c) = state.change_summary.as_ref()
10918        && head == crate::pr_outcome::FinalHead::At(&c.head)
10919    {
10920        body.push_str(&format!(
10921            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10922            c.base, c.head
10923        ));
10924    }
10925    body
10926}
10927
10928/// Local-only preview of the operator's task and candidate summary.
10929fn pr_preview(
10930    state: &RunState,
10931    winner: char,
10932    facts: Option<&BranchFacts>,
10933    head: crate::pr_outcome::FinalHead<'_>,
10934) -> PrMessage {
10935    let summary = state
10936        .candidates
10937        .iter()
10938        .find(|c| c.label == winner)
10939        .map(|c| c.summary.as_str())
10940        .unwrap_or_default();
10941    // The fallback is the operator's own words and may not be English; GitHub
10942    // text always is, so a non-English task gets a neutral title instead.
10943    let review = is_review_run(state);
10944    let title = if review {
10945        facts
10946            .and_then(|f| {
10947                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10948            })
10949            .or_else(|| review_title(state))
10950            .or_else(|| {
10951                state
10952                    .candidates
10953                    .iter()
10954                    .find(|c| c.label == winner)
10955                    .filter(|c| !c.branch.starts_with("magi/"))
10956                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10957            })
10958            .unwrap_or_else(|| neutral_title(state, winner))
10959    } else {
10960        summary_title(summary).unwrap_or_else(|| {
10961            english_title(
10962                &state.instruction,
10963                PR_TITLE_MAX,
10964                state.config.graph.github_english(),
10965            )
10966            .unwrap_or_else(|| neutral_title(state, winner))
10967        })
10968    };
10969
10970    let mut body = String::new();
10971    let what = summary_without_title(summary);
10972    if !what.is_empty() {
10973        body.push_str("## Summary\n\n");
10974        body.push_str(&what);
10975        body.push_str("\n\n");
10976    }
10977
10978    // Review history uses structured facts across all rounds, never reports.
10979    body.push_str(&crate::pr_outcome::review_rounds(state));
10980    body.push_str(&crate::pr_outcome::final_verification(state, head));
10981    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10982
10983    let open = state.open_findings();
10984    if !open.is_empty() {
10985        body.push_str("## Open review findings\n\n");
10986        for f in &open {
10987            body.push_str(&format!(
10988                "- `{}` [{:?}] {}\n",
10989                f.id, f.severity, "Context retained in local review records."
10990            ));
10991        }
10992        body.push('\n');
10993    }
10994
10995    if let Some(fix) = fix
10996        && !fix.rejected.is_empty()
10997    {
10998        body.push_str("## Declined by the fixer\n\n");
10999        for r in &fix.rejected {
11000            body.push_str(&format!(
11001                "- `{}`: {}\n",
11002                r.id, "Reason retained in local review records."
11003            ));
11004        }
11005        body.push('\n');
11006    }
11007
11008    if review {
11009        // The review prompt is not the task; list what the branch carries.
11010        body.push_str("## Commits under review\n\n");
11011        if let Some(facts) = facts {
11012            let mut left = COMMITS_MAX_CHARS;
11013            for (i, (subject, text)) in facts.commits.iter().enumerate() {
11014                let mut entry = format!("- {}\n", markdown_safe(subject));
11015                for l in markdown_safe(text).lines() {
11016                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
11017                }
11018                if left == 0 {
11019                    body.push_str(&format!(
11020                        "- ... {} more commit(s)\n",
11021                        facts.commits.len() - i
11022                    ));
11023                    break;
11024                }
11025                if entry.len() > left {
11026                    // Even the first commit is cut: one huge body must not
11027                    // push the whole description past GitHub's limit.
11028                    let mut end = left;
11029                    while !entry.is_char_boundary(end) {
11030                        end -= 1;
11031                    }
11032                    entry.truncate(end);
11033                    entry.push_str("\n  ... (truncated)\n");
11034                    left = 0;
11035                } else {
11036                    left -= entry.len();
11037                }
11038                body.push_str(&entry);
11039            }
11040            if !facts.stat.trim().is_empty() {
11041                body.push_str(&format!(
11042                    "\n## Diff stat\n\n```\n{}\n```\n",
11043                    markdown_safe(facts.stat.trim())
11044                ));
11045            }
11046        } else {
11047            match &state.reviewed_commits {
11048                Some(subjects) => {
11049                    for s in subjects {
11050                        body.push_str(&format!("- {}\n", s.trim()));
11051                    }
11052                }
11053                None => {
11054                    // An older run kept only the prompt, with the commit list
11055                    // after its first paragraph.
11056                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11057                    body.push_str(rest.trim());
11058                    body.push('\n');
11059                }
11060            }
11061        }
11062    } else {
11063        let task = state.instruction.trim();
11064        let task = if task.is_empty() {
11065            "(empty task)"
11066        } else {
11067            task
11068        };
11069        body.push_str(&format!(
11070            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11071            task.replace("</details>", "&lt;/details&gt;")
11072        ));
11073    }
11074
11075    body.push_str(&format!(
11076        "\n---\nmagi:run/{} magi:candidate-{}\n",
11077        state.id,
11078        winner.to_ascii_lowercase()
11079    ));
11080
11081    PrMessage { title, body }
11082}
11083
11084/// The task with what the repository says about the existing work it names
11085/// appended, so an implementer knows what it started from and what it must
11086/// not redo. Unchanged when the task names nothing.
11087fn seeded_instruction(state: &RunState) -> String {
11088    match refs::describe(&state.seeds) {
11089        Some(facts) => format!(
11090            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11091             Candidates start from the unmerged branch named above, when there \
11092             is one, and carry any unmerged commit named by sha as a \
11093             cherry-pick. Check that this is what the task meant before \
11094             building on it.",
11095            state.instruction
11096        ),
11097        None => state.instruction.clone(),
11098    }
11099}
11100
11101/// Does the winner have no commits ahead of the base it would land on?
11102/// Any failure to find out reads as "not empty": the merge then behaves as it
11103/// always did rather than refusing on a guess.
11104async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11105    let base = &state.base_branch;
11106    let mut against = base.clone();
11107    if mode == MergeMode::Pr {
11108        // A pull request lands on the remote's base, never the local branch
11109        // of the same name: if that cannot be read, "not empty" is the safe
11110        // answer.
11111        let remote = &state.config.merge.remote;
11112        let tracking = format!("{remote}/{base}");
11113        let fetched = git::fetch(repo, remote, base).await;
11114        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11115            against = tracking;
11116        } else {
11117            return false;
11118        }
11119    }
11120    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11121}
11122
11123/// Why nothing was opened for an empty winner, with what the task's own
11124/// references resolved to.
11125fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11126    let mut detail = format!(
11127        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11128         nothing to open a pull request for"
11129    );
11130    match refs::describe(&state.seeds) {
11131        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11132        None => detail.push_str(
11133            "\nThe task names no existing branch or commit; if it means to land work \
11134             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11135        ),
11136    }
11137    detail
11138}
11139
11140/// What the `Pr` merge does once it knows whether the branch already has an
11141/// open pull request.
11142#[derive(Debug, PartialEq, Eq)]
11143enum PrPlan {
11144    Create,
11145    Adopt { url: String, title: String },
11146    Stop(String),
11147}
11148
11149/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
11150/// a failed lookup -> stop with the real reason. Never guesses.
11151fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11152    match found {
11153        Ok(land::OpenPr::None) => PrPlan::Create,
11154        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11155        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11156            "several open pull requests exist for this branch, not picking one: {}",
11157            urls.join(" ")
11158        )),
11159        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11160    }
11161}
11162
11163/// `gh pr create`, returning the PR url.
11164pub(crate) async fn gh_pr_create(
11165    cwd: &Path,
11166    base: &str,
11167    head: &str,
11168    title: &str,
11169    body: &str,
11170) -> Result<String> {
11171    let out = tokio::process::Command::new("gh")
11172        .args([
11173            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11174        ])
11175        .current_dir(cwd)
11176        .quiet()
11177        .stdin(std::process::Stdio::null())
11178        .output()
11179        .await
11180        .context("spawn gh")?;
11181    if out.status.success() {
11182        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11183    } else {
11184        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11185    }
11186}
11187
11188/// Never force deletion of uncommitted work, including unknown legacy trees.
11189async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11190    match std::fs::symlink_metadata(path) {
11191        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11192            // A previous cleanup may have removed the tree before a branch
11193            // deletion or state save failed. Unregister it if still present.
11194            git::worktree_remove_clean(repo, path).await?;
11195            return Ok(());
11196        }
11197        Err(e) => return Err(e.into()),
11198        Ok(_) => {}
11199    }
11200    if !git::is_clean(path).await? {
11201        bail!(
11202            "worktree has uncommitted work; refusing removal: {}",
11203            path.display()
11204        );
11205    }
11206    if !git::worktree_remove_clean(repo, path).await? {
11207        bail!("worktree removal refused: {}", path.display());
11208    }
11209    Ok(())
11210}
11211
11212/// Tear a run's worktrees and branches down.
11213///
11214/// `home` is where the updated `run.json` is saved (via
11215/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
11216/// a housekeeping pass already has its own honest `home` handed to it, and
11217/// falling through to the global here would write back through whichever
11218/// directory some other process or test pinned into that `OnceLock` first,
11219/// not the one the caller actually resolved its `runs` and `state` from.
11220pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11221    if crate::pr_recovery::sources_retained(home, &state.id) {
11222        return Ok(Vec::new());
11223    }
11224    if !state.candidate_preservation.is_empty() {
11225        bail!("candidate preservation incomplete; refusing cleanup");
11226    }
11227    let repo = state.repo.clone();
11228    // CLI, Web and automatic cleanup all enter here, independently of
11229    // Runner::fold_losers. Protect deletion and shared config release too.
11230    let wait = repository_wait(state, None);
11231    let _coordination = match coordinate_operation(state, &repo, &wait).await {
11232        Ok(lease) => lease,
11233        Err(e) => {
11234            state.save_under(home)?;
11235            return Err(e);
11236        }
11237    };
11238    let root = state.worktree_root();
11239    let winner = state.tally.as_ref().map(|t| t.winner);
11240    let mut removed = Vec::new();
11241
11242    for i in 0..state.candidates.len() {
11243        let c = state.candidates[i].clone();
11244        let is_winner = Some(c.label) == winner;
11245        if is_winner && !drop_winner {
11246            continue;
11247        }
11248        let existed = c.worktree.exists();
11249        // A vanished directory can still have a Git registration that keeps
11250        // its branch checked out. Remove that registration before the branch.
11251        remove_preserved_worktree(&repo, &c.worktree).await?;
11252        if existed {
11253            removed.push(c.worktree.to_string_lossy().into_owned());
11254        }
11255        // A branch handed to a later run (and its pull request) is not this
11256        // run's to delete.
11257        let handed_over = state.released_branches.contains(&c.branch);
11258        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11259            if !git::branch_delete(&repo, &c.branch).await? {
11260                bail!("candidate branch removal refused");
11261            }
11262            removed.push(c.branch.clone());
11263        }
11264        state.candidates[i].folded = true;
11265    }
11266
11267    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11268        let path = name.path();
11269        let keep = !drop_winner
11270            && winner.is_some_and(|w| {
11271                path.file_name()
11272                    .is_some_and(|n| n == format!("cand-{w}").as_str())
11273            });
11274        if keep {
11275            continue;
11276        }
11277        remove_preserved_worktree(&repo, &path).await?;
11278        removed.push(path.to_string_lossy().into_owned());
11279    }
11280
11281    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
11282    // judge worktrees, so once the loop above has cleared all of them out,
11283    // the parent is a bare directory nobody else was ever going to remove -
11284    // git only ever managed what was inside it. Left alone, one of these
11285    // accumulates per fully-folded run; the operator's own machine had 74.
11286    // `remove_if_empty` re-checks rather than assuming: a run whose winner
11287    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
11288    // so does anything a run never claimed that happens to share the bay.
11289    remove_if_empty(&root);
11290
11291    if state.enabled_worktree_config && drop_winner {
11292        // A release, not a raw disable: some sibling run in this repository
11293        // may still hold its own reference (see `git::acquire_worktree_config`),
11294        // and only the last release actually turns the setting back off.
11295        git::release_worktree_config(&repo).await.ok();
11296        state.enabled_worktree_config = false;
11297    }
11298    state.save_under(home)?;
11299    Ok(removed)
11300}
11301
11302/// Remove `dir` if it exists and has nothing in it.
11303///
11304/// Best-effort and silent by design: a directory that is not empty (a run
11305/// whose winner is still parked there, a stray file some other process left)
11306/// is exactly the case this must refuse, and a directory that is already gone
11307/// is not a failure worth reporting either. `std::fs::remove_dir` itself
11308/// already refuses a non-empty directory, so the emptiness check below is
11309/// belt, not suspenders - it is what keeps this from ever attempting the
11310/// removal in the case that matters, rather than trusting `remove_dir`'s
11311/// error path to have no side effects if it ever changed.
11312fn remove_if_empty(dir: &Path) {
11313    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11314        std::fs::remove_dir(dir).ok();
11315    }
11316}
11317
11318/// Severity of the worst open finding in the last review round, for reporting.
11319pub fn worst_open(state: &RunState) -> Option<Severity> {
11320    state
11321        .reviews
11322        .last()?
11323        .reviews
11324        .iter()
11325        .flat_map(|r| r.findings.iter())
11326        .map(|f| f.severity)
11327        .max()
11328}
11329
11330/// `[worktree] setup` hides its products per worktree, which needs
11331/// `extensions.worktreeConfig`. Taken once per run and released with the run's
11332/// worktrees, like the hook's; every place that runs setup calls this first.
11333async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11334    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11335        git::acquire_worktree_config(repo).await?;
11336        state.enabled_worktree_config = true;
11337    }
11338    Ok(())
11339}
11340
11341#[cfg(test)]
11342mod tests {
11343    #[tokio::test]
11344    async fn launcher_base_fetch_requires_remote_success() {
11345        let tmp = tempfile::tempdir().unwrap();
11346        let repo = tmp.path().join("repo");
11347        let remote = tmp.path().join("remote");
11348        std::fs::create_dir(&repo).unwrap();
11349        std::fs::create_dir(&remote).unwrap();
11350        git::git(&remote, &["init", "--bare", "-b", "main"])
11351            .await
11352            .unwrap();
11353        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11354        git::git(
11355            &repo,
11356            &[
11357                "-c",
11358                "user.name=Fixture",
11359                "-c",
11360                "user.email=fixture@example.invalid",
11361                "commit",
11362                "--allow-empty",
11363                "-m",
11364                "fixture",
11365            ],
11366        )
11367        .await
11368        .unwrap();
11369        git::git(
11370            &repo,
11371            &["remote", "add", "origin", remote.to_str().unwrap()],
11372        )
11373        .await
11374        .unwrap();
11375        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11376        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11377        assert_eq!(
11378            resolve_base(&repo, "main", "origin", None)
11379                .await
11380                .unwrap()
11381                .commit,
11382            head
11383        );
11384        #[cfg(unix)]
11385        {
11386            let policy: crate::config::Coordination = toml::from_str(
11387                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11388            ).unwrap();
11389            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11390            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11391            let held = crate::repo_lock::SeatLease::operation_with(
11392                &repo,
11393                Some(&policy),
11394                &crate::repo_lock::WaitPolicy::default(),
11395            )
11396            .await
11397            .unwrap();
11398            assert!(held.is_some());
11399            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11400            tokio::pin!(fetch);
11401            assert!(
11402                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11403                    .await
11404                    .is_err()
11405            );
11406            assert!(
11407                !repo.join(".git/FETCH_HEAD").exists(),
11408                "no fetch before admission"
11409            );
11410            drop(held);
11411            let fetched = fetch.await.unwrap();
11412            assert_eq!(fetched.commit, head);
11413            assert_eq!(fetched.remote, "origin");
11414            assert_eq!(fetched.branch, "main");
11415        }
11416        // A surviving tracking ref is not successful fetch evidence.
11417        git::git(
11418            &repo,
11419            &[
11420                "remote",
11421                "set-url",
11422                "origin",
11423                tmp.path().join("missing").to_str().unwrap(),
11424            ],
11425        )
11426        .await
11427        .unwrap();
11428        let error = resolve_base(&repo, "main", "origin", None)
11429            .await
11430            .unwrap_err()
11431            .to_string();
11432        assert!(error.contains("cannot read origin/main"));
11433        assert!(error.contains("refusing to branch off"));
11434    }
11435
11436    #[test]
11437    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11438        let leaked = vec!["chore(deps): update a crate".to_owned()];
11439        let own = "fix(daemon): apply a chosen action";
11440        assert!(should_retitle("", own, &leaked));
11441        assert!(should_retitle(
11442            &format!("{REVIEW_PROMPT_OPENING} `x`"),
11443            own,
11444            &leaked
11445        ));
11446        assert!(should_retitle(
11447            "chore: land candidate A of run 1",
11448            own,
11449            &leaked
11450        ));
11451        assert!(should_retitle(
11452            "magi: candidate A (uncommitted work)",
11453            own,
11454            &leaked
11455        ));
11456        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11457        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11458        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11459    }
11460
11461    #[test]
11462    fn pr_merge_plan_creates_adopts_or_stops() {
11463        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11464        assert_eq!(
11465            pr_merge_plan(Ok(land::OpenPr::One {
11466                url: "u".into(),
11467                title: "t".into()
11468            })),
11469            PrPlan::Adopt {
11470                url: "u".into(),
11471                title: "t".into()
11472            }
11473        );
11474        let PrPlan::Stop(many) =
11475            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11476        else {
11477            panic!("many must stop");
11478        };
11479        assert!(many.contains('a') && many.contains('b'));
11480        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11481            panic!("a failed lookup must stop");
11482        };
11483        assert!(err.contains("bad token"));
11484    }
11485
11486    use super::*;
11487    use crate::run::GateStatus;
11488    use std::collections::BTreeMap;
11489    use std::time::Duration;
11490
11491    fn conductor() -> AgentSpec {
11492        AgentSpec {
11493            id: "conductor".to_owned(),
11494            kind: crate::config::AgentKind::Command,
11495            model: None,
11496            command: vec!["true".to_owned()],
11497            extra_args: Vec::new(),
11498            env: BTreeMap::new(),
11499            prompt_delivery: None,
11500        }
11501    }
11502
11503    fn spec(id: &str) -> AgentSpec {
11504        AgentSpec {
11505            id: id.to_owned(),
11506            kind: crate::config::AgentKind::Command,
11507            model: None,
11508            command: vec!["true".to_owned()],
11509            extra_args: Vec::new(),
11510            env: BTreeMap::new(),
11511            prompt_delivery: None,
11512        }
11513    }
11514
11515    fn ids(xs: &[&str]) -> BTreeSet<String> {
11516        xs.iter().map(|s| (*s).to_owned()).collect()
11517    }
11518
11519    #[test]
11520    fn pick_successor_skips_an_agent_another_seat_holds() {
11521        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
11522        let roster = [spec("a"), spec("b"), spec("c")];
11523        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11524        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11525    }
11526
11527    #[test]
11528    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11529        // The other seat started on c but was handed to d; c is free again.
11530        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11531        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11532        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11533        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11534        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11535    }
11536
11537    #[test]
11538    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11539        let roster = [spec("a"), spec("b")];
11540        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11541        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11542        let carried = ids(&["b"]);
11543        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11544        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11545    }
11546
11547    #[test]
11548    fn pick_successor_returns_none_without_any_untried_successor() {
11549        let roster = [spec("a"), spec("b")];
11550        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11551        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11552        let carried = ids(&["a"]);
11553        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11554    }
11555
11556    #[test]
11557    fn pick_successor_rescue_avoids_another_seats_occupant() {
11558        // b is a carried failure and free; a is held by the other seat.
11559        let roster = [spec("a"), spec("b"), spec("c")];
11560        let carried = ids(&["b", "c"]);
11561        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11562        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11563    }
11564
11565    #[test]
11566    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11567        let roster = [spec("a"), spec("b"), spec("c")];
11568        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11569        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11570    }
11571
11572    #[test]
11573    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11574        let roster = [spec("a"), spec("b"), spec("c")];
11575        let failed = ids(&["a", "b", "c"]);
11576        // Rescue looks at the whole roster, once per id, then runs out.
11577        let mut tried = ids(&["b"]);
11578        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11579        assert_eq!(first.id, "a");
11580        tried.insert(first.id.clone());
11581        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11582        assert_eq!(second.id, "c");
11583        tried.insert(second.id.clone());
11584        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11585    }
11586
11587    #[test]
11588    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11589        let roster = [spec("a"), spec("b")];
11590        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11591        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11592    }
11593
11594    #[test]
11595    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11596        let roster = [spec("a"), spec("b"), spec("c")];
11597        let h = SeatHistory {
11598            failed: ids(&["a"]),
11599            last_ok: Some("b".to_owned()),
11600            last_fail: None,
11601        };
11602        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11603        // A last answerer that left the roster, or later failed, is ignored.
11604        let gone = SeatHistory {
11605            last_ok: Some("zzz".to_owned()),
11606            ..h.clone()
11607        };
11608        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11609        let failed = SeatHistory {
11610            failed: ids(&["a", "b"]),
11611            last_ok: Some("b".to_owned()),
11612            last_fail: None,
11613        };
11614        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11615    }
11616
11617    #[test]
11618    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11619        let first = SeatState::new("review-1", "alpha", 7);
11620        let next = handover_seat("review-1", "gamma", 7);
11621        assert_ne!(first.claude_session, next.claude_session);
11622    }
11623
11624    #[test]
11625    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11626        let mut state = state_with_summary("x", "y");
11627        let a = handover_seat("review-1", "beta", state.next_seat_seed());
11628        let b = handover_seat("review-1", "beta", state.next_seat_seed());
11629        assert_ne!(a.claude_session, b.claude_session);
11630    }
11631
11632    #[test]
11633    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11634        let roster = [spec("a"), spec("b")];
11635        let h = SeatHistory {
11636            failed: ids(&["a", "b"]),
11637            ..SeatHistory::default()
11638        };
11639        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11640        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11641        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11642    }
11643
11644    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
11645    // fallback loop depends on to terminate: it must walk forward from the
11646    // seat's own position, never restart at the front of the roster, and it
11647    // must never hand back an id already tried, however many times that id
11648    // happens to appear.
11649
11650    #[test]
11651    fn failure_signature_ignores_numbers_and_paths() {
11652        assert_eq!(
11653            failure_signature("exited with Some(2) and no usable output"),
11654            failure_signature("exited with Some(137) and no usable output")
11655        );
11656        assert_eq!(
11657            failure_signature("process exited with no usable output (exit code 2)"),
11658            failure_signature("process exited with no usable output (exit code 137)")
11659        );
11660        for (old, new) in [
11661            (
11662                "exited with Some(2) and no usable output",
11663                "process exited with no usable output (exit code 2)",
11664            ),
11665            (
11666                "exited with None and no usable output",
11667                "process exited with no usable output (exit code unavailable)",
11668            ),
11669        ] {
11670            let previous = FailClass::Other(failure_signature(old));
11671            let current = FailClass::Other(failure_signature(new));
11672            assert_eq!(previous, current, "resumption retains the failure class");
11673            assert!(!should_hand_over(Some(&previous), &current));
11674        }
11675        assert_eq!(
11676            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11677            failure_signature("cannot open /var/x.txt: denied")
11678        );
11679        assert_ne!(failure_signature("boom"), failure_signature("bang"));
11680    }
11681
11682    #[test]
11683    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11684        let other = FailClass::Other("x".into());
11685        assert!(should_hand_over(None, &FailClass::Quota));
11686        assert!(should_hand_over(Some(&other), &FailClass::Quota));
11687        assert!(should_hand_over(
11688            Some(&FailClass::Timeout),
11689            &FailClass::Timeout
11690        ));
11691        assert!(should_hand_over(None, &other));
11692        assert!(!should_hand_over(Some(&other), &other));
11693        assert!(should_hand_over(
11694            Some(&other),
11695            &FailClass::Other("y".into())
11696        ));
11697        // A quota or timeout in between ends the run of identical failures.
11698        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11699        assert!(should_hand_over(Some(&FailClass::Quota), &other));
11700    }
11701
11702    #[test]
11703    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11704        let a = SeatState::new("judge-1", "alpha", 7);
11705        let b = handover_seat("judge-1", "beta", 7);
11706        assert_ne!(a.claude_session, b.claude_session);
11707        assert_eq!(b.turns, 0);
11708    }
11709
11710    #[test]
11711    fn a_timeout_is_classified_apart_from_other_failures() {
11712        assert_eq!(
11713            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11714            Some(FailClass::Timeout)
11715        );
11716        assert!(matches!(
11717            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11718            Some(FailClass::Other(_))
11719        ));
11720    }
11721
11722    #[test]
11723    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11724        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11725        let tried = BTreeSet::from(["beta".to_owned()]);
11726        // beta sits at index 1; the next candidate is gamma, never alpha —
11727        // which is very likely a different candidate slot's own agent.
11728        let next = next_untried_in_roster(&roster, 1, &tried);
11729        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11730    }
11731
11732    #[test]
11733    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11734        let roster = vec![spec("alpha"), spec("beta")];
11735        let tried = BTreeSet::from(["beta".to_owned()]);
11736        // beta is the roster's last entry: nothing follows it, and alpha —
11737        // earlier in the roster, almost certainly a different candidate
11738        // slot's own agent — must not be reached by wrapping back to it.
11739        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11740    }
11741
11742    #[test]
11743    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11744        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11745        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11746        // beta (index 1) and gamma (index 2, the only entry after it) have
11747        // both been tried; alpha (index 0) never has, but it comes before
11748        // beta's own position, so there is nothing further for this seat.
11749        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11750    }
11751
11752    #[test]
11753    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11754        let roster = vec![spec("a"), spec("a"), spec("b")];
11755        let tried = BTreeSet::from(["a".to_owned()]);
11756        let next = next_untried_in_roster(&roster, 0, &tried);
11757        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11758    }
11759
11760    #[test]
11761    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11762        let roster = vec![spec("a"), spec("b")];
11763        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11764        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11765    }
11766
11767    #[test]
11768    fn remove_if_empty_only_ever_takes_a_bare_directory() {
11769        let dir = tempfile::tempdir().unwrap();
11770        let bay = dir.path().join("ffff");
11771
11772        // Not there yet: nothing to do, nothing to panic on.
11773        remove_if_empty(&bay);
11774        assert!(!bay.exists());
11775
11776        // Something still inside - the winner's worktree, or a stray file -
11777        // keeps the directory standing.
11778        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11779        remove_if_empty(&bay);
11780        assert!(bay.exists(), "non-empty directory must survive");
11781
11782        // Once the last entry is gone, so is the directory itself.
11783        std::fs::remove_dir(bay.join("cand-A")).unwrap();
11784        remove_if_empty(&bay);
11785        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11786    }
11787
11788    // `round_is_clean` is the exact decision this task fixed: a round with a
11789    // seat that never answered must not read the same as a round every seat
11790    // actually reviewed. These are deterministic and process-free by design —
11791    // the equivalent end-to-end check (a real reviewer timing out under a
11792    // live graph run) is a genuine race against wall-clock contention, and a
11793    // spawn slow enough to blow even a generous budget under a loaded test
11794    // run must not turn this specific regression check flaky.
11795
11796    #[test]
11797    fn a_full_panel_that_found_nothing_is_clean() {
11798        assert!(round_is_clean(
11799            0,
11800            true,
11801            2,
11802            2,
11803            0,
11804            IncompleteReviewPolicy::Block
11805        ));
11806    }
11807
11808    #[test]
11809    fn a_missing_seat_is_never_clean_under_the_default_policy() {
11810        assert!(!round_is_clean(
11811            0,
11812            true,
11813            1,
11814            2,
11815            0,
11816            IncompleteReviewPolicy::Block
11817        ));
11818    }
11819
11820    #[test]
11821    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11822        assert!(!round_is_clean(
11823            1,
11824            true,
11825            1,
11826            2,
11827            0,
11828            IncompleteReviewPolicy::Warn
11829        ));
11830    }
11831
11832    #[test]
11833    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11834        assert!(round_is_clean(
11835            0,
11836            true,
11837            1,
11838            2,
11839            0,
11840            IncompleteReviewPolicy::Warn
11841        ));
11842    }
11843
11844    #[test]
11845    fn a_full_panel_with_an_open_finding_is_not_clean() {
11846        assert!(!round_is_clean(
11847            1,
11848            true,
11849            2,
11850            2,
11851            0,
11852            IncompleteReviewPolicy::Block
11853        ));
11854    }
11855
11856    #[test]
11857    fn a_full_panel_with_a_red_e2e_is_not_clean() {
11858        assert!(!round_is_clean(
11859            0,
11860            false,
11861            2,
11862            2,
11863            0,
11864            IncompleteReviewPolicy::Block
11865        ));
11866    }
11867
11868    // The stall this task closes: under the default `block` policy, a seat
11869    // missing only because it was rate limited must not force a wait for a
11870    // session limit that will not lift by the next round. `round_is_clean`
11871    // is where that quorum carve-out lives; the review loop around it never
11872    // changes what a reviewer's vote or a finding's severity means.
11873
11874    #[test]
11875    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11876        // 1 of 2 answered, and the one missing was quota'd — the exact
11877        // "review-2 rate limited (quota)" shape from the field report.
11878        assert!(round_is_clean(
11879            0,
11880            true,
11881            1,
11882            2,
11883            1,
11884            IncompleteReviewPolicy::Block
11885        ));
11886    }
11887
11888    #[test]
11889    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11890        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
11891        // not a quota loss (`quota_missing` stays 0) — worth another try.
11892        assert!(!round_is_clean(
11893            0,
11894            true,
11895            1,
11896            2,
11897            0,
11898            IncompleteReviewPolicy::Block
11899        ));
11900    }
11901
11902    #[test]
11903    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11904        assert!(!round_is_clean(
11905            1,
11906            true,
11907            1,
11908            2,
11909            1,
11910            IncompleteReviewPolicy::Block
11911        ));
11912        assert!(!round_is_clean(
11913            0,
11914            false,
11915            1,
11916            2,
11917            1,
11918            IncompleteReviewPolicy::Block
11919        ));
11920    }
11921
11922    #[test]
11923    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11924        // Every seat quota'd, nobody answered: there is no panel to decide
11925        // on, so this must fall through to the existing block-and-retry
11926        // fallback rather than call an unreviewed patch clean.
11927        assert!(!round_is_clean(
11928            0,
11929            true,
11930            0,
11931            2,
11932            2,
11933            IncompleteReviewPolicy::Block
11934        ));
11935    }
11936
11937    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11938        CommandOutcome {
11939            command: "test".to_owned(),
11940            code,
11941            output_tail: String::new(),
11942            duration_ms: 0,
11943            resource_blocked,
11944        }
11945    }
11946
11947    #[test]
11948    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11949        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11950        assert!(
11951            !verify_inconclusive(&[outcome(Some(1), false)]),
11952            "an ordinary failure is still evidence about the patch"
11953        );
11954        assert!(verify_inconclusive(&[outcome(None, true)]));
11955        assert!(
11956            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11957            "one inconclusive outcome taints the whole batch"
11958        );
11959        assert!(!verify_inconclusive(&[]));
11960    }
11961
11962    #[tokio::test]
11963    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11964        // Alive for the first two checks, then dead - confirms the loop
11965        // actually re-polls rather than deciding once and sleeping out the
11966        // ceiling regardless.
11967        let calls = std::sync::atomic::AtomicUsize::new(0);
11968        let started = Instant::now();
11969        wait_for_pids_with(
11970            &[123],
11971            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11972            Duration::from_millis(5),
11973            Duration::from_secs(5),
11974        )
11975        .await;
11976        assert!(
11977            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11978            "must keep checking rather than deciding on the first answer"
11979        );
11980        assert!(
11981            started.elapsed() < Duration::from_secs(1),
11982            "must return the moment it is confirmed dead, not wait out the ceiling"
11983        );
11984    }
11985
11986    #[tokio::test]
11987    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11988        let started = Instant::now();
11989        wait_for_pids_with(
11990            &[123],
11991            |_| true, // never reports dead
11992            Duration::from_millis(5),
11993            Duration::from_millis(30),
11994        )
11995        .await;
11996        let elapsed = started.elapsed();
11997        assert!(
11998            elapsed >= Duration::from_millis(30),
11999            "must not give up before its own ceiling: {elapsed:?}"
12000        );
12001        assert!(
12002            elapsed < Duration::from_secs(1),
12003            "must not wait past its own ceiling either: {elapsed:?}"
12004        );
12005    }
12006
12007    #[tokio::test]
12008    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12009        let started = Instant::now();
12010        wait_for_pids_with(
12011            &[],
12012            |_| true,
12013            Duration::from_secs(5),
12014            Duration::from_secs(5),
12015        )
12016        .await;
12017        assert!(
12018            started.elapsed() < Duration::from_millis(200),
12019            "an empty pid list has nothing to confirm"
12020        );
12021    }
12022
12023    // `review_conclusion` is the exact decision the review hand-off task
12024    // fixed: a round budget spent (or a tree that stopped moving) must not
12025    // collapse into `Blocked` regardless of what verification actually
12026    // said. Deterministic and process-free for the same reason the
12027    // `round_is_clean` family above is.
12028    fn review_round(
12029        clean: bool,
12030        blocking: usize,
12031        answered: usize,
12032        expected: usize,
12033        progressed: bool,
12034        e2e_ok: bool,
12035    ) -> ReviewRound {
12036        ReviewRound {
12037            round: 1,
12038            head: "h".to_owned(),
12039            verified_head: None,
12040            verified_at: None,
12041            reviews: Vec::new(),
12042            e2e: vec![CommandOutcome {
12043                command: "test".to_owned(),
12044                code: Some(if e2e_ok { 0 } else { 1 }),
12045                output_tail: String::new(),
12046                duration_ms: 0,
12047                resource_blocked: false,
12048            }],
12049            verify_retried: false,
12050            e2e_deferred: false,
12051            e2e_defer_reason: None,
12052            fix: None,
12053            blocking,
12054            answered,
12055            expected,
12056            clean,
12057            progressed,
12058            vote_split: false,
12059            reconsideration: Vec::new(),
12060            verdict: None,
12061        }
12062    }
12063
12064    #[test]
12065    fn review_conclusion_is_none_when_nothing_has_run() {
12066        assert_eq!(review_conclusion(&[], 3), None);
12067    }
12068
12069    #[test]
12070    fn review_conclusion_is_none_while_rounds_remain() {
12071        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12072        assert_eq!(review_conclusion(&rounds, 3), None);
12073    }
12074
12075    #[test]
12076    fn review_conclusion_is_gating_once_a_round_is_clean() {
12077        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12078        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12079    }
12080
12081    #[test]
12082    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12083        let rounds = vec![
12084            review_round(false, 1, 2, 2, true, true),
12085            review_round(false, 1, 2, 2, true, true),
12086        ];
12087        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12088    }
12089
12090    #[test]
12091    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12092        let rounds = vec![
12093            review_round(false, 1, 2, 2, true, true),
12094            review_round(false, 1, 2, 2, true, false),
12095        ];
12096        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12097    }
12098
12099    #[test]
12100    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12101        // Magi never got a command to run against this round's own head — a
12102        // resource-blocked attempt, not a red one — so this must never
12103        // settle on `Blocked` the way a genuine e2e failure would. `None`
12104        // here is what tells `Runner::review_loop` to retry the check
12105        // itself rather than trust this cheap recomputation with a verdict
12106        // it cannot actually produce.
12107        let mut blocked = review_round(false, 1, 2, 2, true, false);
12108        blocked.e2e[0].resource_blocked = true;
12109        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12110        assert_eq!(review_conclusion(&rounds, 2), None);
12111    }
12112
12113    #[test]
12114    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12115        // Missing input, not a verified tree — never a hand-off candidate.
12116        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12117        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12118    }
12119
12120    #[test]
12121    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12122        let rounds = vec![
12123            review_round(false, 1, 2, 2, false, true),
12124            review_round(false, 1, 2, 2, false, true),
12125        ];
12126        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12127    }
12128
12129    fn secs(n: u64) -> Duration {
12130        Duration::from_secs(n)
12131    }
12132
12133    /// A throwaway repo with one commit on `main`, for tests that need `merge`
12134    /// to make real (and, if it runs at all, real*ly fail*) git calls.
12135    #[tokio::test]
12136    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12137        ask_test_home();
12138        let tmp = tempfile::tempdir().unwrap();
12139        let repo = tmp.path().join("repo");
12140        std::fs::create_dir_all(&repo).unwrap();
12141        init_repo(&repo);
12142        let g = |args: &[&str]| {
12143            let out = std::process::Command::new("git")
12144                .args(args)
12145                .current_dir(&repo)
12146                .quiet()
12147                .output()
12148                .unwrap();
12149            assert!(
12150                out.status.success(),
12151                "{}",
12152                String::from_utf8_lossy(&out.stderr)
12153            );
12154        };
12155        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12156        g(&["checkout", "-b", "historical"]);
12157        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12158        g(&["add", "."]);
12159        g(&["commit", "-m", "old alternative"]);
12160        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12161        g(&["checkout", "-b", "magi/task-selected", &base]);
12162        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12163        g(&["add", "."]);
12164        g(&["commit", "-m", "measurement infrastructure"]);
12165        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12166        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12167        g(&["add", "."]);
12168        g(&["commit", "-m", "dependent implementation"]);
12169        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12170        g(&["checkout", "main"]);
12171        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12172        g(&["add", "."]);
12173        g(&["commit", "-m", "unrelated base change"]);
12174        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12175        let origin = tmp.path().join("origin.git");
12176        g(&[
12177            "clone",
12178            "--bare",
12179            repo.to_str().unwrap(),
12180            origin.to_str().unwrap(),
12181        ]);
12182        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12183        g(&["fetch", "origin"]);
12184        let selection = crate::pr_recovery::Selection {
12185            url: "https://example.test/project/pull/1".into(),
12186            branch: "magi/task-selected".into(),
12187            head: head.clone(),
12188            source_head: None,
12189            disposition: Some(crate::pr_recovery::Disposition::Review),
12190            instruction: "preserve measured behavior".into(),
12191            approved_at: jiff::Timestamp::now(),
12192        };
12193        let mut template = runner_at(RunStatus::Reviewing);
12194        template.state.repo = repo.clone();
12195        template.state.base_commit = current.clone();
12196        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12197        template.state.origin = Some(crate::run::Origin::queue("selected"));
12198        template.state.pr_head = Some(selection.branch.clone());
12199        template.state.pr_recovery = Some(selection.clone());
12200        let mut runner =
12201            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12202                .await
12203                .unwrap();
12204        let wt = runner.state.winner().unwrap().worktree.clone();
12205        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12206        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12207        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12208        assert!(runner.state.seeds.is_empty());
12209        assert_eq!(runner.state.instruction, "preserve measured behavior");
12210        runner.sync_to_base().await.unwrap();
12211        assert_eq!(
12212            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12213            "prerequisite"
12214        );
12215        assert_eq!(
12216            std::fs::read_to_string(wt.join("implementation")).unwrap(),
12217            "measured implementation"
12218        );
12219        assert_eq!(
12220            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12221            "current main change"
12222        );
12223        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12224        assert_eq!(
12225            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12226            head
12227        );
12228
12229        // A failed selected-source setup stays inspectable, never resumable as a new competition.
12230        let mut partial = runner_at(RunStatus::Reviewing);
12231        partial.state.repo = repo.clone();
12232        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12233        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12234            run: Some("exit 1".into()),
12235            ..Default::default()
12236        }];
12237        let selection = runner.state.pr_recovery.clone().unwrap();
12238        partial.state.pr_recovery = Some(selection.clone());
12239        partial.state.pr_head = Some(selection.branch.clone());
12240        partial.state.origin = Some(crate::run::Origin::queue("selected"));
12241        let id = partial.state.id.clone();
12242        let error = match Runner::open_selected(
12243            &repo,
12244            partial.state,
12245            partial.roles,
12246            1,
12247            runner.state.base_commit.clone(),
12248            selection,
12249        )
12250        .await
12251        {
12252            Ok(_) => panic!("setup must fail"),
12253            Err(e) => e,
12254        };
12255        assert_eq!(
12256            error
12257                .downcast_ref::<crate::pr_recovery::StartupFailed>()
12258                .unwrap()
12259                .run,
12260            id
12261        );
12262        let failed = RunState::load(&id).unwrap();
12263        assert_eq!(failed.status, RunStatus::Blocked);
12264        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12265        assert!(Runner::resume(&id).is_err());
12266        runner.state.status = RunStatus::Ready;
12267        runner.state.driver_exited = true;
12268        runner.state.save().unwrap();
12269        let mut retry = runner_at(RunStatus::Reviewing);
12270        retry.state.repo = repo.clone();
12271        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12272        let mut reselected = runner.state.pr_recovery.clone().unwrap();
12273        reselected.approved_at = jiff::Timestamp::now();
12274        retry.state.origin = Some(crate::run::Origin::queue("selected"));
12275        retry.state.pr_head = Some(reselected.branch.clone());
12276        retry.state.pr_recovery = Some(reselected.clone());
12277        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12278            .await
12279            .unwrap();
12280        let restarted = Runner::open_selected(
12281            &repo,
12282            retry.state,
12283            retry.roles,
12284            1,
12285            runner.state.base_commit.clone(),
12286            reselected,
12287        )
12288        .await
12289        .unwrap();
12290        assert_eq!(
12291            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12292                .await
12293                .unwrap(),
12294            head
12295        );
12296        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12297        assert_eq!(
12298            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12299            head
12300        );
12301    }
12302
12303    #[tokio::test]
12304    #[cfg(unix)]
12305    async fn fallback_and_interrupted_restoration_require_coordination() {
12306        use std::os::fd::AsRawFd as _;
12307        ask_test_home();
12308        let tmp = tempfile::tempdir().unwrap();
12309        let mut runner = registered_runner(tmp.path()).await;
12310        runner.state.config.graph.timeout_resource_wait = 1;
12311        let winner = runner.state.candidates[0].clone();
12312        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12313        let leftover = winner.worktree.join("leftover");
12314        std::fs::write(&leftover, "untrusted edits").unwrap();
12315        let next = tmp.path().join("next-fixer");
12316        let mut failed = spec("first");
12317        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12318        let mut successor = spec("second");
12319        successor.command = vec![
12320            "sh".into(),
12321            "-c".into(),
12322            format!("touch '{}'; echo done", next.display()),
12323        ];
12324        let attempts = vec![
12325            (failed.clone(), "first".into()),
12326            (successor.clone(), "second".into()),
12327        ];
12328        let lock_path = runner.state.repo.join(".git/updater.lock");
12329        let external = std::fs::File::create(&lock_path).unwrap();
12330        assert_eq!(
12331            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12332            0
12333        );
12334        let artifacts = tmp.path().join("artifacts");
12335        let build = |spec, seat| SeatJob {
12336            spec,
12337            seat,
12338            cwd: winner.worktree.clone(),
12339            prompt: "fixture".into(),
12340            timeout: Duration::from_secs(10),
12341            allow_write: true,
12342            sessions: false,
12343            artifacts: artifacts.clone(),
12344            stem: "fix".into(),
12345            handover: None,
12346        };
12347        let (_, _, out) = runner
12348            .ask_fixer_among(
12349                attempts.clone(),
12350                "graft",
12351                None,
12352                Some((&winner.worktree, &head)),
12353                build,
12354            )
12355            .await;
12356        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12357        assert!(leftover.exists(), "contention must prevent clean");
12358        assert!(!next.exists());
12359        assert!(runner.state.handovers.is_empty());
12360        // Early interrupted-repair rollback uses the same protected route.
12361        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12362            from_run: "earlier".into(),
12363            generation: 1,
12364            mode: crate::recovery::Mode::VerifyRepair,
12365            reasons: Vec::new(),
12366            reviewed_head: None,
12367            rounds: Vec::new(),
12368            open_findings: Vec::new(),
12369            evidence: None,
12370            criteria_inherited: true,
12371            repair: crate::recovery::Repair::Started {
12372                before: head.clone(),
12373            },
12374        });
12375        assert!(
12376            runner
12377                .recovery_repair(&winner)
12378                .await
12379                .unwrap_err()
12380                .is::<crate::repo_lock::ResourceBlocked>()
12381        );
12382        assert!(matches!(
12383            runner.state.recovery.as_ref().unwrap().repair,
12384            crate::recovery::Repair::Started { .. }
12385        ));
12386        assert!(leftover.exists());
12387        drop(external);
12388        runner.recovery_repair(&winner).await.unwrap();
12389        assert!(!leftover.exists());
12390        assert!(matches!(
12391            runner.state.recovery.as_ref().unwrap().repair,
12392            crate::recovery::Repair::Done { .. }
12393        ));
12394        // A hard lock-open failure also leaves HEAD and untracked work intact.
12395        std::fs::remove_file(&lock_path).unwrap();
12396        std::fs::create_dir(&lock_path).unwrap();
12397        std::fs::write(&leftover, "untrusted edits").unwrap();
12398        let (_, _, out) = runner
12399            .ask_fixer_among(
12400                attempts,
12401                "graft",
12402                None,
12403                Some((&winner.worktree, &head)),
12404                build,
12405            )
12406            .await;
12407        assert!(matches!(out, AgentOutcome::Failed(_)));
12408        assert!(leftover.exists());
12409        assert!(!next.exists());
12410        assert_eq!(
12411            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12412            head
12413        );
12414    }
12415
12416    #[cfg(unix)]
12417    async fn registered_runner(root: &Path) -> Runner {
12418        ask_test_home();
12419        let repo = root.join("repo");
12420        std::fs::create_dir_all(&repo).unwrap();
12421        init_repo(&repo);
12422        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12423        let mut runner = runner_at(RunStatus::Prep);
12424        runner.state.repo = repo.clone();
12425        runner.state.config.coordination = crate::config::test_coordination();
12426        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12427        runner.state.config.graph.worktree_root = Some(root.join("wt"));
12428        runner.state.config.graph.advise = false;
12429        runner.roles.implementers = vec![spec("implement")];
12430        runner.roles.judges = vec![spec("judge")];
12431        runner.prep().await.unwrap();
12432        runner
12433    }
12434
12435    #[tokio::test]
12436    #[cfg(unix)]
12437    async fn parked_implementer_persists_completed_work() {
12438        for registered in [false, true] {
12439            let tmp = tempfile::tempdir().unwrap();
12440            let mut runner = registered_runner(tmp.path()).await;
12441            if !registered {
12442                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12443            }
12444            let marker = tmp.path().join("started");
12445            let release = tmp.path().join("release");
12446            let mut implementer = spec("implement");
12447            implementer.command = vec![
12448                "sh".into(),
12449                "-c".into(),
12450                format!(
12451                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12452                    marker.display(),
12453                    release.display()
12454                ),
12455            ];
12456            runner.roles.implementers = vec![implementer.clone()];
12457            runner.state.config.agents = vec![implementer];
12458            let pause = runner.pause.clone();
12459            let (result, ()) = tokio::join!(runner.implement(), async {
12460                tokio::time::timeout(Duration::from_secs(10), async {
12461                    while !marker.exists() {
12462                        tokio::time::sleep(Duration::from_millis(10)).await;
12463                    }
12464                })
12465                .await
12466                .unwrap();
12467                pause.park();
12468                std::fs::write(&release, "go").unwrap();
12469            });
12470            result.unwrap();
12471            let candidate = &runner.state.candidates[0];
12472            assert!(candidate.commits > 0, "{candidate:?}");
12473            assert!(candidate.summary.contains("saved-result"));
12474            assert_eq!(candidate.files, 1);
12475            let before = candidate.commits;
12476            runner.implement().await.unwrap();
12477            assert_eq!(runner.state.candidates[0].commits, before);
12478        }
12479    }
12480
12481    #[tokio::test]
12482    #[cfg(unix)]
12483    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12484        let tmp = tempfile::tempdir().unwrap();
12485        let mut runner = registered_runner(tmp.path()).await;
12486        let winner = runner.state.candidates[0].clone();
12487        let cache = tmp.path().join("cache");
12488        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12489        runner.state.config.graph.timeout_resource_wait = 0;
12490        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12491        let crate::cache::AcquireOutcome::Acquired(held) =
12492            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12493        else {
12494            panic!("cache must be free")
12495        };
12496        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12497            from_run: "earlier".into(),
12498            generation: 1,
12499            mode: crate::recovery::Mode::VerifyRepair,
12500            reasons: Vec::new(),
12501            reviewed_head: None,
12502            rounds: Vec::new(),
12503            open_findings: Vec::new(),
12504            evidence: None,
12505            criteria_inherited: true,
12506            repair: crate::recovery::Repair::NotStarted,
12507        });
12508        runner.recovery_repair(&winner).await.unwrap();
12509        assert!(matches!(
12510            runner.state.recovery.as_ref().unwrap().repair,
12511            crate::recovery::Repair::WaitingForRestore { .. }
12512        ));
12513        drop(held);
12514        runner.state.resource_block = None;
12515        runner.state.config.graph.timeout_resource_wait = 1;
12516        runner.recovery_repair(&winner).await.unwrap();
12517        assert!(matches!(
12518            runner.state.recovery.as_ref().unwrap().repair,
12519            crate::recovery::Repair::Done { .. }
12520        ));
12521        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12522    }
12523
12524    #[tokio::test]
12525    #[cfg(unix)]
12526    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12527        use std::os::fd::AsRawFd as _;
12528        for graft in [false, true] {
12529            let tmp = tempfile::tempdir().unwrap();
12530            let mut runner = registered_runner(tmp.path()).await;
12531            runner.state.config.graph.timeout_resource_wait = 1;
12532            let winner = runner.state.candidates[0].clone();
12533            let marker = tmp.path().join("fixer-started");
12534            let mut fixer = spec("fixer");
12535            fixer.command = vec![
12536                "sh".into(),
12537                "-c".into(),
12538                format!("echo started >> '{}'; echo done", marker.display()),
12539            ];
12540            runner.roles.fixer = Some(vec![fixer]);
12541            if graft {
12542                let mut loser = winner.clone();
12543                loser.label = 'B';
12544                loser.index = 1;
12545                runner.state.candidates.push(loser);
12546                runner.state.config.graph.graft_runner_up = true;
12547                runner.state.tally = Some(Tally {
12548                    first_choice: BTreeMap::new(),
12549                    borda: BTreeMap::new(),
12550                    winner: winner.label,
12551                    rankings: 1,
12552                    unanimous_initial: true,
12553                    deliberated: false,
12554                    changed_votes: 0,
12555                    unanimous_final: true,
12556                    tie_break: None,
12557                    judges: 1,
12558                    present: 1,
12559                    quorum: 1,
12560                    met_quorum: true,
12561                    uncontested: None,
12562                });
12563                runner.state.judgements.push(crate::run::Judgement {
12564                    judge: 1,
12565                    seat: "judge-1".into(),
12566                    agent: "judge".into(),
12567                    ranking: vec![winner.label, 'B'],
12568                    reasons: BTreeMap::new(),
12569                    confidence: None,
12570                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12571                    order: vec![0, 1],
12572                    failed: None,
12573                    duration_ms: 0,
12574                });
12575            } else {
12576                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12577                    from_run: "earlier".into(),
12578                    generation: 1,
12579                    mode: crate::recovery::Mode::VerifyRepair,
12580                    reasons: Vec::new(),
12581                    reviewed_head: None,
12582                    rounds: Vec::new(),
12583                    open_findings: Vec::new(),
12584                    evidence: None,
12585                    criteria_inherited: true,
12586                    repair: crate::recovery::Repair::NotStarted,
12587                });
12588            }
12589            let reader = crate::repo_lock::SeatLease::for_checkout(
12590                &winner.worktree,
12591                false,
12592                crate::config::test_coordination().as_ref(),
12593            )
12594            .await
12595            .unwrap();
12596            let pause = runner.pause.clone();
12597            let (result, ()) = tokio::join!(
12598                async {
12599                    if graft {
12600                        runner.graft().await
12601                    } else {
12602                        runner.recovery_repair(&winner).await
12603                    }
12604                },
12605                async {
12606                    tokio::time::sleep(Duration::from_millis(100)).await;
12607                    pause.park();
12608                }
12609            );
12610            result.unwrap();
12611            assert!(!marker.exists(), "a blocked seat must not spawn");
12612            // Exercise the persisted state, as a binary replacement/resume does.
12613            runner.state = RunState::load(&runner.state.id).unwrap();
12614            if graft {
12615                assert_eq!(
12616                    runner.state.graft.as_ref().unwrap().outcome,
12617                    GraftOutcome::WaitingForRestore
12618                );
12619            } else {
12620                assert!(matches!(
12621                    runner.state.recovery.as_ref().unwrap().repair,
12622                    crate::recovery::Repair::WaitingForRestore { .. }
12623                ));
12624            }
12625            drop(reader);
12626            runner.pause = Pause::new();
12627            runner.state.resource_block = None;
12628            // Deferred restoration must still respect an external lock owner.
12629            let external =
12630                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12631            assert_eq!(
12632                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12633                0
12634            );
12635            let leftover = winner.worktree.join("leftover");
12636            std::fs::write(&leftover, "earlier fallback edits").unwrap();
12637            let result = if graft {
12638                runner.graft().await
12639            } else {
12640                runner.recovery_repair(&winner).await
12641            };
12642            assert!(
12643                result
12644                    .unwrap_err()
12645                    .is::<crate::repo_lock::ResourceBlocked>()
12646            );
12647            assert!(leftover.exists());
12648            assert!(!marker.exists());
12649            drop(external);
12650            runner.state.resource_block = None;
12651            let result = if graft {
12652                runner.graft().await
12653            } else {
12654                runner.recovery_repair(&winner).await
12655            };
12656            result.unwrap();
12657            assert!(!leftover.exists(), "restore must precede retry");
12658            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12659            if graft {
12660                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12661            } else {
12662                assert!(matches!(
12663                    runner.state.recovery.as_ref().unwrap().repair,
12664                    crate::recovery::Repair::Done { .. }
12665                ));
12666            }
12667        }
12668    }
12669
12670    #[tokio::test]
12671    #[cfg(unix)]
12672    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12673        use std::os::fd::AsRawFd as _;
12674        let tmp = tempfile::tempdir().unwrap();
12675        let mut runner = registered_runner(tmp.path()).await;
12676        runner.state.config.graph.timeout_resource_wait = 1;
12677        let winner = runner.state.candidates[0].clone();
12678        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12679        runner.state.tally = Some(Tally {
12680            first_choice: BTreeMap::new(),
12681            borda: BTreeMap::new(),
12682            winner: winner.label,
12683            rankings: 0,
12684            unanimous_initial: false,
12685            deliberated: false,
12686            changed_votes: 0,
12687            unanimous_final: false,
12688            tie_break: None,
12689            judges: 0,
12690            present: 0,
12691            quorum: 0,
12692            met_quorum: true,
12693            uncontested: Some("fixture".into()),
12694        });
12695        runner.state.graft = Some(GraftRecord {
12696            pre: pre.clone(),
12697            post: None,
12698            lines: None,
12699            limit: None,
12700            agent: None,
12701            outcome: GraftOutcome::Running,
12702            requested: Vec::new(),
12703            adoption: None,
12704            adoption_error: None,
12705        });
12706        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12707        git::commit_all(&winner.worktree, "unfinished graft")
12708            .await
12709            .unwrap();
12710        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12711        let leftover = winner.worktree.join("leftover");
12712        std::fs::write(&leftover, "untrusted edits").unwrap();
12713        let path = runner.state.repo.join(".git/updater.lock");
12714        let external = std::fs::File::create(&path).unwrap();
12715        assert_eq!(
12716            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12717            0
12718        );
12719        assert!(
12720            runner
12721                .graft()
12722                .await
12723                .unwrap_err()
12724                .is::<crate::repo_lock::ResourceBlocked>()
12725        );
12726        assert!(matches!(
12727            runner.state.graft.as_ref().unwrap().outcome,
12728            GraftOutcome::Running
12729        ));
12730        assert_eq!(
12731            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12732            unfinished
12733        );
12734        assert!(leftover.exists());
12735        drop(external);
12736        std::fs::remove_file(&path).unwrap();
12737        std::fs::create_dir(&path).unwrap();
12738        assert!(runner.graft().await.is_err());
12739        assert!(matches!(
12740            runner.state.graft.as_ref().unwrap().outcome,
12741            GraftOutcome::Running
12742        ));
12743        assert_eq!(
12744            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12745            unfinished
12746        );
12747        assert!(leftover.exists());
12748        std::fs::remove_dir(&path).unwrap();
12749        runner.graft().await.unwrap();
12750        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12751        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12752        assert!(!leftover.exists());
12753    }
12754
12755    #[tokio::test]
12756    #[cfg(unix)]
12757    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12758        use std::os::fd::AsRawFd as _;
12759        let tmp = tempfile::tempdir().unwrap();
12760        let mut runner = registered_runner(tmp.path()).await;
12761        runner.state.prep_complete = Some(false);
12762        runner.state.config.graph.timeout_resource_wait = 1;
12763        runner.state.save().unwrap();
12764        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12765        assert_eq!(
12766            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12767            0
12768        );
12769        runner.execute().await.unwrap();
12770        let disk = RunState::load(&runner.state.id).unwrap();
12771        assert!(disk.driver_exited);
12772        assert_eq!(disk.driver_pid, None);
12773        assert_eq!(runner.state.status, RunStatus::Blocked);
12774        assert_eq!(
12775            runner.state.resource_block.as_ref().unwrap().kind,
12776            "repository coordination"
12777        );
12778        assert!(runner.state.prep_failure.is_none());
12779        assert!(runner.state.seats.is_empty());
12780        runner.pause.park();
12781        // Include headroom for Git subprocesses under parallel test load;
12782        // the repository contention deadline itself remains one second.
12783        tokio::time::timeout(Duration::from_secs(10), runner.execute())
12784            .await
12785            .unwrap()
12786            .unwrap();
12787        assert!(runner.state.parked);
12788        assert!(runner.state.prep_failure.is_none());
12789        let disk = RunState::load(&runner.state.id).unwrap();
12790        assert!(disk.driver_exited);
12791        assert_eq!(disk.driver_pid, None);
12792    }
12793
12794    #[tokio::test]
12795    #[cfg(unix)]
12796    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12797        use std::os::fd::AsRawFd as _;
12798        let tmp = tempfile::tempdir().unwrap();
12799        let mut runner = registered_runner(tmp.path()).await;
12800        runner.state.prep_complete = Some(false);
12801        runner.state.config.graph.timeout_resource_wait = 30;
12802        runner.state.save().unwrap();
12803        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12804        assert_eq!(
12805            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12806            0
12807        );
12808        assert!(
12809            tokio::time::timeout(Duration::from_millis(100), runner.execute())
12810                .await
12811                .is_err()
12812        );
12813        let disk = RunState::load(&runner.state.id).unwrap();
12814        assert!(disk.driver_exited);
12815        assert_eq!(disk.driver_pid, None);
12816        assert!(disk.status.resumable());
12817        assert!(runner.state.driver_exited);
12818        assert_eq!(runner.state.driver_pid, None);
12819        runner.state.save().unwrap();
12820        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12821    }
12822
12823    #[tokio::test]
12824    #[cfg(unix)]
12825    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12826        let tmp = tempfile::tempdir().unwrap();
12827        let mut runner = registered_runner(tmp.path()).await;
12828        let winner = runner.state.candidates[0].clone();
12829        let cache = tmp.path().join("cache");
12830        let started = tmp.path().join("pre-gate-started");
12831        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12832        runner.state.config.verify.gate =
12833            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12834        let home = crate::run::home();
12835        let owner =
12836            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12837        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12838            crate::cache::AcquireOutcome::Acquired(g) => g,
12839            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12840        };
12841        let mut gate = Box::pin(runner.run_gate(&winner));
12842        assert!(
12843            tokio::time::timeout(Duration::from_millis(150), &mut gate)
12844                .await
12845                .is_err()
12846        );
12847        assert!(!started.exists(), "pre-gate must wait for the cache too");
12848        // Simulate the wave that already owns the cache entering invoke.
12849        // It must obtain the editor reservation and finish without waiting
12850        // for the gate that is queued behind its cache.
12851        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12852        let editor = crate::repo_lock::SeatLease::checkout_with(
12853            &winner.worktree,
12854            true,
12855            crate::config::test_coordination().as_ref(),
12856            &wait,
12857        )
12858        .await
12859        .unwrap();
12860        drop(editor);
12861        guard.release();
12862        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12863            .await
12864            .unwrap()
12865            .unwrap();
12866        drop(gate);
12867        assert!(outcomes.iter().all(CommandOutcome::ok));
12868        assert!(started.exists());
12869    }
12870
12871    #[tokio::test]
12872    #[cfg(unix)]
12873    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12874        use std::os::fd::AsRawFd as _;
12875
12876        ask_test_home();
12877        let tmp = tempfile::tempdir().unwrap();
12878        let repo = tmp.path().join("repo");
12879        std::fs::create_dir_all(&repo).unwrap();
12880        init_repo(&repo);
12881        let common = repo.join(".git");
12882        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12883        let mut runner = runner_at(RunStatus::Prep);
12884        runner.state.repo = repo.clone();
12885        runner.state.config.coordination = crate::config::test_coordination();
12886        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12887        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12888        runner.state.config.graph.advise = false;
12889        runner.roles.implementers = vec![spec("implement")];
12890        runner.roles.judges = vec![spec("judge")];
12891        runner.prep().await.unwrap();
12892        let winner = runner.state.candidates[0].clone();
12893        // Emulate edits already left by a completed seat. The fixture CLI
12894        // only reports completion; it performs no repository operations.
12895        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12896        let completed = tmp.path().join("seat-completed");
12897        runner.roles.implementers[0].command = vec![
12898            "sh".into(),
12899            "-c".into(),
12900            format!("touch '{}'; echo completed", completed.display()),
12901        ];
12902        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12903        assert_eq!(
12904            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12905            0
12906        );
12907        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12908        let mut implement = Box::pin(runner.implement());
12909        tokio::select! {
12910            result = &mut implement => panic!("rescue must wait: {result:?}"),
12911            result = tokio::time::timeout(Duration::from_secs(10), async {
12912                while !completed.exists() {
12913                    tokio::time::sleep(Duration::from_millis(20)).await;
12914                }
12915            }) => result.unwrap(),
12916        }
12917        assert!(
12918            tokio::time::timeout(Duration::from_millis(200), &mut implement)
12919                .await
12920                .is_err()
12921        );
12922        assert_eq!(
12923            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12924            before
12925        );
12926        assert_eq!(
12927            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12928            0
12929        );
12930        tokio::time::timeout(Duration::from_secs(10), &mut implement)
12931            .await
12932            .unwrap()
12933            .unwrap();
12934        drop(implement);
12935        assert_ne!(
12936            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12937            before
12938        );
12939        assert!(git::is_clean(&winner.worktree).await.unwrap());
12940
12941        let started = tmp.path().join("pre-gate-started");
12942        let proceed = tmp.path().join("pre-gate-proceed");
12943        let checked = tmp.path().join("gate-ran");
12944        runner.state.config.verify.pre_gate = vec![format!(
12945            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12946            started.display(),
12947            proceed.display()
12948        )];
12949        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12950        assert_eq!(
12951            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12952            0
12953        );
12954        let mut gate = Box::pin(runner.run_gate(&winner));
12955        assert!(
12956            tokio::time::timeout(Duration::from_millis(200), &mut gate)
12957                .await
12958                .is_err()
12959        );
12960        assert!(!started.exists());
12961        assert!(!checked.exists());
12962        assert!(git::is_clean(&winner.worktree).await.unwrap());
12963        assert_eq!(
12964            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12965            0
12966        );
12967        tokio::select! {
12968            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12969            result = tokio::time::timeout(Duration::from_secs(10), async {
12970                while !started.exists() {
12971                    tokio::time::sleep(Duration::from_millis(20)).await;
12972                }
12973            }) => result.unwrap(),
12974        }
12975        assert_eq!(
12976            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12977            -1,
12978            "mechanical edits must hold the real lock"
12979        );
12980        std::fs::write(proceed, "ready").unwrap();
12981        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12982            .await
12983            .unwrap()
12984            .unwrap();
12985        drop(gate);
12986        assert!(outcomes.iter().all(CommandOutcome::ok));
12987        assert!(checked.exists());
12988        assert!(runner.state.pre_gate_commit.is_some());
12989        assert!(git::is_clean(&winner.worktree).await.unwrap());
12990        assert_eq!(
12991            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12992            0,
12993            "gate completion must release coordination"
12994        );
12995        drop(outsider);
12996        std::fs::remove_file(common.join("updater.lock")).unwrap();
12997        std::fs::create_dir(common.join("updater.lock")).unwrap();
12998        std::fs::remove_file(&started).unwrap();
12999        std::fs::remove_file(&checked).unwrap();
13000        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13001        let error = runner.run_gate(&winner).await.unwrap_err();
13002        assert!(
13003            error
13004                .to_string()
13005                .contains("acquire mandatory repository lock")
13006        );
13007        assert!(
13008            !started.exists(),
13009            "failed acquisition must prevent pre-gate"
13010        );
13011        assert!(!checked.exists(), "failed acquisition must prevent gate");
13012        assert!(git::is_clean(&winner.worktree).await.unwrap());
13013        assert_eq!(
13014            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13015            head
13016        );
13017    }
13018
13019    #[tokio::test]
13020    #[cfg(unix)]
13021    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13022        use std::os::fd::AsRawFd as _;
13023        use std::time::Duration;
13024
13025        ask_test_home();
13026        let tmp = tempfile::tempdir().unwrap();
13027        let repo = tmp.path().join("repo");
13028        std::fs::create_dir_all(&repo).unwrap();
13029        init_repo(&repo);
13030        let common = repo.join(".git");
13031        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13032        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13033        assert_eq!(
13034            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13035            0
13036        );
13037        let mut runner = runner_at(RunStatus::Prep);
13038        runner.state.repo = repo.clone();
13039        runner.state.config.coordination = crate::config::test_coordination();
13040        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13041        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13042        runner.state.config.graph.advise = true;
13043        runner.state.config.graph.advisors = 1;
13044        runner.roles.implementers = vec![spec("implement")];
13045        runner.roles.judges = vec![spec("judge")];
13046        let root = runner.state.worktree_root();
13047        let started = tmp.path().join("setup-started");
13048        let proceed = tmp.path().join("setup-proceed");
13049        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13050            run: Some(format!(
13051                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13052                started.display(),
13053                proceed.display()
13054            )),
13055            ..Default::default()
13056        }];
13057        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13058        let config = git::git(&repo, &["config", "--local", "--list"])
13059            .await
13060            .unwrap();
13061        let mut prep = Box::pin(runner.prep());
13062        assert!(
13063            tokio::time::timeout(Duration::from_millis(200), &mut prep)
13064                .await
13065                .is_err()
13066        );
13067        assert!(
13068            !root.exists(),
13069            "contention must prevent even checkout creation"
13070        );
13071        assert!(!started.exists(), "contention must prevent setup");
13072        assert_eq!(
13073            git::git(&repo, &["branch", "--list"]).await.unwrap(),
13074            branches
13075        );
13076        assert_eq!(
13077            git::git(&repo, &["config", "--local", "--list"])
13078                .await
13079                .unwrap(),
13080            config
13081        );
13082        drop(lock);
13083        tokio::select! {
13084            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13085            result = tokio::time::timeout(Duration::from_secs(10), async {
13086                while !started.exists() {
13087                    tokio::time::sleep(Duration::from_millis(20)).await;
13088                }
13089            }) => result.unwrap(),
13090        }
13091        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13092        assert_eq!(
13093            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13094            -1
13095        );
13096        assert!(
13097            tokio::time::timeout(
13098                Duration::from_millis(100),
13099                crate::repo_lock::SeatLease::for_checkout(
13100                    &repo,
13101                    true,
13102                    crate::config::test_coordination().as_ref()
13103                )
13104            )
13105            .await
13106            .is_err(),
13107            "preparation must also exclude local editors"
13108        );
13109        std::fs::write(proceed, "ready").unwrap();
13110        tokio::time::timeout(Duration::from_secs(10), &mut prep)
13111            .await
13112            .unwrap()
13113            .unwrap();
13114        drop(prep);
13115        assert_eq!(runner.state.prep_complete, Some(true));
13116        assert!(root.join("judge-1").exists());
13117        assert!(root.join("advisor-1").exists());
13118        assert_eq!(
13119            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13120            0
13121        );
13122    }
13123
13124    #[tokio::test]
13125    #[cfg(unix)]
13126    async fn preparation_lock_failure_prevents_repository_changes() {
13127        ask_test_home();
13128        let tmp = tempfile::tempdir().unwrap();
13129        let repo = tmp.path().join("repo");
13130        std::fs::create_dir_all(&repo).unwrap();
13131        init_repo(&repo);
13132        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13133        // A directory at the required lock path makes opening the real lock
13134        // fail, rather than merely exercising an invalid registration.
13135        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13136        let mut runner = runner_at(RunStatus::Prep);
13137        runner.state.repo = repo.clone();
13138        runner.state.config.coordination = crate::config::test_coordination();
13139        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13140        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13141        runner.roles.implementers = vec![spec("implement")];
13142        let error = runner.prep().await.unwrap_err();
13143        assert!(
13144            error
13145                .to_string()
13146                .contains("acquire mandatory repository lock")
13147        );
13148        assert!(!runner.state.worktree_root().exists());
13149        assert!(runner.state.candidates.is_empty());
13150        assert!(!repo.join(".git/config.worktree").exists());
13151        assert_eq!(
13152            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13153            "* main"
13154        );
13155    }
13156
13157    #[tokio::test]
13158    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13159        ask_test_home();
13160        let tmp = tempfile::tempdir().unwrap();
13161        let repo = tmp.path().join("repo");
13162        std::fs::create_dir_all(&repo).unwrap();
13163        init_repo(&repo);
13164        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13165        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13166        std::fs::write(repo.join("referenced"), "required commit").unwrap();
13167        git::git(&repo, &["add", "."]).await.unwrap();
13168        git::git(&repo, &["commit", "-m", "required seed"])
13169            .await
13170            .unwrap();
13171        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13172        git::git(&repo, &["checkout", "main"]).await.unwrap();
13173        let mut runner = runner_at(RunStatus::Prep);
13174        runner.state.repo = repo.clone();
13175        runner.state.base_commit = base.clone();
13176        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13177        runner.state.config.graph.advise = false;
13178        runner.state.config.blind.commit_msg_hook = false;
13179        runner.state.seeds = vec![refs::Seed {
13180            token: seed.clone(),
13181            sha: seed.clone(),
13182            kind: refs::SeedKind::Unmerged,
13183            branch: false,
13184            detail: String::new(),
13185        }];
13186        runner.roles.implementers = vec![spec("one"), spec("two")];
13187        runner.roles.judges = vec![spec("judge")];
13188        let label = blind::assign_labels(2, runner.state.seed)[0];
13189        let branch = runner.state.branch_for(label);
13190        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13191        git::worktree_add_branch(&repo, &worktree, &branch, &base)
13192            .await
13193            .unwrap();
13194        runner.state.candidates.push(Candidate {
13195            index: 0,
13196            label,
13197            agent: "one".into(),
13198            branch,
13199            worktree,
13200            summary: String::new(),
13201            stat: String::new(),
13202            files: 0,
13203            commits: 0,
13204            empty: false,
13205            failed: None,
13206            verified_noop: None,
13207            duration_ms: 0,
13208            folded: false,
13209        });
13210        runner.state.save().unwrap();
13211        // Reload the checkpoint immediately before its first cherry-pick.
13212        runner.state = RunState::load(&runner.state.id).unwrap();
13213        assert_eq!(runner.state.prep_complete, Some(false));
13214        runner.prep().await.unwrap();
13215        assert_eq!(runner.state.prep_complete, Some(true));
13216        assert_eq!(runner.state.candidates.len(), 2);
13217        for candidate in &runner.state.candidates {
13218            assert_eq!(
13219                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13220                git::tree_of(&repo, &seed).await.unwrap()
13221            );
13222            assert_eq!(
13223                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13224                "required commit"
13225            );
13226        }
13227        assert!(runner.state.worktree_root().join("judge-1").exists());
13228        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13229        runner.prep().await.unwrap();
13230        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13231
13232        runner.state.prep_complete = Some(false);
13233        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13234            run: Some(format!(
13235                "test -f '{}'",
13236                tmp.path().join("dependency-ready").display()
13237            )),
13238            ..Default::default()
13239        }];
13240        assert!(runner.prep().await.is_err());
13241        assert!(runner.state.prep_failure.is_some());
13242        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13243        runner.state = RunState::load(&runner.state.id).unwrap();
13244        assert!(
13245            runner.prep().await.is_err(),
13246            "correcting the cause alone must not auto-retry"
13247        );
13248        let failed_at = runner.state.events.last().unwrap().at;
13249        assert!(
13250            runner.retry_preparation(failed_at).is_err(),
13251            "old approvals cannot unlock the hold"
13252        );
13253        let mut task = crate::queue::Task::new(
13254            "retry".into(),
13255            "retry setup".into(),
13256            repo.clone(),
13257            crate::queue::Source::Human,
13258        );
13259        task.start(runner.state.id.clone());
13260        task.hold_machine(Some("setup failed".into()));
13261        task.release_operator("cli", None);
13262        runner
13263            .retry_preparation(task.resumption.as_ref().unwrap().at)
13264            .unwrap();
13265        runner.prep().await.unwrap();
13266        assert!(runner.state.prep_failure.is_none());
13267        assert_eq!(runner.state.prep_complete, Some(true));
13268        assert_eq!(runner.state.candidates.len(), 2);
13269        assert!(runner.state.worktree_root().join("judge-1").exists());
13270        for candidate in &runner.state.candidates {
13271            assert_eq!(
13272                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13273                git::tree_of(&repo, &seed).await.unwrap()
13274            );
13275        }
13276    }
13277
13278    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13279        let mut heads = Vec::new();
13280        for candidate in candidates {
13281            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13282        }
13283        heads
13284    }
13285
13286    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13287        ask_test_home();
13288        let tmp = tempfile::tempdir().unwrap();
13289        let repo = tmp.path().join("repo");
13290        std::fs::create_dir(&repo).unwrap();
13291        init_repo(&repo);
13292        let mut runner = runner_at(RunStatus::Reviewing);
13293        runner.state.repo = repo.clone();
13294        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13295        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13296        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13297            let branch = format!("candidate-{label}");
13298            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13299            git::worktree_add_branch(&repo, &worktree, &branch, "main")
13300                .await
13301                .unwrap();
13302            if label == 'B' {
13303                std::fs::write(worktree.join("change"), "winner").unwrap();
13304                git::rescue_commit(&worktree, "candidate change")
13305                    .await
13306                    .unwrap();
13307            }
13308            runner.state.candidates.push(Candidate {
13309                index,
13310                label,
13311                agent: "alpha".into(),
13312                branch,
13313                worktree,
13314                summary: "completed reply".into(),
13315                stat: String::new(),
13316                files: usize::from(label == 'B'),
13317                commits: usize::from(label == 'B'),
13318                empty: label == 'A',
13319                failed: None,
13320                verified_noop: None,
13321                duration_ms: 1,
13322                folded: false,
13323            });
13324        }
13325        runner.state.judge_skipped = true;
13326        runner.state.tally = Some(Tally {
13327            first_choice: BTreeMap::new(),
13328            borda: BTreeMap::new(),
13329            winner: 'B',
13330            rankings: 0,
13331            unanimous_initial: false,
13332            deliberated: false,
13333            changed_votes: 0,
13334            unanimous_final: false,
13335            tie_break: None,
13336            judges: 0,
13337            present: 0,
13338            quorum: 0,
13339            met_quorum: true,
13340            uncontested: Some("only B produced a change".into()),
13341        });
13342        runner.state.save().unwrap();
13343        (tmp, runner)
13344    }
13345
13346    #[tokio::test]
13347    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13348        let (_tmp, mut runner) = preservation_resume_fixture().await;
13349        // Schema 15 already confirmed the unfolded no-change result.
13350        runner.implement().await.unwrap();
13351        assert!(runner.state.candidate_preservation.is_empty());
13352        assert!(
13353            !runner
13354                .state
13355                .dir()
13356                .join("artifacts/candidate-A-recovery")
13357                .exists()
13358        );
13359        runner.fold_losers().await.unwrap();
13360        assert!(!runner.state.candidates[0].worktree.exists());
13361        for schema in [14, crate::run::SCHEMA] {
13362            runner.state.schema = schema;
13363            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13364            runner.state.save().unwrap();
13365            runner.state = RunState::load(&runner.state.id).unwrap();
13366            runner.implement().await.unwrap();
13367            assert!(runner.state.candidate_preservation.is_empty());
13368            assert!(runner.state.candidates[0].folded);
13369            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13370        }
13371    }
13372
13373    #[tokio::test]
13374    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13375        let (_tmp, mut runner) = preservation_resume_fixture().await;
13376        let wt = runner.state.candidates[0].worktree.clone();
13377        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13378        runner.state.schema = 14;
13379        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13380        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13381        let tally = serde_json::to_value(&runner.state.tally).unwrap();
13382        for _ in 0..2 {
13383            assert!(runner.implement().await.is_err());
13384            assert_eq!(runner.state.status, RunStatus::Blocked);
13385            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13386            assert!(
13387                runner.state.candidates[0].empty,
13388                "old verdict must remain unusable"
13389            );
13390            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13391            assert!(runner.state.judge_skipped);
13392            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13393            assert!(
13394                crate::run::read_artifact(&runner.state, "cand-A.patch")
13395                    .unwrap()
13396                    .contains("owned edits")
13397            );
13398            assert!(wt.exists());
13399            assert_eq!(
13400                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13401                    .await
13402                    .unwrap(),
13403                1
13404            );
13405            runner.state = RunState::load(&runner.state.id).unwrap();
13406        }
13407    }
13408
13409    #[tokio::test]
13410    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13411        let (_tmp, mut runner) = preservation_resume_fixture().await;
13412        let wt = runner.state.candidates[0].worktree.clone();
13413        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13414        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13415        let index_lock = PathBuf::from(
13416            git::git(
13417                &wt,
13418                &[
13419                    "rev-parse",
13420                    "--path-format=absolute",
13421                    "--git-path",
13422                    "index.lock",
13423                ],
13424            )
13425            .await
13426            .unwrap(),
13427        );
13428        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13429        runner.state.schema = 14;
13430        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13431        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13432        assert!(runner.implement().await.is_err());
13433        assert!(runner.after_implement().is_err());
13434        assert!(runner.fold_losers().await.is_err());
13435        assert_eq!(
13436            std::fs::read_to_string(wt.join("recovered")).unwrap(),
13437            "failed seat edits"
13438        );
13439        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13440        runner.state = RunState::load(&runner.state.id).unwrap();
13441        std::fs::remove_file(index_lock).unwrap();
13442        // Recovery finds changes that the old uncontested result never judged.
13443        assert!(runner.implement().await.is_err());
13444        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13445        assert!(runner.state.candidates[0].empty);
13446        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13447        assert!(
13448            crate::run::read_artifact(&runner.state, "cand-A.patch")
13449                .unwrap()
13450                .contains("failed seat edits")
13451        );
13452        assert!(wt.exists());
13453    }
13454
13455    #[tokio::test]
13456    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13457        let (_tmp, mut runner) = preservation_resume_fixture().await;
13458        let repo = runner.state.repo.clone();
13459        let common = git::git(
13460            &repo,
13461            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13462        )
13463        .await
13464        .unwrap();
13465        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13466        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13467        assert!(runner.fold_losers().await.is_err());
13468        assert!(!runner.state.candidates[0].worktree.exists());
13469        assert!(!runner.state.candidates[0].folded);
13470        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13471        runner.state = RunState::load(&runner.state.id).unwrap();
13472        std::fs::remove_file(branch_lock).unwrap();
13473        let judge = runner.state.worktree_root().join("judge-1");
13474        git::worktree_add_detached(&repo, &judge, "main")
13475            .await
13476            .unwrap();
13477        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13478        runner.roles.judges = vec![spec("judge")];
13479        assert!(runner.fold_losers().await.is_err());
13480        runner.state = RunState::load(&runner.state.id).unwrap();
13481        assert!(
13482            runner.state.candidates[0].folded,
13483            "candidate progress saved before judge failure"
13484        );
13485        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13486        assert!(judge.join("uncommitted").exists());
13487        std::fs::remove_file(judge.join("uncommitted")).unwrap();
13488        runner.fold_losers().await.unwrap();
13489        assert!(!judge.exists());
13490        runner.fold_losers().await.unwrap();
13491    }
13492
13493    #[tokio::test]
13494    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13495        let (_tmp, mut runner) = preservation_resume_fixture().await;
13496        runner.state.candidates.retain(|c| c.label == 'B');
13497        runner.state.tally = None;
13498        runner.state.judge_skipped = false;
13499        runner.state.prep_complete = Some(true);
13500        runner.state.config.graph.advise = false;
13501        runner.state.config.graph.selected_implementer = Some("alpha".into());
13502        let wt = runner.state.candidates[0].worktree.clone();
13503        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13504        runner
13505            .state
13506            .candidate_preservation
13507            .insert('B', "awaiting persistence".into());
13508        let index_lock = git::git(
13509            &wt,
13510            &[
13511                "rev-parse",
13512                "--path-format=absolute",
13513                "--git-path",
13514                "index.lock",
13515            ],
13516        )
13517        .await
13518        .unwrap();
13519        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13520        runner.state.save().unwrap();
13521        assert!(runner.execute_graph().await.is_err());
13522        assert_eq!(runner.state.status, RunStatus::Blocked);
13523        assert!(runner.state.candidate_preservation.contains_key(&'B'));
13524        std::fs::remove_file(index_lock).unwrap();
13525        // Park at the next graph boundary: this tests the real selected-seat
13526        // hold guard without dispatching reviewers or publishing anything.
13527        for _ in 0..2 {
13528            runner.state = RunState::load(&runner.state.id).unwrap();
13529            runner.pause.park();
13530            runner.execute_graph().await.unwrap();
13531            assert_eq!(runner.state.status, RunStatus::Implementing);
13532            assert!(runner.state.candidate_preservation.is_empty());
13533            assert_eq!(runner.state.candidates[0].commits, 2);
13534            assert!(runner.state.tally.is_none());
13535        }
13536        runner.implement().await.unwrap();
13537        assert_eq!(runner.state.status, RunStatus::Judging);
13538        // A genuine selected-implementer hold must still require release.
13539        runner.state.status = RunStatus::Blocked;
13540        runner.state.candidates[0].failed = Some("implementation failed".into());
13541        runner.state.save().unwrap();
13542        let error = runner.execute_graph().await.unwrap_err();
13543        assert!(
13544            error
13545                .to_string()
13546                .contains("selected implementation is held")
13547        );
13548    }
13549
13550    #[tokio::test]
13551    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13552        let (_tmp, mut runner) = preservation_resume_fixture().await;
13553        let repo = runner.state.repo.clone();
13554        let candidate = runner.state.candidates[0].clone();
13555        // Remove only the disposable fixture directory, leaving Git's linked
13556        // worktree registration and branch intact, as in a vanished checkout.
13557        let registered = git::worktree_holding(&repo, &candidate.branch)
13558            .await
13559            .unwrap()
13560            .unwrap();
13561        assert_eq!(
13562            registered.canonicalize().unwrap(),
13563            candidate.worktree.canonicalize().unwrap()
13564        );
13565        std::fs::remove_dir_all(&candidate.worktree).unwrap();
13566        assert_eq!(
13567            git::worktree_holding(&repo, &candidate.branch)
13568                .await
13569                .unwrap(),
13570            Some(registered)
13571        );
13572        assert!(
13573            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13574            "the stale registration initially keeps the branch checked out"
13575        );
13576        runner.fold_losers().await.unwrap();
13577        assert!(runner.state.candidates[0].folded);
13578        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13579        assert_eq!(
13580            git::worktree_holding(&repo, &candidate.branch)
13581                .await
13582                .unwrap(),
13583            None
13584        );
13585        runner.state = RunState::load(&runner.state.id).unwrap();
13586        runner.fold_losers().await.unwrap();
13587    }
13588
13589    #[tokio::test]
13590    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13591        let (_tmp, mut runner) = preservation_resume_fixture().await;
13592        let repo = runner.state.repo.clone();
13593        let candidate = runner.state.candidates[0].clone();
13594        let winner = runner.state.candidates[1].clone();
13595        let registered = git::worktree_holding(&repo, &candidate.branch)
13596            .await
13597            .unwrap()
13598            .unwrap();
13599        assert_eq!(
13600            registered.canonicalize().unwrap(),
13601            candidate.worktree.canonicalize().unwrap()
13602        );
13603        std::fs::remove_dir_all(&candidate.worktree).unwrap();
13604        assert_eq!(
13605            git::worktree_holding(&repo, &candidate.branch)
13606                .await
13607                .unwrap(),
13608            Some(registered)
13609        );
13610        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13611
13612        for _ in 0..2 {
13613            fold_run(&mut runner.state, false, &crate::run::home())
13614                .await
13615                .unwrap();
13616            runner.state = RunState::load(&runner.state.id).unwrap();
13617            assert!(runner.state.candidates[0].folded);
13618            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13619            assert_eq!(
13620                git::worktree_holding(&repo, &candidate.branch)
13621                    .await
13622                    .unwrap(),
13623                None
13624            );
13625            assert!(!runner.state.candidates[1].folded);
13626            assert!(winner.worktree.exists());
13627            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13628        }
13629    }
13630
13631    #[tokio::test]
13632    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13633        ask_test_home();
13634        let tmp = tempfile::tempdir().unwrap();
13635        let repo = tmp.path().join("repo");
13636        std::fs::create_dir(&repo).unwrap();
13637        init_repo(&repo);
13638        let mut runner = runner_at(RunStatus::Implementing);
13639        runner.state.repo = repo.clone();
13640        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13641        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13642        for (index, label) in ['A', 'B'].into_iter().enumerate() {
13643            let branch = format!("candidate-{label}");
13644            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13645            git::worktree_add_branch(&repo, &wt, &branch, "main")
13646                .await
13647                .unwrap();
13648            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13649            git::git(&wt, &["add", "README.md"]).await.unwrap();
13650            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13651            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13652            runner.state.candidates.push(Candidate {
13653                index,
13654                label,
13655                agent: "alpha".to_owned(),
13656                branch,
13657                worktree: wt,
13658                summary: "completed implementation".to_owned(),
13659                stat: String::new(),
13660                files: 0,
13661                commits: 0,
13662                empty: false,
13663                failed: None,
13664                verified_noop: None,
13665                duration_ms: 42,
13666                folded: false,
13667            });
13668            runner
13669                .state
13670                .candidate_preservation
13671                .insert(label, "awaiting persistence".to_owned());
13672        }
13673        let wt = runner.state.candidates[0].worktree.clone();
13674        let index_lock = git::git(
13675            &wt,
13676            &[
13677                "rev-parse",
13678                "--path-format=absolute",
13679                "--git-path",
13680                "index.lock",
13681            ],
13682        )
13683        .await
13684        .unwrap();
13685        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13686        runner.state.save().unwrap();
13687        assert!(runner.persist_candidates().await.is_err());
13688        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13689        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13690        assert_eq!(
13691            runner.state.candidates[1].commits, 1,
13692            "other results are still collected"
13693        );
13694        assert!(
13695            !runner.state.candidates[0].empty,
13696            "failed rescue is not no change"
13697        );
13698        assert_eq!(
13699            runner.state.candidates[0].summary,
13700            "completed implementation"
13701        );
13702        assert!(runner.after_implement().is_err());
13703        let home = tmp.path().join("fold-home");
13704        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13705        assert!(wt.join("new-file").exists());
13706        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13707        assert!(recovery.join("staged.patch").exists());
13708        assert!(recovery.join("unstaged.patch").exists());
13709        assert_eq!(
13710            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13711            "untracked edits\n"
13712        );
13713        runner.state.begin_pass();
13714        runner.state.end_pass_parked(ParkCause::Upgrade {
13715            from: "1.0".into(),
13716            to: Some("1.1".into()),
13717        });
13718        runner.state.save().unwrap();
13719        runner.state = RunState::load(&runner.state.id).unwrap();
13720        assert!(matches!(
13721            runner.state.passes.last().unwrap().end,
13722            Some(crate::run::PassEnd::Parked {
13723                cause: ParkCause::Upgrade { .. }
13724            })
13725        ));
13726        std::fs::remove_file(index_lock).unwrap();
13727        // No implementer is configured: a persistence retry cannot dispatch a seat.
13728        runner.implement().await.unwrap();
13729        assert!(runner.state.candidate_preservation.is_empty());
13730        assert!(
13731            runner
13732                .state
13733                .candidates
13734                .iter()
13735                .all(|c| c.commits == 1 && !c.empty)
13736        );
13737        assert!(git::is_clean(&wt).await.unwrap());
13738        assert!(
13739            crate::run::read_artifact(&runner.state, "cand-A.patch")
13740                .unwrap()
13741                .contains("later tracked edits")
13742        );
13743    }
13744
13745    #[tokio::test]
13746    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13747        ask_test_home();
13748        let tmp = tempfile::tempdir().unwrap();
13749        let repo = tmp.path().join("repo");
13750        std::fs::create_dir(&repo).unwrap();
13751        init_repo(&repo);
13752        let wt = tmp.path().join("candidate");
13753        git::worktree_add_branch(&repo, &wt, "candidate", "main")
13754            .await
13755            .unwrap();
13756        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13757        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13758        assert!(wt.join("uncommitted").exists());
13759        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13760        std::fs::remove_file(wt.join("uncommitted")).unwrap();
13761        let mut runner = runner_at(RunStatus::Implementing);
13762        runner.state.repo = repo.clone();
13763        runner.state.base_commit = "missing-base".to_owned();
13764        runner.state.candidates.push(Candidate {
13765            index: 0,
13766            label: 'A',
13767            agent: "alpha".into(),
13768            branch: "candidate".into(),
13769            worktree: wt.clone(),
13770            summary: "completed".into(),
13771            stat: String::new(),
13772            files: 0,
13773            commits: 0,
13774            empty: false,
13775            failed: None,
13776            verified_noop: None,
13777            duration_ms: 1,
13778            folded: false,
13779        });
13780        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13781        assert!(
13782            fold_run(&mut runner.state, true, &tmp.path().join("home"))
13783                .await
13784                .is_err()
13785        );
13786        assert!(!runner.state.candidates[0].folded);
13787        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13788        std::fs::remove_file(wt.join("uncommitted")).unwrap();
13789        runner
13790            .state
13791            .candidate_preservation
13792            .insert('A', "pending".into());
13793        assert!(runner.persist_candidates().await.is_err());
13794        assert!(!runner.state.candidates[0].empty);
13795        assert!(!runner.state.candidates[0].folded);
13796        assert!(wt.exists());
13797        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13798        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13799        std::fs::create_dir_all(&patch_path).unwrap();
13800        assert!(
13801            runner.persist_candidates().await.is_err(),
13802            "artifact failure stays pending"
13803        );
13804        assert!(runner.state.candidate_preservation.contains_key(&'A'));
13805        assert!(!runner.state.candidates[0].empty);
13806        std::fs::remove_dir(patch_path).unwrap();
13807        runner.persist_candidates().await.unwrap();
13808        assert!(
13809            runner.state.candidates[0].empty,
13810            "only successful inspection confirms no change"
13811        );
13812    }
13813
13814    fn init_repo(dir: &Path) {
13815        let run = |args: &[&str]| {
13816            let out = std::process::Command::new("git")
13817                .args(args)
13818                .current_dir(dir)
13819                .quiet()
13820                .output()
13821                .expect("spawn git");
13822            assert!(
13823                out.status.success(),
13824                "git {args:?} failed: {}",
13825                String::from_utf8_lossy(&out.stderr)
13826            );
13827        };
13828        run(&["init", "-b", "main"]);
13829        run(&["config", "user.name", "magi test"]);
13830        run(&["config", "user.email", "magi@example.com"]);
13831        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13832        run(&["add", "-A"]);
13833        run(&["commit", "-m", "init"]);
13834    }
13835
13836    // `settle_questions` is what closes the ghost the phone showed: a run's
13837    // seat asked something, the run then ended, and nothing was left to
13838    // abandon the question it left `open`. `HOME` is a process-wide
13839    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
13840    // first time it runs in the binary — every test below still reaches the
13841    // same directory whichever call won, and each gets its own run id from
13842    // `RunState::new`, so they never collide there.
13843    fn ask_test_home() {
13844        crate::run::pin_test_home();
13845    }
13846
13847    /// A minimal, git-free `Runner` at a given status — `settle_questions`
13848    /// reads nothing else off it.
13849    fn runner_at(status: RunStatus) -> Runner {
13850        let mut state = RunState::new(
13851            PathBuf::from("/nonexistent/repo"),
13852            "main".to_owned(),
13853            "deadbeef".to_owned(),
13854            "task".to_owned(),
13855            Config::default(),
13856        );
13857        // These fixtures share one home and save concurrently. Short clock
13858        // IDs can collide, making an ownership test read another test's run.
13859        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13860        state.id = format!(
13861            "20000101-000000-{:04x}",
13862            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13863        );
13864        state.status = status;
13865        Runner {
13866            state,
13867            roles: ResolvedRoles {
13868                implementers: Vec::new(),
13869                judges: Vec::new(),
13870                reviewers: Vec::new(),
13871                fixer: None,
13872                conductor: conductor(),
13873                implementer_roster: Vec::new(),
13874                judge_roster: Vec::new(),
13875                reviewer_roster: Vec::new(),
13876            },
13877            sem: Arc::new(Semaphore::new(1)),
13878            pause: Pause::new(),
13879            interrupt: Pause::new(),
13880        }
13881    }
13882
13883    #[test]
13884    fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13885        crate::run::pin_test_home();
13886        let mut runner = runner_at(RunStatus::Gating);
13887        runner.state.save().unwrap();
13888        // run.json can commit, but the subsequent lease metadata write fails.
13889        // Cleanup must run after releasing the short ownership lock.
13890        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13891        assert!(runner.claim_driver().is_err());
13892        assert!(runner.state.driver_exited);
13893        assert_eq!(runner.state.driver_pid, None);
13894        let disk = RunState::load(&runner.state.id).unwrap();
13895        assert!(disk.driver_exited);
13896        assert_eq!(disk.driver_pid, None);
13897        assert_eq!(disk.status, RunStatus::Gating);
13898    }
13899
13900    #[tokio::test]
13901    async fn an_early_graph_error_releases_its_driver() {
13902        crate::run::pin_test_home();
13903        let mut runner = runner_at(RunStatus::Blocked);
13904        runner.state.config.graph.selected_implementer = Some("selected".into());
13905        runner.state.save().unwrap();
13906        assert!(runner.execute().await.is_err());
13907        let disk = RunState::load(&runner.state.id).unwrap();
13908        assert!(disk.driver_exited);
13909        assert_eq!(disk.driver_pid, None);
13910        assert_eq!(disk.status, RunStatus::Blocked);
13911    }
13912
13913    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13914        crate::run::MergeRepair {
13915            question: "repair-question".into(),
13916            head: "old-head".into(),
13917            pr: "https://github.com/o/r/pull/42".into(),
13918            reviews: vec![review_round(false, 1, 1, 1, false, true)],
13919            gate: vec![outcome(Some(0), false)],
13920            gate_head: Some("old-head".into()),
13921            gate_fixes: Vec::new(),
13922            pre_gate: Vec::new(),
13923            phase,
13924            failure: None,
13925        }
13926    }
13927
13928    #[tokio::test]
13929    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13930        crate::run::pin_test_home();
13931        let mut runner = runner_at(RunStatus::Landing);
13932        runner.state.merge_repairs.push(merge_repair_request(
13933            crate::run::MergeRepairPhase::Requested,
13934        ));
13935        runner.state.save().unwrap();
13936        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13937        assert!(
13938            runner
13939                .execute()
13940                .await
13941                .unwrap_err()
13942                .to_string()
13943                .contains("already running")
13944        );
13945        assert_eq!(
13946            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13947            crate::run::MergeRepairPhase::Requested
13948        );
13949        drop(claim);
13950        let mut disk = runner.state.clone();
13951        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13952        disk.save().unwrap();
13953        assert!(
13954            runner
13955                .execute()
13956                .await
13957                .unwrap_err()
13958                .to_string()
13959                .contains("decision changed")
13960        );
13961        assert_eq!(
13962            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13963            crate::run::MergeRepairPhase::Complete
13964        );
13965    }
13966
13967    #[test]
13968    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13969        crate::run::pin_test_home();
13970        let mut runner = runner_at(RunStatus::Landing);
13971        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13972        runner.state.reviews = request.reviews.clone();
13973        runner.state.gate = request.gate.clone();
13974        runner.state.gate_ran = true;
13975        runner.state.land_approval = Some(crate::run::LandApproval {
13976            question: "old".into(),
13977            head: "old-head".into(),
13978        });
13979        runner.state.merge_repairs.push(request);
13980        runner.begin_merge_repair_review(0).unwrap();
13981        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13982        assert!(!runner.state.gate_status().ok());
13983        assert!(runner.state.land_approval.is_none());
13984        let disk = RunState::load(&runner.state.id).unwrap();
13985        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13986        assert!(disk.merge_repairs[0].gate[0].ok());
13987        assert_eq!(
13988            disk.merge_repairs[0].phase,
13989            crate::run::MergeRepairPhase::Reviewing
13990        );
13991        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13992        assert!(disk.reviews.is_empty());
13993    }
13994
13995    #[tokio::test]
13996    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13997        crate::run::pin_test_home();
13998        for phase in [
13999            crate::run::MergeRepairPhase::Fixing,
14000            crate::run::MergeRepairPhase::Requested,
14001        ] {
14002            let mut runner = runner_at(RunStatus::Landing);
14003            runner.state.config.graph.land_rounds = 0;
14004            runner.state.merge_repairs.push(merge_repair_request(phase));
14005            runner.state.save().unwrap();
14006            assert!(!runner.repair_merge_review().await.unwrap());
14007            assert_eq!(runner.state.status, RunStatus::Blocked);
14008            assert_eq!(
14009                runner.state.merge_repairs[0].phase,
14010                crate::run::MergeRepairPhase::Failed
14011            );
14012            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14013            assert!(
14014                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14015                    "interrupted"
14016                } else {
14017                    "budget exhausted"
14018                })
14019            );
14020            assert!(runner.state.merge.is_none());
14021        }
14022    }
14023
14024    /// `park_here` folding in the reason `Pause::park_because` recorded -
14025    /// this is what lets an operator reading a run's events tell an
14026    /// interrupt-driven park from an ordinary shutdown park.
14027    #[test]
14028    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14029        crate::run::pin_test_home();
14030        let mut runner = runner_at(RunStatus::Implementing);
14031        let interrupt = Pause::new();
14032        runner.watch_interrupt(interrupt.clone());
14033
14034        interrupt.park_because("task a1b2 asked to run first");
14035
14036        assert!(runner.park_here().expect("park_here"));
14037        assert!(runner.state.parked);
14038        let last = runner.state.events.last().expect("a park event");
14039        assert_eq!(last.node, "park");
14040        assert!(
14041            last.message.contains("task a1b2 asked to run first"),
14042            "expected the interrupt reason in {:?}",
14043            last.message
14044        );
14045    }
14046
14047    /// The cause is stored on the pass when it parks, and the first cause a
14048    /// shared pause was given is the one that stays.
14049    #[test]
14050    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14051        crate::run::pin_test_home();
14052        let mut runner = runner_at(RunStatus::Implementing);
14053        runner.state.begin_pass();
14054        let shutdown = Pause::new();
14055        runner.on_pause(shutdown.clone());
14056        let upgrade = ParkCause::Upgrade {
14057            from: "0.121.0".to_owned(),
14058            to: Some("0.121.1".to_owned()),
14059        };
14060        shutdown.park_for(upgrade.clone());
14061        shutdown.park_for(ParkCause::Operator);
14062
14063        assert!(runner.park_here().expect("park_here"));
14064        assert_eq!(
14065            runner.state.passes[0].end,
14066            Some(crate::run::PassEnd::Parked { cause: upgrade })
14067        );
14068        let last = runner.state.events.last().expect("a park event");
14069        assert!(last.message.contains("0.121.0"), "{}", last.message);
14070    }
14071
14072    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
14073    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
14074    /// cleared) must not make a *different* run - one only watching its own,
14075    /// unshared interrupt `Pause` - see itself as parked. If a future change
14076    /// ever collapsed these back into one handle, the interrupt scheduler
14077    /// would park every run for the rest of the daemon's life, not just the
14078    /// one it meant to interrupt.
14079    #[test]
14080    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14081        crate::run::pin_test_home();
14082        let mut runner = runner_at(RunStatus::Implementing);
14083        let shutdown = Pause::new();
14084        runner.on_pause(shutdown.clone());
14085        let interrupt = Pause::new();
14086        runner.watch_interrupt(interrupt.clone());
14087
14088        // Nobody has asked for anything yet.
14089        assert!(!runner.park_here().expect("park_here"));
14090        assert!(!runner.state.parked);
14091
14092        // Only the interrupt handle fires; the shutdown handle stays clear.
14093        interrupt.park_because("test");
14094        assert!(!shutdown.parked());
14095        assert!(runner.park_here().expect("park_here"));
14096    }
14097
14098    /// The property every prior attempt at this feature failed to pin down:
14099    /// asking a run to park while one of its nodes has a real, in-flight
14100    /// async operation running (an agent call, in production) must not cut
14101    /// that operation short. `park_here` is only ever consulted *between*
14102    /// `execute`'s node calls - see its own doc - so nothing inside a node
14103    /// can observe a park request until the node itself returns. This proves
14104    /// that structurally, with real `tokio` concurrency and a channel
14105    /// handshake (never a sleep, which would only prove "usually", not
14106    /// "cannot"): the "node" below reports that it has genuinely started,
14107    /// and only then is the park requested; the node still has to be told to
14108    /// finish before `park_here` is ever called, exactly mirroring every
14109    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
14110    /// in `execute`.
14111    #[tokio::test]
14112    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14113        crate::run::pin_test_home();
14114        let mut runner = runner_at(RunStatus::Implementing);
14115        let interrupt = Pause::new();
14116        runner.watch_interrupt(interrupt.clone());
14117
14118        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14119        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14120
14121        // Stands in for one node's in-flight agent call: it proves it has
14122        // genuinely started, then blocks - exactly as a spawned CLI process
14123        // does - until told to finish.
14124        let node = async move {
14125            started_tx.send(()).expect("send started");
14126            finish_rx.await.expect("recv finish");
14127            "node finished"
14128        };
14129
14130        let interrupter = async move {
14131            started_rx.await.expect("recv started");
14132            // The call is now genuinely in flight. Ask it to park.
14133            interrupt.park_because("higher-priority task waiting");
14134            // Nothing the node does can observe this yet - there is no
14135            // check inside it, by construction - so let the executor run
14136            // anything pending and then let the node finish on its own.
14137            tokio::task::yield_now().await;
14138            finish_tx.send(()).expect("send finish");
14139        };
14140
14141        let (node_result, ()) = tokio::join!(node, interrupter);
14142        assert_eq!(
14143            node_result, "node finished",
14144            "the in-flight call ran to completion"
14145        );
14146
14147        // Only now, at the boundary the real `execute` would check right
14148        // after this node, does the park take effect.
14149        assert!(runner.park_here().expect("park_here"));
14150        assert!(runner.state.parked);
14151    }
14152
14153    /// A run parked mid-competition carries every field it had accumulated
14154    /// through the exact same disk round-trip an ordinary resume uses -
14155    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
14156    /// Nothing about parking for an interrupt is a special case of that path;
14157    /// this is what proves it rather than assuming it.
14158    #[test]
14159    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14160        crate::run::pin_test_home();
14161        let mut runner = runner_at(RunStatus::Judging);
14162        // `Runner::resume` re-resolves roles from the saved config, which
14163        // refuses an empty roster - give it the same minimal one `conductor`
14164        // itself uses.
14165        runner.state.config.agents = vec![conductor()];
14166        runner.state.candidates = vec![Candidate {
14167            index: 0,
14168            label: 'A',
14169            agent: "alpha".to_owned(),
14170            branch: "magi/x/A".to_owned(),
14171            worktree: PathBuf::from("/nonexistent/worktree"),
14172            summary: "did the thing".to_owned(),
14173            stat: "1 file changed".to_owned(),
14174            files: 1,
14175            commits: 1,
14176            empty: false,
14177            failed: None,
14178            verified_noop: None,
14179            duration_ms: 1234,
14180            folded: false,
14181        }];
14182        let run_id = runner.state.id.clone();
14183
14184        let interrupt = Pause::new();
14185        runner.watch_interrupt(interrupt.clone());
14186        interrupt.park_because("task c3d4 asked to run first");
14187        assert!(runner.park_here().expect("park_here"));
14188
14189        let resumed = Runner::resume(&run_id).expect("resume");
14190        assert_eq!(resumed.state.candidates.len(), 1);
14191        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14192        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14193        assert_eq!(resumed.state.status, runner.state.status);
14194        assert!(
14195            resumed.state.parked,
14196            "still parked until `execute` actually walks the graph again"
14197        );
14198        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14199    }
14200
14201    /// A fresh open question on `run`, stored and handed back for assertions.
14202    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14203        let mut q = ask::Question::new(
14204            run.to_owned(),
14205            "implement".to_owned(),
14206            "impl-A".to_owned(),
14207            "Which storage backend should the cache use?".to_owned(),
14208            String::new(),
14209            vec!["SQLite".to_owned(), "Redis".to_owned()],
14210        );
14211        store.put(&mut q).unwrap();
14212        q
14213    }
14214
14215    #[test]
14216    fn a_failed_runs_open_question_is_abandoned() {
14217        ask_test_home();
14218        let store = ask::Questions::open();
14219        let mut runner = runner_at(RunStatus::Failed);
14220        let run = runner.state.id.clone();
14221        let q = ask_open_question(&store, &run);
14222
14223        runner.settle_questions();
14224
14225        let back = store.get(&q.id).unwrap();
14226        assert!(
14227            !back.status.open(),
14228            "the seat that asked died with the run; nobody is left to read an answer"
14229        );
14230        assert!(
14231            back.detail.contains(&run) && back.detail.contains("failed"),
14232            "the reason names what the run became, not just that it is gone: {}",
14233            back.detail
14234        );
14235    }
14236
14237    #[test]
14238    fn a_merged_runs_open_question_is_abandoned_too() {
14239        ask_test_home();
14240        let store = ask::Questions::open();
14241        // A run that finishes cleanly still leaves nobody to read an answer -
14242        // this is not only a failure-path cleanup.
14243        for status in [RunStatus::Merged, RunStatus::Ready] {
14244            let mut runner = runner_at(status);
14245            let run = runner.state.id.clone();
14246            let q = ask_open_question(&store, &run);
14247
14248            runner.settle_questions();
14249
14250            let back = store.get(&q.id).unwrap();
14251            assert!(
14252                !back.status.open(),
14253                "{status:?} run's question must not outlive the run"
14254            );
14255        }
14256    }
14257
14258    #[test]
14259    fn a_still_resumable_runs_open_question_is_left_alone() {
14260        ask_test_home();
14261        let store = ask::Questions::open();
14262        // `Blocked` and `Stalled` can still be resumed — the candidates, the
14263        // review round and the seat sessions are all still on disk — so a
14264        // question asked mid-round may yet get a real answer from a real
14265        // resume. Sweeping it here would be exactly the failure mode this
14266        // whole feature exists to avoid on the other side.
14267        for status in [RunStatus::Blocked, RunStatus::Stalled] {
14268            let mut runner = runner_at(status);
14269            let run = runner.state.id.clone();
14270            let q = ask_open_question(&store, &run);
14271
14272            runner.settle_questions();
14273
14274            let back = store.get(&q.id).unwrap();
14275            assert!(
14276                back.status.open(),
14277                "{status:?} is still alive; the question must still be waiting"
14278            );
14279        }
14280    }
14281
14282    #[test]
14283    fn settle_questions_never_touches_an_already_answered_question() {
14284        ask_test_home();
14285        let store = ask::Questions::open();
14286        let mut runner = runner_at(RunStatus::Failed);
14287        let run = runner.state.id.clone();
14288        let mut q = ask_open_question(&store, &run);
14289        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14290            .unwrap();
14291        store.put(&mut q).unwrap();
14292
14293        // Called twice, the way a crash-recovered daemon reclaim and the
14294        // graph's own cleanup both can for the same run — `abandon_for_run`
14295        // only ever touches what is still open, so this must be inert both
14296        // times, not merely the second.
14297        runner.settle_questions();
14298        runner.settle_questions();
14299
14300        let back = store.get(&q.id).unwrap();
14301        assert_eq!(
14302            back.status,
14303            ask::QuestionStatus::Answered,
14304            "a real answer is a decision on record, never overwritten by a sweep"
14305        );
14306    }
14307
14308    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
14309    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
14310    /// without merging, whose winner is still the operator's answer to read.
14311    /// Nothing previously called `fold_run` itself with a real `tally`, so
14312    /// this is the first test to pin down the one distinction the whole
14313    /// automatic-fold feature depends on: the winner's worktree and branch
14314    /// must survive, everything else sharing the run's worktree bay - a
14315    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
14316    /// second sweep treats every non-winner directory under the bay alike -
14317    /// must not.
14318    #[tokio::test]
14319    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14320        crate::run::pin_test_home();
14321        let tmp = tempfile::tempdir().expect("tempdir");
14322        let repo = tmp.path().join("repo");
14323        std::fs::create_dir_all(&repo).unwrap();
14324        init_repo(&repo);
14325
14326        let mut config = Config::default();
14327        config.graph.worktree_root = Some(tmp.path().join("wt"));
14328
14329        let mut state = RunState::new(
14330            repo.clone(),
14331            "main".to_owned(),
14332            "deadbeef".to_owned(),
14333            "task".to_owned(),
14334            config,
14335        );
14336        let root = state.worktree_root();
14337        let wt_a = root.join("cand-A");
14338        let wt_b = root.join("cand-B");
14339        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14340            .await
14341            .expect("worktree A");
14342        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14343            .await
14344            .expect("worktree B");
14345
14346        state.candidates = vec![
14347            Candidate {
14348                index: 0,
14349                label: 'A',
14350                agent: "alpha".to_owned(),
14351                branch: "magi/x/A".to_owned(),
14352                worktree: wt_a.clone(),
14353                summary: String::new(),
14354                stat: String::new(),
14355                files: 0,
14356                commits: 0,
14357                empty: false,
14358                failed: None,
14359                verified_noop: None,
14360                duration_ms: 0,
14361                folded: false,
14362            },
14363            Candidate {
14364                index: 1,
14365                label: 'B',
14366                agent: "beta".to_owned(),
14367                branch: "magi/x/B".to_owned(),
14368                worktree: wt_b.clone(),
14369                summary: String::new(),
14370                stat: String::new(),
14371                files: 0,
14372                commits: 0,
14373                empty: false,
14374                failed: None,
14375                verified_noop: None,
14376                duration_ms: 0,
14377                folded: false,
14378            },
14379        ];
14380        state.tally = Some(Tally {
14381            first_choice: BTreeMap::from([('A', 1)]),
14382            borda: BTreeMap::new(),
14383            winner: 'A',
14384            rankings: 1,
14385            unanimous_initial: true,
14386            deliberated: false,
14387            changed_votes: 0,
14388            unanimous_final: true,
14389            tie_break: None,
14390            judges: 1,
14391            present: 1,
14392            quorum: 1,
14393            met_quorum: true,
14394            uncontested: None,
14395        });
14396        state.status = RunStatus::Ready;
14397
14398        fold_run(&mut state, false, &crate::run::home())
14399            .await
14400            .expect("fold_run");
14401
14402        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14403        assert!(
14404            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14405            "the unmerged winner's branch survives"
14406        );
14407        assert!(
14408            !state.candidates[0].folded,
14409            "the winner is not marked folded"
14410        );
14411
14412        assert!(!wt_b.exists(), "the loser's worktree is removed");
14413        assert!(
14414            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14415            "the loser's branch is removed"
14416        );
14417        assert!(state.candidates[1].folded, "the loser is marked folded");
14418    }
14419
14420    #[tokio::test]
14421    #[cfg(unix)]
14422    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14423        use std::os::fd::AsRawFd as _;
14424        use std::time::Duration;
14425
14426        let tmp = tempfile::tempdir().unwrap();
14427        let repo = tmp.path().join("repo");
14428        std::fs::create_dir_all(&repo).unwrap();
14429        init_repo(&repo);
14430        let common = repo.join(".git");
14431        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14432        let home = tmp.path().join("home");
14433        let mut config = Config::default();
14434        config.coordination = crate::config::test_coordination();
14435        config.graph.worktree_root = Some(tmp.path().join("wt"));
14436        config.graph.timeout_resource_wait = 1;
14437        let mut state = RunState::new(
14438            repo.clone(),
14439            "main".into(),
14440            "deadbeef".into(),
14441            "task".into(),
14442            config,
14443        );
14444        let root = state.worktree_root();
14445        let candidate = root.join("cand-A");
14446        let reviewer = root.join("review-1");
14447        let branch = state.branch_for('A');
14448        git::worktree_add_branch(&repo, &candidate, &branch, "main")
14449            .await
14450            .unwrap();
14451        git::worktree_add_detached(&repo, &reviewer, "main")
14452            .await
14453            .unwrap();
14454        git::acquire_worktree_config(&repo).await.unwrap();
14455        state.enabled_worktree_config = true;
14456        state.status = RunStatus::Ready;
14457        state.candidates.push(Candidate {
14458            index: 0,
14459            label: 'A',
14460            agent: "implement".into(),
14461            branch: branch.clone(),
14462            worktree: candidate.clone(),
14463            summary: String::new(),
14464            stat: String::new(),
14465            files: 0,
14466            commits: 0,
14467            empty: false,
14468            failed: None,
14469            verified_noop: None,
14470            duration_ms: 0,
14471            folded: false,
14472        });
14473        state.save_under(&home).unwrap();
14474        let config_before = std::fs::read(common.join("config")).unwrap();
14475        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14476            .await
14477            .unwrap();
14478        // A separate OS handle bypasses the launcher's local gate, exactly as
14479        // an external updater does, and must still prevent all folding work.
14480        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14481        assert_eq!(
14482            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14483            0
14484        );
14485        let mut fold = Box::pin(fold_run(&mut state, true, &home));
14486        assert!(
14487            tokio::time::timeout(Duration::from_millis(200), &mut fold)
14488                .await
14489                .is_err()
14490        );
14491        assert!(candidate.exists());
14492        assert!(reviewer.exists());
14493        assert!(git::branch_exists(&repo, &branch).await.unwrap());
14494        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14495        assert_eq!(
14496            git::git(&repo, &["worktree", "list", "--porcelain"])
14497                .await
14498                .unwrap(),
14499            worktrees_before
14500        );
14501        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14502            .await
14503            .unwrap()
14504            .unwrap_err();
14505        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14506        drop(fold);
14507        assert!(candidate.exists() && reviewer.exists());
14508        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14509        assert_eq!(
14510            state.resource_block.as_ref().unwrap().kind,
14511            "repository coordination"
14512        );
14513        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14514        drop(external);
14515        let mut fold = Box::pin(fold_run(&mut state, true, &home));
14516        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14517            .await
14518            .unwrap()
14519            .unwrap();
14520        drop(fold);
14521        assert!(removed.contains(&branch));
14522        assert!(!root.exists());
14523        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14524        assert!(state.candidates[0].folded);
14525        assert!(!state.enabled_worktree_config);
14526        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14527        let enabled = git::git_raw(
14528            &repo,
14529            &["config", "--local", "--get", "extensions.worktreeConfig"],
14530        )
14531        .await
14532        .unwrap();
14533        assert!(!enabled.ok(), "the last config reference was released");
14534    }
14535
14536    /// A branch handed to a later run is that run's (and its pull request's):
14537    /// folding the run that released it must not delete it.
14538    #[tokio::test]
14539    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14540        let tmp = tempfile::tempdir().expect("tempdir");
14541        let repo = tmp.path().join("repo");
14542        std::fs::create_dir_all(&repo).unwrap();
14543        init_repo(&repo);
14544        let home = tmp.path().join("home");
14545
14546        let mut config = Config::default();
14547        config.graph.worktree_root = Some(tmp.path().join("wt"));
14548        let mut state = RunState::new(
14549            repo.clone(),
14550            "main".to_owned(),
14551            "deadbeef".to_owned(),
14552            "task".to_owned(),
14553            config,
14554        );
14555        // The worktree is already gone (released); the branch survives.
14556        git::git(&repo, &["branch", "magi/x/A", "main"])
14557            .await
14558            .expect("branch");
14559        state.candidates = vec![Candidate {
14560            index: 0,
14561            label: 'A',
14562            agent: "alpha".to_owned(),
14563            branch: "magi/x/A".to_owned(),
14564            worktree: state.worktree_root().join("cand-A"),
14565            summary: String::new(),
14566            stat: String::new(),
14567            files: 0,
14568            commits: 0,
14569            empty: false,
14570            failed: None,
14571            verified_noop: None,
14572            duration_ms: 0,
14573            folded: true,
14574        }];
14575        state.released_to = Some("20260901-000000-new1".to_owned());
14576        state.released_branches = vec!["magi/x/A".to_owned()];
14577
14578        fold_run(&mut state, true, &home).await.expect("fold_run");
14579
14580        assert!(
14581            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14582            "the handed-over branch survives a fold"
14583        );
14584    }
14585
14586    /// A winner with nothing ahead of the base is caught before `gh` is ever
14587    /// asked for a pull request, and the message carries what the task's
14588    /// references resolved to.
14589    #[tokio::test]
14590    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14591        let tmp = tempfile::tempdir().expect("tempdir");
14592        let repo = tmp.path().join("repo");
14593        std::fs::create_dir_all(&repo).unwrap();
14594        init_repo(&repo);
14595        let run = |args: &[&str]| {
14596            let out = std::process::Command::new("git")
14597                .quiet()
14598                .args(args)
14599                .current_dir(&repo)
14600                .output()
14601                .expect("spawn git");
14602            assert!(out.status.success(), "git {args:?}");
14603        };
14604        run(&["branch", "magi/x/A"]);
14605        run(&["checkout", "-q", "-b", "magi/x/B"]);
14606        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14607        run(&["add", "-A"]);
14608        run(&["commit", "-q", "-m", "work"]);
14609        run(&["checkout", "-q", "main"]);
14610
14611        // A pull request is compared against the remote's base, so the
14612        // fixture needs one. Before it exists nothing can be read, and the
14613        // answer must be "not empty".
14614        let probe = RunState::new(
14615            repo.clone(),
14616            "main".to_owned(),
14617            "deadbeef".to_owned(),
14618            "task".to_owned(),
14619            Config::default(),
14620        );
14621        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14622        let bare = tmp.path().join("origin.git");
14623        let out = std::process::Command::new("git")
14624            .quiet()
14625            .args(["init", "-q", "--bare"])
14626            .arg(&bare)
14627            .output()
14628            .expect("spawn git");
14629        assert!(out.status.success(), "git init --bare");
14630        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14631        run(&["push", "-q", "origin", "main"]);
14632
14633        let mut state = RunState::new(
14634            repo.clone(),
14635            "main".to_owned(),
14636            "deadbeef".to_owned(),
14637            "task".to_owned(),
14638            Config::default(),
14639        );
14640        state.seeds = vec![refs::Seed {
14641            token: "magi/27b2/A".to_owned(),
14642            kind: refs::SeedKind::Unresolved,
14643            sha: String::new(),
14644            branch: true,
14645            detail: "no branch or commit named magi/27b2/A".to_owned(),
14646        }];
14647
14648        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14649        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14650        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14651        let detail = empty_candidate_detail(&state, "main");
14652        assert!(detail.starts_with("empty candidate"), "{detail}");
14653        assert!(detail.contains("magi/27b2/A"), "{detail}");
14654    }
14655
14656    /// `status == Ready` used to be read as "this is the harmless
14657    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
14658    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
14659    /// run's PR was closed without merging — and reentering `merge` with
14660    /// `mode` still `Pr` does not know the difference, so it pushed and
14661    /// opened a second pull request. `mode == Local` reproduces the same
14662    /// blind spot without a network call: reentry must not attempt another
14663    /// git merge once this node has already recorded an outcome.
14664    #[tokio::test]
14665    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14666        let tmp = tempfile::tempdir().expect("tempdir");
14667        let repo = tmp.path().join("repo");
14668        std::fs::create_dir_all(&repo).unwrap();
14669        init_repo(&repo);
14670
14671        let mut config = Config::default();
14672        config.merge.mode = MergeMode::Local;
14673
14674        let mut state = RunState::new(
14675            repo.clone(),
14676            "main".to_owned(),
14677            "deadbeef".to_owned(),
14678            "task".to_owned(),
14679            config,
14680        );
14681        state.candidates = vec![Candidate {
14682            index: 0,
14683            label: 'A',
14684            agent: "alpha".to_owned(),
14685            branch: "does-not-exist".to_owned(),
14686            worktree: repo.clone(),
14687            summary: String::new(),
14688            stat: String::new(),
14689            files: 0,
14690            commits: 0,
14691            empty: false,
14692            failed: None,
14693            verified_noop: None,
14694            duration_ms: 0,
14695            folded: false,
14696        }];
14697        state.tally = Some(Tally {
14698            first_choice: BTreeMap::from([('A', 1)]),
14699            borda: BTreeMap::new(),
14700            winner: 'A',
14701            rankings: 1,
14702            unanimous_initial: true,
14703            deliberated: false,
14704            changed_votes: 0,
14705            unanimous_final: true,
14706            tie_break: None,
14707            judges: 0,
14708            present: 0,
14709            quorum: 0,
14710            met_quorum: true,
14711            uncontested: Some("only candidate A produced a change".to_owned()),
14712        });
14713        state.reviews = vec![ReviewRound {
14714            round: 1,
14715            head: "deadbeef".to_owned(),
14716            verified_head: None,
14717            verified_at: None,
14718            reviews: Vec::new(),
14719            e2e: Vec::new(),
14720            fix: None,
14721            blocking: 0,
14722            answered: 0,
14723            expected: 0,
14724            clean: true,
14725            verify_retried: false,
14726            e2e_deferred: false,
14727            e2e_defer_reason: None,
14728            progressed: false,
14729            vote_split: false,
14730            reconsideration: Vec::new(),
14731            verdict: None,
14732        }];
14733        state.gate = vec![CommandOutcome {
14734            command: "test".to_owned(),
14735            code: Some(0),
14736            output_tail: String::new(),
14737            duration_ms: 0,
14738            resource_blocked: false,
14739        }];
14740        state.gate_ran = true;
14741        // Reached its conclusion already — e.g. `land` closing the PR without
14742        // merging it, which (like the honest `MergeMode::None` path) leaves
14743        // `status` at `Ready`. The recorded outcome is what actually marks
14744        // this node done.
14745        state.status = RunStatus::Ready;
14746        state.merge = Some(MergeOutcome {
14747            mode: MergeMode::Local,
14748            ok: false,
14749            detail: "already concluded".to_owned(),
14750            empty: false,
14751        });
14752
14753        let mut runner = Runner {
14754            state,
14755            roles: ResolvedRoles {
14756                implementers: Vec::new(),
14757                judges: Vec::new(),
14758                reviewers: Vec::new(),
14759                fixer: None,
14760                conductor: conductor(),
14761                implementer_roster: Vec::new(),
14762                judge_roster: Vec::new(),
14763                reviewer_roster: Vec::new(),
14764            },
14765            sem: Arc::new(Semaphore::new(1)),
14766            pause: Pause::new(),
14767            interrupt: Pause::new(),
14768        };
14769
14770        runner.merge().await.expect("merge");
14771
14772        assert_eq!(
14773            runner.state.status,
14774            RunStatus::Ready,
14775            "a concluded run's status must not change on reentry"
14776        );
14777        assert_eq!(
14778            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14779            Some("already concluded"),
14780            "merge must not run again once the node already recorded an outcome"
14781        );
14782    }
14783
14784    /// `gate` leaves `state.gate_ran` false both before it has ever run and
14785    /// when its last attempt was resource-blocked (the shared build cache
14786    /// could not be acquired or confirmed fresh in time - see
14787    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
14788    /// `Vec` this also leaves behind used to read as "nothing failed" and let
14789    /// a run merge a tree the gate never actually checked - exactly the case
14790    /// a contended cache produces on every retry until it clears. `merge`
14791    /// must refuse until `gate` has actually recorded an attempt.
14792    #[tokio::test]
14793    async fn merge_refuses_a_gate_that_has_not_actually_run() {
14794        let tmp = tempfile::tempdir().expect("tempdir");
14795        let repo = tmp.path().join("repo");
14796        std::fs::create_dir_all(&repo).unwrap();
14797        init_repo(&repo);
14798
14799        let mut config = Config::default();
14800        config.merge.mode = MergeMode::Local;
14801
14802        let mut state = RunState::new(
14803            repo.clone(),
14804            "main".to_owned(),
14805            "deadbeef".to_owned(),
14806            "task".to_owned(),
14807            config,
14808        );
14809        state.candidates = vec![Candidate {
14810            index: 0,
14811            label: 'A',
14812            agent: "alpha".to_owned(),
14813            branch: "does-not-exist".to_owned(),
14814            worktree: repo.clone(),
14815            summary: String::new(),
14816            stat: String::new(),
14817            files: 0,
14818            commits: 0,
14819            empty: false,
14820            failed: None,
14821            verified_noop: None,
14822            duration_ms: 0,
14823            folded: false,
14824        }];
14825        state.tally = Some(Tally {
14826            first_choice: BTreeMap::from([('A', 1)]),
14827            borda: BTreeMap::new(),
14828            winner: 'A',
14829            rankings: 1,
14830            unanimous_initial: true,
14831            deliberated: false,
14832            changed_votes: 0,
14833            unanimous_final: true,
14834            tie_break: None,
14835            judges: 0,
14836            present: 0,
14837            quorum: 0,
14838            met_quorum: true,
14839            uncontested: Some("only candidate A produced a change".to_owned()),
14840        });
14841        state.reviews = vec![ReviewRound {
14842            round: 1,
14843            head: "deadbeef".to_owned(),
14844            verified_head: None,
14845            verified_at: None,
14846            reviews: Vec::new(),
14847            e2e: Vec::new(),
14848            fix: None,
14849            blocking: 0,
14850            answered: 0,
14851            expected: 0,
14852            clean: true,
14853            verify_retried: false,
14854            e2e_deferred: false,
14855            e2e_defer_reason: None,
14856            progressed: false,
14857            vote_split: false,
14858            reconsideration: Vec::new(),
14859            verdict: None,
14860        }];
14861        // The point: `gate` has not recorded anything yet.
14862        state.gate = Vec::new();
14863        state.gate_ran = false;
14864        state.status = RunStatus::Gating;
14865
14866        let mut runner = Runner {
14867            state,
14868            roles: ResolvedRoles {
14869                implementers: Vec::new(),
14870                judges: Vec::new(),
14871                reviewers: Vec::new(),
14872                fixer: None,
14873                conductor: conductor(),
14874                implementer_roster: Vec::new(),
14875                judge_roster: Vec::new(),
14876                reviewer_roster: Vec::new(),
14877            },
14878            sem: Arc::new(Semaphore::new(1)),
14879            pause: Pause::new(),
14880            interrupt: Pause::new(),
14881        };
14882
14883        runner.merge().await.expect("merge");
14884
14885        assert!(
14886            runner.state.merge.is_none(),
14887            "an empty gate must never be read as a passing one: {:?}",
14888            runner.state.merge
14889        );
14890    }
14891
14892    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
14893    /// commands configured and `merge.mode` is `none` (a review-only run).
14894    /// `gate` must still record a real attempt — zero commands, vacuously
14895    /// passed — rather than leaving `state.gate` empty in a way `merge`
14896    /// cannot tell apart from "never ran"; otherwise the run reaches
14897    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
14898    #[tokio::test]
14899    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14900        let tmp = tempfile::tempdir().expect("tempdir");
14901        let repo = tmp.path().join("repo");
14902        std::fs::create_dir_all(&repo).unwrap();
14903        init_repo(&repo);
14904
14905        // Default config: `verify.gate` empty, `merge.mode` is `none`.
14906        let config = Config::default();
14907
14908        let mut state = RunState::new(
14909            repo.clone(),
14910            "main".to_owned(),
14911            "deadbeef".to_owned(),
14912            "task".to_owned(),
14913            config,
14914        );
14915        state.candidates = vec![Candidate {
14916            index: 0,
14917            label: 'A',
14918            agent: "alpha".to_owned(),
14919            branch: "does-not-exist".to_owned(),
14920            worktree: repo.clone(),
14921            summary: String::new(),
14922            stat: String::new(),
14923            files: 0,
14924            commits: 0,
14925            empty: false,
14926            failed: None,
14927            verified_noop: None,
14928            duration_ms: 0,
14929            folded: false,
14930        }];
14931        state.tally = Some(Tally {
14932            first_choice: BTreeMap::from([('A', 1)]),
14933            borda: BTreeMap::new(),
14934            winner: 'A',
14935            rankings: 1,
14936            unanimous_initial: true,
14937            deliberated: false,
14938            changed_votes: 0,
14939            unanimous_final: true,
14940            tie_break: None,
14941            judges: 0,
14942            present: 0,
14943            quorum: 0,
14944            met_quorum: true,
14945            uncontested: Some("only candidate A produced a change".to_owned()),
14946        });
14947        state.reviews = vec![ReviewRound {
14948            round: 1,
14949            head: "deadbeef".to_owned(),
14950            verified_head: None,
14951            verified_at: None,
14952            reviews: Vec::new(),
14953            e2e: Vec::new(),
14954            fix: None,
14955            blocking: 0,
14956            answered: 0,
14957            expected: 0,
14958            clean: true,
14959            verify_retried: false,
14960            e2e_deferred: false,
14961            e2e_defer_reason: None,
14962            progressed: false,
14963            vote_split: false,
14964            reconsideration: Vec::new(),
14965            verdict: None,
14966        }];
14967
14968        let mut runner = Runner {
14969            state,
14970            roles: ResolvedRoles {
14971                implementers: Vec::new(),
14972                judges: Vec::new(),
14973                reviewers: Vec::new(),
14974                fixer: None,
14975                conductor: conductor(),
14976                implementer_roster: Vec::new(),
14977                judge_roster: Vec::new(),
14978                reviewer_roster: Vec::new(),
14979            },
14980            sem: Arc::new(Semaphore::new(1)),
14981            pause: Pause::new(),
14982            interrupt: Pause::new(),
14983        };
14984
14985        runner.gate().await.expect("gate");
14986        assert!(
14987            runner.state.gate_ran,
14988            "zero configured commands is still a real attempt, not an unrun gate"
14989        );
14990        assert!(runner.state.gate.is_empty());
14991        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
14992        assert_ne!(
14993            runner.state.status,
14994            RunStatus::Blocked,
14995            "a gate with nothing to check must not read as failed"
14996        );
14997
14998        runner.merge().await.expect("merge");
14999        assert_eq!(
15000            runner.state.status,
15001            RunStatus::Ready,
15002            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15003        );
15004    }
15005
15006    /// `Config::cache_dir` is derived from `verify.e2e` as well as
15007    /// `verify.gate` (so the e2e leg and the final gate never build against
15008    /// different directories). With zero `verify.gate` commands but a
15009    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
15010    /// that lease before discovering it had nothing to run - so a repo with
15011    /// no gate commands could come back `resource_blocked` (and therefore
15012    /// still `gate_ran == false`) on nothing but an unrelated run holding the
15013    /// cache, exactly the contention this run's own zero commands could
15014    /// never have touched. `gate` must recognise there is nothing to check
15015    /// before it ever asks for the lease.
15016    #[tokio::test]
15017    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15018        crate::run::pin_test_home();
15019        let home = crate::run::home();
15020
15021        let tmp = tempfile::tempdir().expect("tempdir");
15022        let repo = tmp.path().join("repo");
15023        std::fs::create_dir_all(&repo).unwrap();
15024        init_repo(&repo);
15025        // Unique to this test, so holding its lease cannot collide with
15026        // another test sharing the same process-wide `home`.
15027        let cache_dir = tmp.path().join("target");
15028
15029        let mut config = Config::default();
15030        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15031        // `verify.gate` stays empty (the default). Bounded so a regression
15032        // that does start waiting fails the test in seconds, not hangs it.
15033        config.graph.timeout_verify = Some(2);
15034        config.graph.timeout_resource_wait = 2;
15035
15036        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15037        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15038            .expect("no io error acquiring directly")
15039        {
15040            crate::cache::AcquireOutcome::Acquired(g) => g,
15041            crate::cache::AcquireOutcome::Busy(b) => {
15042                panic!("expected the direct acquire to win the lease first: {b:?}")
15043            }
15044        };
15045
15046        let mut state = RunState::new(
15047            repo.clone(),
15048            "main".to_owned(),
15049            "deadbeef".to_owned(),
15050            "task".to_owned(),
15051            config,
15052        );
15053        state.candidates = vec![Candidate {
15054            index: 0,
15055            label: 'A',
15056            agent: "alpha".to_owned(),
15057            branch: "does-not-exist".to_owned(),
15058            worktree: repo.clone(),
15059            summary: String::new(),
15060            stat: String::new(),
15061            files: 0,
15062            commits: 0,
15063            empty: false,
15064            failed: None,
15065            verified_noop: None,
15066            duration_ms: 0,
15067            folded: false,
15068        }];
15069        state.tally = Some(Tally {
15070            first_choice: BTreeMap::from([('A', 1)]),
15071            borda: BTreeMap::new(),
15072            winner: 'A',
15073            rankings: 1,
15074            unanimous_initial: true,
15075            deliberated: false,
15076            changed_votes: 0,
15077            unanimous_final: true,
15078            tie_break: None,
15079            judges: 0,
15080            present: 0,
15081            quorum: 0,
15082            met_quorum: true,
15083            uncontested: Some("only candidate A produced a change".to_owned()),
15084        });
15085        state.reviews = vec![ReviewRound {
15086            round: 1,
15087            head: "deadbeef".to_owned(),
15088            verified_head: None,
15089            verified_at: None,
15090            reviews: Vec::new(),
15091            e2e: Vec::new(),
15092            fix: None,
15093            blocking: 0,
15094            answered: 0,
15095            expected: 0,
15096            clean: true,
15097            verify_retried: false,
15098            e2e_deferred: false,
15099            e2e_defer_reason: None,
15100            progressed: false,
15101            vote_split: false,
15102            reconsideration: Vec::new(),
15103            verdict: None,
15104        }];
15105
15106        let mut runner = Runner {
15107            state,
15108            roles: ResolvedRoles {
15109                implementers: Vec::new(),
15110                judges: Vec::new(),
15111                reviewers: Vec::new(),
15112                fixer: None,
15113                conductor: conductor(),
15114                implementer_roster: Vec::new(),
15115                judge_roster: Vec::new(),
15116                reviewer_roster: Vec::new(),
15117            },
15118            sem: Arc::new(Semaphore::new(1)),
15119            pause: Pause::new(),
15120            interrupt: Pause::new(),
15121        };
15122
15123        let started = std::time::Instant::now();
15124        runner.gate().await.expect("gate");
15125        assert!(
15126            started.elapsed() < Duration::from_secs(1),
15127            "a gate with nothing to run must never wait on a lease it never needed"
15128        );
15129        assert!(
15130            runner.state.gate_ran,
15131            "zero commands is still a real, immediate attempt"
15132        );
15133        assert!(runner.state.gate.is_empty());
15134        assert_ne!(
15135            runner.state.status,
15136            RunStatus::Blocked,
15137            "must not read as resource-blocked on a lease it never asked for"
15138        );
15139    }
15140
15141    /// The addendum's second gap: a `verify.gate` command running for real
15142    /// wall-clock time had nothing at all to show for it in `active` before
15143    /// `run_commands` learned to record it — a run could sit in `Gating` for
15144    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
15145    /// what was actually happening. Proven with a genuinely still-running
15146    /// command, not just a before/after check on the final state: a poller
15147    /// task reads the same `run.json` `gate()` is writing, the same way the
15148    /// phone or `magi show` would, while the shell command is still blocked
15149    /// on its own release marker.
15150    #[tokio::test]
15151    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15152        crate::run::pin_test_home();
15153
15154        let tmp = tempfile::tempdir().expect("tempdir");
15155        let repo = tmp.path().join("repo");
15156        std::fs::create_dir_all(&repo).unwrap();
15157        init_repo(&repo);
15158
15159        let mut config = Config::default();
15160        config.verify.gate = vec![
15161            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15162             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15163                .to_owned(),
15164        ];
15165
15166        let mut state = RunState::new(
15167            repo.clone(),
15168            "main".to_owned(),
15169            "deadbeef".to_owned(),
15170            "task".to_owned(),
15171            config,
15172        );
15173        let run_id = state.id.clone();
15174        state.candidates = vec![Candidate {
15175            index: 0,
15176            label: 'A',
15177            agent: "alpha".to_owned(),
15178            branch: "does-not-exist".to_owned(),
15179            worktree: repo.clone(),
15180            summary: String::new(),
15181            stat: String::new(),
15182            files: 0,
15183            commits: 0,
15184            empty: false,
15185            failed: None,
15186            verified_noop: None,
15187            duration_ms: 0,
15188            folded: false,
15189        }];
15190        state.tally = Some(Tally {
15191            first_choice: BTreeMap::from([('A', 1)]),
15192            borda: BTreeMap::new(),
15193            winner: 'A',
15194            rankings: 1,
15195            unanimous_initial: true,
15196            deliberated: false,
15197            changed_votes: 0,
15198            unanimous_final: true,
15199            tie_break: None,
15200            judges: 0,
15201            present: 0,
15202            quorum: 0,
15203            met_quorum: true,
15204            uncontested: Some("only candidate A produced a change".to_owned()),
15205        });
15206        state.reviews = vec![ReviewRound {
15207            round: 1,
15208            head: "deadbeef".to_owned(),
15209            verified_head: None,
15210            verified_at: None,
15211            reviews: Vec::new(),
15212            e2e: Vec::new(),
15213            fix: None,
15214            blocking: 0,
15215            answered: 0,
15216            expected: 0,
15217            clean: true,
15218            verify_retried: false,
15219            e2e_deferred: false,
15220            e2e_defer_reason: None,
15221            progressed: false,
15222            vote_split: false,
15223            reconsideration: Vec::new(),
15224            verdict: None,
15225        }];
15226
15227        let mut runner = Runner {
15228            state,
15229            roles: ResolvedRoles {
15230                implementers: Vec::new(),
15231                judges: Vec::new(),
15232                reviewers: Vec::new(),
15233                fixer: None,
15234                conductor: conductor(),
15235                implementer_roster: Vec::new(),
15236                judge_roster: Vec::new(),
15237                reviewer_roster: Vec::new(),
15238            },
15239            sem: Arc::new(Semaphore::new(1)),
15240            pause: Pause::new(),
15241            interrupt: Pause::new(),
15242        };
15243
15244        let started_marker = repo.join("started.marker");
15245        let release_marker = repo.join("release.marker");
15246        let poller = tokio::spawn(async move {
15247            // Bounded so a regression that never records the task entry
15248            // fails this test in seconds instead of hanging the suite —
15249            // the same shape `a_park_requested_while_a_seat_is_mid_call_
15250            // does_not_cut_it_short` uses for the same reason.
15251            for _ in 0..100 {
15252                if started_marker.exists()
15253                    && let Ok(s) = crate::run::RunState::load(&run_id)
15254                    && let Some(a) = s.active.get("gate")
15255                {
15256                    std::fs::write(&release_marker, b"go").expect("release marker");
15257                    return Some(a.clone());
15258                }
15259                tokio::time::sleep(Duration::from_millis(50)).await;
15260            }
15261            None
15262        });
15263
15264        runner.gate().await.expect("gate");
15265        let captured = poller.await.expect("poller task");
15266        let captured = captured.expect(
15267            "the poller never saw a `gate` task entry in run.json while the command was \
15268             still blocked on its own release marker",
15269        );
15270
15271        assert_eq!(captured.task.as_deref(), Some("gate"));
15272        assert_eq!(captured.node, "gate");
15273        assert_eq!(captured.index, Some(1));
15274        assert_eq!(captured.total, Some(1));
15275        assert!(
15276            captured
15277                .command
15278                .as_deref()
15279                .is_some_and(|c| c.contains("started.marker")),
15280            "{captured:?}"
15281        );
15282
15283        assert!(
15284            runner.state.active.is_empty(),
15285            "the entry must be cleared once the command actually finished: {:?}",
15286            runner.state.active
15287        );
15288        assert!(runner.state.gate_ran);
15289        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15290    }
15291
15292    /// The hand-off over a blocking finding a reviewer rejected on leaves a
15293    /// record for `land`; one with only a Minor, or no reject, leaves none.
15294    #[tokio::test]
15295    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15296        use crate::verdict::{Finding, ReviewVote, Severity};
15297        crate::run::pin_test_home();
15298        let tmp = tempfile::tempdir().expect("tempdir");
15299        let repo = tmp.path().join("repo");
15300        std::fs::create_dir_all(&repo).unwrap();
15301        init_repo(&repo);
15302
15303        for (severity, vote, expect) in [
15304            (Severity::Major, ReviewVote::Reject, true),
15305            (Severity::Minor, ReviewVote::Reject, false),
15306            (Severity::Major, ReviewVote::Approve, false),
15307        ] {
15308            let mut round = review_round(false, 1, 1, 1, false, true);
15309            round.reviews = vec![ReviewRecord {
15310                reviewer: 1,
15311                agent: "alpha".to_owned(),
15312                summary: String::new(),
15313                findings: vec![Finding {
15314                    id: "R1-1-1".to_owned(),
15315                    severity,
15316                    file: None,
15317                    line: None,
15318                    title: "t".to_owned(),
15319                    detail: String::new(),
15320                }],
15321                vote: Some(vote),
15322                failed: None,
15323                duration_ms: 0,
15324                attempts: 0,
15325            }];
15326            let mut state = RunState::new(
15327                repo.clone(),
15328                "main".to_owned(),
15329                "deadbeef".to_owned(),
15330                "task".to_owned(),
15331                Config::default(),
15332            );
15333            state.reviews = vec![round];
15334            let mut runner = Runner {
15335                state,
15336                roles: ResolvedRoles {
15337                    implementers: Vec::new(),
15338                    judges: Vec::new(),
15339                    reviewers: Vec::new(),
15340                    fixer: None,
15341                    conductor: conductor(),
15342                    implementer_roster: Vec::new(),
15343                    judge_roster: Vec::new(),
15344                    reviewer_roster: Vec::new(),
15345                },
15346                sem: Arc::new(Semaphore::new(1)),
15347                pause: Pause::new(),
15348                interrupt: Pause::new(),
15349            };
15350            let shell = runner.state.config.shell();
15351            runner
15352                .stop_reviewing("round budget spent", &shell, &repo)
15353                .await
15354                .expect("stop_reviewing");
15355            assert_eq!(runner.state.status, RunStatus::Gating);
15356            assert_eq!(
15357                runner.state.contested_handoff.is_some(),
15358                expect,
15359                "{severity:?} + {vote:?}"
15360            );
15361        }
15362    }
15363
15364    /// The shape the incident this whole fix responds to actually had: the
15365    /// round budget spent, the last round's own e2e blocked on the shared
15366    /// build cache (held here by a live pid — this test process — exactly
15367    /// `cache`'s own unit tests' pattern for "another owner, still alive"
15368    /// without forking a process). `stop_reviewing` must retry it — not
15369    /// silently leave the round looking untouched (the catch-up-only half of
15370    /// the bug), and not read the contention as a red `e2e` and block the
15371    /// run on it (the other half). Called directly, the same way
15372    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
15373    /// above exercises `gate`, so this never needs a real cargo build to
15374    /// reach: the lease is never released, so `with_cache_lease` never gets
15375    /// past acquiring it into anything that would need a real workspace.
15376    #[tokio::test]
15377    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15378        crate::run::pin_test_home();
15379        let home = crate::run::home();
15380
15381        let tmp = tempfile::tempdir().expect("tempdir");
15382        let repo = tmp.path().join("repo");
15383        std::fs::create_dir_all(&repo).unwrap();
15384        init_repo(&repo);
15385        let head = crate::git::rev_parse(&repo, "HEAD")
15386            .await
15387            .expect("rev-parse");
15388        // Unique to this test, so holding its lease cannot collide with
15389        // another test sharing the same process-wide `home`.
15390        let cache_dir = tmp.path().join("target");
15391
15392        let mut config = Config::default();
15393        config.verify.e2e = vec![format!(
15394            "CARGO_TARGET_DIR='{}' test -f README.md",
15395            cache_dir.display()
15396        )];
15397        config.graph.review_rounds = 1;
15398        // Bounded so a regression that does start waiting fails the test in
15399        // seconds, not hangs it.
15400        config.graph.timeout_verify = Some(2);
15401        config.graph.timeout_resource_wait = 2;
15402
15403        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15404        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15405            .expect("no io error acquiring directly")
15406        {
15407            crate::cache::AcquireOutcome::Acquired(g) => g,
15408            crate::cache::AcquireOutcome::Busy(b) => {
15409                panic!("expected the direct acquire to win the lease first: {b:?}")
15410            }
15411        };
15412
15413        let mut state = RunState::new(
15414            repo.clone(),
15415            "main".to_owned(),
15416            head.clone(),
15417            "task".to_owned(),
15418            config,
15419        );
15420        state.candidates = vec![Candidate {
15421            index: 0,
15422            label: 'A',
15423            agent: "alpha".to_owned(),
15424            branch: "does-not-exist".to_owned(),
15425            worktree: repo.clone(),
15426            summary: String::new(),
15427            stat: String::new(),
15428            files: 0,
15429            commits: 0,
15430            empty: false,
15431            failed: None,
15432            verified_noop: None,
15433            duration_ms: 0,
15434            folded: false,
15435        }];
15436        state.tally = Some(Tally {
15437            first_choice: BTreeMap::from([('A', 1)]),
15438            borda: BTreeMap::new(),
15439            winner: 'A',
15440            rankings: 1,
15441            unanimous_initial: true,
15442            deliberated: false,
15443            changed_votes: 0,
15444            unanimous_final: true,
15445            tie_break: None,
15446            judges: 0,
15447            present: 0,
15448            quorum: 0,
15449            met_quorum: true,
15450            uncontested: Some("only candidate A produced a change".to_owned()),
15451        });
15452        // The round budget's last round, deferred: `needs_catchup_run`'s
15453        // other trigger. `stop_reviewing`'s retry machinery must treat this
15454        // exactly like a resource-blocked attempt once it actually runs.
15455        state.reviews = vec![ReviewRound {
15456            round: 1,
15457            head: head.clone(),
15458            verified_head: None,
15459            verified_at: None,
15460            reviews: Vec::new(),
15461            e2e: Vec::new(),
15462            fix: None,
15463            blocking: 1,
15464            answered: 1,
15465            expected: 1,
15466            clean: false,
15467            verify_retried: false,
15468            e2e_deferred: true,
15469            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15470            progressed: false,
15471            vote_split: false,
15472            reconsideration: Vec::new(),
15473            verdict: None,
15474        }];
15475
15476        let mut runner = Runner {
15477            state,
15478            roles: ResolvedRoles {
15479                implementers: Vec::new(),
15480                judges: Vec::new(),
15481                reviewers: Vec::new(),
15482                fixer: None,
15483                conductor: conductor(),
15484                implementer_roster: Vec::new(),
15485                judge_roster: Vec::new(),
15486                reviewer_roster: Vec::new(),
15487            },
15488            sem: Arc::new(Semaphore::new(1)),
15489            pause: Pause::new(),
15490            interrupt: Pause::new(),
15491        };
15492
15493        let shell = runner.state.config.shell();
15494        runner
15495            .stop_reviewing("round budget spent", &shell, &repo)
15496            .await
15497            .expect("stop_reviewing");
15498
15499        let last = runner.state.reviews.last().expect("round record");
15500        assert_eq!(
15501            last.e2e_status(),
15502            E2eStatus::ResourceBlocked,
15503            "the shared cache is still held; the attempt must read as blocked, not deferred or \
15504             failed: {last:?}"
15505        );
15506        assert_eq!(
15507            last.verified_head.as_deref(),
15508            Some(head.as_str()),
15509            "which commit this attempt targeted is known even though nothing finished checking \
15510             it"
15511        );
15512        let first_attempt_at = last
15513            .verified_at
15514            .expect("when this attempt ran is known too");
15515        assert_ne!(
15516            runner.state.status,
15517            RunStatus::Blocked,
15518            "contention is evidence about the machine, not the patch — it must not settle the \
15519             run as blocked: {:?}",
15520            runner.state.status
15521        );
15522        assert!(
15523            !runner
15524                .state
15525                .events
15526                .iter()
15527                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15528            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15529            runner.state.events
15530        );
15531
15532        // The cache is still held: a later reentry must retry the same
15533        // round's verification again — not leave it looking exactly as
15534        // untouched as the first blocked attempt, which is indistinguishable
15535        // from never having tried again at all.
15536        runner
15537            .stop_reviewing("round budget spent", &shell, &repo)
15538            .await
15539            .expect("stop_reviewing retry");
15540        assert_eq!(
15541            runner.state.reviews.len(),
15542            1,
15543            "no new round was started: {:?}",
15544            runner.state.reviews
15545        );
15546        let last = runner.state.reviews.last().expect("round record");
15547        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15548        assert!(
15549            last.verified_at.expect("still known") > first_attempt_at,
15550            "a second reentry must be a fresh attempt, not a stale copy of the first"
15551        );
15552        assert_ne!(runner.state.status, RunStatus::Blocked);
15553
15554        held.release();
15555    }
15556
15557    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
15558    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
15559    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
15560    /// own top-of-function fast path (`review_conclusion`) correctly reads
15561    /// this shape as `None` rather than guessing `Blocked`, and the loop's
15562    /// own `for` range is empty once the round budget is spent, so
15563    /// `review_loop` must retry the check itself rather than silently doing
15564    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
15565    /// above exercises directly, but through `review_loop`'s own entry point
15566    /// this time, proving the wiring between the two rather than just the
15567    /// retry logic in isolation.
15568    #[tokio::test]
15569    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15570        crate::run::pin_test_home();
15571        let home = crate::run::home();
15572
15573        let tmp = tempfile::tempdir().expect("tempdir");
15574        let repo = tmp.path().join("repo");
15575        std::fs::create_dir_all(&repo).unwrap();
15576        init_repo(&repo);
15577        let head = crate::git::rev_parse(&repo, "HEAD")
15578            .await
15579            .expect("rev-parse");
15580        let cache_dir = tmp.path().join("target");
15581
15582        let mut config = Config::default();
15583        config.verify.e2e = vec![format!(
15584            "CARGO_TARGET_DIR='{}' test -f README.md",
15585            cache_dir.display()
15586        )];
15587        config.graph.review_rounds = 1;
15588        config.graph.timeout_verify = Some(2);
15589        config.graph.timeout_resource_wait = 2;
15590
15591        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15592        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15593            .expect("no io error acquiring directly")
15594        {
15595            crate::cache::AcquireOutcome::Acquired(g) => g,
15596            crate::cache::AcquireOutcome::Busy(b) => {
15597                panic!("expected the direct acquire to win the lease first: {b:?}")
15598            }
15599        };
15600
15601        let mut state = RunState::new(
15602            repo.clone(),
15603            "main".to_owned(),
15604            head.clone(),
15605            "task".to_owned(),
15606            config,
15607        );
15608        state.candidates = vec![Candidate {
15609            index: 0,
15610            label: 'A',
15611            agent: "alpha".to_owned(),
15612            branch: "does-not-exist".to_owned(),
15613            worktree: repo.clone(),
15614            summary: String::new(),
15615            stat: String::new(),
15616            files: 0,
15617            commits: 0,
15618            empty: false,
15619            failed: None,
15620            verified_noop: None,
15621            duration_ms: 0,
15622            folded: false,
15623        }];
15624        state.tally = Some(Tally {
15625            first_choice: BTreeMap::from([('A', 1)]),
15626            borda: BTreeMap::new(),
15627            winner: 'A',
15628            rankings: 1,
15629            unanimous_initial: true,
15630            deliberated: false,
15631            changed_votes: 0,
15632            unanimous_final: true,
15633            tie_break: None,
15634            judges: 0,
15635            present: 0,
15636            quorum: 0,
15637            met_quorum: true,
15638            uncontested: Some("only candidate A produced a change".to_owned()),
15639        });
15640        // The exact shape a prior process's `stop_reviewing` would have left
15641        // on disk: the round budget's last round, a real attempt already
15642        // made and already resource-blocked.
15643        state.reviews = vec![ReviewRound {
15644            round: 1,
15645            head: head.clone(),
15646            verified_head: Some(head.clone()),
15647            verified_at: Some(jiff::Timestamp::now()),
15648            reviews: Vec::new(),
15649            e2e: vec![CommandOutcome {
15650                command: format!(
15651                    "CARGO_TARGET_DIR='{}' test -f README.md",
15652                    cache_dir.display()
15653                ),
15654                code: None,
15655                output_tail: "waiting for the shared build cache".to_owned(),
15656                duration_ms: 0,
15657                resource_blocked: true,
15658            }],
15659            fix: None,
15660            blocking: 1,
15661            answered: 1,
15662            expected: 1,
15663            clean: false,
15664            verify_retried: false,
15665            e2e_deferred: false,
15666            e2e_defer_reason: None,
15667            progressed: false,
15668            vote_split: false,
15669            reconsideration: Vec::new(),
15670            verdict: None,
15671        }];
15672
15673        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15674        let mut runner = Runner {
15675            state,
15676            roles: ResolvedRoles {
15677                implementers: Vec::new(),
15678                judges: Vec::new(),
15679                reviewers: Vec::new(),
15680                fixer: None,
15681                conductor: conductor(),
15682                implementer_roster: Vec::new(),
15683                judge_roster: Vec::new(),
15684                reviewer_roster: Vec::new(),
15685            },
15686            sem: Arc::new(Semaphore::new(1)),
15687            pause: Pause::new(),
15688            interrupt: Pause::new(),
15689        };
15690
15691        // The lease is still held throughout, so this reentry's own retry is
15692        // also contended — proving `review_loop` actually tried again (not
15693        // that it happened to succeed) is what the timestamp comparison
15694        // below is for.
15695        runner.review_loop().await.expect("review_loop");
15696
15697        assert_eq!(
15698            runner.state.reviews.len(),
15699            1,
15700            "no new round was started on top of the unresolved one: {:?}",
15701            runner.state.reviews
15702        );
15703        let last = &runner.state.reviews[0];
15704        assert_eq!(
15705            last.e2e_status(),
15706            E2eStatus::ResourceBlocked,
15707            "still contended: {last:?}"
15708        );
15709        assert!(
15710            last.verified_at.expect("still known") > first_attempt_at,
15711            "review_loop must have actually retried the check, not left it exactly as found"
15712        );
15713        assert_ne!(
15714            runner.state.status,
15715            RunStatus::Blocked,
15716            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15717            runner.state.status
15718        );
15719
15720        held.release();
15721    }
15722
15723    #[tokio::test]
15724    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15725        crate::run::pin_test_home();
15726        let tmp = tempfile::tempdir().expect("tempdir");
15727        let repo = tmp.path().join("repo");
15728        std::fs::create_dir_all(&repo).unwrap();
15729        init_repo(&repo);
15730
15731        let mut config = Config::default();
15732        config.merge.mode = MergeMode::Pr;
15733        config.graph.land = true;
15734        config.graph.land_approval = false;
15735
15736        let mut state = RunState::new(
15737            repo.clone(),
15738            "main".to_owned(),
15739            "deadbeef".to_owned(),
15740            "task".to_owned(),
15741            config,
15742        );
15743        state.candidates = vec![Candidate {
15744            index: 0,
15745            label: 'A',
15746            agent: "alpha".to_owned(),
15747            branch: "does-not-exist".to_owned(),
15748            worktree: repo.clone(),
15749            summary: String::new(),
15750            stat: String::new(),
15751            files: 0,
15752            commits: 0,
15753            empty: false,
15754            failed: None,
15755            verified_noop: None,
15756            duration_ms: 0,
15757            folded: false,
15758        }];
15759        state.tally = Some(Tally {
15760            first_choice: BTreeMap::from([('A', 1)]),
15761            borda: BTreeMap::new(),
15762            winner: 'A',
15763            rankings: 1,
15764            unanimous_initial: true,
15765            deliberated: false,
15766            changed_votes: 0,
15767            unanimous_final: true,
15768            tie_break: None,
15769            judges: 0,
15770            present: 0,
15771            quorum: 0,
15772            met_quorum: true,
15773            uncontested: Some("only candidate A produced a change".to_owned()),
15774        });
15775        state.reviews = vec![ReviewRound {
15776            round: 1,
15777            head: "deadbeef".to_owned(),
15778            verified_head: None,
15779            verified_at: None,
15780            reviews: Vec::new(),
15781            e2e: Vec::new(),
15782            fix: None,
15783            blocking: 0,
15784            answered: 0,
15785            expected: 0,
15786            clean: true,
15787            verify_retried: false,
15788            e2e_deferred: false,
15789            e2e_defer_reason: None,
15790            progressed: false,
15791            vote_split: false,
15792            reconsideration: Vec::new(),
15793            verdict: None,
15794        }];
15795        state.gate = vec![CommandOutcome {
15796            command: "test".to_owned(),
15797            code: Some(0),
15798            output_tail: String::new(),
15799            duration_ms: 0,
15800            resource_blocked: false,
15801        }];
15802        state.gate_ran = true;
15803        // A first pass through `merge` already pushed and opened this pull
15804        // request; `status` is `Landing` because a previous call into `land`
15805        // parked or was interrupted before it reached a terminal outcome.
15806        state.status = RunStatus::Landing;
15807        state.merge = Some(MergeOutcome {
15808            mode: MergeMode::Pr,
15809            ok: true,
15810            detail: "https://example.invalid/x/y/pull/1".to_owned(),
15811            empty: false,
15812        });
15813
15814        // The Landing-resume shortcut calls `run_land` directly rather than
15815        // through `merge`, which is exactly the call site that used to skip
15816        // `settle_questions` - see the fixture below.
15817        ask_test_home();
15818        // A real resume loads a persisted run; driver ownership is checked
15819        // against that record before the Landing shortcut is entered.
15820        state.save().expect("save landing run");
15821        let store = ask::Questions::open();
15822        let q = ask_open_question(&store, &state.id);
15823
15824        let mut runner = Runner {
15825            state,
15826            roles: ResolvedRoles {
15827                implementers: Vec::new(),
15828                judges: Vec::new(),
15829                reviewers: Vec::new(),
15830                fixer: None,
15831                conductor: conductor(),
15832                implementer_roster: Vec::new(),
15833                judge_roster: Vec::new(),
15834                reviewer_roster: Vec::new(),
15835            },
15836            sem: Arc::new(Semaphore::new(1)),
15837            pause: Pause::new(),
15838            interrupt: Pause::new(),
15839        };
15840
15841        // `execute`, not `merge` directly: the Landing-resume shortcut lives
15842        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
15843        // exactly because `review_loop` would otherwise clobber the marker
15844        // first.
15845        runner.execute().await.expect("execute");
15846
15847        assert_eq!(
15848            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15849            Some("https://example.invalid/x/y/pull/1"),
15850            "reentry must not push again or open a second pull request over the \
15851             one `land` is already watching"
15852        );
15853        assert_ne!(
15854            runner.state.status,
15855            RunStatus::Landing,
15856            "land could not actually reach the fake pull request, so it must \
15857             have given up rather than left the run silently parked forever"
15858        );
15859        // `land` could not reach the fake pull request, so it gave up into
15860        // `Blocked` - still resumable, so the question must not have been
15861        // swept just because this branch now also calls `settle_questions`.
15862        assert_eq!(runner.state.status, RunStatus::Blocked);
15863        assert!(
15864            store.get(&q.id).unwrap().status.open(),
15865            "Blocked is still alive; settle_questions must have been a no-op here"
15866        );
15867    }
15868
15869    fn state_with_round(round: ReviewRound) -> RunState {
15870        let mut s = RunState::new(
15871            PathBuf::from("/repo"),
15872            "main".to_owned(),
15873            "abc1234".to_owned(),
15874            "add retries".to_owned(),
15875            Config::default(),
15876        );
15877        s.reviews = vec![round];
15878        s
15879    }
15880
15881    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15882        crate::verdict::Finding {
15883            id: id.to_owned(),
15884            severity,
15885            file: None,
15886            line: None,
15887            title: title.to_owned(),
15888            detail: String::new(),
15889        }
15890    }
15891
15892    #[test]
15893    fn pr_body_names_open_findings_and_declined_ones() {
15894        let round = ReviewRound {
15895            round: 2,
15896            head: "deadbee".to_owned(),
15897            verified_head: None,
15898            verified_at: None,
15899            reviews: vec![ReviewRecord {
15900                attempts: 0,
15901                reviewer: 1,
15902                agent: "alpha".to_owned(),
15903                summary: String::new(),
15904                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15905                vote: None,
15906                failed: None,
15907                duration_ms: 0,
15908            }],
15909            e2e: vec![CommandOutcome {
15910                command: "cargo test".to_owned(),
15911                code: Some(0),
15912                output_tail: String::new(),
15913                duration_ms: 0,
15914                resource_blocked: false,
15915            }],
15916            verify_retried: false,
15917            e2e_deferred: false,
15918            e2e_defer_reason: None,
15919            fix: Some(FixRecord {
15920                agent: "alpha".to_owned(),
15921                addressed: Vec::new(),
15922                rejected: vec![crate::verdict::Rejection {
15923                    id: "R1-1-1".to_owned(),
15924                    why: "not reachable from any caller".to_owned(),
15925                }],
15926                notes: String::new(),
15927                committed: true,
15928                failed: None,
15929                duration_ms: 0,
15930                continuation: None,
15931            }),
15932            blocking: 0,
15933            answered: 1,
15934            expected: 1,
15935            clean: false,
15936            progressed: true,
15937            vote_split: false,
15938            reconsideration: Vec::new(),
15939            verdict: None,
15940        };
15941        let state = state_with_round(round);
15942        let body = pr_message(&state, 'A').body;
15943
15944        assert!(body.contains("add retries"), "the task must still be there");
15945        assert!(body.contains("R2-1-1"), "{body}");
15946        assert!(!body.contains("unused import"), "{body}");
15947        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15948        assert!(
15949            !body.contains("not reachable from any caller"),
15950            "the reason it was declined: {body}"
15951        );
15952    }
15953
15954    #[test]
15955    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15956        let round = ReviewRound {
15957            round: 1,
15958            head: "deadbee".to_owned(),
15959            verified_head: None,
15960            verified_at: None,
15961            reviews: vec![ReviewRecord {
15962                attempts: 0,
15963                reviewer: 1,
15964                agent: "alpha".to_owned(),
15965                summary: String::new(),
15966                findings: Vec::new(),
15967                vote: None,
15968                failed: None,
15969                duration_ms: 0,
15970            }],
15971            e2e: Vec::new(),
15972            verify_retried: false,
15973            e2e_deferred: false,
15974            e2e_defer_reason: None,
15975            fix: None,
15976            blocking: 0,
15977            answered: 1,
15978            expected: 1,
15979            clean: true,
15980            progressed: false,
15981            vote_split: false,
15982            reconsideration: Vec::new(),
15983            verdict: None,
15984        };
15985        let state = state_with_round(round);
15986        let body = pr_message(&state, 'A').body;
15987        assert!(!body.contains("Open review findings"), "{body}");
15988        assert!(!body.contains("Declined"), "{body}");
15989    }
15990
15991    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15992        let mut state = RunState::new(
15993            PathBuf::from("/repo"),
15994            "main".to_owned(),
15995            "abc1234".to_owned(),
15996            instruction.to_owned(),
15997            Config::default(),
15998        );
15999        state.candidates.push(Candidate {
16000            index: 0,
16001            label: 'A',
16002            agent: "alpha".to_owned(),
16003            branch: "magi/x/A".to_owned(),
16004            worktree: PathBuf::from("/wt"),
16005            summary: summary.to_owned(),
16006            stat: String::new(),
16007            files: 1,
16008            commits: 1,
16009            empty: false,
16010            failed: None,
16011            verified_noop: None,
16012            folded: false,
16013            duration_ms: 0,
16014        });
16015        state
16016    }
16017
16018    fn review_state(subjects: &[&str]) -> RunState {
16019        let mut state = state_with_summary(
16020            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16021            "",
16022        );
16023        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16024        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16025        state
16026    }
16027
16028    /// A branch rebased onto a main that moved past the recorded
16029    /// `base_commit` is titled from its own first commit, never main's.
16030    #[tokio::test]
16031    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16032        ask_test_home();
16033        let tmp = tempfile::tempdir().unwrap();
16034        let repo = tmp.path().join("repo");
16035        std::fs::create_dir_all(&repo).unwrap();
16036        init_repo(&repo);
16037        let origin = tmp.path().join("origin.git");
16038        let g = |dir: &Path, args: &[&str]| {
16039            let out = std::process::Command::new("git")
16040                .args(args)
16041                .current_dir(dir)
16042                .quiet()
16043                .output()
16044                .expect("spawn git");
16045            assert!(
16046                out.status.success(),
16047                "git {args:?}: {}",
16048                String::from_utf8_lossy(&out.stderr)
16049            );
16050        };
16051        g(
16052            tmp.path(),
16053            &[
16054                "clone",
16055                "--bare",
16056                "-q",
16057                repo.to_str().unwrap(),
16058                origin.to_str().unwrap(),
16059            ],
16060        );
16061        g(
16062            &repo,
16063            &["remote", "add", "origin", origin.to_str().unwrap()],
16064        );
16065        let c1 = git::rev_parse(&repo, "main").await.unwrap();
16066
16067        // Main moves on; the branch is built on top of the new main.
16068        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16069        g(&repo, &["add", "-A"]);
16070        g(
16071            &repo,
16072            &["commit", "-q", "-m", "chore(deps): update a crate"],
16073        );
16074        g(&repo, &["push", "-q", "origin", "main"]);
16075        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16076        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16077        g(&repo, &["add", "-A"]);
16078        g(
16079            &repo,
16080            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16081        );
16082        g(&repo, &["checkout", "-q", "main"]);
16083
16084        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16085        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16086        // Without a readable tracking ref the recorded base's merge base is used.
16087        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16088        assert_eq!(fallback, c1);
16089
16090        let mut state = review_state(&[
16091            "chore(deps): update a crate",
16092            "fix(daemon): apply a chosen action",
16093        ]);
16094        state.repo = repo.clone();
16095        state.base_branch = "main".to_owned();
16096        state.base_commit = c1;
16097        refresh_reviewed_commits(&mut state, "feat/own").await;
16098        assert_eq!(
16099            state.reviewed_commits,
16100            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16101        );
16102        assert_eq!(
16103            review_title(&state).as_deref(),
16104            Some("fix(daemon): apply a chosen action")
16105        );
16106        assert_eq!(
16107            leaked_subjects(&state, "feat/own").await,
16108            Some(vec!["chore(deps): update a crate".to_owned()])
16109        );
16110        // A stale tracking ref is still used when the fetch fails, but the
16111        // leak list is withheld.
16112        state.config.merge.remote = "nowhere".to_owned();
16113        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16114    }
16115
16116    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
16117    /// with `origin/main` one commit behind it, and a summarizing agent that
16118    /// prints whatever `reply` holds.
16119    struct SummaryFixture {
16120        _tmp: tempfile::TempDir,
16121        repo: PathBuf,
16122        reply: PathBuf,
16123        runner: Runner,
16124    }
16125
16126    fn summary_fixture() -> SummaryFixture {
16127        ask_test_home();
16128        let tmp = tempfile::tempdir().unwrap();
16129        let repo = tmp.path().join("repo");
16130        std::fs::create_dir_all(&repo).unwrap();
16131        init_repo(&repo);
16132        let origin = tmp.path().join("origin.git");
16133        let g = |args: &[&str]| {
16134            let out = std::process::Command::new("git")
16135                .args(args)
16136                .current_dir(&repo)
16137                .quiet()
16138                .output()
16139                .expect("spawn git");
16140            assert!(
16141                out.status.success(),
16142                "git {args:?}: {}",
16143                String::from_utf8_lossy(&out.stderr)
16144            );
16145        };
16146        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16147        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16148        g(&["push", "-q", "origin", "main"]);
16149        g(&["checkout", "-q", "-b", "feat/own"]);
16150        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16151        g(&["add", "-A"]);
16152        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16153        let reply = tmp.path().join("reply.txt");
16154        std::fs::write(
16155            &reply,
16156            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16157        )
16158        .unwrap();
16159        let mut runner = runner_at(RunStatus::Gating);
16160        let mut state = review_state(&["feat(own): add the own file"]);
16161        state.repo = repo.clone();
16162        state.base_branch = "main".to_owned();
16163        state.candidates[0].branch = "feat/own".to_owned();
16164        state.candidates[0].worktree = repo.clone();
16165        let mut author = spec("alpha");
16166        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16167        author
16168            .env
16169            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16170        state.config.agents = vec![author];
16171        runner.state = state;
16172        SummaryFixture {
16173            _tmp: tmp,
16174            repo,
16175            reply,
16176            runner,
16177        }
16178    }
16179
16180    async fn posted(f: &mut SummaryFixture) -> PrMessage {
16181        let winner = f.runner.state.candidates[0].clone();
16182        f.runner
16183            .guarded_pr_message(&winner, None, true)
16184            .await
16185            .unwrap()
16186            .expect("an English description never parks")
16187    }
16188
16189    fn clean_round_with_minor_findings() -> ReviewRound {
16190        let mut round = passing_round(2, None);
16191        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16192        round.reviews = vec![ReviewRecord {
16193            attempts: 0,
16194            reviewer: 1,
16195            agent: "alpha".into(),
16196            summary: "内部レビュー".into(),
16197            findings: vec![
16198                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16199                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16200            ],
16201            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16202            failed: None,
16203            duration_ms: 0,
16204        }];
16205        round
16206    }
16207
16208    #[tokio::test]
16209    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16210        for review_only in [false, true] {
16211            let mut f = summary_fixture();
16212            if !review_only {
16213                f.runner.state.candidates[0].agent = "alpha".into();
16214            }
16215            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16216            assert!(f.runner.state.reviews[0].clean);
16217            assert!(f.runner.state.reviews[0].fix.is_none());
16218            assert!(f.runner.state.open_findings().is_empty());
16219            assert_eq!(
16220                crate::pr_outcome::required_finding_ids(&f.runner.state),
16221                ["R2-1-1", "R2-1-2"]
16222            );
16223            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16224            let evidence = crate::pr_outcome::summary_evidence(
16225                &f.runner.state,
16226                crate::pr_outcome::FinalHead::At(&head),
16227            );
16228            assert!(
16229                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16230            );
16231            // The ordinary reply omits both IDs: no cached description is adopted.
16232            let omitted = posted(&mut f).await;
16233            assert!(omitted.body.contains("No description"));
16234            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16235            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16236            assert!(f.runner.state.change_summary.is_none());
16237            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16238            let described = posted(&mut f).await;
16239            assert!(described.body.contains("improve documentation for callers"));
16240            assert!(described.body.contains("remaining edge case"));
16241            assert!(described.body.contains("## Open review findings"));
16242            assert!(!described.body.contains("未修正"));
16243        }
16244    }
16245
16246    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16247        let mut f = summary_fixture();
16248        if !review_only {
16249            f.runner.state.candidates[0].agent = "alpha".into();
16250        }
16251        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16252        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16253        fixed
16254            .fix
16255            .as_mut()
16256            .unwrap()
16257            .rejected
16258            .push(crate::verdict::Rejection {
16259                id: "R1-1-1".into(),
16260                why: "呼び出されないコードのため辞退".into(),
16261            });
16262        let mut clean = clean_round_with_minor_findings();
16263        clean.verified_head = Some(head.clone());
16264        f.runner.state.reviews = vec![fixed, clean];
16265        ok_gate(&mut f.runner.state, Some(&head));
16266        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16267        f.runner.state.config.agents.push(scripted("beta", reply));
16268        f.runner.state.config.roles.language_judge =
16269            Some(crate::config::AgentChoice::One("beta".into()));
16270        f
16271    }
16272
16273    #[tokio::test]
16274    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16275        for review_only in [false, true] {
16276            let mut f = rewrite_fixture_with_findings(review_only,
16277                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16278            let m = posted(&mut f).await;
16279            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16280            let expected =
16281                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16282            assert!(m.body.contains(&expected));
16283            assert!(m.body.contains("## Review rounds"));
16284            assert!(
16285                m.body
16286                    .contains("Gate (`verify.gate`): passed on the final commit")
16287            );
16288            assert!(
16289                m.body
16290                    .contains("Tests (`verify.e2e`): passed on the final commit")
16291            );
16292            let c = f.runner.state.change_summary.as_ref().unwrap();
16293            assert!(
16294                m.body
16295                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16296            );
16297            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16298            assert!(crate::pr_outcome::covers_required_findings(
16299                &f.runner.state,
16300                &m.body
16301            ));
16302            assert!(m.body.contains("documentation remains incomplete"));
16303            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16304            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16305            assert!(f.runner.state.github_text.is_none());
16306        }
16307    }
16308
16309    #[tokio::test]
16310    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16311        for review_only in [false, true] {
16312            for reply in [
16313                "TITLE: fix: public description\nA short English description.",
16314                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16315            ] {
16316                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16317                let winner = f.runner.state.candidates[0].clone();
16318                let posted = f
16319                    .runner
16320                    .guarded_pr_message(&winner, None, true)
16321                    .await
16322                    .unwrap();
16323                assert!(posted.is_none());
16324                assert!(f.runner.state.parked);
16325                let withheld = f.runner.state.github_text.as_ref().unwrap();
16326                assert!(
16327                    withheld.rewrite.is_none(),
16328                    "an incomplete rewrite must not be selectable"
16329                );
16330                let q = ask::Questions::open()
16331                    .get(withheld.question.as_ref().unwrap())
16332                    .unwrap();
16333                assert!(
16334                    !q.choices
16335                        .contains(&crate::github_text::USE_REWRITE.to_owned())
16336                );
16337                assert!(
16338                    q.detail.contains("R1-1-1")
16339                        && q.detail.contains("R2-1-1")
16340                        && q.detail.contains("R2-1-2")
16341                );
16342            }
16343        }
16344    }
16345
16346    #[tokio::test]
16347    async fn japanese_reports_stay_local_in_both_public_run_modes() {
16348        for review_only in [false, true] {
16349            let mut f = summary_fixture();
16350            if !review_only {
16351                f.runner.state.candidates[0].agent = "alpha".into();
16352            }
16353            f.runner.state.instruction = "内部タスクの説明".into();
16354            f.runner.state.candidates[0].summary = "内部の候補報告".into();
16355            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16356            f.runner.state.reviews = vec![
16357                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16358                round_with_notes(2, Some("二回目の内部修正報告です。")),
16359                passing_round(3, Some(&head)),
16360            ];
16361            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16362            ok_gate(&mut f.runner.state, Some(&head));
16363            let captured = f.repo.join("public-prompt.txt");
16364            let author = &mut f.runner.state.config.agents[0];
16365            author.command = vec![
16366                "sh".into(),
16367                "-c".into(),
16368                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16369            ];
16370            author.env.insert(
16371                "PROMPT_FILE".into(),
16372                captured.to_string_lossy().into_owned(),
16373            );
16374            let m = posted(&mut f).await;
16375            assert!(m.body.contains("Adds `own.txt`"));
16376            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16377            assert!(m.body.contains("R1-1-1"));
16378            assert!(
16379                m.body
16380                    .contains("Gate (`verify.gate`): passed on the final commit")
16381            );
16382            assert!(
16383                m.body
16384                    .contains("Tests (`verify.e2e`): passed on the final commit")
16385            );
16386            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16387            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16388            let prompt = std::fs::read_to_string(captured).unwrap();
16389            assert!(prompt.contains(&format!(
16390                "git diff {} {head}",
16391                f.runner.state.change_summary.as_ref().unwrap().base
16392            )));
16393            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16394            assert_eq!(
16395                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16396                "二回目の内部修正報告です。"
16397            );
16398        }
16399    }
16400
16401    #[tokio::test]
16402    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16403        for review_only in [false, true] {
16404            let mut f = summary_fixture();
16405            if !review_only {
16406                f.runner.state.candidates[0].agent = "alpha".into();
16407            }
16408            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16409            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16410            std::fs::write(&f.reply, "no usable description").unwrap();
16411            let failed = posted(&mut f).await;
16412            assert!(
16413                failed
16414                    .body
16415                    .contains("No description of what this change does could be generated")
16416            );
16417            assert!(failed.body.is_ascii());
16418            assert!(f.runner.state.github_text.is_none());
16419            assert!(f.runner.state.change_summary.is_none());
16420            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16421            let evidence = crate::pr_outcome::summary_evidence(
16422                &f.runner.state,
16423                crate::pr_outcome::FinalHead::At(&head),
16424            );
16425            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16426            assert!(
16427                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16428                    .exists()
16429            );
16430            f.runner.state =
16431                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16432            std::fs::write(
16433                &f.reply,
16434                "TITLE: feat(own): add the own file\nAdds the committed own file.",
16435            )
16436            .unwrap();
16437            let retried = posted(&mut f).await;
16438            assert!(retried.body.contains("Adds the committed own file."));
16439            assert_eq!(
16440                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16441                "日本語の修正報告"
16442            );
16443        }
16444    }
16445
16446    #[tokio::test]
16447    async fn same_head_verification_and_findings_invalidate_public_prose() {
16448        let mut f = summary_fixture();
16449        posted(&mut f).await;
16450        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16451        let original = f
16452            .runner
16453            .state
16454            .change_summary
16455            .as_ref()
16456            .unwrap()
16457            .evidence
16458            .clone();
16459        ok_gate(&mut f.runner.state, Some(&head));
16460        std::fs::write(
16461            &f.reply,
16462            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16463        )
16464        .unwrap();
16465        let verified = posted(&mut f).await;
16466        assert!(verified.body.contains("now verified"));
16467        assert_ne!(
16468            f.runner.state.change_summary.as_ref().unwrap().evidence,
16469            original
16470        );
16471        let mut round = round_with_notes(1, Some("内部報告"));
16472        round.reviews.push(ReviewRecord {
16473            attempts: 0,
16474            reviewer: 1,
16475            agent: "alpha".into(),
16476            summary: "内部レビュー".into(),
16477            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16478            vote: None,
16479            failed: None,
16480            duration_ms: 0,
16481        });
16482        round
16483            .fix
16484            .as_mut()
16485            .unwrap()
16486            .rejected
16487            .push(crate::verdict::Rejection {
16488                id: "R1-1-1".into(),
16489                why: "呼び出し元から到達できない".into(),
16490            });
16491        f.runner.state.reviews = vec![round];
16492        // A description omitting the required ID is unusable, even with a title.
16493        let omitted = posted(&mut f).await;
16494        assert!(omitted.body.contains("No description"));
16495        assert!(omitted.body.contains("R1-1-1"));
16496        assert!(!omitted.body.contains("到達"));
16497        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16498        let described = posted(&mut f).await;
16499        assert!(described.body.contains("unreachable from callers"));
16500        assert!(described.body.contains("R1-1-1"));
16501        assert!(!described.body.contains("到達"));
16502        let evidence = f
16503            .runner
16504            .state
16505            .change_summary
16506            .as_ref()
16507            .unwrap()
16508            .evidence
16509            .clone();
16510        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16511        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16512        assert!(posted(&mut f).await.body.contains("rationale updated"));
16513        assert_ne!(
16514            f.runner.state.change_summary.as_ref().unwrap().evidence,
16515            evidence
16516        );
16517        // Old serialized summaries have no evidence fingerprint and must regenerate.
16518        f.runner
16519            .state
16520            .change_summary
16521            .as_mut()
16522            .unwrap()
16523            .evidence
16524            .clear();
16525        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16526        assert!(posted(&mut f).await.body.contains("current specification"));
16527    }
16528
16529    #[tokio::test]
16530    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16531        let mut f = summary_fixture();
16532        let m = posted(&mut f).await;
16533        assert_eq!(m.title, "feat(own): add the own file");
16534        assert!(
16535            m.body
16536                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16537            "{}",
16538            m.body
16539        );
16540        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16541        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16542        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16543        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16544        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16545        let kept = f.runner.state.change_summary.as_ref().unwrap();
16546        assert_eq!(
16547            (kept.head.as_str(), kept.base.as_str()),
16548            (head.as_str(), base.as_str())
16549        );
16550        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16551    }
16552
16553    #[tokio::test]
16554    async fn review_only_summary_receives_the_configured_github_language() {
16555        let mut f = summary_fixture();
16556        f.runner.state.config.graph.github_language = "ja".into();
16557        let captured = f.repo.join("summary-prompt.txt");
16558        let author = &mut f.runner.state.config.agents[0];
16559        author.command = vec![
16560            "sh".into(),
16561            "-c".into(),
16562            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16563        ];
16564        author.env.insert(
16565            "PROMPT_FILE".into(),
16566            captured.to_string_lossy().into_owned(),
16567        );
16568        std::fs::write(
16569            &f.reply,
16570            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16571        )
16572        .unwrap();
16573        let m = posted(&mut f).await;
16574        let prompt = std::fs::read_to_string(captured).unwrap();
16575        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16576        assert!(!prompt.contains("in English"));
16577        assert_eq!(m.title, "feat(own): ファイルを追加");
16578        assert!(m.body.contains("own.txt を追加します。"));
16579    }
16580
16581    #[tokio::test]
16582    async fn review_fixes_come_after_the_whole_change_summary() {
16583        let mut f = summary_fixture();
16584        f.runner.state.reviews = vec![
16585            round_with_notes(1, Some("handled the PATH case")),
16586            round_with_notes(2, None),
16587        ];
16588        let m = posted(&mut f).await;
16589        let summary = m.body.find("## Summary").expect("summary");
16590        let fixes = m.body.find("## Review rounds").expect("review rounds");
16591        assert!(summary < fixes, "{}", m.body);
16592        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16593        assert!(m.body.contains("## Final verification"), "{}", m.body);
16594    }
16595
16596    #[tokio::test]
16597    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16598        let mut f = summary_fixture();
16599        let first = posted(&mut f).await;
16600        let first_head = f.runner.state.change_summary.clone().unwrap().head;
16601        // The same head reuses the record: a different reply is not read.
16602        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16603        let again = posted(&mut f).await;
16604        assert_eq!((again.title, again.body), (first.title, first.body));
16605
16606        let g = |args: &[&str]| {
16607            let out = std::process::Command::new("git")
16608                .args(args)
16609                .current_dir(&f.repo)
16610                .quiet()
16611                .output()
16612                .expect("spawn git");
16613            assert!(out.status.success(), "git {args:?}");
16614        };
16615        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16616        g(&["add", "-A"]);
16617        g(&["commit", "-q", "-m", "fix(own): handle more"]);
16618        let second = posted(&mut f).await;
16619        let kept = f.runner.state.change_summary.clone().unwrap();
16620        assert_ne!(kept.head, first_head);
16621        assert!(second.body.contains("Not read yet."), "{}", second.body);
16622        assert!(!second.body.contains("so the first summary is read"));
16623        assert_eq!(second.title, "feat(own): other");
16624    }
16625
16626    #[tokio::test]
16627    async fn an_unavailable_summary_is_stated_and_never_invented() {
16628        let mut f = summary_fixture();
16629        std::fs::write(&f.reply, "no title line here").unwrap();
16630        let m = posted(&mut f).await;
16631        assert!(
16632            m.body
16633                .contains("No description of what this change does could be generated"),
16634            "{}",
16635            m.body
16636        );
16637        assert!(m.body.contains("internal reports remain in local records"));
16638        assert!(f.runner.state.change_summary.is_none());
16639        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16640        // A preview asks nobody and claims nothing.
16641        let winner = f.runner.state.candidates[0].clone();
16642        let preview = f
16643            .runner
16644            .guarded_pr_message(&winner, None, false)
16645            .await
16646            .unwrap()
16647            .unwrap();
16648        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16649    }
16650
16651    #[test]
16652    fn pr_message_review_single_commit_uses_its_subject() {
16653        let state = review_state(&["feat(nats): per-role user"]);
16654        let m = pr_message(&state, 'A');
16655        assert_eq!(m.title, "feat(nats): per-role user");
16656        assert!(!m.body.contains("Review the work already"), "{}", m.body);
16657        assert!(m.body.contains("## Commits under review"), "{}", m.body);
16658    }
16659
16660    #[test]
16661    fn pr_message_review_multi_commit_takes_the_oldest() {
16662        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16663        let m = pr_message(&state, 'A');
16664        assert_eq!(m.title, "feat: the change");
16665        for s in ["feat: the change", "fix: typo", "fix: again"] {
16666            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16667        }
16668    }
16669
16670    #[test]
16671    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16672        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16673            let mut state = review_state(&[first, "fix: later fixup"]);
16674            state.candidates[0].branch = "機能/ログイン".to_owned();
16675            let m = pr_message(&state, 'A');
16676            assert!(
16677                m.title.starts_with("chore: land candidate A of run"),
16678                "{}",
16679                m.title
16680            );
16681        }
16682    }
16683
16684    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16685        BranchFacts {
16686            commits: commits
16687                .iter()
16688                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16689                .collect(),
16690            stat: stat.to_owned(),
16691        }
16692    }
16693
16694    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16695        let mut r = review_round(true, 0, 1, 1, true, true);
16696        r.round = round;
16697        r.fix = notes.map(|n| FixRecord {
16698            agent: "fixer".to_owned(),
16699            addressed: Vec::new(),
16700            rejected: Vec::new(),
16701            notes: n.to_owned(),
16702            committed: true,
16703            failed: None,
16704            duration_ms: 0,
16705            continuation: None,
16706        });
16707        r
16708    }
16709
16710    #[test]
16711    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16712        let state = review_state(&["ignored"]);
16713        let f = facts(
16714            &[
16715                (
16716                    "fix(login): resolve PATH on macOS",
16717                    "Login shells skip rc files.",
16718                ),
16719                ("fix: address review", ""),
16720            ],
16721            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16722        );
16723        let m = pr_message_with(&state, 'A', Some(&f));
16724        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16725        assert!(
16726            m.body
16727                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
16728            "{}",
16729            m.body
16730        );
16731        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16732        assert!(m.body.contains("## Diff stat"), "{}", m.body);
16733        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16734        for banned in [
16735            "Review the work already",
16736            "no task statement",
16737            "Original task",
16738        ] {
16739            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16740        }
16741        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16742    }
16743
16744    #[test]
16745    fn pr_message_review_truncates_a_huge_first_commit_body() {
16746        let state = review_state(&["ignored"]);
16747        let f = facts(
16748            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16749            "s",
16750        );
16751        let m = pr_message_with(&state, 'A', Some(&f));
16752        assert!(m.body.len() < 30_000, "{}", m.body.len());
16753        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16754        assert!(
16755            m.body.contains("- ... 1 more commit(s)"),
16756            "{}",
16757            m.body.len()
16758        );
16759        assert!(m.body.ends_with("magi:candidate-a\n"));
16760    }
16761
16762    #[test]
16763    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16764        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16765        assert_eq!(m.title, "feat: x");
16766        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16767        assert!(!m.body.contains("Diff stat"), "{}", m.body);
16768        assert!(!m.body.contains("no task statement"), "{}", m.body);
16769    }
16770
16771    #[test]
16772    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16773        let mut state = review_state(&["日本語の件名"]);
16774        state.candidates[0].branch = "fix/macos-login-path".to_owned();
16775        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16776        state.candidates[0].branch = "機能/ログイン".to_owned();
16777        assert!(
16778            pr_message(&state, 'A')
16779                .title
16780                .starts_with("chore: land candidate A")
16781        );
16782    }
16783
16784    #[test]
16785    fn pr_message_review_fixes_survive_a_clean_final_round() {
16786        let mut state = review_state(&["feat: x"]);
16787        state.reviews = vec![
16788            round_with_notes(1, Some("handled the PATH case")),
16789            round_with_notes(2, None),
16790        ];
16791        let body = pr_message(&state, 'A').body;
16792        assert!(
16793            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16794            "{body}"
16795        );
16796        assert!(body.contains("### Round 1"), "{body}");
16797
16798        state.reviews = vec![
16799            round_with_notes(1, Some("first fix")),
16800            round_with_notes(2, Some("")),
16801            round_with_notes(3, Some("second fix")),
16802            round_with_notes(4, None),
16803        ];
16804        let body = pr_message(&state, 'A').body;
16805        assert!(
16806            body.contains("### Round 1") && !body.contains("first fix"),
16807            "{body}"
16808        );
16809        assert!(
16810            body.contains("### Round 3") && !body.contains("second fix"),
16811            "{body}"
16812        );
16813        assert!(body.contains("### Round 2"), "{body}");
16814    }
16815
16816    const FINAL: &str = "abcdef1234567890";
16817
16818    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16819        let mut r = round_with_notes(round, None);
16820        r.verified_head = verified.map(str::to_owned);
16821        r
16822    }
16823
16824    fn ok_gate(state: &mut RunState, head: Option<&str>) {
16825        state.gate = vec![CommandOutcome {
16826            command: "gate".to_owned(),
16827            code: Some(0),
16828            output_tail: String::new(),
16829            duration_ms: 0,
16830            resource_blocked: false,
16831        }];
16832        state.gate_ran = true;
16833        state.gate_head = head.map(str::to_owned);
16834    }
16835
16836    fn body_at(state: &RunState) -> String {
16837        pr_message_raw(
16838            state,
16839            'A',
16840            None,
16841            crate::pr_outcome::FinalHead::At(FINAL),
16842            ChangeNote::NotAsked,
16843        )
16844        .body
16845    }
16846
16847    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16848
16849    #[test]
16850    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16851        let mut state = review_state(&["feat: x"]);
16852        state.reviews = vec![
16853            round_with_notes(1, Some(SANDBOXED)),
16854            passing_round(2, Some(FINAL)),
16855        ];
16856        ok_gate(&mut state, Some(FINAL));
16857        let body = body_at(&state);
16858        assert!(!body.contains("Handled the PATH case."), "{body}");
16859        assert!(!body.contains("could not commit"), "{body}");
16860        assert!(!body.contains("could not run"), "{body}");
16861        assert!(
16862            body.contains("The fix is committed on the branch"),
16863            "{body}"
16864        );
16865        assert!(!body.contains("sandbox"), "{body}");
16866        assert!(
16867            body.contains("Tests (`verify.e2e`): passed on the final commit"),
16868            "{body}"
16869        );
16870        assert!(
16871            body.contains("Gate (`verify.gate`): passed on the final commit"),
16872            "{body}"
16873        );
16874        // The raw report is untouched in the record.
16875        assert!(
16876            state.reviews[0]
16877                .fix
16878                .as_ref()
16879                .unwrap()
16880                .notes
16881                .contains("could not commit")
16882        );
16883    }
16884
16885    #[test]
16886    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16887        let mut state = review_state(&["feat: x"]);
16888        let mut r1 = round_with_notes(1, Some(SANDBOXED));
16889        r1.fix.as_mut().unwrap().committed = false;
16890        let mut r2 = passing_round(2, Some(FINAL));
16891        r2.e2e[0].code = Some(1);
16892        state.reviews = vec![r1, r2];
16893        let body = body_at(&state);
16894        assert!(!body.contains("could not commit"), "{body}");
16895        assert!(!body.contains("could not run"), "{body}");
16896        assert!(body.contains("This round produced no commit"), "{body}");
16897        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16898        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16899        assert!(!body.contains("passed"), "{body}");
16900    }
16901
16902    #[test]
16903    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16904        let mut state = review_state(&["feat: x"]);
16905        state.reviews = vec![
16906            round_with_notes(1, Some(SANDBOXED)),
16907            passing_round(2, Some("0123456789abcdef")),
16908        ];
16909        ok_gate(&mut state, Some("0123456789abcdef"));
16910        let body = body_at(&state);
16911        assert!(!body.contains("HTTP tests could not run"), "{body}");
16912        // The commit claim is still settled by the structured record.
16913        assert!(!body.contains("could not commit"), "{body}");
16914        assert!(
16915            body.contains("passed on an earlier commit (`0123456`)"),
16916            "{body}"
16917        );
16918        assert!(!body.contains("passed on the final commit"), "{body}");
16919
16920        // A gate recorded before `gate_head` existed is "not recorded".
16921        ok_gate(&mut state, None);
16922        let body = body_at(&state);
16923        assert!(
16924            body.contains("passed on a commit that was not recorded"),
16925            "{body}"
16926        );
16927    }
16928
16929    #[test]
16930    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16931        let state = state_with_summary(
16932            "add retries to the client",
16933            "TITLE: feat: retries\n\nDid it.",
16934        );
16935        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16936        assert_eq!(m.title, "feat: retries");
16937        assert!(
16938            m.body.contains("<summary>Original task</summary>"),
16939            "{}",
16940            m.body
16941        );
16942        assert!(!m.body.contains("Commits under review"), "{}", m.body);
16943        assert!(
16944            m.body
16945                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16946            "{}",
16947            m.body
16948        );
16949    }
16950
16951    #[test]
16952    fn pr_message_review_bounds_a_long_english_subject() {
16953        let long = format!("feat: {}", "word ".repeat(100));
16954        let m = pr_message(&review_state(&[&long]), 'A');
16955        assert!(m.title.starts_with("feat: word"), "{}", m.title);
16956        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16957    }
16958
16959    #[test]
16960    fn pr_message_implementation_run_is_unchanged_by_review_support() {
16961        let state = state_with_summary("add retries\n\ndetails", "- did some things");
16962        let m = pr_message(&state, 'A');
16963        assert_eq!(m.title, "add retries");
16964        assert!(m.body.contains("<summary>Original task</summary>"));
16965        assert!(!m.body.contains("Commits under review"));
16966        assert_eq!(landing_subject_source(&state), state.instruction);
16967    }
16968
16969    #[test]
16970    fn review_run_squash_subject_is_the_change_not_the_prompt() {
16971        let state = review_state(&["feat: the change", "fix: typo"]);
16972        let source = landing_subject_source(&state);
16973        assert_eq!(land::merge_subject("", &source), "feat: the change");
16974        assert_eq!(
16975            land::merge_subject("magi: candidate A (uncommitted work)", &source),
16976            "feat: the change"
16977        );
16978        // An operator's rename still wins.
16979        assert_eq!(
16980            land::merge_subject("feat: renamed by hand", &source),
16981            "feat: renamed by hand"
16982        );
16983        let blank = review_state(&["日本語"]);
16984        assert!(
16985            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16986        );
16987    }
16988
16989    #[test]
16990    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16991        let state = review_state(&["feat: the change"]);
16992        let source = landing_subject_source(&state);
16993        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16994        assert_eq!(
16995            land::merge_subject(landing_title(&state, old), &source),
16996            "feat: the change"
16997        );
16998        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16999        let task = state_with_summary("add retries", "");
17000        assert_eq!(landing_title(&task, old), old);
17001    }
17002
17003    #[test]
17004    fn pr_message_describes_the_change_not_the_task() {
17005        let state = state_with_summary(
17006            "今回やってほしいこと: results projector を直す",
17007            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17008        );
17009        let m = pr_message(&state, 'A');
17010        assert_eq!(m.title, "fix(web): batch the runs list reads");
17011        assert!(
17012            m.body.starts_with("## Summary\n\n- reads run.json once"),
17013            "{}",
17014            m.body
17015        );
17016        assert!(!m.body.contains("TITLE:"), "{}", m.body);
17017        let task_at = m.body.find("今回やってほしいこと").unwrap();
17018        let details_at = m.body.find("<details>").unwrap();
17019        assert!(
17020            details_at < task_at,
17021            "the task lives inside <details>: {}",
17022            m.body
17023        );
17024        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17025        assert!(m.body.contains("magi:candidate-a"));
17026    }
17027
17028    #[test]
17029    fn pr_message_falls_back_to_the_task_without_a_title_line() {
17030        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17031        let m = pr_message(&state, 'A');
17032        assert_eq!(m.title, "add retries");
17033        assert!(
17034            m.body.contains("## Summary\n\n- did some things"),
17035            "{}",
17036            m.body
17037        );
17038
17039        let none = RunState::new(
17040            PathBuf::from("/repo"),
17041            "main".to_owned(),
17042            "abc1234".to_owned(),
17043            "add retries".to_owned(),
17044            Config::default(),
17045        );
17046        let m = pr_message(&none, 'A');
17047        assert_eq!(m.title, "add retries");
17048        assert!(!m.body.contains("## Summary"), "{}", m.body);
17049    }
17050
17051    #[test]
17052    fn pr_message_refuses_the_candidate_commit_subject() {
17053        for bad in [
17054            "TITLE: magi: candidate A (uncommitted work)",
17055            "TITLE: chore: stuff (uncommitted work)",
17056            "TITLE:   ",
17057        ] {
17058            let state = state_with_summary("add retries", bad);
17059            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17060        }
17061    }
17062
17063    #[test]
17064    fn pr_message_bounds_a_very_long_task_and_title() {
17065        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17066        let state = state_with_summary(&long, "- nothing");
17067        let m = pr_message(&state, 'A');
17068        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17069        assert!(!m.title.contains('\n'));
17070
17071        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17072        let m = pr_message(&state, 'A');
17073        assert!(m.title.starts_with("feat: "));
17074        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17075        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17076    }
17077
17078    fn long_title_of(instruction: &str) -> String {
17079        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17080    }
17081
17082    #[test]
17083    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17084        let first = "Make the landing path keep a readable title for long tasks";
17085        let line = format!(
17086            "{first}. {}",
17087            "And then keep going with more words ".repeat(20)
17088        );
17089        let t = long_title_of(&line);
17090        assert_eq!(t, first);
17091        assert!(!t.starts_with("chore: land"));
17092    }
17093
17094    #[test]
17095    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17096        let line = "word ".repeat(200);
17097        let t = long_title_of(&line);
17098        assert!(t.ends_with("word..."), "{t}");
17099        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17100    }
17101
17102    #[test]
17103    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17104        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17105            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17106        }
17107    }
17108
17109    #[test]
17110    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17111        let mut state = state_with_summary("再試行を追加する", "- nothing");
17112        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17113        state.config.graph.github_language = "ja".into();
17114        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17115    }
17116
17117    #[test]
17118    fn pr_message_title_limit_is_exact() {
17119        let at = "a".repeat(PR_TITLE_MAX);
17120        assert_eq!(long_title_of(&at), at);
17121        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17122        assert!(over.ends_with("..."), "{over}");
17123        assert_eq!(over.chars().count(), PR_TITLE_MAX);
17124    }
17125
17126    #[test]
17127    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17128        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17129        let t = long_title_of(&line);
17130        assert!(t.ends_with("..."), "{t}");
17131        assert!(t.is_ascii(), "{t}");
17132    }
17133
17134    #[test]
17135    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17136        let line = format!(
17137            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17138            "plus more filler words ".repeat(20)
17139        );
17140        let t = long_title_of(&line);
17141        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17142    }
17143
17144    #[test]
17145    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17146        let line = format!("\n\n# {}", "title words ".repeat(40));
17147        let state = state_with_summary(&line, "- nothing");
17148        let m = pr_message(&state, 'A');
17149        assert!(m.title.starts_with("title words"), "{}", m.title);
17150        assert_eq!(
17151            land::merge_subject(&m.title, &landing_subject_source(&state)),
17152            m.title
17153        );
17154        // An operator's rename wins untouched.
17155        assert_eq!(
17156            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17157            "feat: renamed by hand"
17158        );
17159    }
17160
17161    #[tokio::test]
17162    async fn github_text_rewrite_is_bounded_and_falls_back() {
17163        crate::run::pin_test_home();
17164        for (reply, accepted) in [
17165            (
17166                "TITLE: fix: retries\nAdd retries for failed requests.",
17167                true,
17168            ),
17169            (
17170                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17171                false,
17172            ),
17173            (
17174                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17175                true,
17176            ),
17177            (
17178                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17179                true,
17180            ),
17181            (
17182                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17183                true,
17184            ),
17185            (
17186                "TITLE: fix: retries\n日本語の説明です。 token=secret",
17187                false,
17188            ),
17189            ("TITLE: fix: retries\n", false),
17190        ] {
17191            let dir = tempfile::tempdir().unwrap();
17192            let mut runner = runner_at(RunStatus::Gating);
17193            runner.state = state_with_summary(
17194                "add retries",
17195                "TITLE: fix: retries\n日本語の説明を書きます。",
17196            );
17197            runner.state.repo = dir.path().to_owned();
17198            runner.state.candidates[0].worktree = dir.path().to_owned();
17199            let mut author = spec("alpha");
17200            author.command = vec![
17201                "sh".into(),
17202                "-c".into(),
17203                "printf '%s' \"$REWRITE_REPLY\"".into(),
17204            ];
17205            author.env.insert("REWRITE_REPLY".into(), reply.into());
17206            runner.state.config.agents = vec![author];
17207            seed_public_summary(&mut runner);
17208            let winner = runner.state.candidates[0].clone();
17209            let first = runner
17210                .guarded_pr_message(&winner, None, true)
17211                .await
17212                .unwrap();
17213            let message = if accepted {
17214                first.expect("a passing rewrite never asks")
17215            } else {
17216                // Withheld: the run parks on a question and posts nothing yet.
17217                assert!(first.is_none());
17218                assert!(runner.state.parked);
17219                let store = ask::Questions::open();
17220                let asked: Vec<_> = store
17221                    .list()
17222                    .into_iter()
17223                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17224                    .collect();
17225                assert_eq!(asked.len(), 1);
17226                assert!(!asked[0].detail.contains("secret"));
17227                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17228                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17229                assert_eq!(
17230                    asked[0]
17231                        .choices
17232                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
17233                    rewrite.is_some()
17234                );
17235                // Silence: the question lapses and the neutral text is used,
17236                // without a second rewrite call or a second question.
17237                store
17238                    .update(&asked[0].id, |q| {
17239                        q.abandon("test");
17240                        Ok(())
17241                    })
17242                    .unwrap();
17243                let again = runner
17244                    .guarded_pr_message(&winner, None, true)
17245                    .await
17246                    .unwrap()
17247                    .expect("an abandoned question falls back");
17248                let n = store
17249                    .list()
17250                    .into_iter()
17251                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17252                    .count();
17253                assert_eq!(n, 1, "asked once per rejected text");
17254                again
17255            };
17256            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17257            assert!(!message.body.contains("secret"));
17258            assert!(!message.body.contains("prefix"));
17259            assert!(!message.title.contains("secret"));
17260            assert_eq!(
17261                message.body.contains("Add retries for failed requests."),
17262                accepted
17263            );
17264            assert_eq!(
17265                runner.state.seats["github-text-rewrite"].turns, 1,
17266                "only one rewrite invocation"
17267            );
17268            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17269            if accepted {
17270                assert!(runner.state.github_text.is_none());
17271                assert!(!runner.state.parked);
17272                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17273                assert!(message.body.contains("## Final verification"));
17274                assert!(message.body.contains("Evidence: final diff"));
17275                assert!(
17276                    message
17277                        .body
17278                        .contains(&format!("magi:run/{}", runner.state.id))
17279                );
17280            }
17281            if !accepted {
17282                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17283                assert!(
17284                    message
17285                        .body
17286                        .contains(&format!("magi:run/{}", runner.state.id))
17287                );
17288            }
17289        }
17290    }
17291
17292    fn scripted(id: &str, reply: &str) -> AgentSpec {
17293        let mut a = spec(id);
17294        a.command = vec![
17295            "sh".into(),
17296            "-c".into(),
17297            "printf '%s' \"$REWRITE_REPLY\"".into(),
17298        ];
17299        a.env.insert("REWRITE_REPLY".into(), reply.into());
17300        a
17301    }
17302
17303    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17304    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17305
17306    /// Gate tests inject generated public output, not internal reports.
17307    fn seed_public_summary(runner: &mut Runner) {
17308        let repo = &runner.state.repo;
17309        let g = |args: &[&str]| {
17310            let out = std::process::Command::new("git")
17311                .args(args)
17312                .current_dir(repo)
17313                .quiet()
17314                .output()
17315                .unwrap();
17316            assert!(out.status.success(), "git {args:?}");
17317            String::from_utf8_lossy(&out.stdout).trim().to_owned()
17318        };
17319        g(&["init", "-q", "-b", "main"]);
17320        g(&[
17321            "-c",
17322            "user.name=Test",
17323            "-c",
17324            "user.email=test@example.invalid",
17325            "commit",
17326            "-q",
17327            "--allow-empty",
17328            "-m",
17329            "feat: fixture",
17330        ]);
17331        let head = g(&["rev-parse", "HEAD"]);
17332        g(&["update-ref", "refs/remotes/origin/main", &head]);
17333        let evidence = crate::pr_outcome::summary_evidence(
17334            &runner.state,
17335            crate::pr_outcome::FinalHead::At(&head),
17336        );
17337        let generated = &runner.state.candidates[0].summary;
17338        runner.state.change_summary = Some(crate::run::ChangeSummary {
17339            base: head.clone(),
17340            head,
17341            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17342            title: summary_title(generated).unwrap(),
17343            body: summary_without_title(generated),
17344        });
17345    }
17346
17347    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17348        let mut runner = runner_at(RunStatus::Gating);
17349        runner.state = state_with_summary("add retries", JA_SUMMARY);
17350        if review_only {
17351            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17352        }
17353        runner.state.repo = dir.to_owned();
17354        runner.state.candidates[0].worktree = dir.to_owned();
17355        runner.state.config.agents = agents;
17356        seed_public_summary(&mut runner);
17357        runner
17358    }
17359
17360    fn gate_questions(runner: &Runner) -> usize {
17361        ask::Questions::open()
17362            .list()
17363            .into_iter()
17364            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17365            .count()
17366    }
17367
17368    #[tokio::test]
17369    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17370        crate::run::pin_test_home();
17371        let dir = tempfile::tempdir().unwrap();
17372        let mut runner = rewrite_runner(
17373            dir.path(),
17374            true,
17375            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17376        );
17377        runner.state.config.roles.language_judge =
17378            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17379        let winner = runner.state.candidates[0].clone();
17380        let msg = runner
17381            .guarded_pr_message(&winner, None, true)
17382            .await
17383            .unwrap()
17384            .expect("an English rewrite posts without asking");
17385        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17386        assert!(msg.body.contains("<details>kept</details>"));
17387        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17388        assert_eq!(gate_questions(&runner), 0);
17389        assert!(runner.state.github_text.is_none());
17390        assert!(!runner.state.seats.contains_key("impl-A"));
17391        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17392    }
17393
17394    #[tokio::test]
17395    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17396        crate::run::pin_test_home();
17397        let dir = tempfile::tempdir().unwrap();
17398        let mut runner = rewrite_runner(
17399            dir.path(),
17400            false,
17401            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17402        );
17403        runner.state.config.roles.language_judge =
17404            Some(crate::config::AgentChoice::One("beta".into()));
17405        runner.state.seats.insert(
17406            "impl-A".into(),
17407            SeatState::new("impl-A", "alpha", runner.state.seed),
17408        );
17409        let winner = runner.state.candidates[0].clone();
17410        let msg = runner
17411            .guarded_pr_message(&winner, None, true)
17412            .await
17413            .unwrap()
17414            .expect("rewritten");
17415        assert!(msg.body.contains("<details>kept</details>"));
17416        assert_eq!(runner.state.seats["impl-A"].turns, 0);
17417        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17418        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17419    }
17420
17421    #[tokio::test]
17422    async fn rewrite_reuses_a_resumable_implementer_session() {
17423        crate::run::pin_test_home();
17424        let dir = tempfile::tempdir().unwrap();
17425        let mut runner = rewrite_runner(
17426            dir.path(),
17427            false,
17428            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17429        );
17430        runner.state.config.roles.language_judge =
17431            Some(crate::config::AgentChoice::One("beta".into()));
17432        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17433        seat.turns = 1;
17434        runner.state.seats.insert("impl-A".into(), seat);
17435        let winner = runner.state.candidates[0].clone();
17436        runner
17437            .guarded_pr_message(&winner, None, true)
17438            .await
17439            .unwrap()
17440            .expect("rewritten");
17441        assert_eq!(runner.state.seats["impl-A"].turns, 2);
17442        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17443    }
17444
17445    #[tokio::test]
17446    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17447        crate::run::pin_test_home();
17448        let dir = tempfile::tempdir().unwrap();
17449        let mut broken = spec("bad");
17450        broken.command = vec!["false".into()];
17451        let mut runner = rewrite_runner(
17452            dir.path(),
17453            true,
17454            vec![
17455                broken,
17456                scripted("noformat", "Add retries."),
17457                scripted("good", EN_REWRITE),
17458            ],
17459        );
17460        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17461            "bad".into(),
17462            "noformat".into(),
17463            "good".into(),
17464        ]));
17465        let winner = runner.state.candidates[0].clone();
17466        let msg = runner
17467            .guarded_pr_message(&winner, None, true)
17468            .await
17469            .unwrap()
17470            .expect("the third agent rewrites");
17471        assert!(msg.body.contains("`src/net.rs`"));
17472        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17473        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17474    }
17475
17476    #[tokio::test]
17477    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17478        crate::run::pin_test_home();
17479        // (agents, role): still Japanese; no TITLE; nothing resolvable.
17480        for (agents, role, saved) in [
17481            (
17482                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17483                Some("beta"),
17484                true,
17485            ),
17486            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17487            (Vec::new(), None, false),
17488        ] {
17489            let dir = tempfile::tempdir().unwrap();
17490            let mut runner = rewrite_runner(dir.path(), true, agents);
17491            runner.state.config.roles.language_judge =
17492                role.map(|r| crate::config::AgentChoice::One(r.into()));
17493            let winner = runner.state.candidates[0].clone();
17494            let first = runner
17495                .guarded_pr_message(&winner, None, true)
17496                .await
17497                .unwrap();
17498            assert!(first.is_none(), "withheld text parks the run");
17499            assert!(runner.state.parked);
17500            assert_eq!(gate_questions(&runner), 1);
17501            let g = runner.state.github_text.as_ref().unwrap();
17502            assert_eq!(g.rewrite.is_some(), saved);
17503            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17504        }
17505    }
17506
17507    #[tokio::test]
17508    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17509        crate::run::pin_test_home();
17510        let dir = tempfile::tempdir().unwrap();
17511        let mut runner = runner_at(RunStatus::Gating);
17512        runner.state =
17513            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17514        runner.state.repo = dir.path().to_owned();
17515        runner.state.candidates[0].worktree = dir.path().to_owned();
17516        let mut author = spec("alpha");
17517        author.command = vec![
17518            "sh".into(),
17519            "-c".into(),
17520            "printf '%s' \"$REWRITE_REPLY\"".into(),
17521        ];
17522        author.env.insert(
17523            "REWRITE_REPLY".into(),
17524            format!(
17525                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17526                "English sentence. ".repeat(600)
17527            ),
17528        );
17529        runner.state.config.agents = vec![author];
17530        seed_public_summary(&mut runner);
17531        let winner = runner.state.candidates[0].clone();
17532        assert!(
17533            runner
17534                .guarded_pr_message(&winner, None, true)
17535                .await
17536                .unwrap()
17537                .is_none()
17538        );
17539        let record = runner.state.github_text.clone().unwrap();
17540        let rewrite = record.rewrite.clone().unwrap();
17541        assert_eq!(rewrite.failed, ["body-language"]);
17542        assert!(
17543            rewrite
17544                .body
17545                .ends_with(&format!("magi:run/{}", runner.state.id))
17546        );
17547        let store = ask::Questions::open();
17548        let q = store.get(record.question.as_ref().unwrap()).unwrap();
17549        assert!(
17550            q.choices
17551                .contains(&crate::github_text::USE_REWRITE.to_owned())
17552        );
17553        for label in [
17554            "ORIGINAL",
17555            "FALLBACK",
17556            "REWRITE",
17557            "Rules still failed: body-language",
17558            "truncated",
17559        ] {
17560            assert!(q.detail.contains(label), "{label}");
17561        }
17562        let artifact = std::fs::read_to_string(crate::run::artifact_path(
17563            &runner.state,
17564            &crate::github_text::artifact_name(&record.fingerprint),
17565        ))
17566        .unwrap();
17567        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17568        // Restart from serialized state, then retry after a bad owner title.
17569        runner.state =
17570            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17571        store
17572            .update(&q.id, |q| {
17573                q.say("まだ日本語")?;
17574                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17575            })
17576            .unwrap();
17577        assert!(
17578            runner
17579                .guarded_pr_message(&winner, None, true)
17580                .await
17581                .unwrap()
17582                .is_none()
17583        );
17584        let retry = runner.state.github_text.as_ref().unwrap();
17585        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17586        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17587        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17588        assert!(
17589            q.posting_comparison
17590                .as_ref()
17591                .unwrap()
17592                .matches_detail(&q.detail)
17593        );
17594        store
17595            .update(&q.id, |q| {
17596                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17597            })
17598            .unwrap();
17599        let message = runner
17600            .guarded_pr_message(&winner, None, true)
17601            .await
17602            .unwrap()
17603            .unwrap();
17604        assert_eq!(message.title, rewrite.title);
17605        assert_eq!(message.body, rewrite.body);
17606        assert_eq!(
17607            runner.state.github_text.as_ref().unwrap().choice,
17608            Some(GithubTextChoice::Rewrite)
17609        );
17610        let again = runner
17611            .guarded_pr_message(&winner, None, true)
17612            .await
17613            .unwrap()
17614            .unwrap();
17615        assert_eq!(again.title, message.title);
17616        assert_eq!(again.body, message.body);
17617        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17618        assert!(
17619            !runner
17620                .state
17621                .events
17622                .iter()
17623                .any(|e| e.message.contains(&rewrite.title)
17624                    || e.message.contains("English sentence."))
17625        );
17626    }
17627
17628    #[tokio::test]
17629    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17630        crate::run::pin_test_home();
17631        let dir = tempfile::tempdir().unwrap();
17632        let mut runner = runner_at(RunStatus::Gating);
17633        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17634        runner.state.repo = dir.path().to_owned();
17635        runner.state.candidates[0].worktree = dir.path().to_owned();
17636        seed_public_summary(&mut runner);
17637        let winner = runner.state.candidates[0].clone();
17638        assert!(
17639            runner
17640                .guarded_pr_message(&winner, None, true)
17641                .await
17642                .unwrap()
17643                .is_none()
17644        );
17645        let record = runner.state.github_text.as_ref().unwrap();
17646        assert!(record.rewrite.is_none());
17647        // Records written before the additive field deserialize with no rewrite.
17648        let mut old = serde_json::to_value(record).unwrap();
17649        old.as_object_mut().unwrap().remove("rewrite");
17650        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17651        assert!(old.rewrite.is_none());
17652        let id = record.question.clone().unwrap();
17653        ask::Questions::open()
17654            .update(&id, |q| {
17655                q.choices.push(crate::github_text::USE_REWRITE.into());
17656                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17657            })
17658            .unwrap();
17659        let message = runner
17660            .guarded_pr_message(&winner, None, true)
17661            .await
17662            .unwrap()
17663            .unwrap();
17664        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17665        assert_eq!(
17666            runner.state.github_text.as_ref().unwrap().choice,
17667            Some(GithubTextChoice::Fallback)
17668        );
17669    }
17670
17671    #[tokio::test]
17672    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17673        crate::run::pin_test_home();
17674        let dir = tempfile::tempdir().unwrap();
17675        let mut runner = runner_at(RunStatus::Gating);
17676        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17677        runner.state.repo = dir.path().to_owned();
17678        runner.state.candidates[0].worktree = dir.path().to_owned();
17679        seed_public_summary(&mut runner);
17680        let winner = runner.state.candidates[0].clone();
17681        assert!(
17682            runner
17683                .guarded_pr_message(&winner, None, true)
17684                .await
17685                .unwrap()
17686                .is_none()
17687        );
17688        let store = ask::Questions::open();
17689        let find = |run: &str| {
17690            store
17691                .list()
17692                .into_iter()
17693                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17694                .unwrap()
17695        };
17696        let run = runner.state.id.clone();
17697        let q = find(&run);
17698        let fp = runner
17699            .state
17700            .github_text
17701            .as_ref()
17702            .unwrap()
17703            .fingerprint
17704            .clone();
17705        let name = crate::github_text::artifact_name(&fp);
17706        let path = crate::run::artifact_path(&runner.state, &name);
17707        assert!(q.detail.contains(&path.display().to_string()));
17708        assert!(q.detail.contains("title-language: title"));
17709        let saved = std::fs::read_to_string(&path).expect("artifact written");
17710        assert!(saved.contains("日本語のタイトル"));
17711        assert!(q.cwd.is_none());
17712        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17713        // A bad replacement is refused and asked about once more.
17714        let say = |id: &str, text: &str, choice: &str| {
17715            store
17716                .update(id, |q| {
17717                    q.thread.push(ask::Turn {
17718                        who: ask::Who::Operator,
17719                        body: text.to_owned(),
17720                        at: jiff::Timestamp::now(),
17721                        note: None,
17722                    });
17723                    q.answer(ask::Answer::Choice(choice.to_owned()))
17724                })
17725                .unwrap();
17726        };
17727        say(&q.id, "まだ日本語", "use my text");
17728        assert!(
17729            runner
17730                .guarded_pr_message(&winner, None, true)
17731                .await
17732                .unwrap()
17733                .is_none()
17734        );
17735        let second = find(&run);
17736        assert_ne!(second.id, q.id);
17737        assert!(second.detail.contains("did not pass"));
17738        // The retry still compares the original text; the rejected
17739        // replacement is named by category only.
17740        assert!(second.detail.contains("日本語のタイトル"));
17741        assert!(!second.detail.contains("まだ日本語"));
17742        assert!(second.detail.contains(&path.display().to_string()));
17743        say(&second.id, "fix: retry failed requests", "use my text");
17744        let message = runner
17745            .guarded_pr_message(&winner, None, true)
17746            .await
17747            .unwrap()
17748            .expect("a vetted replacement is posted");
17749        assert_eq!(message.title, "fix: retry failed requests");
17750        assert_eq!(
17751            runner
17752                .state
17753                .github_text
17754                .as_ref()
17755                .unwrap()
17756                .chosen_title
17757                .as_deref(),
17758            Some("fix: retry failed requests")
17759        );
17760        assert!(runner.state.github_text.as_ref().unwrap().resolved);
17761        // Resumed again: the saved decision is applied, nothing is re-asked.
17762        let again = runner
17763            .guarded_pr_message(&winner, None, true)
17764            .await
17765            .unwrap()
17766            .unwrap();
17767        assert_eq!(again.title, message.title);
17768        assert_eq!(
17769            store
17770                .list()
17771                .into_iter()
17772                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17773                .count(),
17774            2
17775        );
17776    }
17777
17778    #[test]
17779    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17780        // What magi itself writes stays English under any configured language,
17781        // so a future localisation of these headings fails here. (The agents'
17782        // own text is also checked by the posting gate.)
17783        let mut state = state_with_summary(
17784            "add retries",
17785            "TITLE: fix(web): batch reads\n- reads run.json once",
17786        );
17787        state.config.graph.language = "ja".to_owned();
17788        let m = pr_message(&state, 'A');
17789        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17790        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17791
17792        // The task is the operator's own text: it goes in untouched, and the
17793        // fallback title (no summary) may be in its language too.
17794        let task = "今回やってほしいこと: results projector を直す";
17795        let mut state = state_with_summary(task, "- no title line");
17796        state.config.graph.language = "ja".to_owned();
17797        let m = pr_message(&state, 'A');
17798        assert_eq!(
17799            m.title,
17800            format!("chore: land candidate A of run {}", state.id)
17801        );
17802        assert!(
17803            m.body.contains(&format!(
17804                "<summary>Original task</summary>\n\n{task}\n\n</details>"
17805            )),
17806            "{}",
17807            m.body
17808        );
17809    }
17810
17811    #[test]
17812    fn pr_message_scrubs_home_paths_and_addresses() {
17813        let state = state_with_summary(
17814            "fix it in /Users/someone/src/x",
17815            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17816        );
17817        let m = pr_message(&state, 'A');
17818        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17819            assert!(!m.body.contains(leak), "{}", m.body);
17820        }
17821        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17822    }
17823
17824    #[test]
17825    fn pr_message_survives_a_task_that_closes_details() {
17826        let state = state_with_summary("a </details> b", "TITLE: fix: x");
17827        let m = pr_message(&state, 'A');
17828        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17829    }
17830
17831    #[test]
17832    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17833        let cmd = manual_merge_command(
17834            MergeStyle::Squash,
17835            Path::new("/repo"),
17836            "b",
17837            "fix: \"quoted\" $(x) `y`\n\nbody",
17838        );
17839        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17840    }
17841
17842    #[test]
17843    fn manual_merge_command_matches_the_configured_style() {
17844        let repo = Path::new("/repo");
17845        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17846
17847        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17848        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17849
17850        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17851        assert_eq!(
17852            squash,
17853            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17854             \"Merge magi run 0832 (candidate A)\""
17855        );
17856
17857        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17858        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17859    }
17860
17861    #[test]
17862    fn a_nudge_gets_a_quarter_of_the_budget() {
17863        // The judge and implement budgets magi ships with.
17864        assert_eq!(retry_budget(secs(1200), true), secs(300));
17865        assert_eq!(retry_budget(secs(3600), true), secs(900));
17866    }
17867
17868    #[test]
17869    fn a_resent_prompt_keeps_the_whole_budget() {
17870        // The seat kept no context, so the retry is the original job again and
17871        // shortening it would only guarantee a second failure.
17872        assert_eq!(retry_budget(secs(1200), false), secs(1200));
17873        assert_eq!(retry_budget(secs(60), false), secs(60));
17874    }
17875
17876    #[test]
17877    fn the_floor_never_exceeds_the_original_budget() {
17878        // A short configured timeout must not be *raised* by the floor: the
17879        // operator asked for a bound, and a retry may not outlast the attempt
17880        // it is retrying.
17881        assert_eq!(retry_budget(secs(60), true), secs(60));
17882        assert_eq!(retry_budget(secs(480), true), secs(120));
17883        assert_eq!(retry_budget(secs(0), true), secs(0));
17884    }
17885
17886    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17887        agent::CommandEvidence {
17888            id: "item1".to_owned(),
17889            description: "cargo test".to_owned(),
17890            exit_code,
17891            result_summary: String::new(),
17892            source: "codex".to_owned(),
17893        }
17894    }
17895
17896    #[test]
17897    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17898        // No evidence is not the same fact as unconfirmed evidence: a
17899        // backend with no adapter, or a reply that ran no commands at all,
17900        // must not be misread as carrying a dangling job.
17901        assert!(!has_unconfirmed_command(&[]));
17902    }
17903
17904    #[test]
17905    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17906        // Deliberately not a check on the exit code's *value*: a fixer
17907        // legitimately runs something that fails mid-iteration before it
17908        // succeeds, and that must never by itself reopen a valid report.
17909        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17910        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17911        assert!(!has_unconfirmed_command(&[
17912            evidence(Some(0)),
17913            evidence(Some(101))
17914        ]));
17915    }
17916
17917    #[test]
17918    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17919        assert!(has_unconfirmed_command(&[
17920            evidence(Some(0)),
17921            evidence(None)
17922        ]));
17923    }
17924
17925    #[test]
17926    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17927        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17928        assert_eq!(
17929            verified_noop_claim(true, &[], text).as_deref(),
17930            Some("already fixed by b32cfc4, on main.")
17931        );
17932    }
17933
17934    #[test]
17935    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17936        // A timeout or a bad exit code reads as the ordinary loss it is,
17937        // whatever the reply's own prose claims.
17938        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17939        assert!(verified_noop_claim(false, &[], text).is_none());
17940    }
17941
17942    #[test]
17943    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17944        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17945        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17946        // A confirmed command alongside the marker is fine.
17947        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17948    }
17949
17950    #[test]
17951    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17952        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17953    }
17954
17955    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
17956    /// pair, in order, labelled A, B, C, ...
17957    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17958        runner.state.candidates = shape
17959            .iter()
17960            .enumerate()
17961            .map(|(i, &(empty, verified))| Candidate {
17962                index: i,
17963                label: (b'A' + i as u8) as char,
17964                agent: "sonnet".to_owned(),
17965                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17966                worktree: PathBuf::from(format!("/wt/{i}")),
17967                summary: String::new(),
17968                stat: String::new(),
17969                files: 0,
17970                commits: 0,
17971                empty,
17972                failed: None,
17973                verified_noop: verified.map(str::to_owned),
17974                duration_ms: 0,
17975                folded: false,
17976            })
17977            .collect();
17978    }
17979
17980    #[test]
17981    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17982        ask_test_home();
17983        let mut runner = runner_at(RunStatus::Implementing);
17984        set_candidates(
17985            &mut runner,
17986            &[
17987                (true, Some("already on main at b32cfc4")),
17988                (true, Some("same fix, see the existing test")),
17989            ],
17990        );
17991
17992        runner
17993            .after_implement()
17994            .expect("a verified no-op is not an error");
17995
17996        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17997    }
17998
17999    #[test]
18000    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18001        ask_test_home();
18002        let mut runner = runner_at(RunStatus::Implementing);
18003        // Candidate A declares a verified no-op; candidate B simply wrote
18004        // nothing and said nothing about why. One candidate's claim is not
18005        // the whole run's agreement.
18006        set_candidates(
18007            &mut runner,
18008            &[(true, Some("already on main at b32cfc4")), (true, None)],
18009        );
18010
18011        let err = runner
18012            .after_implement()
18013            .expect_err("an unverified empty candidate must still fail the run");
18014
18015        assert!(
18016            err.to_string().contains("no candidate produced a change"),
18017            "{err}"
18018        );
18019        assert_eq!(runner.state.status, RunStatus::Failed);
18020    }
18021
18022    #[test]
18023    fn after_implement_still_fails_an_ordinary_all_empty_run() {
18024        ask_test_home();
18025        let mut runner = runner_at(RunStatus::Implementing);
18026        set_candidates(&mut runner, &[(true, None), (true, None)]);
18027
18028        let err = runner
18029            .after_implement()
18030            .expect_err("no candidate declared anything; this is an ordinary failure");
18031
18032        assert!(
18033            err.to_string().contains("no candidate produced a change"),
18034            "{err}"
18035        );
18036        assert_eq!(runner.state.status, RunStatus::Failed);
18037    }
18038}
18039
18040/// A graft rollback whose restore could not be confirmed (see
18041/// [`Runner::graft`]).
18042#[derive(Debug)]
18043struct GraftRestoreFailed(String);
18044
18045impl std::fmt::Display for GraftRestoreFailed {
18046    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18047        write!(
18048            f,
18049            "could not restore the winner to its pre-graft commit: {}",
18050            self.0
18051        )
18052    }
18053}
18054
18055impl std::error::Error for GraftRestoreFailed {}
18056
18057/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
18058/// not the agent: they never draw on an agent's own timeout, and running out of
18059/// them never reads as an agent failure.
18060#[cfg(test)]
18061mod resource_wait_tests {
18062    use super::*;
18063    use std::collections::BTreeMap;
18064
18065    fn sh(id: &str, script: &str) -> AgentSpec {
18066        AgentSpec {
18067            id: id.to_owned(),
18068            kind: crate::config::AgentKind::Command,
18069            model: None,
18070            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18071            extra_args: Vec::new(),
18072            env: BTreeMap::new(),
18073            prompt_delivery: None,
18074        }
18075    }
18076
18077    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18078        SeatJob {
18079            seat: SeatState::new("impl-A", &spec.id, 1),
18080            spec,
18081            cwd: dir.to_path_buf(),
18082            prompt: "do it".to_owned(),
18083            timeout,
18084            allow_write: true,
18085            sessions: false,
18086            artifacts: dir.join("art"),
18087            stem: "impl-A".to_owned(),
18088            handover: None,
18089        }
18090    }
18091
18092    fn state(resource_wait_secs: u64) -> RunState {
18093        let mut config = Config::default();
18094        config.graph.timeout_resource_wait = resource_wait_secs;
18095        RunState::new(
18096            PathBuf::from("/repo"),
18097            "main".to_owned(),
18098            "abc1234".to_owned(),
18099            "task".to_owned(),
18100            config,
18101        )
18102    }
18103
18104    fn hold(cache: &Path) -> crate::cache::Guard {
18105        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18106        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18107            crate::cache::AcquireOutcome::Acquired(g) => g,
18108            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18109        }
18110    }
18111
18112    async fn run(
18113        jobs: Vec<SeatJob>,
18114        sem: Arc<Semaphore>,
18115        cache: Option<&Path>,
18116        st: &mut RunState,
18117    ) -> Vec<(usize, SeatState, AgentOutcome)> {
18118        let prompts = Prompts::default();
18119        let ctx = WaveCtx {
18120            run: "r",
18121            node: "implement",
18122            prompts: &prompts,
18123            cache,
18124            round: None,
18125            carry_seats: false,
18126            pauses: None,
18127        };
18128        wave(jobs, sem, &ctx, st, 0).await
18129    }
18130
18131    #[tokio::test]
18132    async fn verification_cache_wait_preserves_the_command_budget() {
18133        crate::run::pin_test_home();
18134        let tmp = tempfile::tempdir().unwrap();
18135        let cache = tmp.path().join("target");
18136        let guard = hold(&cache);
18137        let release = tokio::spawn(async move {
18138            tokio::time::sleep(Duration::from_millis(100)).await;
18139            guard.release();
18140        });
18141        let mut st = state(10);
18142        let budget = Duration::from_millis(50);
18143        let wait = repository_wait(&st, None);
18144        let (outcomes, _) = with_cache_lease(
18145            &mut st,
18146            Some(&cache),
18147            "gate",
18148            "gate",
18149            tmp.path(),
18150            "abc1234",
18151            budget,
18152            "test gate",
18153            &wait,
18154            |_, available| async move {
18155                assert_eq!(
18156                    available, budget,
18157                    "contention must not shorten verification"
18158                );
18159                (Vec::new(), false, Vec::new())
18160            },
18161        )
18162        .await
18163        .unwrap();
18164        release.await.unwrap();
18165        assert!(outcomes.is_empty(), "{outcomes:?}");
18166        assert!(
18167            st.events
18168                .iter()
18169                .any(|event| event.message.contains("waiting for"))
18170        );
18171        assert!(st.resource_wait.is_none());
18172        // The verification body no longer owns the lease after returning.
18173        hold(&cache).release();
18174    }
18175
18176    #[tokio::test]
18177    async fn verification_cache_wait_expires_without_starting_commands() {
18178        crate::run::pin_test_home();
18179        let tmp = tempfile::tempdir().unwrap();
18180        let cache = tmp.path().join("target");
18181        let _held = hold(&cache);
18182        let mut st = state(0);
18183        let wait = repository_wait(&st, None);
18184        let (outcomes, _) = with_cache_lease(
18185            &mut st,
18186            Some(&cache),
18187            "gate",
18188            "gate",
18189            tmp.path(),
18190            "abc1234",
18191            Duration::from_secs(60),
18192            "test gate",
18193            &wait,
18194            |_, _| async { panic!("commands must not start without the cache lease") },
18195        )
18196        .await
18197        .unwrap();
18198        assert_eq!(outcomes.len(), 1);
18199        assert!(outcomes[0].resource_blocked);
18200        assert!(st.resource_wait.is_none());
18201    }
18202
18203    #[tokio::test]
18204    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18205        crate::run::pin_test_home();
18206        let tmp = tempfile::tempdir().unwrap();
18207        let cache = tmp.path().join("target-a");
18208        let guard = hold(&cache);
18209        // Held for longer than the agent's whole 1s budget.
18210        tokio::spawn(async move {
18211            tokio::time::sleep(Duration::from_millis(1500)).await;
18212            guard.release();
18213        });
18214        let mut st = state(60);
18215        let results = run(
18216            vec![job(
18217                sh("alpha", "echo ok"),
18218                tmp.path(),
18219                Duration::from_secs(1),
18220            )],
18221            Arc::new(Semaphore::new(1)),
18222            Some(&cache),
18223            &mut st,
18224        )
18225        .await;
18226        assert!(
18227            matches!(results[0].2, AgentOutcome::Ok(_)),
18228            "the wait must not be charged to the agent"
18229        );
18230        assert!(st.resource_block.is_none());
18231        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18232        assert!(st.resource_wait.is_none() && st.active.is_empty());
18233    }
18234
18235    #[tokio::test]
18236    #[cfg(unix)]
18237    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18238        use std::os::fd::AsRawFd as _;
18239        crate::run::pin_test_home();
18240        let tmp = tempfile::tempdir().unwrap();
18241        crate::git::git(tmp.path(), &["init"]).await.unwrap();
18242        let common = tmp.path().join(".git");
18243        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18244        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18245        assert_eq!(
18246            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18247            0
18248        );
18249        let mut st = state(1);
18250        st.config.coordination = crate::config::test_coordination();
18251        let mut reader = job(
18252            sh("reader", "echo must-not-run"),
18253            tmp.path(),
18254            Duration::from_secs(30),
18255        );
18256        reader.allow_write = false;
18257        let results = run(
18258            vec![reader.clone()],
18259            Arc::new(Semaphore::new(1)),
18260            None,
18261            &mut st,
18262        )
18263        .await;
18264        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18265        assert_eq!(results[0].1.turns, 0);
18266        assert!(FailClass::of(&results[0].2).is_none());
18267        assert_eq!(
18268            st.resource_block.as_ref().unwrap().kind,
18269            "repository coordination"
18270        );
18271        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18272        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18273        drop(external);
18274        let editor = crate::repo_lock::SeatLease::for_checkout(
18275            tmp.path(),
18276            true,
18277            crate::config::test_coordination().as_ref(),
18278        )
18279        .await
18280        .unwrap();
18281        let pause = Pause::new();
18282        let interrupt = Pause::new();
18283        let prompts = Prompts::default();
18284        let ctx = WaveCtx {
18285            run: "r",
18286            node: "review",
18287            prompts: &prompts,
18288            cache: None,
18289            round: None,
18290            carry_seats: false,
18291            pauses: Some((&pause, &interrupt)),
18292        };
18293        let (results, ()) = tokio::join!(
18294            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18295            async {
18296                tokio::time::sleep(Duration::from_millis(50)).await;
18297                pause.park();
18298            }
18299        );
18300        assert!(
18301            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18302        );
18303        assert_eq!(results[0].1.turns, 0);
18304        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18305        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18306        drop(editor);
18307    }
18308
18309    #[tokio::test]
18310    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18311        crate::run::pin_test_home();
18312        let tmp = tempfile::tempdir().unwrap();
18313        let cache = tmp.path().join("target-b");
18314        let _held = hold(&cache);
18315        let mut st = state(1);
18316        let results = run(
18317            vec![job(
18318                sh("alpha", "echo ok"),
18319                tmp.path(),
18320                Duration::from_secs(1),
18321            )],
18322            Arc::new(Semaphore::new(1)),
18323            Some(&cache),
18324            &mut st,
18325        )
18326        .await;
18327        let (_, seat, out) = &results[0];
18328        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18329        assert!(wave_blocked(&results));
18330        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18331        assert_eq!(seat.turns, 0, "nothing was asked");
18332        assert!(
18333            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18334            "no agent was spawned"
18335        );
18336        assert!(FailClass::of(out).is_none(), "not a failure class");
18337        let block = st.resource_block.as_ref().expect("recorded for the resume");
18338        assert_eq!(block.kind, "shared build cache");
18339        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18340        assert!(st.active.is_empty());
18341    }
18342
18343    #[tokio::test]
18344    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18345        crate::run::pin_test_home();
18346        let tmp = tempfile::tempdir().unwrap();
18347        let mut st = state(1);
18348        let results = run(
18349            vec![job(
18350                sh("alpha", "echo ok"),
18351                tmp.path(),
18352                Duration::from_secs(1),
18353            )],
18354            Arc::new(Semaphore::new(0)),
18355            None,
18356            &mut st,
18357        )
18358        .await;
18359        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18360        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18361        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18362    }
18363
18364    #[tokio::test]
18365    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18366        crate::run::pin_test_home();
18367        let tmp = tempfile::tempdir().unwrap();
18368        let mut st = state(60);
18369        let results = run(
18370            vec![job(
18371                sh("alpha", "sleep 30"),
18372                tmp.path(),
18373                Duration::from_secs(1),
18374            )],
18375            Arc::new(Semaphore::new(1)),
18376            None,
18377            &mut st,
18378        )
18379        .await;
18380        let out = &results[0].2;
18381        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18382        let class = FailClass::of(out).expect("a failure");
18383        assert_eq!(class, FailClass::Timeout);
18384        assert!(should_hand_over(None, &class), "existing handover policy");
18385        assert!(st.resource_block.is_none());
18386    }
18387}