magi/graph.rs
1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//! │ │
6//! └──── unanimous ───────────┤
7//! ▼
8//! merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47 DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, JobRecord, JobStatus, Judgement,
48 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, QuotaLoss,
49 ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord, tail,
50 write_artifact,
51};
52use crate::verdict::{
53 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
54 ReviewVote, Severity,
55};
56
57/// How much verification output is kept and fed back to the fixer.
58const OUTPUT_TAIL: usize = 8_000;
59
60/// Bytes of a failing command's output kept in an event, so the reason a run
61/// stopped is readable from the report without opening `run.json`.
62const EVENT_OUTPUT_TAIL: usize = 2_000;
63
64/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
65/// command's pid before releasing the build cache's lease.
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
67
68/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
69/// command's pid to actually exit before giving up and releasing anyway.
70///
71/// A timeout means the process was asked to die (`kill_on_drop`,
72/// `start_kill`), not that it already has — on Windows in particular that can
73/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
74/// the instant the command returns would let the very next acquirer (this
75/// run's own next round, another run's verification, the janitor's prune)
76/// start touching the same directory while it might still be writing to it,
77/// so this polls the actual pid — real confirmation, not a fixed guess —
78/// until it is gone or this ceiling is reached. It is still not full
79/// process-tree reaping: a grandchild the timed-out process spawned and that
80/// outlives it independently is invisible to a pid check, and continuing to
81/// observe and collect *that* stays a different piece of work with its own
82/// owner. Set generously because the common case returns early the moment
83/// the pid is confirmed gone, not because every timeout pays this in full.
84const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
85
86/// Consecutive review rounds with no tree progress (see
87/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
88/// instead of spending the rest of the round budget.
89///
90/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
91/// legitimately finds nothing left to change (its previous round's fix already
92/// covered it, and this round's reviewers re-raised only nits) looks the same
93/// as one that is spinning, for exactly one round. Two in a row is where the
94/// two stop being distinguishable, and a review round on this workload has
95/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
96/// third attempt at a tree that has not moved twice running is pure cost.
97/// This does not touch `review_rounds` itself, which stays the operator's
98/// call.
99pub(crate) const STAGNANT_LIMIT: usize = 2;
100
101/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
102/// a base that moved before giving up and leaving the run `Blocked` for a
103/// person.
104///
105/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
106/// that keeps moving faster than a run can catch it is not something more
107/// rebasing fixes, it is a person's call. Not the same *number as*
108/// `land_rounds` - this budget is spent before a pull request exists, land's
109/// after - but bounded for the identical reason, so it uses the same
110/// default. Counted across both call sites in [`Runner::finish_after_tally`]
111/// (once before review, once before the gate), because either one finding
112/// the base still moving is the same signal.
113const BASE_SYNC_ROUNDS: usize = 4;
114
115/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
116/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
117/// reply held no [`FixReport`].
118///
119/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
120/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
121/// "I'll pause here until the `cargo make check` background run reports
122/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
123/// No `FixReport` was ever collected from that seat, and the run moved on to
124/// the next review round regardless.
125///
126/// Bounded independently of `review_rounds` and `graph.retries`: this
127/// recovers one seat's missing report mid-round, not a new round of review or
128/// an ordinary parse retry, and must not itself become the unbounded wait the
129/// rest of this module exists to avoid.
130const MAX_FIX_CONTINUATIONS: usize = 2;
131
132/// One queued agent invocation.
133///
134/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
135/// CLI hung up on its own stream is asked again from the same job rather than
136/// rebuilt from scratch. See [`Runner::resume_undelivered`].
137#[derive(Clone)]
138struct SeatJob {
139 spec: AgentSpec,
140 seat: SeatState,
141 cwd: PathBuf,
142 prompt: String,
143 timeout: Duration,
144 allow_write: bool,
145 sessions: bool,
146 artifacts: PathBuf,
147 stem: String,
148}
149
150/// How the graph reads one agent invocation.
151///
152/// Quota is split out from an ordinary failure on purpose: a rate-limited call
153/// is known to fail again if retried now, so the retry loop must not spend an
154/// attempt on it. `Dropped` is split out for the opposite reason: unlike
155/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
156/// error JSON, never the agent's answer — a caller that matched only
157/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
158/// left to read that JSON as if it were usable output. `resume_undelivered`
159/// is the only caller that acts on it; everywhere else it is reported like an
160/// ordinary failure.
161enum AgentOutcome {
162 /// A usable output.
163 Ok(AgentOutput),
164 /// The CLI ran out of quota / rate limit. Retrying now is pointless.
165 Quota(AgentOutput),
166 /// The CLI hung up on its own stream after billed work. See
167 /// [`agent::AgentOutput::work_undelivered`].
168 Dropped(AgentOutput),
169 /// Any other failure: a timeout, a bad exit code, an empty reply.
170 Failed(String),
171}
172
173/// A request to park the run at its next node boundary.
174///
175/// Cloning is how the request travels: the loop keeps one handle and hands a
176/// clone to each [`Runner`], and every clone points at the same flag. There
177/// is no channel because there is nothing to send - the only message is
178/// "park", it is idempotent, and a flag cannot be missed by a receiver that
179/// was not listening yet.
180///
181/// The boundary is what makes this cheap. Every node writes the run's state
182/// before the next one starts, and every node skips what is already recorded:
183/// `prep` returns early once candidates exist, `implement` asks only the seats
184/// with nothing on disk, `judge` returns early once judgements exist. So a
185/// parked run resumes into exactly the node it stopped before, and no agent
186/// work is thrown away. Killing the process mid-node, by contrast, loses
187/// whatever the seats in flight had not yet written - which for an implement
188/// wave is an hour of paid work.
189///
190/// A [`Runner`] watches two independent handles of this type - see
191/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
192/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
193/// clone covering the whole daemon's lifetime and is never asked to un-park,
194/// which is correct exactly because nothing is dispatched after it fires.
195/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
196/// that parks for an interrupted task must go on to run other tasks
197/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
198/// reusing the daemon-wide one.
199#[derive(Debug, Clone, Default)]
200pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
201
202impl Pause {
203 /// A pause nobody has asked for yet.
204 #[must_use]
205 pub fn new() -> Self {
206 Self::default()
207 }
208
209 /// Ask the run to park at its next node boundary. Idempotent.
210 pub fn park(&self) {
211 self.0.store(true, Ordering::SeqCst);
212 }
213
214 /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
215 /// fold into the run's own `park` event - so an operator reading the run
216 /// later knows this was a deliberate interrupt rather than a shutdown or
217 /// a binary swap. The first reason recorded wins; a park already in
218 /// flight is not relabelled by a second, unrelated request.
219 pub fn park_because(&self, reason: impl Into<String>) {
220 let mut reason_guard = self
221 .1
222 .lock()
223 .unwrap_or_else(std::sync::PoisonError::into_inner);
224 if reason_guard.is_none() {
225 *reason_guard = Some(reason.into());
226 }
227 drop(reason_guard);
228 self.park();
229 }
230
231 /// Has a park been asked for?
232 #[must_use]
233 pub fn parked(&self) -> bool {
234 self.0.load(Ordering::SeqCst)
235 }
236
237 /// Why the park was asked for, when the caller used [`Pause::park_because`].
238 #[must_use]
239 pub fn reason(&self) -> Option<String> {
240 self.1
241 .lock()
242 .unwrap_or_else(std::sync::PoisonError::into_inner)
243 .clone()
244 }
245}
246
247/// Drives one run.
248pub struct Runner {
249 /// Run state; public so the CLI can report on it.
250 pub state: RunState,
251 roles: ResolvedRoles,
252 sem: Arc<Semaphore>,
253 /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
254 /// run parked at its next node boundary. See [`Pause`]'s own doc for why
255 /// this is never the same handle as `interrupt`.
256 pause: Pause,
257 /// Set when `magi serve`'s interrupt scheduler wants this specific run
258 /// parked at its next node boundary, to let a task marked
259 /// [`crate::queue::Task::interrupt`] run alone before this one carries
260 /// on. Unlike `pause`, a fresh, unshared handle per run - see
261 /// [`Runner::watch_interrupt`].
262 interrupt: Pause,
263}
264
265/// The commit a run branches from: the base branch as the remote has it.
266///
267/// Two failures this replaces. A run used to branch off `HEAD` and so refused
268/// to start on a dirty tree, which made `magi serve` decline every task for as
269/// long as the operator had work in progress - most of the time. Branching off
270/// the *local* base branch fixed that and introduced a worse one: `land` merges
271/// the winner on GitHub, nothing updates the local ref, and the next run
272/// branches off a base missing everything the previous runs landed. Two tasks
273/// in a row from a phone would have had the second silently re-implementing
274/// against stale code and opening a pull request that reverted the first.
275///
276/// Only refs move here - no checkout, no local branch, no merge - so it is safe
277/// with uncommitted work in the tree. A machine with no network still starts:
278/// the fetch may fail and the local tip is used with a warning, because
279/// refusing to run offline is a worse failure than running against a base the
280/// operator can see for themselves.
281///
282/// One function, called by both entry points. Two answers to "where does a run
283/// branch from" is the kind of drift nobody notices until a diff is wrong.
284async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
285 let tracking = format!("{remote}/{base_branch}");
286 let fetched = git::fetch(repo, remote, base_branch).await;
287 if let Ok(out) = &fetched
288 && out.ok()
289 && git::rev_exists(repo, &tracking).await
290 {
291 return git::rev_parse(repo, &tracking).await;
292 }
293 let why = match &fetched {
294 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
295 Ok(_) => format!("{remote} has no {base_branch}"),
296 Err(e) => e.to_string(),
297 };
298 tracing::warn!(
299 "could not read {tracking} ({why}); branching off the local \
300 {base_branch} instead, which may be behind"
301 );
302 git::rev_parse(repo, base_branch).await.with_context(|| {
303 format!(
304 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
305 branch that exists"
306 )
307 })
308}
309
310/// Exclusive claim on one run's `magi fix` step, released on drop — including
311/// on an early return or a panic.
312///
313/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
314/// manual `magi fix` invocations against the same run are otherwise
315/// invisible to each other and would race to remove and recreate the same
316/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
317/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
318/// which is only ever reclaimed later, out of band, by
319/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
320/// `magi fix` invocation is not necessarily running under either of those, so
321/// nothing would ever sweep a lock a killed or crashed process left behind.
322/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
323/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
324/// lease use: an unreadable or unparsable pid, or a liveness query the
325/// platform cannot answer, reads as alive and the lock is left in place.
326struct FixClaim {
327 path: PathBuf,
328}
329
330impl FixClaim {
331 fn acquire(dir: &Path) -> Result<Self> {
332 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
333 let path = dir.join("fix.lock");
334 match Self::create(&path) {
335 Ok(claim) => Ok(claim),
336 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
337 if Self::reclaim_if_dead(&path) {
338 Self::create(&path).with_context(|| format!("lock {}", path.display()))
339 } else {
340 bail!(
341 "another `magi fix` is already running for this run ({} exists)",
342 path.display()
343 )
344 }
345 }
346 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
347 }
348 }
349
350 fn create(path: &Path) -> std::io::Result<Self> {
351 let mut f = std::fs::OpenOptions::new()
352 .write(true)
353 .create_new(true)
354 .open(path)?;
355 use std::io::Write as _;
356 // Read back by `reclaim_if_dead` on a later, stuck invocation.
357 writeln!(f, "{}", std::process::id())?;
358 Ok(Self {
359 path: path.to_owned(),
360 })
361 }
362
363 /// True if the lock named a process confirmed dead, in which case it was
364 /// also removed. Never true on an unreadable file, an unparsable pid, or
365 /// a liveness query the platform cannot answer — see this type's own doc.
366 fn reclaim_if_dead(path: &Path) -> bool {
367 let dead = std::fs::read_to_string(path)
368 .ok()
369 .and_then(|body| body.trim().parse::<u32>().ok())
370 .is_some_and(|pid| !crate::proc::pid_alive(pid));
371 dead && std::fs::remove_file(path).is_ok()
372 }
373}
374
375impl Drop for FixClaim {
376 fn drop(&mut self) {
377 let _ = std::fs::remove_file(&self.path);
378 }
379}
380
381impl Runner {
382 /// Start a fresh run against `repo`.
383 pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
384 let repo = git::toplevel(repo).await?;
385 let missing = agent::missing_programs(&config.agents);
386 if !missing.is_empty() {
387 bail!(
388 "these agent programs are not on PATH: {}. Fix the roster in \
389 magi.toml or install them.",
390 missing.join(", ")
391 );
392 }
393 let base_branch = match config.merge.base.clone() {
394 Some(b) => b,
395 None => git::current_branch(&repo)
396 .await?
397 .context("HEAD is detached; set [merge] base in magi.toml")?,
398 };
399 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
400 // Still worth saying out loud. The operator's uncommitted work is not
401 // part of this run, and someone watching a candidate fail to use a
402 // change they just made deserves to know why.
403 if !git::is_clean(&repo).await? {
404 tracing::warn!(
405 "{} has uncommitted changes; they are not part of this run, \
406 which branches off {base_branch} ({})",
407 repo.display(),
408 &base_commit[..base_commit.len().min(8)]
409 );
410 }
411 let roles = config.resolve_roles()?;
412 let max_parallel = config.graph.max_parallel.max(1);
413 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
414 state.event("start", format!("run {} created", state.id));
415 state.save()?;
416 Ok(Self {
417 state,
418 roles,
419 sem: Arc::new(Semaphore::new(max_parallel)),
420 pause: Pause::new(),
421 interrupt: Pause::new(),
422 })
423 }
424
425 /// Open a review-only run against work that already exists on `branch`.
426 ///
427 /// The expensive half of the graph is the implement wave — measured at
428 /// 111 and 134 internal tool-loop turns on this repository, against a
429 /// handful for a judge or a reviewer. The cheap half is worth running on
430 /// hand-written work too, and there was no way to reach it.
431 ///
432 /// No new state and no schema change are needed: a run with **one** viable
433 /// candidate and a tally already decided degrades `execute` to exactly
434 /// review → gate → merge, because `judge` skips a single-candidate field,
435 /// `deliberate` has fewer than two first choices to reconcile, `vote`
436 /// returns early, `tally` is already present and `fold_losers` has no
437 /// losers. Resuming such a run therefore does the right thing as well.
438 pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
439 let repo = git::toplevel(repo).await?;
440 let missing = agent::missing_programs(&config.agents);
441 if !missing.is_empty() {
442 bail!(
443 "these agent programs are not on PATH: {}. Fix the roster in \
444 magi.toml or install them.",
445 missing.join(", ")
446 );
447 }
448 if !git::branch_exists(&repo, branch).await? {
449 bail!("no branch `{branch}` in {}", repo.display());
450 }
451 let base_branch = match config.merge.base.clone() {
452 Some(b) => b,
453 None => git::current_branch(&repo)
454 .await?
455 .context("HEAD is detached; set [merge] base in magi.toml")?,
456 };
457 if base_branch == branch {
458 bail!("`{branch}` is the base branch; there is nothing to review against");
459 }
460 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
461
462 let roles = config.resolve_roles()?;
463 let max_parallel = config.graph.max_parallel.max(1);
464 // The commit subjects are the closest thing to a task statement that
465 // existing work carries, and the reviewers are told as much.
466 let log = git::log_oneline(&repo, &base_commit, branch)
467 .await
468 .unwrap_or_default();
469 let instruction = format!(
470 "Review the work already on branch `{branch}`. There is no task \
471 statement: what the change claims to do is whatever its commits \
472 say.\n\n{}",
473 if log.trim().is_empty() {
474 "(no commit messages)"
475 } else {
476 log.trim()
477 }
478 );
479 let mut state = RunState::new(
480 repo.clone(),
481 base_branch,
482 base_commit.clone(),
483 instruction,
484 config,
485 );
486
487 // An attached worktree, so the fixer's commits land on the branch under
488 // review rather than on a detached head nobody will look at again.
489 let worktree = state.worktree_root().join("under-review");
490 if let Some(parent) = worktree.parent() {
491 tokio::fs::create_dir_all(parent).await.ok();
492 }
493 let path = worktree.to_string_lossy().to_string();
494 git::git(&repo, &["worktree", "add", &path, branch])
495 .await
496 .with_context(|| {
497 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
498 })?;
499
500 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
501 .await
502 .unwrap_or(0);
503 if commits == 0 {
504 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
505 }
506 let files = git::changed_files(&worktree, &base_commit, "HEAD")
507 .await
508 .map(|f| f.len())
509 .unwrap_or(0);
510 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
511 .await
512 .unwrap_or_default();
513
514 state.candidates.push(Candidate {
515 index: 0,
516 label: 'A',
517 // Not an agent id on purpose: nothing in the roster wrote this, and
518 // the stats tables must not credit anyone with a win for it.
519 agent: "(existing branch)".to_owned(),
520 branch: branch.to_owned(),
521 worktree,
522 summary: String::new(),
523 stat,
524 files,
525 commits,
526 empty: false,
527 failed: None,
528 verified_noop: None,
529 duration_ms: 0,
530 folded: false,
531 });
532 state.tally = Some(Tally {
533 first_choice: BTreeMap::from([('A', 0)]),
534 borda: BTreeMap::new(),
535 winner: 'A',
536 rankings: 0,
537 unanimous_initial: false,
538 deliberated: false,
539 changed_votes: 0,
540 unanimous_final: false,
541 tie_break: None,
542 // No panel sat, so no quorum applies. Zero judges is the correct
543 // number for work that never competed, and must not be reported as
544 // a collapsed panel.
545 judges: 0,
546 present: 0,
547 quorum: 0,
548 met_quorum: true,
549 uncontested: Some("review-only run: nothing competed".to_owned()),
550 });
551 state.status = RunStatus::Reviewing;
552 state.event(
553 "start",
554 format!(
555 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
556 state.id
557 ),
558 );
559 state.save()?;
560 Ok(Self {
561 state,
562 roles,
563 sem: Arc::new(Semaphore::new(max_parallel)),
564 pause: Pause::new(),
565 interrupt: Pause::new(),
566 })
567 }
568
569 /// Reopen an existing run.
570 pub fn resume(id: &str) -> Result<Self> {
571 let state = RunState::load(id)?;
572 let roles = state.config.resolve_roles()?;
573 let max_parallel = state.config.graph.max_parallel.max(1);
574 Ok(Self {
575 state,
576 roles,
577 sem: Arc::new(Semaphore::new(max_parallel)),
578 pause: Pause::new(),
579 interrupt: Pause::new(),
580 })
581 }
582
583 /// Walk the graph to a terminal state, skipping nodes already recorded.
584 pub async fn execute(&mut self) -> Result<()> {
585 // Moving again, so it is no longer parked. Set before the walk rather
586 // than in `resume`, so every way of re-entering the graph clears it
587 // and a card cannot claim a run is waiting to be resumed while the
588 // agents are already working.
589 self.state.parked = false;
590 // Any seat this state still lists as answering belongs to whatever
591 // process last drove this run — this one included, if it crashed
592 // mid-wave. Cleared and flushed immediately, before anything else
593 // runs, so a resume can never show a seat as live when nothing is
594 // asking it anything yet; the node that actually dispatches the next
595 // wave repopulates it.
596 self.state.clear_active();
597 // Recorded in the same spot, and flushed together with the clear
598 // above: this is the pid a reader checks (`RunState::liveness`) when
599 // no daemon claim exists to answer "is a process still driving this
600 // run" — a plain `magi run` / `magi review` typed into a terminal
601 // claims nothing there. Always overwritten, never only-if-absent, so
602 // a resumed run's stale pid from a previous, possibly-dead process
603 // can never survive into this one's own report. Unlike
604 // `clear_active`, this changes on every single `execute()` call, so
605 // the save below is now unconditional rather than only-if-cleared.
606 //
607 // `driver_started_at` is recorded in the same breath, from this same
608 // pid, so `liveness` can tell a live pid that is genuinely still us
609 // apart from one the OS has since handed to an unrelated process —
610 // see that field's own doc for why the pid alone is not enough.
611 let pid = std::process::id();
612 self.state.driver_pid = Some(pid);
613 self.state.driver_started_at = crate::proc::process_started_at(pid);
614 self.state.save()?;
615 // A run that already lost its quorum never resumes into the verdict
616 // machinery: `deliberate` and `vote` would otherwise clobber the
617 // stalled marker back to Voting and the run would keep going past a
618 // verdict that is no longer trustworthy. Everything already recorded is
619 // kept, so the run stays resumable (or foldable) for a human to pick up.
620 //
621 // On --resume the run gets one chance to repair itself: the seats a
622 // rate limit took out are re-asked. If their quota has since reset and
623 // the quorum is restored, the run picks up and finishes; otherwise it
624 // stays stale and still-resumable for a later retry. If it does not
625 // recover, the returned status stays `Stalled` and nothing was
626 // clobbered (the recovery only mutates entries for the lost seats).
627 if self.state.status == RunStatus::Stalled {
628 if self.recover_stall().await? {
629 self.finish_after_tally().await?;
630 } else {
631 // Still below quorum: persist the marker and stay resumable.
632 self.state.save()?;
633 }
634 return Ok(());
635 }
636 // A run parked inside `land` - watching CI, mid fix-round, or
637 // waiting on the owner's merge approval - resumes directly into it,
638 // never back through `prep`. Everything before `merge` already
639 // concluded; that is the only way `status` reaches `Landing` in the
640 // first place. Re-walking `review_loop` first would also be actively
641 // wrong: its own status recomputation (see its doc) treats any
642 // clean round as reason to set `status` to `Gating`, which would
643 // clobber this marker before `merge` ever ran, and this run would
644 // never find its way back into `land` at all.
645 if self.state.status == RunStatus::Landing {
646 self.run_land().await?;
647 // `run_land` may have settled the run right here - CI came back
648 // green and the PR merged, say - without ever passing back
649 // through `merge`'s own trailing call. Whatever it left `status`
650 // as is what this has to read.
651 self.settle_questions();
652 return Ok(());
653 }
654 self.prep().await?;
655 if self.park_here()? {
656 return Ok(());
657 }
658 self.advise().await?;
659 if self.park_here()? {
660 return Ok(());
661 }
662 self.implement().await?;
663 if self.park_here()? {
664 return Ok(());
665 }
666 // `after_implement` already saved the state and settled any open
667 // questions when it set this; nothing later in the graph has
668 // anything to judge.
669 if self.state.status == RunStatus::VerifiedNoop {
670 return Ok(());
671 }
672 self.judge().await?;
673 if self.park_here()? {
674 return Ok(());
675 }
676 self.deliberate().await?;
677 if self.park_here()? {
678 return Ok(());
679 }
680 self.vote().await?;
681 if self.park_here()? {
682 return Ok(());
683 }
684 self.tally()?;
685 // A verdict that lost its quorum is not trustworthy: do not review,
686 // gate, or merge on it. Everything already done is kept, so the run
687 // stays resumable (or foldable); the human can replace the agent that
688 // ran out of quota and pick it up.
689 if self.state.status == RunStatus::Stalled {
690 // Persist the stalled marker now — the normal end-of-execute save
691 // below is below this early return, and without it a resumed run
692 // would reload a pre-tally status and keep going.
693 self.state.save()?;
694 return Ok(());
695 }
696 self.finish_after_tally().await?;
697 Ok(())
698 }
699
700 /// Park here if asked to, recording it in the run's own timeline.
701 ///
702 /// Returns whether the caller should stop walking the graph. The state is
703 /// saved either way by the node that just finished; this adds the event so
704 /// the operator's card says why a run that is neither finished nor moving
705 /// is sitting where it is.
706 fn park_here(&mut self) -> Result<bool> {
707 // Either handle asking is enough - see `Pause`'s own doc for why
708 // they are never the same one. `interrupt` is checked second so a
709 // reason it carries is preferred in the message below over a plain
710 // shutdown park racing it at the same boundary.
711 if !self.pause.parked() && !self.interrupt.parked() {
712 return Ok(false);
713 }
714 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
715 Some(reason) => format!(
716 "parked after `{}` ({reason}) — resume to carry on from here",
717 self.state.status.as_str()
718 ),
719 None => format!(
720 "parked after `{}` — resume to carry on from here",
721 self.state.status.as_str()
722 ),
723 };
724 self.state.event("park", why);
725 self.state.parked = true;
726 self.state.save()?;
727 Ok(true)
728 }
729
730 /// Hand the runner the pause `magi serve`'s own shutdown watches.
731 pub fn on_pause(&mut self, pause: Pause) {
732 self.pause = pause;
733 }
734
735 /// Hand the runner a second, independent pause: `magi serve`'s interrupt
736 /// scheduler asking this one run - and no other - to park so a task
737 /// marked [`crate::queue::Task::interrupt`] can run alone. See
738 /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
739 /// handle.
740 pub fn watch_interrupt(&mut self, pause: Pause) {
741 self.interrupt = pause;
742 }
743
744 /// Abandon this run's own open questions, once `status` has actually
745 /// settled rather than merely paused.
746 ///
747 /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
748 /// either back up with the candidates, the review round and the seat
749 /// sessions already on disk, so a question an implementer asked mid-round
750 /// may still get a real answer read by a real resume. Only the statuses
751 /// `resumable` excludes are actually final: the run merged, it reached
752 /// `Ready` with nothing left to do, it failed outright with no
753 /// established point to continue from, or every candidate agreed, with
754 /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
755 /// every one of those the seat that asked is gone for good, exactly like
756 /// the run being deleted under `magi run rm` - so the same cleanup
757 /// applies, worded for what actually happened instead of "the run was
758 /// deleted".
759 ///
760 /// Best-effort and silent on success: called from every place `status`
761 /// can land on one of those three, including ones a resumed run revisits,
762 /// so it must cost nothing when there was nothing open to begin with.
763 fn settle_questions(&mut self) {
764 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
765 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
766 }
767 }
768
769 /// The tail of the graph after a trustworthy tally: fold losers, review,
770 /// gate, merge, and persist.
771 async fn finish_after_tally(&mut self) -> Result<()> {
772 self.fold_losers().await?;
773 // Before review starts, and again right before the gate: a run's
774 // review rounds can themselves take long enough for the base to move
775 // a second time, and the gate is the one node whose "green" gets
776 // acted on.
777 self.sync_to_base().await?;
778 self.review_loop().await?;
779 self.sync_to_base().await?;
780 self.gate().await?;
781 self.merge().await?;
782 self.state.save()?;
783 Ok(())
784 }
785
786 // ---------------------------------------------------------------- prep
787
788 async fn prep(&mut self) -> Result<()> {
789 if !self.state.candidates.is_empty() {
790 return Ok(());
791 }
792 self.state.status = RunStatus::Prep;
793 let repo = self.state.repo.clone();
794 let base = self.state.base_commit.clone();
795 let root = self.state.worktree_root();
796 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
797
798 // The hook is the write-time half of the blindness contract; the
799 // presentation filter in `blind` is the half that cannot be bypassed.
800 let hooks_dir = self.state.dir().join("hooks");
801 if self.state.config.blind.commit_msg_hook {
802 std::fs::create_dir_all(&hooks_dir)
803 .with_context(|| format!("create {}", hooks_dir.display()))?;
804 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
805 let path = hooks_dir.join("commit-msg");
806 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
807 make_executable(&path)?;
808 // Ref-counted rather than a plain idempotent set: with more than
809 // one run able to be in flight in the same repository at once
810 // (see `Config::daemon.max_concurrent_runs`), a bare "already
811 // true?" check cannot tell "another run of mine still needs
812 // this" from "nobody does", and the run that happens to finish
813 // first would disable the hook out from under a sibling still
814 // relying on it.
815 git::acquire_worktree_config(&repo).await?;
816 self.state.enabled_worktree_config = true;
817 }
818
819 for (index, (spec, label)) in self
820 .roles
821 .implementers
822 .clone()
823 .into_iter()
824 .zip(labels)
825 .enumerate()
826 {
827 let branch = self.state.branch_for(label);
828 let worktree = root.join(format!("cand-{label}"));
829 git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
830 if self.state.config.blind.commit_msg_hook {
831 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
832 }
833 git::local_exclude(&worktree, "/.magi/").await?;
834 self.state.candidates.push(Candidate {
835 index,
836 label,
837 agent: spec.id.clone(),
838 branch,
839 worktree,
840 summary: String::new(),
841 stat: String::new(),
842 files: 0,
843 commits: 0,
844 empty: false,
845 failed: None,
846 verified_noop: None,
847 duration_ms: 0,
848 folded: false,
849 });
850 }
851
852 for j in 1..=self.roles.judges.len() {
853 let wt = root.join(format!("judge-{j}"));
854 if !wt.exists() {
855 git::worktree_add_detached(&repo, &wt, &base).await?;
856 }
857 }
858
859 // Disposable, detached checkouts for the design-deliberation stage's
860 // advisor seats — the same shape as the judges' above, at the same
861 // base commit, since advisors also only ever read. Sized off the
862 // configured count directly rather than a resolved roster: unlike
863 // `implementers`/`judges`/`reviewers`, advisor seats are resolved
864 // lazily inside `advise` itself (see `Config::advisors`'s doc), so
865 // `prep` has no `ResolvedRoles` field to read a count from here.
866 if self.state.config.graph.advise {
867 for k in 1..=self.state.config.graph.advisors {
868 let wt = root.join(format!("advisor-{k}"));
869 if !wt.exists() {
870 git::worktree_add_detached(&repo, &wt, &base).await?;
871 }
872 }
873 }
874
875 // A judge cannot tell it is looking at its own patch — the seats keep
876 // separate conversations — but a panel that shares agents with the
877 // field is less independent than it looks, and that is worth saying out
878 // loud once per run rather than leaving it in the config.
879 let authors: Vec<&str> = self
880 .roles
881 .implementers
882 .iter()
883 .map(|a| a.id.as_str())
884 .collect();
885 let overlap: Vec<String> = self
886 .roles
887 .judges
888 .iter()
889 .enumerate()
890 .filter(|(_, j)| authors.contains(&j.id.as_str()))
891 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
892 .collect();
893 if !overlap.is_empty() {
894 let note = format!(
895 "{} also authored a candidate; blind, but the panel is less \
896 independent than {} distinct agents would be",
897 overlap.join(", "),
898 self.roles.judges.len()
899 );
900 self.state.event("prep", note);
901 }
902
903 self.state.event(
904 "prep",
905 format!(
906 "{} candidates, {} judges, base {} ({})",
907 self.state.candidates.len(),
908 self.roles.judges.len(),
909 &self.state.base_commit[..7.min(self.state.base_commit.len())],
910 self.state.base_branch
911 ),
912 );
913 self.state.status = RunStatus::Implementing;
914 self.state.save()?;
915 Ok(())
916 }
917
918 // -------------------------------------------------------------- advise
919
920 /// The design-deliberation stage: independent, read-only advisor seats
921 /// each sketch a design before any implementer touches the repository,
922 /// and (when at least one produced a usable proposal) a synthesis seat
923 /// blends them into a brief `implement` carries in every candidate's
924 /// prompt.
925 ///
926 /// `[graph] advise` is the on/off switch, on by default; `[graph]
927 /// advisors` is the proposal count. Everything here is best-effort and
928 /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
929 /// that cannot reach quota, or a synthesis seat that produced nothing
930 /// usable all leave `implement` exactly as it was before this stage
931 /// existed — the task instruction alone — rather than failing the whole
932 /// competition over an enrichment stage. Every outcome is still recorded
933 /// as an event, so a run that got nothing from this stage says why.
934 ///
935 /// [`RunState::advise_attempted`] is this node's idempotency marker, the
936 /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
937 /// resumed run whose stage failed would re-run it, and re-spend the
938 /// agent calls, on every reentry before `implement`.
939 ///
940 /// Also skipped once any candidate shows implementation progress — the
941 /// exact predicate `implement` itself uses to decide a candidate is no
942 /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
943 /// is not enough: a run created by an older binary that predates this
944 /// field deserializes it as `false` (`#[serde(default)]`), so resuming
945 /// an already-`Implementing`-or-later run under this build would
946 /// otherwise walk straight back through `prep` (a no-op once candidates
947 /// exist) into this node and spawn every advisor seat against worktrees
948 /// `prep` never recreated — after implementation has already started,
949 /// which is exactly the invariant this stage exists to guarantee.
950 async fn advise(&mut self) -> Result<()> {
951 let implement_untouched = self
952 .state
953 .candidates
954 .iter()
955 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
956 if !self.state.config.graph.advise || self.state.advise_attempted {
957 return Ok(());
958 }
959 if !implement_untouched {
960 self.state.event(
961 "advise",
962 "skipping the design-deliberation stage: at least one \
963 candidate already shows implementation progress, so this \
964 run is past the point the stage exists to run before"
965 .to_owned(),
966 );
967 self.state.advise_attempted = true;
968 self.state.save()?;
969 return Ok(());
970 }
971 let run_id = self.state.id.clone();
972 let prompts = self.state.config.prompts.clone();
973 let instruction = self.state.instruction.clone();
974 let language = self.state.config.graph.language.clone();
975 let root = self.state.worktree_root();
976 let n = self.state.config.graph.advisors;
977 let where_recorded = self.state.dir().join("run.json");
978
979 let seats = match self.state.config.advisors() {
980 Ok(seats) if !seats.is_empty() => seats,
981 Ok(_) => {
982 self.state.event(
983 "advise",
984 format!(
985 "[graph] advisors is 0; skipping the design-deliberation \
986 stage and continuing without a synthesis brief (see {})",
987 where_recorded.display()
988 ),
989 );
990 self.state.advise_attempted = true;
991 self.state.save()?;
992 return Ok(());
993 }
994 Err(e) => {
995 self.state.event(
996 "advise",
997 format!(
998 "could not resolve advisor seats ({e:#}); continuing \
999 without a design-deliberation brief (see {})",
1000 where_recorded.display()
1001 ),
1002 );
1003 self.state.advise_attempted = true;
1004 self.state.save()?;
1005 return Ok(());
1006 }
1007 };
1008
1009 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1010 let artifacts = agent::artifacts_dir(&self.state.dir());
1011 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1012
1013 let mut jobs = Vec::new();
1014 for (i, spec) in seats.iter().cloned().enumerate() {
1015 let seat_key = format!("advisor-{}", i + 1);
1016 let seat = self.seat(&seat_key, &spec.id);
1017 jobs.push(SeatJob {
1018 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1019 spec,
1020 seat,
1021 cwd: worktrees[i % worktrees.len()].clone(),
1022 timeout,
1023 allow_write: false,
1024 sessions: false,
1025 artifacts: artifacts.clone(),
1026 stem: seat_key,
1027 });
1028 }
1029
1030 self.state.event(
1031 "advise",
1032 format!(
1033 "{} advisor seat(s) sketching a design in parallel",
1034 jobs.len()
1035 ),
1036 );
1037 let mut quota_losses = Vec::new();
1038 let cache = self.state.config.cache_dir();
1039 let ctx = WaveCtx {
1040 run: &run_id,
1041 node: "advise",
1042 prompts: &prompts,
1043 cache: cache.as_deref(),
1044 round: None,
1045 };
1046 let results = ask_json_wave::<Proposal>(
1047 jobs,
1048 Arc::clone(&self.sem),
1049 self.state.config.graph.retries,
1050 &ctx,
1051 &mut quota_losses,
1052 &mut self.state,
1053 &|p: &Proposal| p.validate(),
1054 )
1055 .await;
1056 self.state.quota.extend(quota_losses);
1057
1058 let mut records = Vec::with_capacity(results.len());
1059 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1060 let agent_id = seat.agent.clone();
1061 self.state.seats.insert(seat.key.clone(), seat);
1062 match res {
1063 Ok((proposal, out)) => {
1064 self.state
1065 .event("advise", format!("advisor-{} proposed a design", i + 1));
1066 records.push(advise::AdvisorRecord::proposed(
1067 i + 1,
1068 agent_id,
1069 proposal,
1070 out.duration_ms,
1071 ));
1072 }
1073 Err(e) => {
1074 self.state.event(
1075 "advise",
1076 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1077 );
1078 records.push(advise::AdvisorRecord::failed(
1079 i + 1,
1080 agent_id,
1081 e.to_string(),
1082 ));
1083 }
1084 }
1085 }
1086
1087 let mut advice = advise::Advice {
1088 records,
1089 synthesis: None,
1090 };
1091 if advice.proposals().is_empty() {
1092 self.state.event(
1093 "advise",
1094 "no advisor produced a usable proposal; continuing without a \
1095 synthesis brief"
1096 .to_owned(),
1097 );
1098 } else {
1099 match self
1100 .synthesize_brief(
1101 &advice,
1102 &instruction,
1103 &language,
1104 &worktrees[0],
1105 &artifacts,
1106 &run_id,
1107 &prompts,
1108 cache.as_deref(),
1109 )
1110 .await
1111 {
1112 Ok(Some(text)) => {
1113 self.state.event(
1114 "advise",
1115 "synthesized a design brief for the implementer".to_owned(),
1116 );
1117 advice.synthesis = Some(text);
1118 }
1119 Ok(None) => {
1120 self.state.event(
1121 "advise",
1122 "the synthesis seat produced nothing usable; continuing \
1123 without a design brief"
1124 .to_owned(),
1125 );
1126 }
1127 Err(e) => {
1128 self.state.event(
1129 "advise",
1130 format!("could not synthesize a design brief: {e:#}"),
1131 );
1132 }
1133 }
1134 }
1135 advise::apply_reflection(&mut advice);
1136
1137 self.state.advice = Some(advice);
1138 self.state.advise_attempted = true;
1139 self.state.save()?;
1140 Ok(())
1141 }
1142
1143 /// The synthesis seat: reads every advisor's proposal and blends them
1144 /// into the design brief `advise` stores on [`RunState::advice`]. Split
1145 /// out of [`Runner::advise`] only for readability — it is not called
1146 /// anywhere else.
1147 ///
1148 /// Picked the same way [`crate::talk`]'s standing conversation and
1149 /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1150 /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1151 /// order (a claude seat, else the first runnable agent in roster order)
1152 /// used when that field is unset — see `[roles] synthesizer`'s own doc
1153 /// in [`crate::config`] for why a dedicated field exists here at all.
1154 #[allow(clippy::too_many_arguments)]
1155 async fn synthesize_brief(
1156 &mut self,
1157 advice: &advise::Advice,
1158 instruction: &str,
1159 language: &str,
1160 cwd: &Path,
1161 artifacts: &Path,
1162 run_id: &str,
1163 prompts: &Prompts,
1164 cache: Option<&Path>,
1165 ) -> Result<Option<String>> {
1166 let want = self.state.config.roles.synthesizer.as_deref();
1167 let spec = agent::pick(&self.state.config.agents, want, &agent::installed)?;
1168 let mut seat = self.seat("advise-synthesis", &spec.id);
1169 let proposals = advice.proposals();
1170 let mut prompt = prompt::with_overlay(
1171 prompt::synthesize_brief(instruction, &proposals, language),
1172 prompts.overlay("advise"),
1173 );
1174 if cache.is_some() {
1175 // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1176 // below — see `prompt::build_cache_note`'s doc for why telling a
1177 // read-only seat to build through the shared cache is exactly how
1178 // a sandbox's write refusal gets misread as a defect.
1179 prompt.push('\n');
1180 prompt.push_str(&prompt::build_cache_note("advise", false));
1181 }
1182 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1183 let out = agent::invoke(
1184 &spec,
1185 &mut seat,
1186 &Invocation {
1187 cwd,
1188 prompt: &prompt,
1189 timeout,
1190 allow_write: false,
1191 sessions: false,
1192 artifacts,
1193 stem: "advise-synthesis",
1194 run: run_id,
1195 node: "advise",
1196 cache_dir: None,
1197 attachments: &[],
1198 },
1199 )
1200 .await?;
1201 self.state.seats.insert(seat.key.clone(), seat);
1202 if !out.usable() {
1203 return Ok(None);
1204 }
1205 let text =
1206 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1207 Ok((!text.trim().is_empty()).then_some(text))
1208 }
1209
1210 // ----------------------------------------------------------- implement
1211
1212 async fn implement(&mut self) -> Result<()> {
1213 // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1214 // agent files with `magi task add` name the run that paid for it. The
1215 // prompt overlay is cloned alongside it because the waves borrow it
1216 // while `self` is mutably borrowed by the node's own bookkeeping.
1217 let run_id = self.state.id.clone();
1218 let prompts = self.state.config.prompts.clone();
1219 let todo: Vec<usize> = self
1220 .state
1221 .candidates
1222 .iter()
1223 .enumerate()
1224 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1225 .map(|(i, _)| i)
1226 .collect();
1227 if todo.is_empty() {
1228 return self.after_implement();
1229 }
1230 self.state.status = RunStatus::Implementing;
1231
1232 let language = self.state.config.graph.language.clone();
1233 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1234 let sessions = self.state.config.graph.sessions;
1235 let artifacts = agent::artifacts_dir(&self.state.dir());
1236 // The design-deliberation stage's blended brief, when `advise` found
1237 // one — carried into every implementer's prompt the same way
1238 // regardless of which candidate it is.
1239 let brief = self
1240 .state
1241 .advice
1242 .as_ref()
1243 .and_then(|a| a.synthesis.as_deref())
1244 .map(str::to_owned);
1245
1246 let mut jobs = Vec::new();
1247 for &i in &todo {
1248 let (index, label, worktree) = {
1249 let c = &self.state.candidates[i];
1250 (c.index, c.label, c.worktree.clone())
1251 };
1252 let spec = self.roles.implementers[index].clone();
1253 let seat_key = format!("impl-{label}");
1254 let seat = self.seat(&seat_key, &spec.id);
1255 let instruction = self.state.instruction.clone();
1256 jobs.push(SeatJob {
1257 spec,
1258 seat,
1259 prompt: prompt::implement(
1260 &instruction,
1261 &worktree.to_string_lossy(),
1262 &language,
1263 brief.as_deref(),
1264 ),
1265 cwd: worktree,
1266 timeout,
1267 allow_write: true,
1268 sessions,
1269 artifacts: artifacts.clone(),
1270 stem: format!("impl-{label}"),
1271 });
1272 }
1273
1274 self.state.event(
1275 "implement",
1276 format!("{} candidates in parallel", jobs.len()),
1277 );
1278 // Kept so a seat whose CLI hung up can be asked again from the same
1279 // job: `wave` consumes what it is given. Mutable so `resume_quota_losses`
1280 // can update a seat's own entry once a fallback agent takes it over —
1281 // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1282 // whichever agent actually answered, not the one that quota'd out.
1283 let mut sent = jobs.clone();
1284 let cache = self.state.config.cache_dir();
1285 let ctx = WaveCtx {
1286 run: &run_id,
1287 node: "implement",
1288 prompts: &prompts,
1289 cache: cache.as_deref(),
1290 round: None,
1291 };
1292 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1293 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1294 .await;
1295 self.resume_quota_losses(&mut results, &mut sent, &prompts, &run_id)
1296 .await;
1297 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1298 .await;
1299
1300 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1301 let seat_key = seat.key.clone();
1302 // A quota fallback (`resume_quota_losses`) may have handed this
1303 // seat to a different agent than the one `prep` recorded on the
1304 // candidate; the stats tables and any later fixer-defaults-to-
1305 // winner's-author lookup must credit whoever actually answered —
1306 // unless every fallback also quota'd out, in which case nobody
1307 // actually answered and crediting the last agent tried would
1308 // erase every earlier agent's own quota loss from the stats
1309 // tables instead of just this one seat's.
1310 let agent = seat.agent.clone();
1311 let exhausted_the_fallback_chain = matches!(&out, AgentOutcome::Quota(_));
1312 self.state.seats.insert(seat.key.clone(), seat);
1313 let label = self.state.candidates[i].label;
1314 let worktree = self.state.candidates[i].worktree.clone();
1315 let base = self.state.base_commit.clone();
1316
1317 let (summary, duration, failed, verified_claim) = match out {
1318 AgentOutcome::Ok(o) => {
1319 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1320 let failed = (!o.usable()).then(|| {
1321 if o.timed_out {
1322 "agent timed out".to_owned()
1323 } else {
1324 format!("agent exited with {:?}", o.exit_code)
1325 }
1326 });
1327 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1328 (text, o.duration_ms, failed, verified_claim)
1329 }
1330 // Left un-resumed by `resume_undelivered` (a dirty tree
1331 // already rescues the work, or there was no session left to
1332 // resume into) — reported like the ordinary failure it is,
1333 // never as if `o.text` (the CLI's raw error JSON) were an
1334 // answer.
1335 AgentOutcome::Dropped(o) => {
1336 let why = o
1337 .dropped
1338 .as_ref()
1339 .map(|d| d.why.as_str())
1340 .unwrap_or("the CLI ended the stream without delivering its answer");
1341 (
1342 String::new(),
1343 o.duration_ms,
1344 Some(format!("the CLI dropped the stream ({why})")),
1345 None,
1346 )
1347 }
1348 AgentOutcome::Quota(o) => {
1349 self.state.quota.push(QuotaLoss {
1350 seat: seat_key,
1351 node: "implement".to_owned(),
1352 at: Timestamp::now(),
1353 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1354 });
1355 (
1356 String::new(),
1357 o.duration_ms,
1358 Some("rate limited (quota); produced no change".to_owned()),
1359 None,
1360 )
1361 }
1362 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1363 };
1364
1365 // Rescue anything the agent edited but never committed: an
1366 // uncommitted candidate would silently be an empty one.
1367 let rescued = match git::rescue_commit(
1368 &worktree,
1369 &format!("magi: candidate {label} (uncommitted work)"),
1370 )
1371 .await
1372 {
1373 Ok(r) => {
1374 self.state.note_withheld("implement", &r.withheld);
1375 r.committed
1376 }
1377 Err(_) => false,
1378 };
1379 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1380 .await
1381 .unwrap_or(0);
1382 let patch = git::diff(&worktree, &base, "HEAD")
1383 .await
1384 .unwrap_or_default();
1385 let stat = git::diff_stat(&worktree, &base, "HEAD")
1386 .await
1387 .unwrap_or_default();
1388 let files = git::changed_files(&worktree, &base, "HEAD")
1389 .await
1390 .map(|f| f.len())
1391 .unwrap_or(0);
1392 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1393
1394 let c = &mut self.state.candidates[i];
1395 if !exhausted_the_fallback_chain {
1396 c.agent = agent;
1397 }
1398 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1399 c.stat = stat;
1400 c.files = files;
1401 c.commits = commits;
1402 c.duration_ms = duration;
1403 c.empty = commits == 0 || patch.trim().is_empty();
1404 // An agent that failed but still produced a committed change stays
1405 // in the running: the patch is what gets judged, not the exit code.
1406 c.failed = match failed {
1407 Some(_) if c.empty => failed,
1408 _ => None,
1409 };
1410 // Only an empty candidate can be a verified no-op: a claim next
1411 // to a real patch is not what the marker is for, and `c.failed`
1412 // being `Some` here already implies `verified_claim` was never
1413 // set (see the guard above the match that produced it).
1414 c.verified_noop = if c.empty { verified_claim } else { None };
1415 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1416 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1417 (None, true, Some(_), _) => {
1418 format!("candidate {label}: no change produced (agent-verified no-op)")
1419 }
1420 (None, true, None, _) => format!("candidate {label}: no change produced"),
1421 (None, false, _, true) => {
1422 format!(
1423 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1424 )
1425 }
1426 (None, false, _, false) => {
1427 format!("candidate {label}: {files} files, {commits} commits")
1428 }
1429 };
1430 self.state.event("implement", note);
1431 self.state.save()?;
1432 }
1433
1434 self.after_implement()
1435 }
1436
1437 /// Ask again, once, for work a CLI did and then failed to hand over.
1438 ///
1439 /// [`agent::dropped_stream`] recognises the one shape observed: an error
1440 /// status with an empty response and a usage report showing output tokens,
1441 /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1442 /// seven minutes and 14,267 output tokens that arrived as an empty
1443 /// candidate, because `agy`'s own subscriber fell behind and hung up.
1444 ///
1445 /// Two conditions, and both matter:
1446 ///
1447 /// - **Only when the tree is untouched.** Often the agent has already
1448 /// written its files and only the closing message was lost; the rescue
1449 /// commit below picks that up and there is nothing to ask for. Re-asking
1450 /// then would pay for a second implementation of work already on disk.
1451 /// - **Once.** A CLI that drops one stream can drop the next, and this
1452 /// node is the most expensive in the graph.
1453 ///
1454 /// The re-ask is a resume, not a re-run: `has_context` is true because the
1455 /// dropped reply still carried its `conversation_id`, so the seat is asked
1456 /// to finish what it was doing rather than sent the whole task again. It
1457 /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1458 /// node's - for the same reason a re-ranked judge does: restating finished
1459 /// work is not the work.
1460 ///
1461 /// Unlike a quota this is worth retrying at all: a rate limit fails the
1462 /// same way until it resets, while an abandoned conversation is still
1463 /// there to be picked up.
1464 async fn resume_undelivered(
1465 &mut self,
1466 results: &mut [(usize, SeatState, AgentOutcome)],
1467 sent: &[SeatJob],
1468 prompts: &Prompts,
1469 run_id: &str,
1470 ) {
1471 for (wi, seat, out) in results.iter_mut() {
1472 let Some(dropped) = (match &*out {
1473 AgentOutcome::Dropped(o) => o.dropped.clone(),
1474 _ => None,
1475 }) else {
1476 continue;
1477 };
1478 let Some(job) = sent.get(*wi) else { continue };
1479 // Already on disk? Then only the closing message was lost.
1480 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1481 self.state.event(
1482 "implement",
1483 format!(
1484 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1485 work is in the tree",
1486 seat.key, dropped.output_tokens, dropped.why
1487 ),
1488 );
1489 continue;
1490 }
1491 // The re-ask only makes sense as a resume: `resume_after_drop`
1492 // says nothing about the task, trusting the seat to still hold it.
1493 // Without a session to resume — sessions disabled, or this CLI's
1494 // drop shape happened not to carry a session id — that prompt
1495 // would open a brand-new conversation with no context at all,
1496 // which is worse than leaving this as the ordinary failure it
1497 // already is.
1498 if !has_context(&job.spec, seat, job.sessions) {
1499 self.state.event(
1500 "implement",
1501 format!(
1502 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1503 is no session left to resume",
1504 seat.key, dropped.output_tokens, dropped.why
1505 ),
1506 );
1507 continue;
1508 }
1509 self.state.event(
1510 "implement",
1511 format!(
1512 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1513 conversation",
1514 seat.key, dropped.output_tokens, dropped.why
1515 ),
1516 );
1517 let mut retry = job.clone();
1518 retry.seat = seat.clone();
1519 retry.prompt = prompt::resume_after_drop(&dropped.why);
1520 retry.timeout = retry_budget(job.timeout, true);
1521 retry.stem = format!("{}-resume", job.stem);
1522 let cache = self.state.config.cache_dir();
1523 let ctx = WaveCtx {
1524 run: run_id,
1525 node: "implement",
1526 prompts,
1527 cache: cache.as_deref(),
1528 round: None,
1529 };
1530 let (resumed_seat, resumed) =
1531 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1532 *seat = resumed_seat;
1533 *out = resumed;
1534 }
1535 }
1536
1537 /// Fall an implement seat through to the next untried agent in the
1538 /// implementer roster when it lost to quota, instead of leaving the
1539 /// seat's loss final the moment one agent's account runs dry.
1540 ///
1541 /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1542 /// are the motivating case: `Config::resolve_roles`'s `implementers`
1543 /// truncates to the single slot rotation picked, so a solo task whose one
1544 /// implementer hits quota mid-run used to have nothing else to try. This
1545 /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1546 /// unrotated roster — which is the only place the *other* candidates in
1547 /// the machine's roster still exist once `implementers` has been cut down
1548 /// to size.
1549 ///
1550 /// Walks forward from just past the seat's own original position in the
1551 /// roster, never wrapping back to the front: a later candidate slot (say
1552 /// `beta`, the roster's second entry) must fall through to the *next*
1553 /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1554 /// almost certainly a different candidate's own agent already — and once
1555 /// the roster's tail is exhausted there is nothing left to fall through
1556 /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1557 /// whole [`AgentSpec`]: a roster with the same id named twice must not
1558 /// let this retry that id forever. The loop keeps falling through until
1559 /// an attempt lands something other than `Quota` or the roster's tail
1560 /// runs out of untried ids, at which point the seat is left exactly as
1561 /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1562 /// `QuotaLoss` recorded, the candidate failed/empty.
1563 ///
1564 /// `sent` is taken mutably and updated with the fallback agent's spec:
1565 /// `resume_unconfirmed_commands`, which runs after this and also reads
1566 /// `sent`, must see whichever agent actually ended up answering the seat
1567 /// — reading the stale, original spec there would check session
1568 /// eligibility against the wrong CLI and could hand a fallback agent's
1569 /// session id to the agent that just lost the seat to quota.
1570 ///
1571 /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1572 /// — `self.seat` only reuses state when the agent id is unchanged, so
1573 /// handing it a different id already gets this for free. Reusing the old
1574 /// seat would resume a different CLI's session as if it were a
1575 /// continuation of this one.
1576 ///
1577 /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1578 /// a quota loss cuts an agent off mid-turn, so anything already in the
1579 /// tree is unfinished work, not a completed candidate a re-ask would pay
1580 /// for twice. A dirty tree is rescued into a commit first (the same
1581 /// neutral-identity rescue `implement`'s own outcome loop gives every
1582 /// candidate) so the next agent starts clean.
1583 ///
1584 /// The new agent gets the implementer's full prompt and full
1585 /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1586 /// it has no session and no context, and is implementing the task from
1587 /// nothing, unlike a resumed drop which is only restating work already
1588 /// done.
1589 ///
1590 /// Every intermediate `Quota` this loop absorbs is folded into a plain
1591 /// `implement` event, never into `self.state.quota` — that is what
1592 /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1593 /// go unspent, and a seat that ultimately recovered on its second or
1594 /// third agent is not the stalled panel that check exists to catch. Only
1595 /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1596 /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1597 /// outcome loop already has — this helper never pushes to it itself.
1598 async fn resume_quota_losses(
1599 &mut self,
1600 results: &mut [(usize, SeatState, AgentOutcome)],
1601 sent: &mut [SeatJob],
1602 prompts: &Prompts,
1603 run_id: &str,
1604 ) {
1605 let instruction = self.state.instruction.clone();
1606 let language = self.state.config.graph.language.clone();
1607 let brief = self
1608 .state
1609 .advice
1610 .as_ref()
1611 .and_then(|a| a.synthesis.as_deref())
1612 .map(str::to_owned);
1613 for (wi, seat, out) in results.iter_mut() {
1614 let Some(job) = sent.get_mut(*wi) else {
1615 continue;
1616 };
1617 // Where the seat's own original agent sits in the roster — the
1618 // fallback walk starts just past here, never at the front, so a
1619 // later candidate slot's quota loss does not fall back onto an
1620 // earlier slot's own agent.
1621 let start = self
1622 .roles
1623 .implementer_roster
1624 .iter()
1625 .position(|s| s.id == job.spec.id)
1626 .unwrap_or(0);
1627 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1628 let mut fallback_attempt = 0usize;
1629 while matches!(&*out, AgentOutcome::Quota(_)) {
1630 let Some(next) =
1631 next_untried_implementer(&self.roles.implementer_roster, start, &tried)
1632 .cloned()
1633 else {
1634 break;
1635 };
1636 tried.insert(next.id.clone());
1637 fallback_attempt += 1;
1638
1639 if let Ok(r) = git::rescue_commit(
1640 &job.cwd,
1641 &format!(
1642 "magi: candidate {} (uncommitted work before quota fallback)",
1643 seat.key
1644 ),
1645 )
1646 .await
1647 {
1648 self.state.note_withheld("implement", &r.withheld);
1649 }
1650
1651 self.state.event(
1652 "implement",
1653 format!(
1654 "{}: rate limited (quota) on {}; retrying with {}",
1655 seat.key, seat.agent, next.id
1656 ),
1657 );
1658
1659 let new_seat = self.seat(&seat.key, &next.id);
1660 // Kept in sync on `sent` itself, not just the local retry: a
1661 // later helper (`resume_unconfirmed_commands`) reads `sent`
1662 // after this one returns and must see whichever agent is now
1663 // occupying the seat, not the one that just quota'd out —
1664 // otherwise it would judge session/continuation eligibility
1665 // by the wrong CLI and could resend a fallback's session id
1666 // to the agent that lost it the seat in the first place.
1667 job.spec = next.clone();
1668 let mut retry = job.clone();
1669 retry.seat = new_seat;
1670 retry.prompt = prompt::implement(
1671 &instruction,
1672 &job.cwd.to_string_lossy(),
1673 &language,
1674 brief.as_deref(),
1675 );
1676 retry.stem = format!("{}-quota-{}", job.stem, next.id);
1677 let cache = self.state.config.cache_dir();
1678 let ctx = WaveCtx {
1679 run: run_id,
1680 node: "implement",
1681 prompts,
1682 cache: cache.as_deref(),
1683 round: None,
1684 };
1685 let (fallback_seat, fallback_out) = run_one(
1686 retry,
1687 Arc::clone(&self.sem),
1688 &ctx,
1689 &mut self.state,
1690 fallback_attempt,
1691 )
1692 .await;
1693 *seat = fallback_seat;
1694 *out = fallback_out;
1695 }
1696 }
1697 }
1698
1699 /// Ask an implement seat's own CLI to confirm what it started, once, when
1700 /// its reply reported a command whose completion status it never
1701 /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1702 /// that does and does not mean.
1703 ///
1704 /// The completion contract this task asks for, extended to `implement`
1705 /// with the same signal `continue_fix_report` reads for the fixer,
1706 /// rather than a keyword search over the reply or a hard requirement on
1707 /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1708 /// a candidate's CLI turn ended cleanly while a test run it had started
1709 /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1710 /// named in it at all is untouched by this: `commands` is empty, so
1711 /// there is nothing to be unconfirmed.
1712 ///
1713 /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1714 /// this is not about recovering edits that might already be on disk, it
1715 /// is about a result the seat itself never vouched for, which resuming
1716 /// asks for regardless of what the tree already holds. Bounded to one
1717 /// attempt for the same reason `resume_undelivered` is — this is the
1718 /// most expensive node in the graph — and a seat that still cannot
1719 /// confirm on that attempt is left as whatever its (possibly still
1720 /// unconfirmed) reply says; this does not invent a new "failed" reason
1721 /// for a candidate that otherwise produced a real, committed change.
1722 async fn resume_unconfirmed_commands(
1723 &mut self,
1724 results: &mut [(usize, SeatState, AgentOutcome)],
1725 sent: &[SeatJob],
1726 prompts: &Prompts,
1727 run_id: &str,
1728 ) {
1729 for (wi, seat, out) in results.iter_mut() {
1730 let AgentOutcome::Ok(o) = &*out else {
1731 continue;
1732 };
1733 if !has_unconfirmed_command(&o.commands) {
1734 continue;
1735 }
1736 let Some(job) = sent.get(*wi) else { continue };
1737 if !has_context(&job.spec, seat, job.sessions) {
1738 self.state.event(
1739 "implement",
1740 format!(
1741 "{}: the reply named a command whose own CLI never confirmed the exit \
1742 status of, but there is no session left to resume",
1743 seat.key
1744 ),
1745 );
1746 continue;
1747 }
1748 self.state.event(
1749 "implement",
1750 format!(
1751 "{}: the reply named a command whose own CLI never confirmed the exit \
1752 status of; resuming the conversation",
1753 seat.key
1754 ),
1755 );
1756 let mut retry = job.clone();
1757 retry.seat = seat.clone();
1758 retry.prompt = prompt::resume_incomplete(
1759 "a command in your last reply had no confirmed exit status",
1760 );
1761 retry.timeout = retry_budget(job.timeout, true);
1762 retry.stem = format!("{}-confirm", job.stem);
1763 let cache = self.state.config.cache_dir();
1764 let ctx = WaveCtx {
1765 run: run_id,
1766 node: "implement",
1767 prompts,
1768 cache: cache.as_deref(),
1769 round: None,
1770 };
1771 let (resumed_seat, resumed) =
1772 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1773 *seat = resumed_seat;
1774 *out = resumed;
1775 }
1776 }
1777
1778 /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1779 /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1780 /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1781 /// that motivated this.
1782 ///
1783 /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1784 /// own nudge loop already covers for judge/review/vote seats, and not a
1785 /// dropped stream, which [`Runner::resume_undelivered`] covers for
1786 /// implement seats: here the CLI turn genuinely finished while the node's
1787 /// own work — the fixer's account of what it did — had not. Gated purely
1788 /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1789 /// reply, never on any wording in it, so a fixer whose valid, first-try
1790 /// `FixReport` happens to mention having waited on a background test is
1791 /// never resumed — the `Ok(report)` branch at the call site returns
1792 /// before this is ever invoked.
1793 ///
1794 /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1795 /// attempt ([`retry_budget`]), nothing attempted once the session is
1796 /// gone, and a quota hit ends the loop immediately rather than retrying a
1797 /// rate limit that fails the same way again.
1798 async fn continue_fix_report(
1799 &mut self,
1800 mut seat: SeatState,
1801 parse_err: String,
1802 job: &SeatJob,
1803 prompts: &Prompts,
1804 run_id: &str,
1805 round: usize,
1806 ) -> (
1807 SeatState,
1808 Option<FixReport>,
1809 Option<String>,
1810 ContinuationRecord,
1811 ) {
1812 let mut last_err = parse_err;
1813 let mut cumulative_wait_ms = 0u64;
1814 let mut attempts = 0usize;
1815 loop {
1816 if !has_context(&job.spec, &seat, job.sessions) {
1817 self.state.event(
1818 "fix",
1819 format!(
1820 "round {round}: fixer's reply had no adoption report ({last_err}); no \
1821 session left to resume into"
1822 ),
1823 );
1824 let outcome = if attempts == 0 {
1825 ContinuationOutcome::NoSession
1826 } else {
1827 ContinuationOutcome::Exhausted
1828 };
1829 return (
1830 seat,
1831 None,
1832 Some(format!("unparsable fix report: {last_err}")),
1833 ContinuationRecord {
1834 attempts,
1835 cumulative_wait_ms,
1836 outcome,
1837 },
1838 );
1839 }
1840 if attempts >= MAX_FIX_CONTINUATIONS {
1841 self.state.event(
1842 "fix",
1843 format!(
1844 "round {round}: fixer's reply still had no adoption report after \
1845 {attempts} continuation(s) ({last_err}); giving up"
1846 ),
1847 );
1848 return (
1849 seat,
1850 None,
1851 Some(format!(
1852 "unparsable fix report after {attempts} continuation(s): {last_err}"
1853 )),
1854 ContinuationRecord {
1855 attempts,
1856 cumulative_wait_ms,
1857 outcome: ContinuationOutcome::Exhausted,
1858 },
1859 );
1860 }
1861 attempts += 1;
1862 self.state.event(
1863 "fix",
1864 format!(
1865 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
1866 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
1867 ),
1868 );
1869 let mut retry = job.clone();
1870 retry.seat = seat.clone();
1871 retry.prompt = prompt::resume_incomplete(&last_err);
1872 retry.timeout = retry_budget(job.timeout, true);
1873 retry.stem = format!("{}-continue{attempts}", job.stem);
1874 let cache = self.state.config.cache_dir();
1875 let ctx = WaveCtx {
1876 run: run_id,
1877 node: "fix",
1878 prompts,
1879 cache: cache.as_deref(),
1880 round: Some(round),
1881 };
1882 let (resumed_seat, resumed_out) = run_one(
1883 retry,
1884 Arc::clone(&self.sem),
1885 &ctx,
1886 &mut self.state,
1887 attempts,
1888 )
1889 .await;
1890 seat = resumed_seat;
1891 match resumed_out {
1892 AgentOutcome::Ok(o) => {
1893 cumulative_wait_ms += o.duration_ms;
1894 match verdict::extract_json::<FixReport>(&o.text) {
1895 Ok(report) if !has_unconfirmed_command(&o.commands) => {
1896 self.state.event(
1897 "fix",
1898 format!(
1899 "round {round}: fixer's adoption report recovered after \
1900 {attempts} continuation(s)"
1901 ),
1902 );
1903 return (
1904 seat,
1905 Some(report),
1906 None,
1907 ContinuationRecord {
1908 attempts,
1909 cumulative_wait_ms,
1910 outcome: ContinuationOutcome::Resumed,
1911 },
1912 );
1913 }
1914 // The report parsed, but this same reply's own
1915 // CommandEvidence — the identical record `state.jobs`
1916 // renders — names a command whose CLI never
1917 // confirmed an exit status. Read together, that is
1918 // not a resolved answer: keep nudging rather than
1919 // accept a report standing next to a command the
1920 // seat's own CLI cannot vouch for.
1921 Ok(_) => {
1922 last_err = "the reply parsed, but it reported a command whose own CLI \
1923 never confirmed an exit status"
1924 .to_owned();
1925 }
1926 Err(e) => last_err = e.to_string(),
1927 }
1928 }
1929 AgentOutcome::Quota(o) => {
1930 cumulative_wait_ms += o.duration_ms;
1931 self.state.quota.push(QuotaLoss {
1932 seat: seat.key.clone(),
1933 node: "fix".to_owned(),
1934 at: Timestamp::now(),
1935 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1936 });
1937 self.state.event(
1938 "fix",
1939 format!(
1940 "round {round}: continuation rate limited (quota); not retrying now"
1941 ),
1942 );
1943 return (
1944 seat,
1945 None,
1946 Some("rate limited (quota) while recovering the fix report".to_owned()),
1947 ContinuationRecord {
1948 attempts,
1949 cumulative_wait_ms,
1950 outcome: ContinuationOutcome::QuotaLost,
1951 },
1952 );
1953 }
1954 AgentOutcome::Dropped(o) => {
1955 cumulative_wait_ms += o.duration_ms;
1956 let why = o
1957 .dropped
1958 .as_ref()
1959 .map(|d| d.why.as_str())
1960 .unwrap_or("the CLI ended the stream without delivering its answer");
1961 last_err = format!("the CLI dropped the stream ({why})");
1962 }
1963 AgentOutcome::Failed(e) => last_err = e,
1964 }
1965 }
1966 }
1967
1968 fn after_implement(&mut self) -> Result<()> {
1969 // Scan every candidate patch once the set is complete.
1970 if self.state.leaks.is_empty() {
1971 let cfg = self.state.config.blind.clone();
1972 let mut leaks = Vec::new();
1973 for c in &self.state.candidates {
1974 let Some(patch) =
1975 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1976 else {
1977 continue;
1978 };
1979 leaks.extend(blind::scan(
1980 &format!("candidate {} patch", c.label),
1981 &patch,
1982 &cfg.vendor_tokens,
1983 ));
1984 }
1985 if !leaks.is_empty() {
1986 let summary = leaks
1987 .iter()
1988 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
1989 .collect::<Vec<_>>()
1990 .join(", ");
1991 match cfg.on_leak {
1992 LeakPolicy::Fail => {
1993 self.state.status = RunStatus::Failed;
1994 self.state
1995 .event("blind", format!("vendor text in a patch: {summary}"));
1996 self.state.leaks = leaks;
1997 self.state.save()?;
1998 self.settle_questions();
1999 bail!(
2000 "blind.on_leak = \"fail\" and vendor text reached a \
2001 judged patch: {summary}"
2002 );
2003 }
2004 LeakPolicy::Redact => self.state.event(
2005 "blind",
2006 format!("redacting vendor text for judging: {summary}"),
2007 ),
2008 LeakPolicy::Warn => self.state.event(
2009 "blind",
2010 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2011 ),
2012 }
2013 self.state.leaks = leaks;
2014 }
2015 }
2016
2017 if self.state.viable().is_empty() {
2018 if self.state.all_candidates_verified_noop() {
2019 // Every candidate agreed, with evidence the adoption guard
2020 // accepted, that nothing belongs in this worktree. That is
2021 // not the same fact as a candidate that simply failed to
2022 // write anything, and settling it as an ordinary `Failed`
2023 // (see `SCHEMA`'s doc for schema 10) is what let two of
2024 // task 391f's attempts burn a retry each re-discovering the
2025 // same already-landed fix. Terminal either way, so `judge`
2026 // must never run over an empty candidate set — unlike the
2027 // `Failed` branch below this returns `Ok`, not an error:
2028 // nothing here failed.
2029 self.state.status = RunStatus::VerifiedNoop;
2030 self.state.save()?;
2031 self.settle_questions();
2032 return Ok(());
2033 }
2034 self.state.status = RunStatus::Failed;
2035 self.state.save()?;
2036 self.settle_questions();
2037 bail!("no candidate produced a change; nothing to judge");
2038 }
2039 self.state.status = RunStatus::Judging;
2040 self.state.save()?;
2041 Ok(())
2042 }
2043
2044 // --------------------------------------------------------------- judge
2045
2046 async fn judge(&mut self) -> Result<()> {
2047 // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2048 // agent files with `magi task add` name the run that paid for it. The
2049 // prompt overlay is cloned alongside it because the waves borrow it
2050 // while `self` is mutably borrowed by the node's own bookkeeping.
2051 let run_id = self.state.id.clone();
2052 let prompts = self.state.config.prompts.clone();
2053 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2054 return Ok(());
2055 }
2056 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2057 if viable.len() == 1 {
2058 // Recorded so this is a one-time event: `judgements` stays empty
2059 // either way, which without this flag is indistinguishable from
2060 // "not yet judged" on the next reentry — and status is left
2061 // untouched, so a later node's conclusion (e.g. `Blocked` after
2062 // the review budget ran out) survives a resume instead of being
2063 // clobbered back to `Judging` by this node running again.
2064 self.state.judge_skipped = true;
2065 self.state.event(
2066 "judge",
2067 format!(
2068 "only candidate {} produced a change; judging skipped",
2069 viable[0].label
2070 ),
2071 );
2072 self.state.save()?;
2073 return Ok(());
2074 }
2075 self.state.status = RunStatus::Judging;
2076
2077 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2078 let language = self.state.config.graph.language.clone();
2079 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2080 let sessions = self.state.config.graph.sessions;
2081 let artifacts = agent::artifacts_dir(&self.state.dir());
2082 let root = self.state.worktree_root();
2083 let base_short = short(&self.state.base_commit);
2084
2085 let mut jobs = Vec::new();
2086 let mut orders = Vec::new();
2087 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2088 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2089 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2090 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2091 let seat_key = format!("judge-{}", j + 1);
2092 let seat = self.seat(&seat_key, &spec.id);
2093 jobs.push(SeatJob {
2094 prompt: prompt::judge(
2095 &self.state.instruction,
2096 &views,
2097 self.roles.judges.len(),
2098 &base_short,
2099 &language,
2100 ),
2101 spec,
2102 seat,
2103 cwd: root.join(format!("judge-{}", j + 1)),
2104 timeout,
2105 allow_write: false,
2106 sessions,
2107 artifacts: artifacts.clone(),
2108 stem: format!("judge-{}", j + 1),
2109 });
2110 }
2111
2112 self.state.event(
2113 "judge",
2114 format!(
2115 "{} judges ranking {} candidates blind",
2116 jobs.len(),
2117 viable.len()
2118 ),
2119 );
2120 let labels_for_check = labels.clone();
2121 let mut quota_losses = Vec::new();
2122 let cache = self.state.config.cache_dir();
2123 let ctx = WaveCtx {
2124 run: &run_id,
2125 node: "judge",
2126 prompts: &prompts,
2127 cache: cache.as_deref(),
2128 round: None,
2129 };
2130 let results = ask_json_wave::<Ranking>(
2131 jobs,
2132 Arc::clone(&self.sem),
2133 self.state.config.graph.retries,
2134 &ctx,
2135 &mut quota_losses,
2136 &mut self.state,
2137 &move |r: &Ranking| r.validate(&labels_for_check),
2138 )
2139 .await;
2140 self.state.quota.extend(quota_losses);
2141
2142 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2143 let agent_id = seat.agent.clone();
2144 self.state.seats.insert(seat.key.clone(), seat);
2145 let mut record = Judgement {
2146 judge: j + 1,
2147 seat: format!("judge-{}", j + 1),
2148 agent: agent_id,
2149 ranking: Vec::new(),
2150 reasons: BTreeMap::new(),
2151 confidence: None,
2152 order: orders[j].clone(),
2153 failed: None,
2154 duration_ms: 0,
2155 };
2156 match res {
2157 Ok((ranking, out)) => {
2158 record.ranking = ranking.normalized();
2159 record.reasons = ranking.reasons;
2160 record.confidence = ranking.confidence;
2161 record.duration_ms = out.duration_ms;
2162 self.state.event(
2163 "judge",
2164 format!(
2165 "judge {} ranked {}",
2166 j + 1,
2167 record.ranking.iter().collect::<String>()
2168 ),
2169 );
2170 }
2171 Err(e) => {
2172 record.failed = Some(e.to_string());
2173 self.state
2174 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2175 }
2176 }
2177 self.state.judgements.push(record);
2178 self.state.save()?;
2179 }
2180 Ok(())
2181 }
2182
2183 // ---------------------------------------------------------- deliberate
2184
2185 async fn deliberate(&mut self) -> Result<()> {
2186 // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2187 // agent files with `magi task add` name the run that paid for it. The
2188 // prompt overlay is cloned alongside it because the waves borrow it
2189 // while `self` is mutably borrowed by the node's own bookkeeping.
2190 let run_id = self.state.id.clone();
2191 let prompts = self.state.config.prompts.clone();
2192 if !self.state.deliberation.is_empty() {
2193 return Ok(());
2194 }
2195 let tops: Vec<char> = self
2196 .state
2197 .judgements
2198 .iter()
2199 .filter_map(|j| j.ranking.first().copied())
2200 .collect();
2201 let rounds = self.state.config.graph.deliberate_rounds;
2202 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2203 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2204 self.state.event(
2205 "deliberate",
2206 format!("judges agreed on {} outright; no deliberation", tops[0]),
2207 );
2208 }
2209 self.state.status = RunStatus::Voting;
2210 self.state.save()?;
2211 return Ok(());
2212 }
2213
2214 self.state.status = RunStatus::Deliberating;
2215 self.state.event(
2216 "deliberate",
2217 format!(
2218 "split: first choices were {} — opening {rounds} round(s)",
2219 tops.iter().collect::<String>()
2220 ),
2221 );
2222
2223 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2224 let language = self.state.config.graph.language.clone();
2225 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2226 let sessions = self.state.config.graph.sessions;
2227 let artifacts = agent::artifacts_dir(&self.state.dir());
2228 let root = self.state.worktree_root();
2229 let base_short = short(&self.state.base_commit);
2230
2231 // Judges argue in sequence so that a turn can answer the one before it;
2232 // that is the difference between deliberation and three parallel
2233 // monologues.
2234 for round in 1..=rounds {
2235 let mut turns: Vec<DeliberationTurn> = Vec::new();
2236 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2237 if self.state.judgements[j].failed.is_some() {
2238 continue;
2239 }
2240 let seat_key = format!("judge-{}", j + 1);
2241 let mut seat = self.seat(&seat_key, &spec.id);
2242 let transcript = self.transcript(&turns, j);
2243 let context = if has_context(&spec, &seat, sessions) {
2244 None
2245 } else {
2246 Some(self.candidate_block(&viable, &base_short))
2247 };
2248 let text = prompt::deliberate(
2249 &self.state.instruction,
2250 context.as_deref(),
2251 &transcript,
2252 round,
2253 rounds,
2254 &language,
2255 );
2256 let job = SeatJob {
2257 spec,
2258 seat: seat.clone(),
2259 prompt: text,
2260 cwd: root.join(format!("judge-{}", j + 1)),
2261 timeout,
2262 allow_write: false,
2263 sessions,
2264 artifacts: artifacts.clone(),
2265 stem: format!("delib-{round}-judge-{}", j + 1),
2266 };
2267 let cache = self.state.config.cache_dir();
2268 let ctx = WaveCtx {
2269 run: &run_id,
2270 node: "deliberate",
2271 prompts: &prompts,
2272 cache: cache.as_deref(),
2273 round: None,
2274 };
2275 let (updated, out) =
2276 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2277 seat = updated;
2278 let agent_id = seat.agent.clone();
2279 let seat_key = seat.key.clone();
2280 self.state.seats.insert(seat.key.clone(), seat);
2281 let body = match out {
2282 AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2283 // Never read the CLI's raw error JSON as this judge's
2284 // position — skip the seat instead, the same as any other
2285 // failed turn.
2286 AgentOutcome::Dropped(o) => {
2287 let why =
2288 o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2289 "the CLI ended the stream without delivering its answer",
2290 );
2291 self.state.event(
2292 "deliberate",
2293 format!(
2294 "judge {} skipped: the CLI dropped the stream ({why})",
2295 j + 1
2296 ),
2297 );
2298 continue;
2299 }
2300 AgentOutcome::Quota(o) => {
2301 self.state.quota.push(QuotaLoss {
2302 seat: seat_key,
2303 node: "deliberate".to_owned(),
2304 at: Timestamp::now(),
2305 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2306 });
2307 self.state.event(
2308 "deliberate",
2309 format!("judge {} skipped: rate limited (quota)", j + 1),
2310 );
2311 continue;
2312 }
2313 AgentOutcome::Failed(e) => {
2314 self.state
2315 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2316 continue;
2317 }
2318 };
2319 let tentative = verdict::extract_json::<Position>(&body)
2320 .ok()
2321 .and_then(|p| p.tentative)
2322 .and_then(|s| s.trim().chars().next())
2323 .map(|c| c.to_ascii_uppercase());
2324 self.state.event(
2325 "deliberate",
2326 format!(
2327 "round {round}: judge {} now favours {}",
2328 j + 1,
2329 tentative.map_or("—".to_owned(), |c| c.to_string())
2330 ),
2331 );
2332 turns.push(DeliberationTurn {
2333 judge: j + 1,
2334 agent: agent_id,
2335 body: blind::sanitize_prose(&body, &self.state.config.blind),
2336 tentative,
2337 });
2338 }
2339 self.state
2340 .deliberation
2341 .push(DeliberationRound { round, turns });
2342 self.state.save()?;
2343 }
2344
2345 self.state.status = RunStatus::Voting;
2346 self.state.save()?;
2347 Ok(())
2348 }
2349
2350 // ---------------------------------------------------------------- vote
2351
2352 async fn vote(&mut self) -> Result<()> {
2353 // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2354 // agent files with `magi task add` name the run that paid for it. The
2355 // prompt overlay is cloned alongside it because the waves borrow it
2356 // while `self` is mutably borrowed by the node's own bookkeeping.
2357 let run_id = self.state.id.clone();
2358 let prompts = self.state.config.prompts.clone();
2359 if !self.state.votes.is_empty() {
2360 return Ok(());
2361 }
2362 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2363 if viable.len() == 1 {
2364 return Ok(());
2365 }
2366 self.state.status = RunStatus::Voting;
2367
2368 let language = self.state.config.graph.language.clone();
2369 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2370 let sessions = self.state.config.graph.sessions;
2371 let artifacts = agent::artifacts_dir(&self.state.dir());
2372 let root = self.state.worktree_root();
2373 let base_short = short(&self.state.base_commit);
2374 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2375
2376 let mut jobs = Vec::new();
2377 let mut seats_at = Vec::new();
2378 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2379 if self
2380 .state
2381 .judgements
2382 .get(j)
2383 .is_some_and(|r| r.failed.is_some())
2384 {
2385 continue;
2386 }
2387 let seat_key = format!("judge-{}", j + 1);
2388 let seat = self.seat(&seat_key, &spec.id);
2389 let mut text = prompt::final_vote(&viable, &language);
2390 if !has_context(&spec, &seat, sessions) {
2391 text = format!(
2392 "{}\n\n# Candidates\n\n{}",
2393 text,
2394 self.candidate_block(&candidates, &base_short)
2395 );
2396 }
2397 jobs.push(SeatJob {
2398 spec,
2399 seat,
2400 prompt: text,
2401 cwd: root.join(format!("judge-{}", j + 1)),
2402 timeout,
2403 allow_write: false,
2404 sessions,
2405 artifacts: artifacts.clone(),
2406 stem: format!("vote-judge-{}", j + 1),
2407 });
2408 seats_at.push(j);
2409 }
2410
2411 self.state.event(
2412 "vote",
2413 format!(
2414 "collecting {} final votes one by one, privately",
2415 jobs.len()
2416 ),
2417 );
2418 let allowed = viable.clone();
2419 let mut quota_losses = Vec::new();
2420 let cache = self.state.config.cache_dir();
2421 let ctx = WaveCtx {
2422 run: &run_id,
2423 node: "vote",
2424 prompts: &prompts,
2425 cache: cache.as_deref(),
2426 round: None,
2427 };
2428 let results = ask_json_wave::<FinalVote>(
2429 jobs,
2430 Arc::clone(&self.sem),
2431 self.state.config.graph.retries,
2432 &ctx,
2433 &mut quota_losses,
2434 &mut self.state,
2435 &move |v: &FinalVote| match v.label() {
2436 Some(c) if allowed.contains(&c) => Ok(()),
2437 other => bail!("vote {other:?} is not one of {allowed:?}"),
2438 },
2439 )
2440 .await;
2441 self.state.quota.extend(quota_losses);
2442
2443 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2444 let agent_id = seat.agent.clone();
2445 self.state.seats.insert(seat.key.clone(), seat);
2446 let initial = self
2447 .state
2448 .judgements
2449 .get(j)
2450 .and_then(|r| r.ranking.first().copied());
2451 let mut record = VoteRecord {
2452 judge: j + 1,
2453 agent: agent_id,
2454 vote: None,
2455 reason: String::new(),
2456 changed: false,
2457 };
2458 match res {
2459 Ok((v, _)) => {
2460 record.vote = v.label();
2461 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2462 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2463 self.state.event(
2464 "vote",
2465 format!(
2466 "judge {} voted {}{}",
2467 j + 1,
2468 record.vote.unwrap_or('?'),
2469 if record.changed { " (changed)" } else { "" }
2470 ),
2471 );
2472 }
2473 Err(e) => {
2474 self.state
2475 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2476 }
2477 }
2478 self.state.votes.push(record);
2479 self.state.save()?;
2480 }
2481 Ok(())
2482 }
2483
2484 // --------------------------------------------------------------- tally
2485
2486 fn tally(&mut self) -> Result<()> {
2487 if self.state.tally.is_some() {
2488 return Ok(());
2489 }
2490 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2491 let tops: Vec<char> = self
2492 .state
2493 .judgements
2494 .iter()
2495 .filter_map(|j| j.ranking.first().copied())
2496 .collect();
2497 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2498
2499 // A judge whose private vote failed still counted once, in the initial
2500 // ranking; using it beats discarding a whole seat.
2501 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2502 let mut cast: Vec<char> = Vec::new();
2503 for (i, j) in self.state.judgements.iter().enumerate() {
2504 let vote = self
2505 .state
2506 .votes
2507 .iter()
2508 .find(|v| v.judge == i + 1)
2509 .and_then(|v| v.vote)
2510 .or_else(|| j.ranking.first().copied());
2511 if let Some(v) = vote {
2512 *first_choice.entry(v).or_insert(0) += 1;
2513 cast.push(v);
2514 }
2515 }
2516
2517 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2518 for j in &self.state.judgements {
2519 let n = j.ranking.len();
2520 for (pos, label) in j.ranking.iter().enumerate() {
2521 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2522 }
2523 }
2524
2525 let best = first_choice.values().copied().max().unwrap_or(0);
2526 let mut leaders: Vec<char> = first_choice
2527 .iter()
2528 .filter(|(_, v)| **v == best)
2529 .map(|(k, _)| *k)
2530 .collect();
2531 let mut tie_break = None;
2532 if leaders.len() > 1 {
2533 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2534 let borda_leaders: Vec<char> = leaders
2535 .iter()
2536 .copied()
2537 .filter(|l| borda[l] == top_borda)
2538 .collect();
2539 tie_break = Some(if borda_leaders.len() == 1 {
2540 format!(
2541 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2542 leaders.len()
2543 )
2544 } else {
2545 format!(
2546 "{} way tie on both first-choice votes and Borda points, broken by label order",
2547 leaders.len()
2548 )
2549 });
2550 leaders = borda_leaders;
2551 leaders.sort_unstable();
2552 }
2553 let winner = *leaders
2554 .first()
2555 .or(viable.first())
2556 .context("no candidate to declare a winner from")?;
2557
2558 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2559 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2560 let deliberated = !self.state.deliberation.is_empty();
2561
2562 // Whose verdict is this? A rate-limited seat is absent even if it
2563 // ranked before the limit hit, so presence is measured against the
2564 // recorded losses, not just "did a ranking ever appear".
2565 let quota_seats: std::collections::BTreeSet<&str> =
2566 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2567 let mut present = 0usize;
2568 for (i, j) in self.state.judgements.iter().enumerate() {
2569 if quota_seats.contains(j.seat.as_str()) {
2570 continue;
2571 }
2572 let ranked = !j.ranking.is_empty() && j.failed.is_none();
2573 let voted = self
2574 .state
2575 .votes
2576 .iter()
2577 .any(|v| v.judge == i + 1 && v.vote.is_some());
2578 if ranked || voted {
2579 present += 1;
2580 }
2581 }
2582 // Strict majority of the configured panel. A bare majority is real
2583 // signal we can act on, while a minority verdict must never stand in
2584 // for a healthy one. A one-candidate run needs no panel at all, and
2585 // `judges` stays `0` rather than the roster size a panel that never
2586 // sat would otherwise be credited with.
2587 let needs_quorum = viable.len() > 1;
2588 let judges_total = if needs_quorum {
2589 self.roles.judges.len()
2590 } else {
2591 0
2592 };
2593 let quorum = if needs_quorum {
2594 judges_total / 2 + 1
2595 } else {
2596 0
2597 };
2598 let met_quorum = !needs_quorum || present >= quorum;
2599 let uncontested = (!needs_quorum).then(|| {
2600 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2601 });
2602
2603 self.state.event(
2604 "tally",
2605 match &uncontested {
2606 Some(reason) => format!("winner {winner} — {reason}"),
2607 None => format!(
2608 "winner {winner} — votes {} | initial {} | {} changed | \
2609 {present}/{judges_total} judges{}",
2610 first_choice
2611 .iter()
2612 .map(|(k, v)| format!("{k}:{v}"))
2613 .collect::<Vec<_>>()
2614 .join(" "),
2615 if unanimous_initial {
2616 "unanimous"
2617 } else {
2618 "split"
2619 },
2620 changed_votes,
2621 if met_quorum {
2622 String::new()
2623 } else {
2624 format!(" — below quorum ({quorum} required)")
2625 },
2626 ),
2627 },
2628 );
2629 if !met_quorum {
2630 self.state.event(
2631 "stall",
2632 format!(
2633 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2634 the run stops here, resumable"
2635 ),
2636 );
2637 }
2638 self.state.tally = Some(Tally {
2639 first_choice,
2640 borda,
2641 winner,
2642 rankings: tops.len(),
2643 unanimous_initial,
2644 deliberated,
2645 changed_votes,
2646 unanimous_final,
2647 tie_break,
2648 judges: judges_total,
2649 present,
2650 quorum,
2651 met_quorum,
2652 uncontested,
2653 });
2654 self.state.status = if met_quorum {
2655 RunStatus::Reviewing
2656 } else {
2657 RunStatus::Stalled
2658 };
2659 self.state.save()?;
2660 Ok(())
2661 }
2662
2663 // ------------------------------------------------------------- recover
2664
2665 /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2666 /// resumed toward completion once the transient cause clears.
2667 ///
2668 /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2669 /// it toward the quorum, which is exactly the set of seats whose absence
2670 /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2671 /// not depend on which node happened to hit the limit), or an ordinary
2672 /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2673 /// seat is never disturbed.
2674 ///
2675 /// A seat that now answers with a usable ranking is "recovered": its
2676 /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2677 /// `tally` counts it present again), and its vote re-collected. A seat that
2678 /// still fails keeps its loss and stays absent.
2679 ///
2680 /// Returns `true` when the re-tally restores the quorum (the run may proceed
2681 /// to review/gate/merge), `false` when it is still below quorum (the run
2682 /// stays `Stalled`, still resumable for a later retry).
2683 #[allow(clippy::too_many_lines)]
2684 async fn recover_stall(&mut self) -> Result<bool> {
2685 // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2686 // agent files with `magi task add` name the run that paid for it. The
2687 // prompt overlay is cloned alongside it because the waves borrow it
2688 // while `self` is mutably borrowed by the node's own bookkeeping.
2689 let run_id = self.state.id.clone();
2690 let prompts = self.state.config.prompts.clone();
2691 // Absent seats = quota-lost at any node, or failed outright. Mirroring
2692 // `tally`'s presence test (rather than the old quota-judge/vote filter)
2693 // is what keeps a non-quota collapse — or a quota loss recorded at the
2694 // deliberate node — from being a permanent dead-end on `--resume`.
2695 let quota_seats: BTreeSet<&str> =
2696 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2697 let absent: Vec<String> = self
2698 .state
2699 .judgements
2700 .iter()
2701 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2702 .map(|j| j.seat.clone())
2703 .collect();
2704 if absent.is_empty() {
2705 return Ok(false);
2706 }
2707 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2708 if viable.len() <= 1 {
2709 return Ok(false);
2710 }
2711 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2712 let language = self.state.config.graph.language.clone();
2713 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2714 let sessions = self.state.config.graph.sessions;
2715 let artifacts = agent::artifacts_dir(&self.state.dir());
2716 let root = self.state.worktree_root();
2717 let base_short = short(&self.state.base_commit);
2718 let candidates: Vec<Candidate> = viable.clone();
2719
2720 // Map each absent seat key to its 0-based position in `roles.judges`.
2721 let mut positions: Vec<usize> = absent
2722 .iter()
2723 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2724 .collect();
2725 if positions.is_empty() {
2726 return Ok(false);
2727 }
2728 positions.sort_unstable();
2729 positions.dedup();
2730
2731 // Re-rank the lost seats, one blind prompt each.
2732 let mut judge_jobs = Vec::new();
2733 for &j in &positions {
2734 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2735 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2736 let seat_key = format!("judge-{}", j + 1);
2737 let spec = self.roles.judges[j].clone();
2738 let seat = self.seat(&seat_key, &spec.id);
2739 judge_jobs.push(SeatJob {
2740 spec,
2741 seat,
2742 prompt: prompt::judge(
2743 &self.state.instruction,
2744 &views,
2745 self.roles.judges.len(),
2746 &base_short,
2747 &language,
2748 ),
2749 cwd: root.join(seat_key),
2750 timeout,
2751 allow_write: false,
2752 sessions,
2753 artifacts: artifacts.clone(),
2754 stem: format!("judge-{}-recover", j + 1),
2755 });
2756 }
2757
2758 let labels_for_check = labels.clone();
2759 let mut judge_losses = Vec::new();
2760 let retries = self.state.config.graph.retries;
2761 let cache = self.state.config.cache_dir();
2762 let ctx = WaveCtx {
2763 run: &run_id,
2764 node: "judge",
2765 prompts: &prompts,
2766 cache: cache.as_deref(),
2767 round: None,
2768 };
2769 let results = ask_json_wave::<Ranking>(
2770 judge_jobs,
2771 Arc::clone(&self.sem),
2772 retries,
2773 &ctx,
2774 &mut judge_losses,
2775 &mut self.state,
2776 &move |r: &Ranking| r.validate(&labels_for_check),
2777 )
2778 .await;
2779
2780 // Refresh the judgement of every seat that ranked again.
2781 let mut recovered: BTreeSet<usize> = BTreeSet::new();
2782 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
2783 self.state.seats.insert(seat.key.clone(), seat);
2784 let record = &mut self.state.judgements[j];
2785 match res {
2786 Ok((ranking, out)) => {
2787 record.ranking = ranking.normalized();
2788 record.reasons = ranking.reasons;
2789 record.confidence = ranking.confidence;
2790 record.failed = None;
2791 record.duration_ms = out.duration_ms;
2792 recovered.insert(j);
2793 self.state.event(
2794 "recover",
2795 format!("judge {} ranked again after the limit", j + 1),
2796 );
2797 }
2798 Err(e) => {
2799 self.state
2800 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2801 }
2802 }
2803 }
2804
2805 // Re-ask the votes of the seats that recovered a ranking.
2806 let mut vote_jobs = Vec::new();
2807 let mut vote_pos: Vec<usize> = Vec::new();
2808 for &j in &recovered {
2809 let seat_key = format!("judge-{}", j + 1);
2810 let spec = self.roles.judges[j].clone();
2811 let seat = self.seat(&seat_key, &spec.id);
2812 let mut text = prompt::final_vote(&labels, &language);
2813 if !has_context(&spec, &seat, sessions) {
2814 text = format!(
2815 "{}\n\n# Candidates\n\n{}",
2816 text,
2817 self.candidate_block(&candidates, &base_short)
2818 );
2819 }
2820 vote_jobs.push(SeatJob {
2821 spec,
2822 seat,
2823 prompt: text,
2824 cwd: root.join(seat_key),
2825 timeout,
2826 allow_write: false,
2827 sessions,
2828 artifacts: artifacts.clone(),
2829 stem: format!("vote-judge-{}-recover", j + 1),
2830 });
2831 vote_pos.push(j);
2832 }
2833 let allowed = labels.clone();
2834 let mut vote_losses = Vec::new();
2835 let vote_retries = self.state.config.graph.retries;
2836 let vote_cache = self.state.config.cache_dir();
2837 let ctx = WaveCtx {
2838 run: &run_id,
2839 node: "vote",
2840 prompts: &prompts,
2841 cache: vote_cache.as_deref(),
2842 round: None,
2843 };
2844 let votes = ask_json_wave::<FinalVote>(
2845 vote_jobs,
2846 Arc::clone(&self.sem),
2847 vote_retries,
2848 &ctx,
2849 &mut vote_losses,
2850 &mut self.state,
2851 &move |v: &FinalVote| match v.label() {
2852 Some(c) if allowed.contains(&c) => Ok(()),
2853 other => bail!("vote {other:?} is not one of {allowed:?}"),
2854 },
2855 )
2856 .await;
2857 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
2858 let agent_id = seat.agent.clone();
2859 self.state.seats.insert(seat.key.clone(), seat);
2860 match res {
2861 Ok((v, _)) => {
2862 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2863 rec.vote = v.label();
2864 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2865 } else {
2866 self.state.votes.push(VoteRecord {
2867 judge: j + 1,
2868 agent: agent_id,
2869 vote: v.label(),
2870 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2871 changed: false,
2872 });
2873 }
2874 self.state.event(
2875 "recover",
2876 format!("judge {} voted again after the limit", j + 1),
2877 );
2878 }
2879 Err(e) => {
2880 self.state
2881 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2882 }
2883 }
2884 }
2885
2886 // A seat that ranked again is present even if its re-vote failed —
2887 // `tally` falls back to the initial ranking's first choice — so clear
2888 // its quota loss. Seats that still fail keep theirs and stay absent.
2889 let recovered_keys: BTreeSet<String> = recovered
2890 .iter()
2891 .map(|&j| format!("judge-{}", j + 1))
2892 .collect();
2893 self.state
2894 .quota
2895 .retain(|q| !recovered_keys.contains(&q.seat));
2896 // A seat that hit the limit again is a fresh loss, not the old one:
2897 // replace the stale entry so the history stays one-per-seat and the
2898 // daemon can tell this attempt's loss from a previous session's.
2899 for loss in judge_losses.into_iter().chain(vote_losses) {
2900 if recovered_keys.contains(&loss.seat) {
2901 continue;
2902 }
2903 self.state.quota.retain(|q| q.seat != loss.seat);
2904 self.state.quota.push(loss);
2905 }
2906
2907 // Recompute the verdict from the refreshed panel.
2908 self.state.tally = None;
2909 self.tally()?;
2910 Ok(self
2911 .state
2912 .tally
2913 .as_ref()
2914 .map(|t| t.met_quorum)
2915 .unwrap_or(false))
2916 }
2917
2918 // ----------------------------------------------------------------- fold
2919
2920 async fn fold_losers(&mut self) -> Result<()> {
2921 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2922 return Ok(());
2923 };
2924 let repo = self.state.repo.clone();
2925 let mut folded = Vec::new();
2926 for i in 0..self.state.candidates.len() {
2927 let c = &self.state.candidates[i];
2928 if c.label == winner || c.folded {
2929 continue;
2930 }
2931 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2932 git::worktree_remove(&repo, &wt).await.ok();
2933 git::branch_delete(&repo, &branch).await.ok();
2934 self.state.candidates[i].folded = true;
2935 folded.push(label.to_string());
2936 }
2937 // The judges are finished; their checkouts are pure cost from here.
2938 let root = self.state.worktree_root();
2939 for j in 1..=self.roles.judges.len() {
2940 let wt = root.join(format!("judge-{j}"));
2941 if wt.exists() {
2942 git::worktree_remove(&repo, &wt).await.ok();
2943 }
2944 }
2945 // The design-deliberation stage is finished by the time a tally
2946 // exists — same reasoning as the judges above.
2947 if self.state.config.graph.advise {
2948 for k in 1..=self.state.config.graph.advisors {
2949 let wt = root.join(format!("advisor-{k}"));
2950 if wt.exists() {
2951 git::worktree_remove(&repo, &wt).await.ok();
2952 }
2953 }
2954 }
2955 if !folded.is_empty() {
2956 self.state
2957 .event("fold", format!("folded candidates {}", folded.join(", ")));
2958 self.state.save()?;
2959 }
2960 Ok(())
2961 }
2962
2963 // ------------------------------------------------------------ base sync
2964
2965 /// Land the winner's tree on the current tip of `<remote>/<base>` before
2966 /// anything verifies it.
2967 ///
2968 /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
2969 /// read whatever is checked out in the winner's worktree. Left alone that
2970 /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
2971 /// it when the run *branched* - and a run takes long enough that the base
2972 /// has usually moved by the time it gets here. A gate that ran there
2973 /// answers "green on the commit this run started from", not "green on
2974 /// what is about to land", and the difference showed up three times in
2975 /// one day as a green run whose merge would have reverted a file another
2976 /// pull request had already landed.
2977 ///
2978 /// Reuses [`git::rebase_branch_in_temp`] rather than a second
2979 /// implementation of the same idea: `land::Step::Rebase` already worked
2980 /// out the rules - throwaway worktree, conflict stops and reports rather
2981 /// than feeding a fixer, nothing runs in the primary tree - and a second
2982 /// rebase path is exactly the kind of drift `resolve_base`'s own doc
2983 /// warns about ("two answers to a question nobody notices until a diff is
2984 /// wrong").
2985 ///
2986 /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
2987 /// so it survives a park/resume. A conflict or a push failure sets
2988 /// `state.base_sync.conflict` and leaves the branch and worktree exactly
2989 /// as they were - untouched, for a person to look at - which is also what
2990 /// makes re-entering this function afterwards a no-op instead of a second
2991 /// attempt at the same wall.
2992 async fn sync_to_base(&mut self) -> Result<()> {
2993 if self
2994 .state
2995 .base_sync
2996 .as_ref()
2997 .is_some_and(|s| s.conflict.is_some())
2998 {
2999 return Ok(());
3000 }
3001 let Some(winner) = self.state.winner().cloned() else {
3002 return Ok(());
3003 };
3004
3005 let repo = self.state.repo.clone();
3006 let remote = self.state.config.merge.remote.clone();
3007 let base_branch = self.state.base_branch.clone();
3008 let tracking = format!("{remote}/{base_branch}");
3009
3010 git::fetch(&repo, &remote, &base_branch).await.ok();
3011 // No network, or the remote never had this branch: `resolve_base`
3012 // already treats that as non-fatal at branch time, and a run that got
3013 // this far must not be blocked by it here either.
3014 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3015 return Ok(());
3016 };
3017
3018 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3019 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3020 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3021
3022 if behind == 0 {
3023 self.state.base_sync = Some(BaseSync {
3024 tip,
3025 behind: 0,
3026 attempts,
3027 conflict: None,
3028 });
3029 self.state.save()?;
3030 return Ok(());
3031 }
3032
3033 if attempts >= BASE_SYNC_ROUNDS {
3034 let why = format!(
3035 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3036 rebase(s); rebasing again would only race it",
3037 winner.branch
3038 );
3039 self.state.status = RunStatus::Blocked;
3040 self.state.base_sync = Some(BaseSync {
3041 tip,
3042 behind,
3043 attempts,
3044 conflict: Some(why.clone()),
3045 });
3046 self.state.event("land", why);
3047 self.state.save()?;
3048 return Ok(());
3049 }
3050
3051 self.state.event(
3052 "land",
3053 format!(
3054 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3055 winner.branch
3056 ),
3057 );
3058 self.state.save()?;
3059
3060 let scratch = self.state.dir().join("base-sync");
3061 let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
3062 let attempts = attempts + 1;
3063 match rebased {
3064 Ok(None) => {
3065 // The branch ref moved, but a worktree that already had it
3066 // checked out (the winner's) was not told; sync its index and
3067 // files before anything reads them.
3068 git::sync_to_head(&winner.worktree).await?;
3069 self.state.base_sync = Some(BaseSync {
3070 tip: tip.clone(),
3071 behind: 0,
3072 attempts,
3073 conflict: None,
3074 });
3075 self.state
3076 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3077 }
3078 Ok(Some(conflict)) => {
3079 let why = format!(
3080 "{} conflicts with {tracking} and did not rebase: {}",
3081 winner.branch,
3082 conflict.chars().take(600).collect::<String>()
3083 );
3084 self.state.status = RunStatus::Blocked;
3085 self.state.base_sync = Some(BaseSync {
3086 tip,
3087 behind,
3088 attempts,
3089 conflict: Some(why.clone()),
3090 });
3091 self.state.event("land", why);
3092 }
3093 Err(e) => {
3094 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3095 self.state.status = RunStatus::Blocked;
3096 self.state.base_sync = Some(BaseSync {
3097 tip,
3098 behind,
3099 attempts,
3100 conflict: Some(why.clone()),
3101 });
3102 self.state.event("land", why);
3103 }
3104 }
3105 self.state.save()?;
3106 Ok(())
3107 }
3108
3109 /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3110 /// last landed the winner on, once it has run, else the commit the run
3111 /// branched from.
3112 ///
3113 /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3114 /// and `vote` all happen before there is a winner to rebase, so they
3115 /// compare every candidate against the branch point on purpose, and a
3116 /// base that moves after they are already done cannot change an answer
3117 /// they already gave.
3118 fn landing_base(&self) -> String {
3119 self.state
3120 .base_sync
3121 .as_ref()
3122 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3123 }
3124
3125 // ------------------------------------------------------- operator fix
3126
3127 /// Route specific, already-recorded review findings to a fixer for a
3128 /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3129 /// entry point.
3130 ///
3131 /// Distinct from `review_loop`'s own fix step in three ways: it never
3132 /// runs a reviewer wave, it never spends review-round budget, and what
3133 /// happened is recorded as an [`OperatorFixRequest`] appended to
3134 /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3135 /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3136 /// and vote must never be rewritten to look like a manufactured blocking
3137 /// verdict.
3138 ///
3139 /// Only meaningful once review has actually concluded: `Ready` (handed
3140 /// off with findings still open, or simply concluded clean while minor
3141 /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3142 /// gate failed). Everything else is refused: a run still in progress
3143 /// should simply be resumed, and a `Merged` run's branch has already
3144 /// landed — reopening *this* run's own record cannot change that, so the
3145 /// answer there is a fresh `magi review <branch>`.
3146 ///
3147 /// A real commit here re-verifies through a fresh, ordinary review-only
3148 /// run on the same branch ([`Self::review`]) rather than reopening this
3149 /// run's own `review_loop`: once any round in this run's history went
3150 /// clean, `review_conclusion` treats that as permanent by design (the
3151 /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3152 /// way to force one more genuine reviewer wave out of *this* run without
3153 /// either rewriting history or weakening that guarantee for every other
3154 /// caller. A review-only run costs nothing extra — no implementation, no
3155 /// judging, no vote — and exercises the exact same review → verify →
3156 /// gate → (human) merge path, unmodified.
3157 pub async fn fix_selected(
3158 &mut self,
3159 ids: &[String],
3160 reason: &str,
3161 allow_stale: bool,
3162 ) -> Result<()> {
3163 let reason = reason.trim();
3164 if reason.is_empty() {
3165 bail!("a fix request needs a reason — that is the operator's own record of why");
3166 }
3167 if ids.is_empty() {
3168 bail!("no finding id given");
3169 }
3170 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3171 bail!(
3172 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3173 has already concluded — can be given a targeted fix. A run still \
3174 in progress should simply be resumed; a `merged` run's branch has \
3175 already landed, so its answer is a fresh `magi review <branch>`, \
3176 not reopening this run's own record",
3177 self.state.id,
3178 self.state.status.as_str()
3179 );
3180 }
3181 let Some(winner) = self.state.winner().cloned() else {
3182 bail!("run {} has no winning candidate to fix", self.state.id);
3183 };
3184 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3185 bail!(
3186 "branch `{}` no longer exists; this run cannot be extended",
3187 winner.branch
3188 );
3189 }
3190 let home = crate::run::home();
3191 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3192 bail!(
3193 "run {} is currently being worked on by another magi process",
3194 self.state.id
3195 );
3196 }
3197 // Held for the rest of this call, including the follow-up review
3198 // below: two `magi fix` invocations against the same run must not
3199 // both reach the worktree manipulation further down, which would
3200 // otherwise race to remove and recreate the same directory — see
3201 // [`FixClaim`]'s own doc.
3202 let _claim = FixClaim::acquire(&self.state.dir())?;
3203
3204 // Resolve every id before spending anything — an unknown id refuses
3205 // the whole request rather than silently dropping it — and dedup
3206 // while keeping the operator's own order.
3207 let mut seen = BTreeSet::new();
3208 let mut findings = Vec::new();
3209 let mut missing = Vec::new();
3210 for id in ids {
3211 if !seen.insert(id.clone()) {
3212 continue;
3213 }
3214 match self.state.finding(id) {
3215 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3216 id: f.id.clone(),
3217 severity: f.severity,
3218 reviewer_vote: rec.vote,
3219 round: round.round,
3220 round_head: round.head.clone(),
3221 reviewer: rec.reviewer,
3222 agent: rec.agent.clone(),
3223 file: f.file.clone(),
3224 line: f.line,
3225 title: f.title.clone(),
3226 detail: f.detail.clone(),
3227 outcome: OperatorFixOutcome::Pending,
3228 }),
3229 None => missing.push(id.clone()),
3230 }
3231 }
3232 if !missing.is_empty() {
3233 bail!(
3234 "unknown finding id(s): {}; nothing was changed",
3235 missing.join(", ")
3236 );
3237 }
3238
3239 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3240 let stale_details: Vec<(String, String)> = findings
3241 .iter()
3242 .filter(|f| f.round_head != head_at_request)
3243 .map(|f| (f.id.clone(), f.round_head.clone()))
3244 .collect();
3245 let stale = !stale_details.is_empty();
3246 if stale && !allow_stale {
3247 bail!(
3248 "the branch has moved since some finding(s) were raised — {} — now \
3249 at {}; pass --allow-stale to fix anyway, or re-run review first",
3250 stale_details
3251 .iter()
3252 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3253 .collect::<Vec<_>>()
3254 .join(", "),
3255 short(&head_at_request)
3256 );
3257 }
3258
3259 let request = OperatorFixRequest {
3260 requested_at: Timestamp::now(),
3261 reason: reason.to_owned(),
3262 findings,
3263 head_at_request: head_at_request.clone(),
3264 allow_stale,
3265 stale,
3266 fix: None,
3267 result_head: None,
3268 follow_up_review_run: None,
3269 };
3270 self.state.event(
3271 "fix",
3272 format!(
3273 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3274 request.findings.len(),
3275 request
3276 .findings
3277 .iter()
3278 .map(|f| f.id.as_str())
3279 .collect::<Vec<_>>()
3280 .join(", "),
3281 ),
3282 );
3283 // Recorded now, before any worktree work or the fixer call itself —
3284 // and re-saved at each checkpoint below: a crash at any point after
3285 // this (mid fixer call, mid follow-up review) must not lose the fact
3286 // that this was requested, for which findings, and why. Everything
3287 // past this point reads and writes through `request_index` rather
3288 // than a local variable, since `request` itself is moved here.
3289 self.state.operator_fixes.push(request);
3290 self.state.save()?;
3291 let request_index = self.state.operator_fixes.len() - 1;
3292
3293 // A fresh, dedicated worktree for this one call, never the winner's
3294 // own worktree in place: that one may already be gone (folded away),
3295 // and reusing it in place would leave the branch checked out there
3296 // when the follow-up review below tries to check it out again. Freed
3297 // immediately after, either way — but only once confirmed clean:
3298 // `worktree_remove` is a `git worktree remove --force`, which would
3299 // otherwise discard uncommitted work left there by the operator or
3300 // another process before this had a chance to even look at it.
3301 if winner.worktree.exists() {
3302 // Lockfiles a rescue commit withheld stay untracked on purpose and
3303 // are already recorded; they are not the operator's work to protect.
3304 let dirty = git::git(
3305 &winner.worktree,
3306 &["status", "--porcelain", "--untracked-files=all"],
3307 )
3308 .await?;
3309 let only_withheld = dirty.lines().all(|l| {
3310 l.strip_prefix("?? ")
3311 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3312 });
3313 if !only_withheld {
3314 bail!(
3315 "`{}` has uncommitted changes; refusing to touch it — commit or \
3316 discard them first",
3317 winner.worktree.display()
3318 );
3319 }
3320 git::worktree_remove(&self.state.repo, &winner.worktree)
3321 .await
3322 .ok();
3323 }
3324 let fix_worktree = self.state.worktree_root().join("operator-fix");
3325 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3326 git::git(
3327 &self.state.repo,
3328 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3329 )
3330 .await
3331 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3332 if !git::is_clean(&fix_worktree).await? {
3333 git::worktree_remove(&self.state.repo, &fix_worktree)
3334 .await
3335 .ok();
3336 bail!(
3337 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3338 winner.branch
3339 );
3340 }
3341
3342 let run_id = self.state.id.clone();
3343 let prompts = self.state.config.prompts.clone();
3344 let language = self.state.config.graph.language.clone();
3345 let sessions = self.state.config.graph.sessions;
3346 let artifacts = agent::artifacts_dir(&self.state.dir());
3347 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3348 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3349 _ => (
3350 self.state
3351 .config
3352 .agent(&winner.agent)
3353 .cloned()
3354 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3355 format!("impl-{}", winner.label),
3356 ),
3357 };
3358 let seat = self.seat(&fix_seat_key, &fix_spec.id);
3359 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3360 .findings
3361 .iter()
3362 .map(|f| Finding {
3363 id: f.id.clone(),
3364 severity: f.severity,
3365 file: f.file.clone(),
3366 line: f.line,
3367 title: f.title.clone(),
3368 detail: f.detail.clone(),
3369 })
3370 .collect();
3371 let job = SeatJob {
3372 prompt: prompt::operator_fix(
3373 &self.state.instruction,
3374 &finding_list,
3375 reason,
3376 &stale_details,
3377 &head_at_request,
3378 &language,
3379 ),
3380 spec: fix_spec.clone(),
3381 seat,
3382 cwd: fix_worktree.clone(),
3383 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3384 allow_write: true,
3385 sessions,
3386 artifacts: artifacts.clone(),
3387 stem: "operator-fix".to_owned(),
3388 };
3389 let cache = self.state.config.cache_dir();
3390 let ctx = WaveCtx {
3391 run: &run_id,
3392 node: "fix",
3393 prompts: &prompts,
3394 cache: cache.as_deref(),
3395 round: None,
3396 };
3397 let (seat, out) =
3398 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3399 let agent_id = seat.agent.clone();
3400
3401 let mut fix = FixRecord {
3402 agent: agent_id,
3403 addressed: Vec::new(),
3404 rejected: Vec::new(),
3405 notes: String::new(),
3406 committed: false,
3407 failed: None,
3408 duration_ms: 0,
3409 continuation: None,
3410 };
3411 let mut final_seat = seat.clone();
3412 match out {
3413 AgentOutcome::Ok(o) => {
3414 fix.duration_ms = o.duration_ms;
3415 let parsed = verdict::extract_json::<FixReport>(&o.text);
3416 let incomplete_reason = match &parsed {
3417 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3418 "the reply parsed, but it reported a command whose own CLI \
3419 never confirmed an exit status"
3420 .to_owned(),
3421 ),
3422 Ok(_) => None,
3423 Err(e) => Some(e.to_string()),
3424 };
3425 match incomplete_reason {
3426 None => {
3427 let report = parsed.expect("checked Ok above");
3428 fix.addressed = report.addressed;
3429 fix.rejected = report.rejected;
3430 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
3431 }
3432 Some(reason) => {
3433 let (resumed_seat, resolved, failure, cont) = self
3434 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
3435 .await;
3436 fix.duration_ms += cont.cumulative_wait_ms;
3437 fix.continuation = Some(cont);
3438 final_seat = resumed_seat;
3439 match resolved {
3440 Some(report) => {
3441 fix.addressed = report.addressed;
3442 fix.rejected = report.rejected;
3443 fix.notes =
3444 blind::sanitize_prose(&report.notes, &self.state.config.blind);
3445 }
3446 None => fix.failed = failure,
3447 }
3448 }
3449 }
3450 }
3451 AgentOutcome::Dropped(o) => {
3452 fix.duration_ms = o.duration_ms;
3453 let why = o
3454 .dropped
3455 .as_ref()
3456 .map(|d| d.why.as_str())
3457 .unwrap_or("the CLI ended the stream without delivering its answer");
3458 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3459 }
3460 AgentOutcome::Quota(o) => {
3461 self.state.quota.push(QuotaLoss {
3462 seat: final_seat.key.clone(),
3463 node: "fix".to_owned(),
3464 at: Timestamp::now(),
3465 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3466 });
3467 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3468 }
3469 AgentOutcome::Failed(e) => fix.failed = Some(e),
3470 }
3471 if fix.continuation.is_none() {
3472 fix.continuation = Some(ContinuationRecord::not_needed());
3473 }
3474 self.state.seats.insert(final_seat.key.clone(), final_seat);
3475
3476 let rescue_message = format!(
3477 "magi: operator-selected fix ({}) (uncommitted work)",
3478 self.state.operator_fixes[request_index]
3479 .findings
3480 .iter()
3481 .map(|f| f.id.as_str())
3482 .collect::<Vec<_>>()
3483 .join(", ")
3484 );
3485 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
3486 self.state.note_withheld("fix", &r.withheld);
3487 }
3488 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
3489 fix.committed = after != head_at_request;
3490 git::worktree_remove(&self.state.repo, &fix_worktree)
3491 .await
3492 .ok();
3493
3494 self.state.event(
3495 "fix",
3496 match &fix.failed {
3497 Some(reason) => format!(
3498 "operator fix: adoption report was lost ({reason}); {}",
3499 if fix.committed {
3500 "committed"
3501 } else {
3502 "NO new commit"
3503 }
3504 ),
3505 None => format!(
3506 "operator fix: {} addressed, {} rejected, {}",
3507 fix.addressed.len(),
3508 fix.rejected.len(),
3509 if fix.committed {
3510 "committed"
3511 } else {
3512 "NO new commit"
3513 }
3514 ),
3515 },
3516 );
3517
3518 // Every selected finding gets an outcome — never left `Pending` once
3519 // the fixer's own turn is over. A report that never came back at all
3520 // marks every one of them `Unreported`, not silently "not addressed":
3521 // quota, a dropped stream, or an exhausted continuation are gaps in
3522 // the report, not evidence about the finding itself (see [`SCHEMA`]'s
3523 // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
3524 for f in &mut self.state.operator_fixes[request_index].findings {
3525 f.outcome = if fix.failed.is_some() {
3526 OperatorFixOutcome::Unreported
3527 } else if fix.addressed.contains(&f.id) {
3528 OperatorFixOutcome::Addressed
3529 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
3530 OperatorFixOutcome::Rejected { why: r.why.clone() }
3531 } else {
3532 OperatorFixOutcome::Unreported
3533 };
3534 }
3535
3536 let committed = fix.committed;
3537 if committed {
3538 self.state.operator_fixes[request_index].result_head = Some(after.clone());
3539 }
3540 self.state.operator_fixes[request_index].fix = Some(fix);
3541 // Saved again now that the fixer's own outcome is final, on top of
3542 // the save right after the request was first pushed above.
3543 self.state.save()?;
3544
3545 if committed {
3546 self.state.event(
3547 "fix",
3548 format!(
3549 "operator fix committed {}; opening a follow-up review-only run",
3550 short(&after)
3551 ),
3552 );
3553 match Self::review(&self.state.repo, &winner.branch, self.state.config.clone()).await {
3554 Ok(mut follow_up) => {
3555 follow_up.state.event(
3556 "start",
3557 format!(
3558 "requested by an operator fix on run {} for finding(s) {}",
3559 self.state.id,
3560 self.state.operator_fixes[request_index]
3561 .findings
3562 .iter()
3563 .map(|f| f.id.as_str())
3564 .collect::<Vec<_>>()
3565 .join(", "),
3566 ),
3567 );
3568 follow_up.state.save()?;
3569 let follow_up_id = follow_up.state.id.clone();
3570 if let Err(e) = follow_up.execute().await {
3571 self.state.event(
3572 "fix",
3573 format!(
3574 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
3575 ),
3576 );
3577 }
3578 self.state.operator_fixes[request_index].follow_up_review_run =
3579 Some(follow_up_id);
3580 }
3581 Err(e) => {
3582 self.state.event(
3583 "fix",
3584 format!("committed the fix but could not open a follow-up review: {e:#}"),
3585 );
3586 }
3587 }
3588 self.state.save()?;
3589 }
3590
3591 Ok(())
3592 }
3593
3594 // --------------------------------------------------------------- review
3595
3596 async fn review_loop(&mut self) -> Result<()> {
3597 // A base that would not rebase is a person's decision, not a review
3598 // round: nothing here would change the answer, and reviewers and a
3599 // fixer would be spending real budget on a tree that cannot land
3600 // regardless of what they find.
3601 if self
3602 .state
3603 .base_sync
3604 .as_ref()
3605 .is_some_and(|s| s.conflict.is_some())
3606 {
3607 return Ok(());
3608 }
3609 // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3610 // agent files with `magi task add` name the run that paid for it. The
3611 // prompt overlay is cloned alongside it because the waves borrow it
3612 // while `self` is mutably borrowed by the node's own bookkeeping.
3613 let run_id = self.state.id.clone();
3614 let prompts = self.state.config.prompts.clone();
3615 let Some(winner) = self.state.winner().cloned() else {
3616 return Ok(());
3617 };
3618 let max_rounds = self.state.config.graph.review_rounds;
3619 // A clean round, an exhausted round budget, or a stalled tree (see
3620 // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
3621 // they are recorded — recomputed here, not read off `status`, so a
3622 // reentry into a run that already stopped restates the identical
3623 // verdict instead of silently handing back whatever an earlier node
3624 // in this same walk clobbered `status` to (a solo-candidate
3625 // `judge`/`deliberate` skip rewrites it on every reentry). The loop
3626 // below runs an empty range once the budget is spent, and would
3627 // otherwise fall through without touching `status` at all.
3628 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
3629 self.state.status = status;
3630 self.state.save()?;
3631 return Ok(());
3632 }
3633 self.state.status = RunStatus::Reviewing;
3634 // A last recorded round whose own verification never resolved
3635 // (`ResourceBlocked` — the shared build cache, not the patch) is
3636 // never a concluded round, whatever the round budget says: starting
3637 // a fresh round on top of it would spend a whole new reviewer wave
3638 // re-reading an unchanged patch instead of just retrying the one
3639 // check that actually needs it, and once the budget is spent the
3640 // loop below has nothing left to do at all (its range is empty).
3641 // Retry that check directly instead, exactly the same retry
3642 // `stop_reviewing` already does for its own catch-up case.
3643 if self
3644 .state
3645 .reviews
3646 .last()
3647 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
3648 {
3649 let shell = self.state.config.shell();
3650 return self
3651 .stop_reviewing(
3652 "the last round's own verification never resolved",
3653 &shell,
3654 &winner.worktree,
3655 )
3656 .await;
3657 }
3658
3659 let repo = self.state.repo.clone();
3660 let root = self.state.worktree_root();
3661 let language = self.state.config.graph.language.clone();
3662 let sessions = self.state.config.graph.sessions;
3663 let artifacts = agent::artifacts_dir(&self.state.dir());
3664 let base = self.landing_base();
3665 let base_short = short(&base);
3666 let reviewers = self.roles.reviewers.clone();
3667 let shell = self.state.config.shell();
3668
3669 for round in (self.state.reviews.len() + 1)..=max_rounds {
3670 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3671 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
3672 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
3673 // The prior round's own record, already persisted — never a
3674 // hand-carried variable of just its failing output: that is
3675 // exactly what let a round's e2e result drift out of sync with
3676 // which commit it was actually about (see `SCHEMA`'s doc for
3677 // schema 8). Judged against `head`, the commit reviewers are
3678 // about to look at now, so the summary always reads as "an
3679 // earlier head" here — this round's own patch has not been
3680 // checked yet.
3681 let prev_verification = self
3682 .state
3683 .reviews
3684 .last()
3685 .and_then(|r| r.verification_summary(&head));
3686
3687 // Each reviewer gets its own detached checkout of exactly this
3688 // commit: nobody can perturb the winner's tree, and the fixer can
3689 // keep working without racing a reviewer.
3690 let mut jobs = Vec::new();
3691 for (r, spec) in reviewers.iter().cloned().enumerate() {
3692 let wt = root.join(format!("review-{}", r + 1));
3693 if wt.exists() {
3694 git::reset_detached(&wt, &head).await?;
3695 } else {
3696 git::worktree_add_detached(&repo, &wt, &head).await?;
3697 }
3698 let seat_key = format!("review-{}", r + 1);
3699 let seat = self.seat(&seat_key, &spec.id);
3700 jobs.push(SeatJob {
3701 prompt: prompt::review(&prompt::ReviewCtx {
3702 instruction: &self.state.instruction,
3703 branch: &winner.branch,
3704 base_short: &base_short,
3705 stat: &stat,
3706 patch: &patch,
3707 verification: prev_verification.as_ref(),
3708 reviewers: reviewers.len(),
3709 round,
3710 rounds: max_rounds,
3711 // A review-only run has no rankings, so nothing
3712 // competed for this patch and the reviewer is told so.
3713 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
3714 lens: Lens::for_seat(r),
3715 language: &language,
3716 }),
3717 spec,
3718 seat,
3719 cwd: wt,
3720 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3721 allow_write: false,
3722 sessions,
3723 artifacts: artifacts.clone(),
3724 stem: format!("review-{round}-{}", r + 1),
3725 });
3726 }
3727
3728 self.state.event(
3729 "review",
3730 format!(
3731 "round {round}: {} reviewers on {}",
3732 jobs.len(),
3733 short(&head)
3734 ),
3735 );
3736 let mut quota_losses = Vec::new();
3737 let review_retries = self.state.config.graph.retries;
3738 let review_cache = self.state.config.cache_dir();
3739 let ctx = WaveCtx {
3740 run: &run_id,
3741 node: "review",
3742 prompts: &prompts,
3743 cache: review_cache.as_deref(),
3744 round: Some(round),
3745 };
3746 let results = ask_json_wave::<Review>(
3747 jobs,
3748 Arc::clone(&self.sem),
3749 review_retries,
3750 &ctx,
3751 &mut quota_losses,
3752 &mut self.state,
3753 &|_: &Review| Ok(()),
3754 )
3755 .await;
3756 // Counted before the move below: how many of *this* round's
3757 // reviewer seats were lost to their own rate limit, as opposed to
3758 // a crash, a timeout, or unparsable output — see `round_is_clean`.
3759 let round_quota_missing = quota_losses.len();
3760 self.state.quota.extend(quota_losses);
3761
3762 let mut records = Vec::new();
3763 let mut all_findings = Vec::new();
3764 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
3765 let agent_id = seat.agent.clone();
3766 self.state.seats.insert(seat.key.clone(), seat);
3767 let mut record = ReviewRecord {
3768 reviewer: r + 1,
3769 agent: agent_id,
3770 summary: String::new(),
3771 findings: Vec::new(),
3772 vote: None,
3773 failed: None,
3774 duration_ms: 0,
3775 // Set for both outcomes: `failed: Some(_)` with
3776 // `attempts > 0` is a seat every retry still lost, not a
3777 // recovered one — only `failed: None` with `attempts > 0`
3778 // reads as "answered after a nudge" (see this field's own
3779 // doc).
3780 attempts,
3781 };
3782 match res {
3783 Ok((review, out)) => {
3784 // Sanitized here, at the point every other piece of
3785 // agent prose in this file is (candidate summaries,
3786 // deliberation turns, vote reasons): a reviewer's own
3787 // words are the one thing about it that could name
3788 // it, and reconsideration below broadcasts this same
3789 // summary and these same findings to every other
3790 // seat on the panel.
3791 record.summary =
3792 blind::sanitize_prose(&review.summary, &self.state.config.blind);
3793 record.vote = Some(review.vote);
3794 record.duration_ms = out.duration_ms;
3795 for (n, mut f) in review.findings.into_iter().enumerate() {
3796 // ids are magi's, never the agent's: the fixer's
3797 // adoption report is keyed by them.
3798 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
3799 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
3800 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
3801 // `file` is agent-supplied prose too, never
3802 // checked against the real tree — the same
3803 // exposure `title`/`detail` above have, just in
3804 // a field easy to forget because it looks like a
3805 // path rather than free text.
3806 f.file = f
3807 .file
3808 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
3809 all_findings.push(f.clone());
3810 record.findings.push(f);
3811 }
3812 self.state.event(
3813 "review",
3814 format!(
3815 "round {round}: reviewer {} voted {} with {} finding(s)",
3816 r + 1,
3817 review.vote.label(),
3818 record.findings.len()
3819 ),
3820 );
3821 }
3822 Err(e) => {
3823 record.failed = Some(e.to_string());
3824 self.state.event(
3825 "review",
3826 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
3827 );
3828 }
3829 }
3830 records.push(record);
3831 }
3832
3833 // Tally the round's votes and, if they split, spend the one
3834 // round of reconsideration the split -> deliberate -> revote
3835 // shape `judge`/`vote` use for the panel, sized down to what a
3836 // read-only review round can afford: one round, and a revote
3837 // rather than an argument, because the panel already wrote its
3838 // reasoning down as findings the first time around.
3839 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
3840 let vote_split =
3841 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
3842 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
3843 if vote_split {
3844 self.state.event(
3845 "review",
3846 format!(
3847 "round {round}: votes split ({}) — one round of reconsideration",
3848 initial_votes
3849 .iter()
3850 .map(|v| v.label())
3851 .collect::<Vec<_>>()
3852 .join(", ")
3853 ),
3854 );
3855 // Seats read every seat's findings and votes, still numbered
3856 // and never named — the same anonymity `review` itself keeps.
3857 let panel: Vec<ReviewSeatReport<'_>> = records
3858 .iter()
3859 .filter_map(|r| {
3860 r.vote.map(|vote| ReviewSeatReport {
3861 reviewer: r.reviewer,
3862 vote,
3863 summary: &r.summary,
3864 findings: &r.findings,
3865 })
3866 })
3867 .collect();
3868
3869 let mut jobs = Vec::new();
3870 let mut seats_at = Vec::new();
3871 for (r, spec) in reviewers.iter().cloned().enumerate() {
3872 // A seat with no initial vote has nothing to reconsider
3873 // from and stays absent, the same as it stayed absent
3874 // from `panel` above.
3875 if records[r].vote.is_none() {
3876 continue;
3877 }
3878 let wt = root.join(format!("review-{}", r + 1));
3879 let seat_key = format!("review-{}", r + 1);
3880 let seat = self.seat(&seat_key, &spec.id);
3881 // A seat with no live session has already forgotten the
3882 // initial review's prompt — restate the patch it is
3883 // voting on, the same as `deliberate`/`vote` do for a
3884 // judge in the same position.
3885 let patch_ctx = if has_context(&spec, &seat, sessions) {
3886 None
3887 } else {
3888 Some(ReviewPatch {
3889 branch: &winner.branch,
3890 base_short: &base_short,
3891 stat: &stat,
3892 patch: &patch,
3893 })
3894 };
3895 let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
3896 instruction: &self.state.instruction,
3897 reviewer: r + 1,
3898 lens: Lens::for_seat(r),
3899 panel: &panel,
3900 patch: patch_ctx,
3901 round,
3902 rounds: max_rounds,
3903 language: &language,
3904 });
3905 jobs.push(SeatJob {
3906 prompt,
3907 spec,
3908 seat,
3909 cwd: wt,
3910 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3911 allow_write: false,
3912 sessions,
3913 artifacts: artifacts.clone(),
3914 stem: format!("review-{round}-reconsider-{}", r + 1),
3915 });
3916 seats_at.push(r);
3917 }
3918
3919 let mut recon_quota_losses = Vec::new();
3920 let recon_cache = self.state.config.cache_dir();
3921 let recon_ctx = WaveCtx {
3922 run: &run_id,
3923 node: "review",
3924 prompts: &prompts,
3925 cache: recon_cache.as_deref(),
3926 round: Some(round),
3927 };
3928 let recon_results = ask_json_wave::<ReviewRevote>(
3929 jobs,
3930 Arc::clone(&self.sem),
3931 review_retries,
3932 &recon_ctx,
3933 &mut recon_quota_losses,
3934 &mut self.state,
3935 &|_: &ReviewRevote| Ok(()),
3936 )
3937 .await;
3938 self.state.quota.extend(recon_quota_losses);
3939
3940 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
3941 let agent_id = seat.agent.clone();
3942 self.state.seats.insert(seat.key.clone(), seat);
3943 let mut rec = ReviewRevoteRecord {
3944 reviewer: r + 1,
3945 agent: agent_id,
3946 vote: None,
3947 reason: String::new(),
3948 failed: None,
3949 };
3950 match res {
3951 Ok((rv, _)) => {
3952 rec.vote = Some(rv.vote);
3953 rec.reason =
3954 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
3955 self.state.event(
3956 "review",
3957 format!(
3958 "round {round}: reviewer {} revoted {}",
3959 r + 1,
3960 rv.vote.label()
3961 ),
3962 );
3963 }
3964 Err(e) => {
3965 rec.failed = Some(e.to_string());
3966 self.state.event(
3967 "review",
3968 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
3969 );
3970 }
3971 }
3972 reconsideration.push(rec);
3973 }
3974 } else if initial_votes.len() > 1 {
3975 self.state.event(
3976 "review",
3977 format!(
3978 "round {round}: votes agreed ({}) — no reconsideration",
3979 initial_votes[0].label()
3980 ),
3981 );
3982 }
3983
3984 // The final vote per seat is its revote where reconsideration
3985 // ran and answered, its initial vote otherwise — the same
3986 // fallback `tally` uses for a judge whose private vote failed.
3987 let final_votes: Vec<ReviewVote> = records
3988 .iter()
3989 .filter_map(|r| {
3990 reconsideration
3991 .iter()
3992 .find(|rv| rv.reviewer == r.reviewer)
3993 .and_then(|rv| rv.vote)
3994 .or(r.vote)
3995 })
3996 .collect();
3997 let round_verdict = ReviewVote::worst(final_votes);
3998
3999 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4000 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4001 // A round that already has a blocking finding and a round left to
4002 // try is going back to the fixer no matter what `verify.e2e`
4003 // says, so running it first only spends the loop's slowest step
4004 // (minutes, for a Rust repo's full test suite) on a head about
4005 // to be rewritten. Deferred, never skipped: `verify.e2e` still
4006 // runs once a round has no blocking findings left (see
4007 // `round_is_clean`, which a deferred — empty — `e2e` can never
4008 // satisfy since `blocking` is nonzero whenever this branch is
4009 // taken), and `stop_reviewing` forces a real run before it will
4010 // ever read a deferred round as green.
4011 let defer_e2e =
4012 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4013 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4014 let reason =
4015 format!("{blocking} blocking finding(s) already required a fix this round");
4016 self.state.event(
4017 "verify",
4018 format!(
4019 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4020 {}); it will run once a round has none left",
4021 short(&head)
4022 ),
4023 );
4024 (Vec::new(), false, true, Some(reason))
4025 } else {
4026 let e2e_commands = self.state.config.verify.e2e.clone();
4027 let cache_dir = self.state.config.cache_dir();
4028 let context = format!("round {round}");
4029 let (e2e, verify_retried) = with_cache_lease(
4030 &mut self.state,
4031 cache_dir.as_deref(),
4032 "e2e",
4033 "e2e",
4034 &winner.worktree,
4035 &head,
4036 verify_timeout,
4037 &context,
4038 |state, budget| {
4039 let shell = shell.clone();
4040 let e2e_commands = e2e_commands.clone();
4041 let worktree = winner.worktree.clone();
4042 let context = context.clone();
4043 async move {
4044 run_e2e_with_retry(
4045 state,
4046 &shell,
4047 &e2e_commands,
4048 &worktree,
4049 budget,
4050 &context,
4051 )
4052 .await
4053 }
4054 },
4055 )
4056 .await;
4057 (e2e, verify_retried, false, None)
4058 };
4059
4060 let expected = records.len();
4061 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4062 let incomplete = answered < expected;
4063 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4064 let policy = self.state.config.graph.incomplete_review;
4065 let clean = round_is_clean(
4066 blocking,
4067 e2e_ok,
4068 answered,
4069 expected,
4070 round_quota_missing,
4071 policy,
4072 );
4073
4074 let mut round_record = ReviewRound {
4075 round,
4076 head: head.clone(),
4077 verified_head: None,
4078 verified_at: None,
4079 reviews: records,
4080 e2e,
4081 verify_retried,
4082 e2e_deferred,
4083 e2e_defer_reason,
4084 fix: None,
4085 blocking,
4086 answered,
4087 expected,
4088 clean,
4089 progressed: false,
4090 vote_split,
4091 reconsideration,
4092 verdict: round_verdict,
4093 };
4094 // Which commit and when magi actually attempted to check —
4095 // known the moment a command was dispatched against `head`,
4096 // whether or not it finished: a resource-blocked attempt still
4097 // targeted a specific commit at a specific time, and leaving
4098 // that unrecorded is exactly what made `verification_summary`
4099 // report a fresh attempt as "commit unknown ... recorded before
4100 // this was tracked", indistinguishable from a genuinely old,
4101 // untracked record. Only a deferred or unconfigured round never
4102 // ran at all and has nothing to record — see
4103 // `ReviewRound::verified_head`'s own doc.
4104 if !matches!(
4105 round_record.e2e_status(),
4106 E2eStatus::Deferred | E2eStatus::NotConfigured
4107 ) {
4108 round_record.verified_head = Some(head.clone());
4109 round_record.verified_at = Some(Timestamp::now());
4110 }
4111 let this_round_verification = round_record.verification_summary(&head);
4112
4113 if incomplete {
4114 let missing: Vec<String> = round_record
4115 .reviews
4116 .iter()
4117 .filter(|r| r.failed.is_some())
4118 .map(|r| format!("review-{}", r.reviewer))
4119 .collect();
4120 self.state.event(
4121 "review",
4122 format!(
4123 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4124 missing.join(", ")
4125 ),
4126 );
4127 }
4128
4129 if clean {
4130 self.state.event(
4131 "review",
4132 if incomplete && policy == IncompleteReviewPolicy::Warn {
4133 format!(
4134 "round {round}: clean (warn policy, incomplete panel) — no \
4135 blocking findings from the seats that answered, verification green"
4136 )
4137 } else if incomplete {
4138 format!(
4139 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4140 quorum) — no blocking findings from the seats that answered, \
4141 verification green",
4142 expected - answered
4143 )
4144 } else {
4145 format!("round {round}: clean — no blocking findings, verification green")
4146 },
4147 );
4148 self.state.reviews.push(round_record);
4149 self.state.status = RunStatus::Gating;
4150 self.state.save()?;
4151 return Ok(());
4152 }
4153
4154 // Nothing was raised and verification passed, but not every seat
4155 // answered and `round_is_clean` still refused to call it clean —
4156 // either a seat is missing for a reason other than its own quota
4157 // (a crash, a timeout, unparsable output — worth another try), or
4158 // every seat that could have answered lost its quota and nobody
4159 // is left to decide on: re-review rather than send the fixer
4160 // after a round with nothing to fix.
4161 if incomplete && blocking == 0 && e2e_ok {
4162 self.state.reviews.push(round_record);
4163 self.state.save()?;
4164 if round == max_rounds {
4165 self.state.status = RunStatus::Blocked;
4166 self.state.event(
4167 "review",
4168 format!(
4169 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4170 refusing to call it clean",
4171 expected - answered
4172 ),
4173 );
4174 return Ok(());
4175 }
4176 continue;
4177 }
4178
4179 // Nothing for the fixer to act on (`blocking == 0`) and the only
4180 // reason this round is not clean is that magi itself never got
4181 // a command to run — the shared build cache, not the patch (see
4182 // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4183 // the fixer would invite a change to appease contention that has
4184 // nothing to do with the diff, and would leave this attempt
4185 // sitting in the next round's prompt as if it were about an
4186 // earlier, superseded commit rather than what it actually is:
4187 // the same head, still waiting to be checked. Wait for it the
4188 // same way the final round's own contention is already handled,
4189 // whatever round this happens to be.
4190 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4191 self.state.reviews.push(round_record);
4192 return self
4193 .stop_reviewing(
4194 "the round's own verification could not run",
4195 &shell,
4196 &winner.worktree,
4197 )
4198 .await;
4199 }
4200
4201 if round == max_rounds {
4202 self.state.reviews.push(round_record);
4203 return self
4204 .stop_reviewing(
4205 &format!(
4206 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4207 ),
4208 &shell,
4209 &winner.worktree,
4210 )
4211 .await;
4212 }
4213
4214 // Fix. The winner's own implementer seat continues its conversation:
4215 // the competition is over, so context is pure benefit now.
4216 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
4217 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4218 _ => (
4219 self.state
4220 .config
4221 .agent(&winner.agent)
4222 .cloned()
4223 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4224 format!("impl-{}", winner.label),
4225 ),
4226 };
4227 let seat = self.seat(&fix_seat_key, &fix_spec.id);
4228 let blocking_findings: Vec<_> = all_findings
4229 .iter()
4230 .filter(|f| f.severity.blocks())
4231 .cloned()
4232 .collect();
4233 let job = SeatJob {
4234 prompt: prompt::fix(
4235 &self.state.instruction,
4236 &blocking_findings,
4237 this_round_verification.as_ref(),
4238 round,
4239 max_rounds,
4240 &language,
4241 ),
4242 spec: fix_spec.clone(),
4243 seat,
4244 cwd: winner.worktree.clone(),
4245 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4246 allow_write: true,
4247 sessions,
4248 artifacts: artifacts.clone(),
4249 stem: format!("fix-{round}"),
4250 };
4251 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4252 let cache = self.state.config.cache_dir();
4253 let ctx = WaveCtx {
4254 run: &run_id,
4255 node: "fix",
4256 prompts: &prompts,
4257 cache: cache.as_deref(),
4258 round: Some(round),
4259 };
4260 let (seat, out) =
4261 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4262 let agent_id = seat.agent.clone();
4263
4264 let mut fix = FixRecord {
4265 agent: agent_id,
4266 addressed: Vec::new(),
4267 rejected: Vec::new(),
4268 notes: String::new(),
4269 committed: false,
4270 failed: None,
4271 duration_ms: 0,
4272 continuation: None,
4273 };
4274 let mut continuation = ContinuationRecord::not_needed();
4275 let mut final_seat = seat.clone();
4276 match out {
4277 AgentOutcome::Ok(o) => {
4278 fix.duration_ms = o.duration_ms;
4279 let parsed = verdict::extract_json::<FixReport>(&o.text);
4280 // A parsed report standing next to a command this same
4281 // reply's own CLI never confirmed the exit status of is
4282 // not a resolved answer — the identical `CommandEvidence`
4283 // `state.jobs` renders, read here instead of only on
4284 // display, per the completion judgment and the shown
4285 // record needing to agree.
4286 let incomplete_reason = match &parsed {
4287 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4288 "the reply parsed, but it reported a command whose own CLI never \
4289 confirmed an exit status"
4290 .to_owned(),
4291 ),
4292 Ok(_) => None,
4293 Err(e) => Some(e.to_string()),
4294 };
4295 match incomplete_reason {
4296 None => {
4297 let report = parsed.expect("checked Ok above");
4298 fix.addressed = report.addressed;
4299 fix.rejected = report.rejected;
4300 fix.notes =
4301 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4302 }
4303 Some(reason) => {
4304 let (resumed_seat, resolved, failure, cont) = self
4305 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4306 .await;
4307 fix.duration_ms += cont.cumulative_wait_ms;
4308 continuation = cont;
4309 final_seat = resumed_seat;
4310 match resolved {
4311 Some(report) => {
4312 fix.addressed = report.addressed;
4313 fix.rejected = report.rejected;
4314 fix.notes = blind::sanitize_prose(
4315 &report.notes,
4316 &self.state.config.blind,
4317 );
4318 }
4319 None => fix.failed = failure,
4320 }
4321 }
4322 }
4323 }
4324 // The CLI's raw error JSON is not a fix report to parse.
4325 AgentOutcome::Dropped(o) => {
4326 fix.duration_ms = o.duration_ms;
4327 let why = o
4328 .dropped
4329 .as_ref()
4330 .map(|d| d.why.as_str())
4331 .unwrap_or("the CLI ended the stream without delivering its answer");
4332 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4333 }
4334 AgentOutcome::Quota(o) => {
4335 self.state.quota.push(QuotaLoss {
4336 seat: final_seat.key.clone(),
4337 node: "fix".to_owned(),
4338 at: Timestamp::now(),
4339 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4340 });
4341 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4342 }
4343 AgentOutcome::Failed(e) => fix.failed = Some(e),
4344 }
4345 fix.continuation = Some(continuation);
4346 self.state.seats.insert(final_seat.key.clone(), final_seat);
4347 if let Ok(r) = git::rescue_commit(
4348 &winner.worktree,
4349 &format!("magi: review round {round} fixes (uncommitted work)"),
4350 )
4351 .await
4352 {
4353 self.state.note_withheld("fix", &r.withheld);
4354 }
4355 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4356 fix.committed = after != before;
4357 // Judged by what `git` says moved against base, never by the
4358 // fixer's own `addressed`/`rejected` count — see
4359 // `ReviewRound::progressed`. Propagated with `?`, the same as the
4360 // `patch` snapshot above: swallowing this error would default
4361 // `diff_after` to empty, which almost always differs from a
4362 // non-empty `patch` and reads as "progressed" — exactly backwards
4363 // for a `git` failure the stagnation check cannot see through.
4364 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4365 let progressed = diff_after != patch;
4366 let commit_note = if fix.committed {
4367 "committed"
4368 } else {
4369 "NO new commit"
4370 };
4371 let tree_note = if progressed {
4372 "changed vs base"
4373 } else {
4374 "unchanged vs base"
4375 };
4376 self.state.event(
4377 "fix",
4378 match &fix.failed {
4379 // Distinct on purpose from "0 addressed, 0 rejected": the
4380 // fixer's own diff still landed (blocking counts do keep
4381 // falling round over round), only its adoption report did
4382 // not come back, so this must never read like every
4383 // finding was reviewed and declined.
4384 Some(reason) => {
4385 format!(
4386 "round {round}: fixer's adoption report was lost ({reason}); \
4387 {commit_note}, tree {tree_note}"
4388 )
4389 }
4390 None => format!(
4391 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4392 {tree_note}{}",
4393 fix.addressed.len(),
4394 fix.rejected.len(),
4395 if continuation.outcome == ContinuationOutcome::Resumed {
4396 format!(
4397 " (adoption report recovered after {} continuation(s))",
4398 continuation.attempts
4399 )
4400 } else {
4401 String::new()
4402 },
4403 ),
4404 },
4405 );
4406 round_record.fix = Some(fix);
4407 round_record.progressed = progressed;
4408 self.state.reviews.push(round_record);
4409 self.state.save()?;
4410
4411 // The fixer's own report never came back this round, even after
4412 // `continue_fix_report`'s own budget was spent on it — not an
4413 // ordinary "no report" (dropped stream, quota, plain failure),
4414 // which already reads that way and is left to the existing round
4415 // budget. Stopping here, rather than opening another round, is
4416 // what keeps a next reviewer/fixer wave from ever being
4417 // dispatched onto `winner.worktree` while whatever the seat's
4418 // last call may still have running there is unaccounted for: no
4419 // process liveness check exists (and none is being added — see
4420 // AGENTS.md/this task's own scope), so the only way to honour
4421 // "nothing starts before a valid report returns" is to not start
4422 // anything further on this worktree from this run at all.
4423 if matches!(
4424 continuation.outcome,
4425 ContinuationOutcome::Exhausted
4426 | ContinuationOutcome::QuotaLost
4427 | ContinuationOutcome::NoSession
4428 ) {
4429 return self
4430 .stop_reviewing(
4431 "the fixer's adoption report never came back, even after resuming its \
4432 own seat; refusing to start another round against the same worktree \
4433 while that is unresolved",
4434 &shell,
4435 &winner.worktree,
4436 )
4437 .await;
4438 }
4439
4440 let streak = self
4441 .state
4442 .reviews
4443 .iter()
4444 .rev()
4445 .take_while(|r| !r.progressed)
4446 .count();
4447 if streak >= STAGNANT_LIMIT {
4448 return self
4449 .stop_reviewing(
4450 &format!(
4451 "the tree has not moved against base for {streak} round(s) in a row"
4452 ),
4453 &shell,
4454 &winner.worktree,
4455 )
4456 .await;
4457 }
4458 }
4459 Ok(())
4460 }
4461
4462 /// Decide, from the last recorded round's own verification, whether
4463 /// stopping the review loop is a hand-off or a genuine block.
4464 ///
4465 /// Called once the loop has given up trying — the round budget is spent,
4466 /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
4467 /// findings still open, never while a round is still clean or the
4468 /// incomplete-panel case handled inline above. Gate and e2e are facts
4469 /// about the tree; a lingering review finding is an opinion, and this
4470 /// workload's own `magi stats` puts reviewer precision low enough
4471 /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
4472 /// must not by itself stand between a green, verified change and the
4473 /// human who decides what to do with it. A red e2e is not an opinion, so
4474 /// that case still blocks, with the failing command and a tail of its
4475 /// output recorded here rather than left in `run.json` for someone to go
4476 /// find.
4477 ///
4478 /// A round that deferred its own e2e (see [`Config::graph`]'s
4479 /// `e2e_every_round`) is never read as that green: its `e2e` is empty
4480 /// only because nothing ran, and treating an empty list as a passing one
4481 /// here is exactly the "deferred painted green" bug this function exists
4482 /// to not have. When the last round's own verification never resolved —
4483 /// deferred on purpose, or a real attempt the shared build cache blocked
4484 /// — this makes (or retries) the real run, on the actual worktree this
4485 /// loop is about to stop touching, before deciding anything. A
4486 /// resource-blocked attempt is likewise never read as either green or
4487 /// red: it is evidence about the machine, not the patch (see
4488 /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
4489 /// blocked cache leaves this call without deciding rather than guessing
4490 /// — the caller retries on a later reentry.
4491 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
4492 let round_idx = self.state.reviews.len() - 1;
4493 // A deferred round and a resource-blocked one are the same shape
4494 // here: neither has a real result yet, and both get one more
4495 // attempt. Read off `e2e_status` — the single source for this —
4496 // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
4497 // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
4498 // still retries instead of being read as a settled result the
4499 // instant it stops being empty.
4500 let needs_catchup_run = matches!(
4501 self.state.reviews[round_idx].e2e_status(),
4502 E2eStatus::Deferred | E2eStatus::ResourceBlocked
4503 );
4504 if needs_catchup_run {
4505 let round = self.state.reviews[round_idx].round;
4506 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4507 let commands = self.state.config.verify.e2e.clone();
4508 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
4509 let cache_dir = self.state.config.cache_dir();
4510 let context = format!(
4511 "round {round}: verification unresolved, catching up before the final decision"
4512 );
4513 let (outcomes, verify_retried) = with_cache_lease(
4514 &mut self.state,
4515 cache_dir.as_deref(),
4516 "e2e",
4517 "e2e",
4518 worktree,
4519 &attempted_head,
4520 timeout,
4521 &context,
4522 |state, budget| {
4523 let shell = shell.to_vec();
4524 let commands = commands.clone();
4525 let context = context.clone();
4526 async move {
4527 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
4528 .await
4529 }
4530 },
4531 )
4532 .await;
4533 let last = &mut self.state.reviews[round_idx];
4534 last.e2e = outcomes;
4535 last.verify_retried = verify_retried;
4536 // Always the commit and time this attempt actually targeted,
4537 // whether or not it happens to equal the reviewed `head` and
4538 // whether or not a command finished — see
4539 // `ReviewRound::verified_head`'s own doc. A still-inconclusive
4540 // attempt is recorded too, so a later reader sees "attempted
4541 // again at T2" rather than silence.
4542 last.verified_head = Some(attempted_head);
4543 last.verified_at = Some(Timestamp::now());
4544 if verify_inconclusive(&last.e2e) {
4545 // Still not a real result: `e2e_deferred` is left exactly
4546 // as it was, so `needs_catchup_run` above reads
4547 // `ResourceBlocked` (via `e2e_status`, which checks
4548 // `resource_blocked` before `e2e_deferred`) and retries
4549 // again on the next reentry, rather than recording
4550 // contention as a red e2e and blocking the run on it.
4551 self.state.save()?;
4552 return Ok(());
4553 }
4554 last.e2e_deferred = false;
4555 }
4556 let last = &self.state.reviews[round_idx];
4557 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
4558
4559 match last.e2e_status() {
4560 E2eStatus::Failed => {
4561 let red: Vec<String> = last
4562 .e2e
4563 .iter()
4564 .filter(|o| !o.ok())
4565 .map(|o| {
4566 format!(
4567 "`{}` -> {:?}\n{}",
4568 o.command,
4569 o.code,
4570 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4571 )
4572 })
4573 .collect();
4574 self.state
4575 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
4576 self.state.status = RunStatus::Blocked;
4577 }
4578 // `needs_catchup_run` above already retried once this call; if
4579 // it is still blocked, this is magi's own admission it could
4580 // not get a command to run, never a verdict on the patch — the
4581 // run is left exactly where a later reentry can retry again.
4582 E2eStatus::ResourceBlocked => {
4583 self.state.event(
4584 "review",
4585 format!(
4586 "{why}; e2e could not run (shared build cache unavailable); not \
4587 deciding yet"
4588 ),
4589 );
4590 }
4591 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
4592 self.state.event(
4593 "review",
4594 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
4595 );
4596 self.state.status = RunStatus::Gating;
4597 }
4598 }
4599 self.state.save()?;
4600 Ok(())
4601 }
4602
4603 // ----------------------------------------------------------------- gate
4604
4605 async fn gate(&mut self) -> Result<()> {
4606 // Judged by the review record itself, not by `status`: a solo
4607 // candidate's `judge`/`deliberate` skip rewrites `status` on every
4608 // reentry (see `judge`), and trusting it here is exactly how a run
4609 // that exhausted its review budget got gated and merged a second
4610 // time around. `review_conclusion` recomputes the review loop's own
4611 // verdict from the round records themselves — `Gating` for a clean
4612 // round or a hand-off (see `stop_reviewing`), anything else means the
4613 // loop is still going or genuinely blocked.
4614 // A base the winner could not be replayed onto is a decision, not a
4615 // round: there is no landing tree to gate. Read as its own record for
4616 // the same reason the review verdict is.
4617 if self.state.status == RunStatus::Failed
4618 || self
4619 .state
4620 .base_sync
4621 .as_ref()
4622 .is_some_and(|s| s.conflict.is_some())
4623 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4624 != Some(RunStatus::Gating)
4625 {
4626 return Ok(());
4627 }
4628 if self.state.gate_ran {
4629 // `review_loop` derives its conclusion from the clean review
4630 // record on every reentry and therefore puts a completed run back
4631 // in `Gating`. A recorded gate is a stronger, terminal fact:
4632 // retain its original command output (or lack of any, for a repo
4633 // with no `verify.gate` commands — see `RunState::gate_ran`'s own
4634 // doc) and restore `Blocked` on a real failure rather than
4635 // pretending the command is still running or running it a second
4636 // time. `gate_ran == false` remains the only shape — unattempted,
4637 // or a resource-blocked retry — that may still need to execute a
4638 // command.
4639 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
4640 self.state.status = RunStatus::Blocked;
4641 self.state.save()?;
4642 }
4643 return Ok(());
4644 }
4645 let Some(winner) = self.state.winner().cloned() else {
4646 return Ok(());
4647 };
4648 self.state.status = RunStatus::Gating;
4649 let shell = self.state.config.shell();
4650 let gate_commands = self.state.config.verify.gate.clone();
4651 // Zero commands has nothing to run and nothing that could touch the
4652 // shared build cache, so it never needs a lease: `Config::cache_dir`
4653 // is derived from `verify.e2e` too, so a repo with no `verify.gate`
4654 // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
4655 // otherwise queue behind an unrelated run's lease and come back
4656 // resource-blocked - `gate_ran` would stay false on nothing but
4657 // cache contention, for a step that had nothing to check in the
4658 // first place.
4659 let outcomes = if gate_commands.is_empty() {
4660 Vec::new()
4661 } else {
4662 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4663 let cache_dir = self.state.config.cache_dir();
4664 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4665 let (outcomes, _) = with_cache_lease(
4666 &mut self.state,
4667 cache_dir.as_deref(),
4668 "gate",
4669 "gate",
4670 &winner.worktree,
4671 &head,
4672 timeout,
4673 "final gate",
4674 |state, budget| {
4675 let shell = shell.clone();
4676 let gate_commands = gate_commands.clone();
4677 let worktree = winner.worktree.clone();
4678 async move {
4679 let (outcomes, timed_out_pids) = run_commands(
4680 state,
4681 "gate",
4682 "gate",
4683 0,
4684 &shell,
4685 &gate_commands,
4686 &worktree,
4687 budget,
4688 )
4689 .await;
4690 (outcomes, false, timed_out_pids)
4691 }
4692 },
4693 )
4694 .await;
4695 outcomes
4696 };
4697 if outcomes.is_empty() {
4698 // Nothing configured to check — distinct from every other
4699 // silence in this run's event log, since an empty `gate` alone
4700 // no longer says whether the gate ran at all (see
4701 // `RunState::gate_ran`'s own doc).
4702 self.state.event(
4703 "gate",
4704 "no gate commands configured; nothing to check, passing",
4705 );
4706 }
4707 for o in &outcomes {
4708 self.state.event(
4709 "gate",
4710 format!(
4711 "`{}` -> {}",
4712 o.command,
4713 if o.ok() {
4714 "pass".to_owned()
4715 } else {
4716 format!(
4717 "FAIL ({:?})\n{}",
4718 o.code,
4719 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4720 )
4721 }
4722 ),
4723 );
4724 }
4725 // A resource-blocked outcome means the gate command never actually
4726 // ran - the shared build cache could not be acquired or confirmed
4727 // fresh in time - which is evidence about the machine, not about the
4728 // tree (see `CommandOutcome::resource_blocked`'s own doc). Recording
4729 // it as a red gate would mark a run `Blocked` on nothing but
4730 // contention magi has already logged above; leaving `self.state.gate`
4731 // empty and `self.state.gate_ran` false instead keeps the shape this
4732 // function already treats as "still needs to run" (see the
4733 // early-return above), so the next call retries the command rather
4734 // than concluding anything.
4735 if verify_inconclusive(&outcomes) {
4736 self.state.save()?;
4737 return Ok(());
4738 }
4739 let passed = outcomes.iter().all(CommandOutcome::ok);
4740 self.state.gate = outcomes;
4741 self.state.gate_ran = true;
4742 if !passed {
4743 self.state.status = RunStatus::Blocked;
4744 self.state.event("gate", "gate failed; not merging");
4745 }
4746 self.state.save()?;
4747 Ok(())
4748 }
4749
4750 // ---------------------------------------------------------------- merge
4751
4752 async fn merge(&mut self) -> Result<()> {
4753 // Same reasoning as `gate`: ask the review and gate records directly
4754 // rather than `status`, which a solo-candidate `judge`/`deliberate`
4755 // skip can rewrite on reentry to something that no longer says
4756 // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
4757 // so a hand-off (open findings, green verification) reaches merge
4758 // exactly like a genuinely clean round does.
4759 //
4760 // A run resumed mid-`land` never reaches here at all: `execute`
4761 // recognises `RunStatus::Landing` before it even calls `prep`, and
4762 // routes straight to `run_land` instead. That has to happen a level
4763 // up from this function, not with a check in here, because
4764 // `review_loop`'s own status recomputation (see its doc) runs
4765 // *before* `merge` on every reentry and would otherwise overwrite
4766 // the `Landing` marker with `Gating` before this node ever saw it.
4767 if self
4768 .state
4769 .base_sync
4770 .as_ref()
4771 .is_some_and(|s| s.conflict.is_some())
4772 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4773 != Some(RunStatus::Gating)
4774 // `gate_ran == false` is not "passed" - `gate` leaves it false
4775 // both before it has ever run and when its last attempt was
4776 // resource-blocked (see `Runner::gate`'s own doc), and neither is
4777 // permission to merge on nothing but the review record. Only a
4778 // gate that actually ran - zero commands configured and
4779 // vacuously passed, or one or more that all exited 0 - may
4780 // proceed; `RunState::gate_status` is the single place that
4781 // reading is computed.
4782 || !self.state.gate_status().ok()
4783 {
4784 return Ok(());
4785 }
4786 // This node's own record, not `status`: `status == Ready` is not
4787 // unique to the harmless `MergeMode::None` path this line was
4788 // written for. `land` (below) sets it too, when a `MergeMode::Pr`
4789 // run's PR was closed without merging — and on that run `mode` is
4790 // still `Pr`, so a reentry that fell through here would push and
4791 // open a second pull request. `self.state.merge` is set exactly once
4792 // this node (or `land`) has already produced a verdict, under every
4793 // mode, which is what "already done" actually means here.
4794 if self.state.merge.is_some() {
4795 return Ok(());
4796 }
4797 let Some(winner) = self.state.winner().cloned() else {
4798 return Ok(());
4799 };
4800 let repo = self.state.repo.clone();
4801 let base = self.state.base_branch.clone();
4802 let mode = self.state.config.merge.mode;
4803 let style = self.state.config.merge.style;
4804 let pr = pr_message(&self.state, winner.label);
4805 let message = pr.commit_message();
4806
4807 let outcome = match mode {
4808 MergeMode::None => MergeOutcome {
4809 mode,
4810 ok: true,
4811 detail: manual_merge_command(style, &repo, &winner.branch, &message),
4812 },
4813 MergeMode::Local => {
4814 let on = git::current_branch(&repo).await?;
4815 if on.as_deref() != Some(base.as_str()) {
4816 MergeOutcome {
4817 mode,
4818 ok: false,
4819 detail: format!(
4820 "{} has {} checked out, not the base branch {base}",
4821 repo.display(),
4822 on.unwrap_or_else(|| "a detached HEAD".to_owned())
4823 ),
4824 }
4825 } else if !git::is_clean(&repo).await? {
4826 MergeOutcome {
4827 mode,
4828 ok: false,
4829 detail: format!("{} is dirty; refusing to merge", repo.display()),
4830 }
4831 } else {
4832 let out = match style {
4833 MergeStyle::Merge => {
4834 git::merge_no_ff(&repo, &winner.branch, &message).await?
4835 }
4836 MergeStyle::Squash => {
4837 git::merge_squash(&repo, &winner.branch, &message).await?
4838 }
4839 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
4840 };
4841 MergeOutcome {
4842 mode,
4843 ok: out.ok(),
4844 detail: if out.ok() { out.stdout } else { out.stderr },
4845 }
4846 }
4847 }
4848 MergeMode::Pr => {
4849 let remote = self.state.config.merge.remote.clone();
4850 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
4851 if !pushed.ok() {
4852 MergeOutcome {
4853 mode,
4854 ok: false,
4855 detail: pushed.stderr,
4856 }
4857 } else {
4858 let out =
4859 gh_pr_create(&winner.worktree, &base, &winner.branch, &pr.title, &pr.body)
4860 .await;
4861 match out {
4862 Ok(url) => MergeOutcome {
4863 mode,
4864 ok: true,
4865 detail: url,
4866 },
4867 Err(e) => MergeOutcome {
4868 mode,
4869 ok: false,
4870 detail: e.to_string(),
4871 },
4872 }
4873 }
4874 }
4875 };
4876
4877 self.state.status = match (mode, outcome.ok) {
4878 (MergeMode::None, _) => RunStatus::Ready,
4879 (_, true) => RunStatus::Merged,
4880 (_, false) => RunStatus::Blocked,
4881 };
4882 self.state.event(
4883 "merge",
4884 format!(
4885 "{:?}: {}",
4886 mode,
4887 outcome.detail.lines().next().unwrap_or("")
4888 ),
4889 );
4890 self.state.merge = Some(outcome);
4891 self.state.save()?;
4892
4893 // The PR is open and the run would historically stop here, leaving the
4894 // operator to watch checks, feed review comments back to a fixer, and
4895 // merge. That was done by hand six times in one session before this
4896 // existed. Opt-in, because merging is the one irreversible thing magi
4897 // can do to a repository.
4898 if self.state.config.graph.land
4899 && mode == MergeMode::Pr
4900 && self.state.status == RunStatus::Merged
4901 {
4902 self.run_land().await?;
4903 }
4904 // `run_land` may have left `status` at `Landing` - still waiting on
4905 // CI or the owner's approval, not actually settled - so this has to
4906 // read whatever `status` ended up as here, not the `Merged` this
4907 // function set a few lines up.
4908 self.settle_questions();
4909 Ok(())
4910 }
4911
4912 /// Enter `land`.
4913 ///
4914 /// Shared between a fresh run's first pass through [`Runner::merge`] and
4915 /// a resumed run's re-entry. `land::land` itself is what serialises the
4916 /// two git-mutating moments inside the loop — the rebase push and
4917 /// `gh pr merge` — per repository (see its own doc); nothing here needs
4918 /// to hold a lock across the whole call, and doing so would serialise
4919 /// this run's CI wait against a *different* run's land-approval resume
4920 /// in the same repository, which is exactly the "must not wait on
4921 /// another task" property the daemon's slot-freeing exists to give.
4922 async fn run_land(&mut self) -> Result<()> {
4923 let url = self
4924 .state
4925 .merge
4926 .as_ref()
4927 .map(|m| m.detail.clone())
4928 .unwrap_or_default();
4929 let url = url.lines().next().unwrap_or("").trim().to_owned();
4930 if !url.starts_with("http") {
4931 return Ok(());
4932 }
4933 // A land failure is not a lost run: the work is on a branch and the
4934 // pull request is open, which is exactly where a human takes over.
4935 match land::land(&mut self.state, &url).await {
4936 Ok(pr) if self.state.parked => {
4937 // `land` already saved the parked marker; nothing here
4938 // overrides `status` back to a terminal value while an
4939 // approval is still outstanding.
4940 let _ = pr;
4941 }
4942 Ok(pr) => {
4943 self.state.status = match pr.state {
4944 land::PrLifecycle::Merged => RunStatus::Merged,
4945 _ => RunStatus::Blocked,
4946 };
4947 // Downstream of a confirmed merge only - see
4948 // `bump::should_release_bump`'s own doc for why this one
4949 // check covers all three of `land`'s success paths.
4950 // Best-effort: the run already landed, so a failure here
4951 // (the decision call, `gh`, `cargo`) is recorded and never
4952 // turns a landed run into a failed one.
4953 if bump::should_release_bump(self.state.status)
4954 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
4955 {
4956 self.state
4957 .event("bump", format!("release bump skipped: {e:#}"));
4958 }
4959 self.state.save()?;
4960 }
4961 Err(e) => {
4962 self.state.status = RunStatus::Blocked;
4963 self.state.event("land", format!("gave up: {e}"));
4964 self.state.save()?;
4965 }
4966 }
4967 Ok(())
4968 }
4969
4970 // -------------------------------------------------------------- helpers
4971
4972 /// Fetch or create a seat, keeping its conversation across nodes.
4973 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
4974 if let Some(existing) = self.state.seats.get(key)
4975 && existing.agent == agent
4976 {
4977 return existing.clone();
4978 }
4979 let fresh = SeatState::new(key, agent, self.state.seed);
4980 self.state.seats.insert(key.to_owned(), fresh.clone());
4981 fresh
4982 }
4983
4984 /// A candidate rendered for judging, with the leak policy applied.
4985 fn view(&self, c: &Candidate) -> CandidateView {
4986 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4987 .unwrap_or_default();
4988 let (patch, _) = blind::sanitize_patch(
4989 &format!("candidate {} patch", c.label),
4990 &raw,
4991 &self.state.config.blind,
4992 );
4993 CandidateView {
4994 label: c.label,
4995 branch: c.branch.clone(),
4996 summary: c.summary.clone(),
4997 stat: c.stat.clone(),
4998 patch,
4999 }
5000 }
5001
5002 /// The full candidate set as prompt text, for seats with no live session.
5003 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
5004 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
5005 prompt::judge(
5006 "(see above)",
5007 &views,
5008 self.roles.judges.len(),
5009 base_short,
5010 "en",
5011 )
5012 }
5013
5014 /// Anonymised transcript for judge `self_idx`.
5015 ///
5016 /// The initial rankings are always the opening statements. Seeding them
5017 /// only when no turn had been taken yet meant every judge after the first
5018 /// argued against a single voice instead of against the actual split — the
5019 /// disagreement is the information, so it is always on the table.
5020 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
5021 let mut turns = Vec::new();
5022 for j in &self.state.judgements {
5023 if j.ranking.is_empty() {
5024 continue;
5025 }
5026 let reasons = j
5027 .reasons
5028 .iter()
5029 .map(|(k, v)| format!("- {k}: {v}"))
5030 .collect::<Vec<_>>()
5031 .join("\n");
5032 turns.push(Turn {
5033 who: format!("Judge {} (opening ranking)", j.judge),
5034 is_self: j.judge == self_idx + 1,
5035 body: format!(
5036 "Ranked {}{}{reasons}",
5037 j.ranking.iter().collect::<String>(),
5038 if reasons.is_empty() {
5039 ""
5040 } else {
5041 ", because:\n"
5042 }
5043 ),
5044 });
5045 }
5046 for t in self
5047 .state
5048 .deliberation
5049 .iter()
5050 .flat_map(|r| r.turns.iter())
5051 .chain(current)
5052 {
5053 turns.push(Turn {
5054 who: format!("Judge {}", t.judge),
5055 is_self: t.judge == self_idx + 1,
5056 body: t.body.clone(),
5057 });
5058 }
5059 turns
5060 }
5061}
5062
5063/// Does this seat still hold the context a follow-up prompt would rely on?
5064fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
5065 agent::has_session(spec.kind, seat, sessions)
5066}
5067
5068/// The next entry in `roster` after `start`, never wrapping back to the
5069/// front, whose id is not in `tried` yet.
5070///
5071/// Starts one past `start` rather than at the front of `roster`: `start` is
5072/// the seat's own original position, and a seat whose candidate slot already
5073/// sits on the roster's second entry must fall through to the third next, not
5074/// restart at the first — which is very likely a different candidate's own
5075/// agent already. Never wraps back past `start`, for the same reason: an
5076/// entry earlier in the roster than the seat's own position is almost
5077/// certainly some *other* candidate slot's own agent, and once the tail of
5078/// the roster is exhausted there are no more untried agents for *this* seat
5079/// to fall through to — the caller's fallback chain ends there, exactly as
5080/// "no further untried agents remain in the list for that seat" asks for.
5081///
5082/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
5083/// the same id twice (an operator's `roles.implementers` typo, or a
5084/// `[[agents]]` list reused across roles) must not let
5085/// [`Runner::resume_quota_losses`] retry that id forever — one forward pass
5086/// over `roster` either finds an untried id or runs out, so this always
5087/// terminates regardless of duplicates.
5088fn next_untried_implementer<'a>(
5089 roster: &'a [AgentSpec],
5090 start: usize,
5091 tried: &BTreeSet<String>,
5092) -> Option<&'a AgentSpec> {
5093 roster
5094 .get(start + 1..)?
5095 .iter()
5096 .find(|s| !tried.contains(&s.id))
5097}
5098
5099/// Did this reply report running a command whose own CLI never confirmed an
5100/// exit status?
5101///
5102/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
5103/// command *finished* (see that type's own doc), so this can only be `true`
5104/// for a command whose completion event carried no readable exit code — not
5105/// for one that simply is not mentioned at all. That is the one signal this
5106/// crate can read, from the same record `state.jobs` renders, about a reply
5107/// standing next to work its own CLI cannot vouch for finishing; it is
5108/// deliberately not a check on the exit code's *value* (a fixer legitimately
5109/// runs a command that fails mid-iteration before it succeeds) and not a
5110/// guess at a command still running in the background (which emits no event
5111/// at all, and so leaves no evidence here to find).
5112fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
5113 commands.iter().any(|c| c.exit_code.is_none())
5114}
5115
5116/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
5117/// trusted as a verified no-op — the adoption guard's own text-level half.
5118///
5119/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
5120/// timed out): a marker only earns the benefit of the doubt from a turn the
5121/// CLI itself vouches for finishing properly, the same house style
5122/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
5123/// *fix* report to for `commands`. A candidate that timed out, exited
5124/// non-zero, or left a command unconfirmed is read as the ordinary loss it
5125/// is, whatever prose it wrote — this returns `None` before it ever looks at
5126/// `text`. The remaining guards (the tree really is empty, the evidence is
5127/// non-empty) are the caller's: this only reads what the reply *claimed*.
5128fn verified_noop_claim(
5129 usable: bool,
5130 commands: &[agent::CommandEvidence],
5131 text: &str,
5132) -> Option<String> {
5133 (usable && !has_unconfirmed_command(commands))
5134 .then(|| verdict::verified_noop(text))
5135 .flatten()
5136}
5137
5138fn short(commit: &str) -> String {
5139 commit.chars().take(7).collect()
5140}
5141
5142fn make_executable(path: &Path) -> Result<()> {
5143 #[cfg(unix)]
5144 {
5145 use std::os::unix::fs::PermissionsExt as _;
5146 let mut perms = std::fs::metadata(path)?.permissions();
5147 perms.set_mode(0o755);
5148 std::fs::set_permissions(path, perms)?;
5149 }
5150 #[cfg(not(unix))]
5151 {
5152 let _ = path;
5153 }
5154 Ok(())
5155}
5156
5157/// What every seat in one batch shares: where the answers are attributed, the
5158/// prompt overlay they inherit, and the build cache they are told to use.
5159///
5160/// A struct rather than four more parameters: `wave` also needs the run's
5161/// state (to record who is answering right now) and the attempt number, and
5162/// eight positional arguments is both unreadable and a clippy error.
5163struct WaveCtx<'a> {
5164 /// Exported as `MAGI_RUN`, so a task an agent files names the run that
5165 /// paid for it.
5166 run: &'a str,
5167 /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
5168 node: &'a str,
5169 prompts: &'a Prompts,
5170 /// The shared `CARGO_TARGET_DIR`, when the config declares one.
5171 cache: Option<&'a Path>,
5172 /// The review round this wave belongs to, for `"review"`/`"fix"` — see
5173 /// `JobRecord::round`. `None` for every other node.
5174 round: Option<usize>,
5175}
5176
5177/// Run one job, honouring the parallelism budget.
5178async fn run_one(
5179 job: SeatJob,
5180 sem: Arc<Semaphore>,
5181 ctx: &WaveCtx<'_>,
5182 state: &mut RunState,
5183 attempt: usize,
5184) -> (SeatState, AgentOutcome) {
5185 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
5186 .await
5187 .pop()
5188 .expect("one job in, one result out");
5189 (seat, out)
5190}
5191
5192/// Run every job concurrently, capped by the semaphore, preserving order.
5193///
5194/// Every seat in the batch is recorded into [`RunState::active`] before the
5195/// wave starts and cleared as each answer lands, so the run's own record says
5196/// who is still being waited on rather than only who finished.
5197async fn wave(
5198 jobs: Vec<SeatJob>,
5199 sem: Arc<Semaphore>,
5200 ctx: &WaveCtx<'_>,
5201 state: &mut RunState,
5202 attempt: usize,
5203) -> Vec<(usize, SeatState, AgentOutcome)> {
5204 let WaveCtx {
5205 run,
5206 node,
5207 prompts,
5208 cache,
5209 round,
5210 } = *ctx;
5211 for job in &jobs {
5212 state.seat_started(node, &job.seat.key, job.timeout, attempt);
5213 }
5214 if let Err(e) = state.save() {
5215 // A failed persist of "who is answering right now" must not abort the
5216 // wave: the seats are already being asked, and the alternative is
5217 // losing the answers to save a status line nobody may even be
5218 // watching.
5219 tracing::warn!("could not persist in-progress seats: {e:#}");
5220 }
5221 // Hold the shared build cache's lease for the whole batch, not per job:
5222 // several candidates (an implement wave) or a fixer legitimately share
5223 // one cache concurrently within this run, and that stays untouched — a
5224 // single lease taken once for the whole wave and released once it is
5225 // done is what stops a *different* borrower (another run's own wave, its
5226 // e2e/gate, a human's `magi review`) from interleaving a build into the
5227 // same directory while this one is in flight. Best-effort, not
5228 // all-or-nothing: a wave that cannot get the lease within its own
5229 // longest job's budget still runs — an hour of paid implementer calls is
5230 // not thrown away over cache contention — but every write-allowed seat
5231 // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
5232 // below), the same fallback a read-only seat always gets, rather than
5233 // building into a directory this run was never granted. The identity
5234 // record is still invalidated below either way, so the next tracked
5235 // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
5236 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
5237 let wait_started = Instant::now();
5238 let cache_guard = if let Some(cache_dir) = cache {
5239 if jobs_had_a_writer {
5240 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
5241 let budget = jobs
5242 .iter()
5243 .map(|j| j.timeout)
5244 .max()
5245 .unwrap_or(Duration::from_secs(60));
5246 acquire_cache_lease(state, cache_dir, &owner, budget, node)
5247 .await
5248 .ok()
5249 } else {
5250 None
5251 }
5252 } else {
5253 None
5254 };
5255 // Carved out of each job's own budget, not added on top of it: a seat
5256 // that waited behind the lease must not also get its full timeout
5257 // afterward, or a run contended on the cache could double the time it
5258 // spends per wave. `saturating_sub` floors at zero rather than
5259 // wrapping - a job whose whole budget was spent waiting starts with
5260 // none left, which is the honest number, not a free minimum.
5261 let waited_for_lease = wait_started.elapsed();
5262 let mut set = tokio::task::JoinSet::new();
5263 let overlay = prompts.overlay(node);
5264 for (i, mut job) in jobs.into_iter().enumerate() {
5265 job.timeout = job.timeout.saturating_sub(waited_for_lease);
5266 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
5267 if cache.is_some() {
5268 job.prompt.push('\n');
5269 job.prompt
5270 .push_str(&prompt::build_cache_note(node, job.allow_write));
5271 }
5272 let sem = Arc::clone(&sem);
5273 let run = run.to_owned();
5274 let node = node.to_owned();
5275 // A read-only seat is never handed `CARGO_TARGET_DIR` — see
5276 // `prompt::build_cache_note`'s doc for why setting it anyway is
5277 // exactly how a sandboxed reviewer's write refusal got reported as a
5278 // defect in the patch, not a property of its own seat. And a
5279 // write-allowed one is handed it only when the lease above was
5280 // actually acquired: a wave that could not get it (`cache_guard` is
5281 // `None`, see its own comment) must not send seats to build into a
5282 // directory this run does not hold - that is the exact concurrent,
5283 // unmanaged-write race this module exists to prevent, not something
5284 // "proceeding anyway" is allowed to reintroduce.
5285 let cache = cache
5286 .filter(|_| job.allow_write && cache_guard.is_some())
5287 .map(Path::to_path_buf);
5288 set.spawn(async move {
5289 let _permit = sem.acquire().await;
5290 let mut seat = job.seat;
5291 let out = agent::invoke(
5292 &job.spec,
5293 &mut seat,
5294 &Invocation {
5295 cwd: &job.cwd,
5296 prompt: &job.prompt,
5297 timeout: job.timeout,
5298 allow_write: job.allow_write,
5299 sessions: job.sessions,
5300 artifacts: &job.artifacts,
5301 stem: &job.stem,
5302 run: &run,
5303 node: &node,
5304 cache_dir: cache.as_deref(),
5305 attachments: &[],
5306 },
5307 )
5308 .await;
5309 let out = match out {
5310 Ok(o) if o.usable() => AgentOutcome::Ok(o),
5311 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
5312 // Billed work the CLI failed to hand over is not an ordinary
5313 // failure, but its text is the CLI's raw error JSON, not an
5314 // answer — `Dropped` keeps it out of `Ok` so a caller cannot
5315 // read it as one by forgetting to check. `usable()` is always
5316 // false here (dropped implies an empty response), so this has
5317 // to be checked before the catch-all `Failed` below or the
5318 // one shape this exists for is lost with the rest.
5319 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
5320 Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
5321 Ok(o) => AgentOutcome::Failed(format!(
5322 "exited with {:?} and no usable output",
5323 o.exit_code
5324 )),
5325 Err(e) => AgentOutcome::Failed(e.to_string()),
5326 };
5327 (i, seat, out)
5328 });
5329 }
5330 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
5331 while let Some(joined) = set.join_next().await {
5332 let (i, seat, out) = match joined {
5333 Ok(v) => v,
5334 // No seat to clear: a panicked task never reported which one it
5335 // was. The defensive sweep below this loop is what stops that
5336 // seat's `active` entry from surviving forever.
5337 Err(e) => {
5338 tracing::error!("agent task panicked: {e}");
5339 continue;
5340 }
5341 };
5342 state.seat_finished(&seat.key);
5343 record_jobs(state, node, round, &seat.key, &out);
5344 if let Err(e) = state.save() {
5345 tracing::warn!("could not persist a seat's completion: {e:#}");
5346 }
5347 if collected.len() <= i {
5348 collected.resize_with(i + 1, || None);
5349 }
5350 collected[i] = Some((i, seat, out));
5351 }
5352 // Belt-and-braces for the panic branch above: every seat this exact batch
5353 // started shares this `(node, attempt)` pair, and every seat that finished
5354 // normally already cleared itself, so anything left tagged with it here
5355 // can only be a panicked task's leftover. Cleared unconditionally rather
5356 // than left to read as still answering forever.
5357 if state
5358 .active
5359 .values()
5360 .any(|a| a.node == node && a.attempt == attempt)
5361 {
5362 state
5363 .active
5364 .retain(|_, a| !(a.node == node && a.attempt == attempt));
5365 if let Err(e) = state.save() {
5366 tracing::warn!("could not persist the end of a wave: {e:#}");
5367 }
5368 }
5369 // Whether or not the lease above was actually held, several worktrees
5370 // may just have built into the cache with nothing here able to name one
5371 // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
5372 // own doc. Forgetting the old record costs the next `e2e`/`gate` one
5373 // clean it might not have strictly needed; trusting a stale match would
5374 // cost it a wrong answer.
5375 if let Some(cache_dir) = cache
5376 && jobs_had_a_writer
5377 {
5378 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
5379 }
5380 if let Some(guard) = cache_guard {
5381 guard.release();
5382 }
5383 collected.into_iter().flatten().collect()
5384}
5385
5386/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
5387/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
5388/// this is data collection, not the fix-specific completion contract in
5389/// [`Runner::continue_fix_report`], and applies regardless of which node
5390/// asked.
5391///
5392/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
5393/// evidence from; `Failed` does not, and correctly contributes nothing — a
5394/// timeout or crash is not itself evidence about a command the seat may have
5395/// started.
5396fn record_jobs(
5397 state: &mut RunState,
5398 node: &str,
5399 round: Option<usize>,
5400 seat: &str,
5401 out: &AgentOutcome,
5402) {
5403 let commands: &[agent::CommandEvidence] = match out {
5404 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
5405 AgentOutcome::Failed(_) => &[],
5406 };
5407 let checked_at = Timestamp::now();
5408 for c in commands {
5409 state.jobs.push(JobRecord {
5410 node: node.to_owned(),
5411 round,
5412 seat: seat.to_owned(),
5413 id: c.id.clone(),
5414 description: c.description.clone(),
5415 checked_at,
5416 status: match c.exit_code {
5417 Some(0) => JobStatus::Completed,
5418 Some(_) => JobStatus::Failed,
5419 None => JobStatus::Unknown,
5420 },
5421 exit_code: c.exit_code,
5422 result_summary: c.result_summary.clone(),
5423 source: c.source.clone(),
5424 });
5425 }
5426}
5427
5428/// Is a review round clean, given how many reviewer seats answered against
5429/// how many the round expected?
5430///
5431/// A seat that never answered (timeout, crash, unparsable output) is not a
5432/// seat that read the patch and found nothing — treating it as such is
5433/// exactly the bug this function exists to close. Under the default `block`
5434/// policy a missing seat can never be clean; `warn` still requires the seats
5435/// that *did* answer to have found nothing blocking and verification to be
5436/// green.
5437///
5438/// `quota_missing` narrows that `block` default for exactly one cause of
5439/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
5440/// a session limit lifts by the very next round buys nothing — the seat is
5441/// asked again with the same quota — so once every missing seat is accounted
5442/// for by a quota loss (and at least one seat *did* answer, so a decision has
5443/// something to rest on) the round is decided on the panel that could answer,
5444/// same as `warn` would. A panel that lost every seat to quota is not
5445/// decided here: `answered == 0` falls through to the existing `block`
5446/// fallback so a fully collapsed panel still waits rather than landing on no
5447/// review at all.
5448fn round_is_clean(
5449 blocking: usize,
5450 e2e_ok: bool,
5451 answered: usize,
5452 expected: usize,
5453 quota_missing: usize,
5454 policy: IncompleteReviewPolicy,
5455) -> bool {
5456 if blocking != 0 || !e2e_ok {
5457 return false;
5458 }
5459 if answered == expected || policy == IncompleteReviewPolicy::Warn {
5460 return true;
5461 }
5462 answered > 0 && expected - answered <= quota_missing
5463}
5464
5465/// The review loop's own conclusion, derived entirely from its persisted
5466/// round records and the round budget that produced them — never from
5467/// `status`, so a reentry (or `gate`/`merge` reading it independently)
5468/// recomputes the identical answer regardless of what an earlier node in the
5469/// same walk, or a previous walk, did to `status`.
5470///
5471/// `None` while more rounds remain to try, including when review never ran
5472/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
5473/// gone clean, or the budget is spent, or the tree has stopped moving (see
5474/// [`STAGNANT_LIMIT`]), the answer is one of two things:
5475///
5476/// - An incomplete panel that raised nothing is missing input, not a
5477/// verified tree — never a hand-off candidate, whatever verification said
5478/// (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
5479/// - Otherwise, green e2e on the last round hands off (see
5480/// [`Runner::stop_reviewing`]); red e2e blocks.
5481///
5482/// A last round whose own verification is still `ResourceBlocked` — magi
5483/// itself never got a command to run, not evidence the patch is broken —
5484/// is neither: this returns `None` for it too, the same as "more rounds
5485/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
5486/// handling of that shape) instead of this cheap recomputation guessing a
5487/// verdict a real attempt never produced.
5488fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
5489 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
5490 return Some(RunStatus::Gating);
5491 }
5492 let last = reviews.last()?;
5493 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
5494 if reviews.len() < max_rounds && !stagnant {
5495 return None;
5496 }
5497 if last.incomplete() && last.blocking == 0 {
5498 return Some(RunStatus::Blocked);
5499 }
5500 if last.e2e_status() == E2eStatus::ResourceBlocked {
5501 return None;
5502 }
5503 Some(if last.e2e.iter().all(CommandOutcome::ok) {
5504 RunStatus::Gating
5505 } else {
5506 RunStatus::Blocked
5507 })
5508}
5509
5510/// How long a re-ask may take, given the budget the first attempt had.
5511///
5512/// A `nudged` retry is a request to restate an answer the seat has already
5513/// worked out: it carries no new work, so it does not deserve the original
5514/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
5515/// 133 seconds while a third sat for over ten minutes on a resumed session
5516/// holding 410 KB of prior output - and because the retry had inherited the
5517/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
5518/// judging round whose other seats were long finished.
5519///
5520/// A quarter of the budget, with a floor so that a deliberately short timeout
5521/// does not collapse to nothing. A retry that re-sends the whole prompt
5522/// (because the seat kept no context) is the original job again, and keeps the
5523/// original budget.
5524fn retry_budget(full: Duration, nudged: bool) -> Duration {
5525 if nudged {
5526 (full / 4).max(Duration::from_secs(120)).min(full)
5527 } else {
5528 full
5529 }
5530}
5531
5532/// Run a wave and parse each reply, re-asking the seats whose reply was
5533/// unusable.
5534///
5535/// The re-ask is a nudge rather than the whole prompt again when the seat still
5536/// holds its conversation, which is the difference between a cheap retry and
5537/// paying for the entire candidate set twice.
5538///
5539/// A seat that hits a rate limit is **not** re-asked: the same call will fail
5540/// the same way until the limit resets, so spending a retry attempt on it is
5541/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
5542/// like any other absent seat — the caller decides whether the panel still has
5543/// a quorum.
5544#[allow(clippy::too_many_arguments)]
5545async fn ask_json_wave<T>(
5546 jobs: Vec<SeatJob>,
5547 sem: Arc<Semaphore>,
5548 retries: usize,
5549 ctx: &WaveCtx<'_>,
5550 losses: &mut Vec<QuotaLoss>,
5551 state: &mut RunState,
5552 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
5553) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
5554where
5555 T: serde::de::DeserializeOwned + Send + 'static,
5556{
5557 let n = jobs.len();
5558 let originals: Vec<SeatJob> = jobs;
5559 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
5560 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
5561 // Which attempt each seat's `done[i]` reflects — 0 for a first-ask
5562 // answer, N once it has gone through N nudges. Read back once this
5563 // returns, so a caller building a history record (`ReviewRecord`) can
5564 // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
5565 // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
5566 // field's own doc.
5567 let mut attempts_used: Vec<usize> = vec![0; n];
5568 let mut pending: Vec<usize> = (0..n).collect();
5569
5570 for attempt in 0..=retries {
5571 if pending.is_empty() {
5572 break;
5573 }
5574 let mut batch = Vec::with_capacity(pending.len());
5575 for &i in &pending {
5576 let src = &originals[i];
5577 // The prompt and the budget are one decision: a nudge restates
5578 // finished work, a re-sent prompt redoes it.
5579 let (prompt, timeout) = if attempt == 0 {
5580 (src.prompt.clone(), src.timeout)
5581 } else {
5582 let why = done[i]
5583 .as_ref()
5584 .and_then(|r| r.as_ref().err().map(ToString::to_string))
5585 .unwrap_or_else(|| "no parsable answer".to_owned());
5586 let nudge = prompt::nudge(&why);
5587 let nudged = has_context(&src.spec, &seats[i], src.sessions);
5588 let prompt = if nudged {
5589 nudge
5590 } else {
5591 format!("{}\n\n---\n\n{}", src.prompt, nudge)
5592 };
5593 (prompt, retry_budget(src.timeout, nudged))
5594 };
5595 batch.push(SeatJob {
5596 spec: src.spec.clone(),
5597 seat: seats[i].clone(),
5598 cwd: src.cwd.clone(),
5599 prompt,
5600 timeout,
5601 allow_write: src.allow_write,
5602 sessions: src.sessions,
5603 artifacts: src.artifacts.clone(),
5604 stem: if attempt == 0 {
5605 src.stem.clone()
5606 } else {
5607 format!("{}-retry{attempt}", src.stem)
5608 },
5609 });
5610 }
5611
5612 if attempt > 0 {
5613 let seats_out: Vec<&str> = pending
5614 .iter()
5615 .map(|&i| originals[i].seat.key.as_str())
5616 .collect();
5617 state.event(
5618 ctx.node,
5619 format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
5620 );
5621 }
5622 let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
5623 let mut still = Vec::new();
5624 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
5625 seats[i] = seat;
5626 let (parsed, quota) = match out {
5627 AgentOutcome::Ok(o) => (
5628 match verdict::extract_json::<T>(&o.text) {
5629 Ok(v) => match validate(&v) {
5630 Ok(()) => Ok((v, o)),
5631 Err(e) => Err(e),
5632 },
5633 Err(e) => Err(e),
5634 },
5635 false,
5636 ),
5637 AgentOutcome::Quota(o) => {
5638 losses.push(QuotaLoss {
5639 seat: originals[i].seat.key.clone(),
5640 node: ctx.node.to_owned(),
5641 at: Timestamp::now(),
5642 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5643 });
5644 (
5645 Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
5646 true,
5647 )
5648 }
5649 // Not a parseable answer, but also not worth a special-cased
5650 // retry here: the nudge loop above already re-asks anything
5651 // that fails to parse, which is exactly what a dropped stream
5652 // needs. Just don't hand its raw error JSON to `extract_json`.
5653 AgentOutcome::Dropped(o) => {
5654 let why = o
5655 .dropped
5656 .as_ref()
5657 .map(|d| d.why.as_str())
5658 .unwrap_or("the CLI ended the stream without delivering its answer");
5659 (
5660 Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
5661 false,
5662 )
5663 }
5664 AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
5665 };
5666 let failed = parsed.is_err();
5667 done[i] = Some(parsed);
5668 attempts_used[i] = attempt;
5669 // Do not re-ask a rate-limited seat (quota) — a retry is known to
5670 // fail the same way; and never re-ask a seat that already parsed.
5671 if failed && !quota {
5672 still.push(i);
5673 }
5674 }
5675 pending = still;
5676 }
5677
5678 seats
5679 .into_iter()
5680 .zip(done)
5681 .zip(attempts_used)
5682 .map(|((seat, res), attempts)| {
5683 (
5684 seat,
5685 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
5686 attempts,
5687 )
5688 })
5689 .collect()
5690}
5691
5692/// Acquire the shared build cache's lease, waiting out contention within
5693/// `budget` (never past it — see AGENTS.md's build-cache section on why an
5694/// unbounded wait is never acceptable).
5695///
5696/// A first, non-blocking check happens before ever waiting; if it finds the
5697/// lease busy, that fact is logged as a `verify` event *and* flushed with
5698/// [`RunState::save`] immediately — not only once the wait finally succeeds
5699/// or gives up — so a `magi show` run by a different process while this one
5700/// is still waiting reads a `run.json` that says so, rather than whatever it
5701/// looked like before the wait started. The same applies to the terminal
5702/// failure: logged and saved before this returns `Err`, so a caller that
5703/// could not get the lease at all still leaves a legible record of why.
5704async fn acquire_cache_lease(
5705 state: &mut RunState,
5706 cache_dir: &Path,
5707 owner: &crate::cache::Owner,
5708 budget: Duration,
5709 context: &str,
5710) -> Result<crate::cache::Guard> {
5711 let home = crate::run::home();
5712 let started = Instant::now();
5713 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
5714 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
5715 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
5716 Err(e) => {
5717 state.event(
5718 "verify",
5719 format!("{context}: could not check the shared build cache: {e:#}"),
5720 );
5721 if let Err(e2) = state.save() {
5722 tracing::warn!("could not persist a cache-check failure: {e2:#}");
5723 }
5724 return Err(e);
5725 }
5726 };
5727 state.event(
5728 "verify",
5729 format!(
5730 "{context}: waiting for the shared build cache at {} ({})",
5731 cache_dir.display(),
5732 busy.describe()
5733 ),
5734 );
5735 if let Err(e) = state.save() {
5736 tracing::warn!("could not persist a cache wait: {e:#}");
5737 }
5738 let remaining = budget.saturating_sub(started.elapsed());
5739 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
5740 Ok(g) => Ok(g),
5741 Err(e) => {
5742 state.event("verify", format!("{context}: {e:#}"));
5743 if let Err(e2) = state.save() {
5744 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
5745 }
5746 Err(e)
5747 }
5748 }
5749}
5750
5751/// Run `body` — a verify command batch — while holding the shared build
5752/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
5753/// never interleave with another borrower's build against the same
5754/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
5755/// timeout, or a human's own `magi review`. See the `cache` module doc for
5756/// why this matters more than Cargo's own per-target locking covers — two
5757/// *different* worktrees building the same package name/version into one
5758/// cache directory is a staleness bug, not a lock contention one.
5759///
5760/// The wait for the lease is carved out of `budget`, never on top of it —
5761/// `body` is handed whatever is left, so a caller's own node timeout is the
5762/// only clock involved, exactly what AGENTS.md's build-cache section asks
5763/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
5764/// cache configured at all — this is a pass-through: `body` runs with the
5765/// full budget and nothing is leased.
5766///
5767/// A lease that cannot be acquired within `budget` is reported as a single
5768/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
5769/// verification — the same shape a spawn failure already takes in
5770/// [`run_commands`], so a caller need not special-case it.
5771#[allow(clippy::too_many_arguments)]
5772async fn with_cache_lease<'s, F, Fut>(
5773 state: &'s mut RunState,
5774 cache_dir: Option<&Path>,
5775 node: &str,
5776 seat: &str,
5777 worktree: &Path,
5778 head: &str,
5779 budget: Duration,
5780 context: &str,
5781 body: F,
5782) -> (Vec<CommandOutcome>, bool)
5783where
5784 F: FnOnce(&'s mut RunState, Duration) -> Fut,
5785 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
5786{
5787 let Some(cache_dir) = cache_dir else {
5788 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
5789 return (outcomes, retried);
5790 };
5791 let home = crate::run::home();
5792 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
5793 let started = Instant::now();
5794 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
5795 Ok(g) => g,
5796 Err(e) => {
5797 return (
5798 vec![CommandOutcome {
5799 command: "(waiting for the shared build cache)".to_owned(),
5800 code: None,
5801 output_tail: e.to_string(),
5802 duration_ms: started.elapsed().as_millis() as u64,
5803 resource_blocked: true,
5804 }],
5805 false,
5806 );
5807 }
5808 };
5809 let identity = crate::cache::Identity::new(worktree, head);
5810 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
5811 // A failed freshness check means this process cannot vouch for what
5812 // is sitting in the cache right now - on Windows this is exactly the
5813 // "a stale test executable is still locked, `cargo clean -p` cannot
5814 // remove it" case the evidence log records. Running verify anyway
5815 // and reporting whatever it says would let a result nobody can trust
5816 // stand for the tree it claims to have checked; fail the step
5817 // instead of the patch.
5818 state.event(
5819 "verify",
5820 format!(
5821 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
5822 worktree.display(),
5823 short(head)
5824 ),
5825 );
5826 guard.release();
5827 return (
5828 vec![CommandOutcome {
5829 command: "(confirming the shared build cache is fresh)".to_owned(),
5830 code: None,
5831 output_tail: e.to_string(),
5832 duration_ms: started.elapsed().as_millis() as u64,
5833 resource_blocked: true,
5834 }],
5835 false,
5836 );
5837 }
5838 let remaining = budget.saturating_sub(started.elapsed());
5839 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
5840 // A timed-out command's process was only *asked* to die (`kill_on_drop`,
5841 // `start_kill`); confirm it actually has before handing the directory to
5842 // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
5843 // for what this can and cannot see.
5844 if !timed_out_pids.is_empty() {
5845 wait_for_timed_out_children_to_die(&timed_out_pids).await;
5846 }
5847 guard.release();
5848 (outcomes, retried)
5849}
5850
5851/// Poll `pids` — commands [`run_commands`] reports as still running when its
5852/// own timeout elapsed — until every one is confirmed gone, or
5853/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
5854///
5855/// Real confirmation where confirmation is possible, not a substitute for
5856/// full process-tree observation: a grandchild the timed-out process spawned
5857/// and that survives independently of it is invisible to a pid check the
5858/// same way it always was, and continuing to observe and collect *that*
5859/// stays a different piece of work with its own owner. This only narrows a
5860/// fixed blind wait into an actual check of the pids this process does know
5861/// about.
5862async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
5863 wait_for_pids_with(
5864 pids,
5865 crate::proc::pid_alive,
5866 LEASE_RELEASE_POLL,
5867 LEASE_RELEASE_MAX_WAIT,
5868 )
5869 .await;
5870}
5871
5872/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
5873/// interval and ceiling supplied by the caller, so the polling *logic* -
5874/// returns as soon as every pid reports dead, gives up at the ceiling
5875/// otherwise - is testable on millisecond durations without asking the real
5876/// OS about a pid at all.
5877async fn wait_for_pids_with<F: Fn(u32) -> bool>(
5878 pids: &[u32],
5879 alive: F,
5880 poll: Duration,
5881 max_wait: Duration,
5882) {
5883 let deadline = Instant::now() + max_wait;
5884 loop {
5885 if pids.iter().all(|&pid| !alive(pid)) {
5886 return;
5887 }
5888 if Instant::now() >= deadline {
5889 return;
5890 }
5891 tokio::time::sleep(poll).await;
5892 }
5893}
5894
5895/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
5896/// admission that it could not even get a verify command to run, as opposed
5897/// to evidence the command actually produced? A caller that would otherwise
5898/// read a resource-blocked outcome as a red command must check this first:
5899/// see [`Runner::gate`], which retries rather than records `Blocked` when
5900/// this is true.
5901fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
5902 outcomes.iter().any(|o| o.resource_blocked)
5903}
5904
5905/// Describe one verify command's outcome for the event log, distinguishing a
5906/// build/link failure — the toolchain never produced a binary to run — from
5907/// an actual test failure, since only the latter is a verdict on the patch.
5908fn e2e_outcome_label(o: &CommandOutcome) -> String {
5909 if o.ok() {
5910 return "pass".to_owned();
5911 }
5912 let reason = if o.build_failed() {
5913 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
5914 } else {
5915 format!("FAIL ({:?})", o.code)
5916 };
5917 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
5918}
5919
5920/// Run `verify.e2e`, retrying once if the first attempt could not build or
5921/// link — a build/link failure is frequently a race against a shared
5922/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
5923/// `verify` event per command, tagged with `context` (normally `"round N"`)
5924/// so the two call sites that need this — the ordinary per-round leg in
5925/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
5926/// it will ever call a round green — read identically in the event log.
5927async fn run_e2e_with_retry(
5928 state: &mut RunState,
5929 shell: &[String],
5930 commands: &[String],
5931 worktree: &Path,
5932 timeout: Duration,
5933 context: &str,
5934) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
5935 let (mut e2e, mut timed_out_pids) = run_commands(
5936 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
5937 )
5938 .await;
5939 for o in &e2e {
5940 state.event(
5941 "verify",
5942 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
5943 );
5944 }
5945 // A build/link failure is not a verdict on the patch — it is frequently a
5946 // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
5947 // one retry before letting a red like that decide the round.
5948 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
5949 if verify_retried {
5950 state.event(
5951 "verify",
5952 format!(
5953 "{context}: verify could not build/link, not a test result — retrying once \
5954 before concluding"
5955 ),
5956 );
5957 let retried = run_commands(
5958 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
5959 )
5960 .await;
5961 e2e = retried.0;
5962 // Both attempts' timeouts matter, not just the last one: the first
5963 // attempt's descendants may still be alive alongside the retry's.
5964 timed_out_pids.extend(retried.1);
5965 for o in &e2e {
5966 state.event(
5967 "verify",
5968 format!(
5969 "{context}: retry `{}` -> {}",
5970 o.command,
5971 e2e_outcome_label(o)
5972 ),
5973 );
5974 }
5975 }
5976 (e2e, verify_retried, timed_out_pids)
5977}
5978
5979/// Run configured shell commands in `cwd`, in order. The second element is
5980/// the pid of every command that hit `timeout` and was still running when
5981/// this stopped waiting on it (best-effort: `None` when the platform did not
5982/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
5983/// that releases a shared resource afterward needs to know.
5984///
5985/// Records `task` into [`RunState::active`] at every command boundary
5986/// (`RunState::task_command`) and clears it once the whole list has run
5987/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
5988/// for minutes with no seat and no output of its own to show for it (see
5989/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
5990/// apart from "not yet run" without this), and this is the only place that
5991/// knows which command is running right now and how many are left. Three
5992/// saves per command — start, not per second — matching the same "only at a
5993/// boundary" rule [`wave`] already follows for seats.
5994#[allow(clippy::too_many_arguments)]
5995async fn run_commands(
5996 state: &mut RunState,
5997 node: &str,
5998 task: &str,
5999 attempt: usize,
6000 shell: &[String],
6001 commands: &[String],
6002 cwd: &Path,
6003 timeout: Duration,
6004) -> (Vec<CommandOutcome>, Vec<u32>) {
6005 if commands.is_empty() {
6006 // Nothing to mark as running and nothing to clear — an empty list
6007 // means "not configured", and touching `active` (or the disk) over
6008 // that would be a write for every round of a repo with no
6009 // `verify.e2e` / `verify.gate` commands at all.
6010 return (Vec::new(), Vec::new());
6011 }
6012 let mut out = Vec::new();
6013 let mut timed_out_pids = Vec::new();
6014 let total = commands.len();
6015 for (idx, command) in commands.iter().enumerate() {
6016 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
6017 if let Err(e) = state.save() {
6018 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
6019 }
6020 let started = Instant::now();
6021 let mut cmd = tokio::process::Command::new(&shell[0]);
6022 cmd.quiet();
6023 cmd.args(&shell[1..])
6024 .arg(command)
6025 .current_dir(cwd)
6026 .stdin(std::process::Stdio::null())
6027 .stdout(std::process::Stdio::piped())
6028 .stderr(std::process::Stdio::piped())
6029 .kill_on_drop(true);
6030 let spawned = cmd.spawn();
6031 let (code, body) = match spawned {
6032 Ok(child) => {
6033 // Captured before the child is consumed below: `kill_on_drop`
6034 // only *asks* the process to die when the timeout branch
6035 // drops it, and the pid is the only way anyone downstream can
6036 // later check whether that request actually took.
6037 let pid = child.id();
6038 match tokio::time::timeout(timeout, child.wait_with_output()).await {
6039 Ok(Ok(o)) => {
6040 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
6041 body.push_str(&String::from_utf8_lossy(&o.stderr));
6042 (o.status.code(), body)
6043 }
6044 Ok(Err(e)) => (None, format!("failed to run: {e}")),
6045 Err(_) => {
6046 if let Some(pid) = pid {
6047 timed_out_pids.push(pid);
6048 }
6049 (None, format!("timed out after {}s", timeout.as_secs()))
6050 }
6051 }
6052 }
6053 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
6054 };
6055 out.push(CommandOutcome {
6056 command: command.clone(),
6057 code,
6058 output_tail: tail(&body, OUTPUT_TAIL),
6059 duration_ms: started.elapsed().as_millis() as u64,
6060 resource_blocked: false,
6061 });
6062 }
6063 state.task_finished(task);
6064 if let Err(e) = state.save() {
6065 tracing::warn!("could not persist the end of {task}: {e:#}");
6066 }
6067 (out, timed_out_pids)
6068}
6069
6070/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
6071/// section (`report::run`) and in the `merge` event this node records — for
6072/// the operator to run by hand.
6073///
6074/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
6075/// branch whose ruleset forbids merge commits (GitHub's "must not contain
6076/// merge commits", or "require linear history") rejects the push a `--no-ff`
6077/// merge would produce, which is exactly the guidance this function replaces.
6078/// `message`'s first line becomes the squash commit's subject, matching the
6079/// note `report::run` prints alongside this command — see that function for
6080/// why an explicit subject is not optional there.
6081fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
6082 let repo = repo.display();
6083 match style {
6084 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
6085 MergeStyle::Squash => {
6086 // The subject sits inside double quotes, and a title an agent
6087 // wrote may carry the characters that break out of them.
6088 let subject = message
6089 .lines()
6090 .next()
6091 .unwrap_or(branch)
6092 .replace(['\\', '"', '$', '`'], "");
6093 format!(
6094 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
6095 )
6096 }
6097 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
6098 }
6099}
6100
6101/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
6102/// under the hood, rejects a `title` over 256 characters and the whole
6103/// command fails — no PR at all, for a run whose body was otherwise fine
6104/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
6105/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
6106/// always how GitHub counts, plus one character for the trailing ellipsis
6107/// `title_from` may add. It is a margin, not a guarantee — a title packed
6108/// with multi-unit characters could still in principle land close to the
6109/// edge, but a real task title's occasional emoji or accented letter fits
6110/// comfortably inside it.
6111const PR_TITLE_MAX: usize = 240;
6112
6113/// What `merge = "pr"` (and the merge commit of the other modes) says about a
6114/// change: a title and a body describing what was *implemented*, not the task
6115/// that asked for it. A task reads as a request; a reader of the merged
6116/// history wants the change.
6117struct PrMessage {
6118 title: String,
6119 body: String,
6120}
6121
6122impl PrMessage {
6123 /// Title, blank line, body. The first line is the squash/merge commit
6124 /// subject (`manual_merge_command` takes it via `lines().next()`), so it
6125 /// has to stay one sensible line.
6126 fn commit_message(&self) -> String {
6127 format!("{}\n\n{}", self.title, self.body)
6128 }
6129}
6130
6131/// The text after a leading `TITLE:` (any case) on `line`.
6132fn title_marker(line: &str) -> Option<&str> {
6133 let line = line.trim();
6134 let head = line.get(..6)?;
6135 head.eq_ignore_ascii_case("title:")
6136 .then(|| line[6..].trim())
6137}
6138
6139/// The implementer's own one-line title: the `TITLE:` line the implement
6140/// prompt asks for at the top of its SUMMARY. Candidate commits are all
6141/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
6142/// source, and a title that says as much is refused here too.
6143fn summary_title(summary: &str) -> Option<String> {
6144 let first = summary.lines().find(|l| !l.trim().is_empty())?;
6145 let raw = title_marker(first)?;
6146 if raw.is_empty() {
6147 return None;
6148 }
6149 let title = queue::title_from(raw, PR_TITLE_MAX);
6150 let lower = title.to_ascii_lowercase();
6151 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
6152 return None;
6153 }
6154 Some(title)
6155}
6156
6157/// `summary` without its `TITLE:` line, which the pull request title already
6158/// carries.
6159fn summary_without_title(summary: &str) -> String {
6160 let mut lines = summary.trim().lines().peekable();
6161 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
6162 lines.next();
6163 }
6164 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
6165}
6166
6167/// The pull request title and body for the winning candidate.
6168///
6169/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
6170/// the task's own opening line via [`queue::title_from`] when there is none.
6171/// `state.instruction` can open with blank lines (`task_text` only rejects a
6172/// body that is blank *entirely*), which `title_from` skips.
6173///
6174/// Body: the implementer's summary and the fixer's notes, then — when the
6175/// winning review round was not clean — the findings still open and whatever
6176/// the fixer declined, so `merge = "pr"` hands the reader the same material
6177/// `magi show` does. The task follows inside a collapsed block, and the
6178/// footer repeats the run and candidate as plain tags for a reader holding
6179/// only the merged commit or the PR body.
6180fn pr_message(state: &RunState, winner: char) -> PrMessage {
6181 let summary = state
6182 .candidates
6183 .iter()
6184 .find(|c| c.label == winner)
6185 .map(|c| c.summary.as_str())
6186 .unwrap_or_default();
6187 // The fallback is the operator's own words, not something generated, so it
6188 // may be in the task's language; only the summary path is prompted English.
6189 let title = summary_title(summary)
6190 .unwrap_or_else(|| queue::title_from(&state.instruction, PR_TITLE_MAX));
6191
6192 let mut body = String::new();
6193 let what = summary_without_title(summary);
6194 if !what.is_empty() {
6195 body.push_str("## Summary\n\n");
6196 body.push_str(&what);
6197 body.push_str("\n\n");
6198 }
6199
6200 let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
6201 if let Some(fix) = fix
6202 && !fix.notes.trim().is_empty()
6203 {
6204 body.push_str("## Review fixes\n\n");
6205 body.push_str(fix.notes.trim());
6206 body.push_str("\n\n");
6207 }
6208
6209 let open = state.open_findings();
6210 if !open.is_empty() {
6211 body.push_str("## Open review findings\n\n");
6212 for f in &open {
6213 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
6214 }
6215 body.push('\n');
6216 }
6217
6218 if let Some(fix) = fix
6219 && !fix.rejected.is_empty()
6220 {
6221 body.push_str("## Declined by the fixer\n\n");
6222 for r in &fix.rejected {
6223 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
6224 }
6225 body.push('\n');
6226 }
6227
6228 let task = state.instruction.trim();
6229 let task = if task.is_empty() {
6230 "(empty task)"
6231 } else {
6232 task
6233 };
6234 body.push_str(&format!(
6235 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
6236 task.replace("</details>", "</details>")
6237 ));
6238
6239 body.push_str(&format!(
6240 "\n---\nmagi:run/{} magi:candidate-{}\n",
6241 state.id,
6242 winner.to_ascii_lowercase()
6243 ));
6244
6245 PrMessage { title, body }
6246}
6247
6248/// `gh pr create`, returning the PR url.
6249async fn gh_pr_create(
6250 cwd: &Path,
6251 base: &str,
6252 head: &str,
6253 title: &str,
6254 body: &str,
6255) -> Result<String> {
6256 let out = tokio::process::Command::new("gh")
6257 .args([
6258 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
6259 ])
6260 .current_dir(cwd)
6261 .quiet()
6262 .stdin(std::process::Stdio::null())
6263 .output()
6264 .await
6265 .context("spawn gh")?;
6266 if out.status.success() {
6267 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
6268 } else {
6269 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
6270 }
6271}
6272
6273/// Tear a run's worktrees and branches down.
6274///
6275/// `home` is where the updated `run.json` is saved (via
6276/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
6277/// a housekeeping pass already has its own honest `home` handed to it, and
6278/// falling through to the global here would write back through whichever
6279/// directory some other process or test pinned into that `OnceLock` first,
6280/// not the one the caller actually resolved its `runs` and `state` from.
6281pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
6282 let repo = state.repo.clone();
6283 let root = state.worktree_root();
6284 let winner = state.tally.as_ref().map(|t| t.winner);
6285 let mut removed = Vec::new();
6286
6287 for i in 0..state.candidates.len() {
6288 let c = state.candidates[i].clone();
6289 let is_winner = Some(c.label) == winner;
6290 if is_winner && !drop_winner {
6291 continue;
6292 }
6293 if c.worktree.exists() {
6294 git::worktree_remove(&repo, &c.worktree).await.ok();
6295 removed.push(c.worktree.to_string_lossy().into_owned());
6296 }
6297 if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
6298 git::branch_delete(&repo, &c.branch).await.ok();
6299 removed.push(c.branch.clone());
6300 }
6301 state.candidates[i].folded = true;
6302 }
6303
6304 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
6305 let path = name.path();
6306 let keep = !drop_winner
6307 && winner.is_some_and(|w| {
6308 path.file_name()
6309 .is_some_and(|n| n == format!("cand-{w}").as_str())
6310 });
6311 if keep {
6312 continue;
6313 }
6314 git::worktree_remove(&repo, &path).await.ok();
6315 removed.push(path.to_string_lossy().into_owned());
6316 }
6317
6318 // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
6319 // judge worktrees, so once the loop above has cleared all of them out,
6320 // the parent is a bare directory nobody else was ever going to remove -
6321 // git only ever managed what was inside it. Left alone, one of these
6322 // accumulates per fully-folded run; the operator's own machine had 74.
6323 // `remove_if_empty` re-checks rather than assuming: a run whose winner
6324 // was kept (`!drop_winner`) leaves its directory behind on purpose, and
6325 // so does anything a run never claimed that happens to share the bay.
6326 remove_if_empty(&root);
6327
6328 if state.enabled_worktree_config && drop_winner {
6329 // A release, not a raw disable: some sibling run in this repository
6330 // may still hold its own reference (see `git::acquire_worktree_config`),
6331 // and only the last release actually turns the setting back off.
6332 git::release_worktree_config(&repo).await.ok();
6333 state.enabled_worktree_config = false;
6334 }
6335 state.save_under(home)?;
6336 Ok(removed)
6337}
6338
6339/// Remove `dir` if it exists and has nothing in it.
6340///
6341/// Best-effort and silent by design: a directory that is not empty (a run
6342/// whose winner is still parked there, a stray file some other process left)
6343/// is exactly the case this must refuse, and a directory that is already gone
6344/// is not a failure worth reporting either. `std::fs::remove_dir` itself
6345/// already refuses a non-empty directory, so the emptiness check below is
6346/// belt, not suspenders - it is what keeps this from ever attempting the
6347/// removal in the case that matters, rather than trusting `remove_dir`'s
6348/// error path to have no side effects if it ever changed.
6349fn remove_if_empty(dir: &Path) {
6350 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
6351 std::fs::remove_dir(dir).ok();
6352 }
6353}
6354
6355/// Severity of the worst open finding in the last review round, for reporting.
6356pub fn worst_open(state: &RunState) -> Option<Severity> {
6357 state
6358 .reviews
6359 .last()?
6360 .reviews
6361 .iter()
6362 .flat_map(|r| r.findings.iter())
6363 .map(|f| f.severity)
6364 .max()
6365}
6366
6367#[cfg(test)]
6368mod tests {
6369 use super::*;
6370 use crate::run::GateStatus;
6371 use std::collections::BTreeMap;
6372 use std::time::Duration;
6373
6374 fn conductor() -> AgentSpec {
6375 AgentSpec {
6376 id: "conductor".to_owned(),
6377 kind: crate::config::AgentKind::Command,
6378 model: None,
6379 command: vec!["true".to_owned()],
6380 extra_args: Vec::new(),
6381 env: BTreeMap::new(),
6382 prompt_delivery: None,
6383 }
6384 }
6385
6386 fn spec(id: &str) -> AgentSpec {
6387 AgentSpec {
6388 id: id.to_owned(),
6389 kind: crate::config::AgentKind::Command,
6390 model: None,
6391 command: vec!["true".to_owned()],
6392 extra_args: Vec::new(),
6393 env: BTreeMap::new(),
6394 prompt_delivery: None,
6395 }
6396 }
6397
6398 // `next_untried_implementer` is the property `resume_quota_losses`'s own
6399 // fallback loop depends on to terminate: it must walk forward from the
6400 // seat's own position, never restart at the front of the roster, and it
6401 // must never hand back an id already tried, however many times that id
6402 // happens to appear.
6403
6404 #[test]
6405 fn next_untried_implementer_walks_forward_from_the_seats_own_position() {
6406 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6407 let tried = BTreeSet::from(["beta".to_owned()]);
6408 // beta sits at index 1; the next candidate is gamma, never alpha —
6409 // which is very likely a different candidate slot's own agent.
6410 let next = next_untried_implementer(&roster, 1, &tried);
6411 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
6412 }
6413
6414 #[test]
6415 fn next_untried_implementer_does_not_wrap_back_past_its_own_start() {
6416 let roster = vec![spec("alpha"), spec("beta")];
6417 let tried = BTreeSet::from(["beta".to_owned()]);
6418 // beta is the roster's last entry: nothing follows it, and alpha —
6419 // earlier in the roster, almost certainly a different candidate
6420 // slot's own agent — must not be reached by wrapping back to it.
6421 assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6422 }
6423
6424 #[test]
6425 fn next_untried_implementer_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
6426 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6427 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
6428 // beta (index 1) and gamma (index 2, the only entry after it) have
6429 // both been tried; alpha (index 0) never has, but it comes before
6430 // beta's own position, so there is nothing further for this seat.
6431 assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6432 }
6433
6434 #[test]
6435 fn next_untried_implementer_skips_ids_already_tried_even_when_duplicated() {
6436 let roster = vec![spec("a"), spec("a"), spec("b")];
6437 let tried = BTreeSet::from(["a".to_owned()]);
6438 let next = next_untried_implementer(&roster, 0, &tried);
6439 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
6440 }
6441
6442 #[test]
6443 fn next_untried_implementer_returns_none_once_every_id_is_tried() {
6444 let roster = vec![spec("a"), spec("b")];
6445 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
6446 assert!(next_untried_implementer(&roster, 0, &tried).is_none());
6447 }
6448
6449 #[test]
6450 fn remove_if_empty_only_ever_takes_a_bare_directory() {
6451 let dir = tempfile::tempdir().unwrap();
6452 let bay = dir.path().join("ffff");
6453
6454 // Not there yet: nothing to do, nothing to panic on.
6455 remove_if_empty(&bay);
6456 assert!(!bay.exists());
6457
6458 // Something still inside - the winner's worktree, or a stray file -
6459 // keeps the directory standing.
6460 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
6461 remove_if_empty(&bay);
6462 assert!(bay.exists(), "non-empty directory must survive");
6463
6464 // Once the last entry is gone, so is the directory itself.
6465 std::fs::remove_dir(bay.join("cand-A")).unwrap();
6466 remove_if_empty(&bay);
6467 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
6468 }
6469
6470 // `round_is_clean` is the exact decision this task fixed: a round with a
6471 // seat that never answered must not read the same as a round every seat
6472 // actually reviewed. These are deterministic and process-free by design —
6473 // the equivalent end-to-end check (a real reviewer timing out under a
6474 // live graph run) is a genuine race against wall-clock contention, and a
6475 // spawn slow enough to blow even a generous budget under a loaded test
6476 // run must not turn this specific regression check flaky.
6477
6478 #[test]
6479 fn a_full_panel_that_found_nothing_is_clean() {
6480 assert!(round_is_clean(
6481 0,
6482 true,
6483 2,
6484 2,
6485 0,
6486 IncompleteReviewPolicy::Block
6487 ));
6488 }
6489
6490 #[test]
6491 fn a_missing_seat_is_never_clean_under_the_default_policy() {
6492 assert!(!round_is_clean(
6493 0,
6494 true,
6495 1,
6496 2,
6497 0,
6498 IncompleteReviewPolicy::Block
6499 ));
6500 }
6501
6502 #[test]
6503 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
6504 assert!(!round_is_clean(
6505 1,
6506 true,
6507 1,
6508 2,
6509 0,
6510 IncompleteReviewPolicy::Warn
6511 ));
6512 }
6513
6514 #[test]
6515 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
6516 assert!(round_is_clean(
6517 0,
6518 true,
6519 1,
6520 2,
6521 0,
6522 IncompleteReviewPolicy::Warn
6523 ));
6524 }
6525
6526 #[test]
6527 fn a_full_panel_with_an_open_finding_is_not_clean() {
6528 assert!(!round_is_clean(
6529 1,
6530 true,
6531 2,
6532 2,
6533 0,
6534 IncompleteReviewPolicy::Block
6535 ));
6536 }
6537
6538 #[test]
6539 fn a_full_panel_with_a_red_e2e_is_not_clean() {
6540 assert!(!round_is_clean(
6541 0,
6542 false,
6543 2,
6544 2,
6545 0,
6546 IncompleteReviewPolicy::Block
6547 ));
6548 }
6549
6550 // The stall this task closes: under the default `block` policy, a seat
6551 // missing only because it was rate limited must not force a wait for a
6552 // session limit that will not lift by the next round. `round_is_clean`
6553 // is where that quorum carve-out lives; the review loop around it never
6554 // changes what a reviewer's vote or a finding's severity means.
6555
6556 #[test]
6557 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
6558 // 1 of 2 answered, and the one missing was quota'd — the exact
6559 // "review-2 rate limited (quota)" shape from the field report.
6560 assert!(round_is_clean(
6561 0,
6562 true,
6563 1,
6564 2,
6565 1,
6566 IncompleteReviewPolicy::Block
6567 ));
6568 }
6569
6570 #[test]
6571 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
6572 // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
6573 // not a quota loss (`quota_missing` stays 0) — worth another try.
6574 assert!(!round_is_clean(
6575 0,
6576 true,
6577 1,
6578 2,
6579 0,
6580 IncompleteReviewPolicy::Block
6581 ));
6582 }
6583
6584 #[test]
6585 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
6586 assert!(!round_is_clean(
6587 1,
6588 true,
6589 1,
6590 2,
6591 1,
6592 IncompleteReviewPolicy::Block
6593 ));
6594 assert!(!round_is_clean(
6595 0,
6596 false,
6597 1,
6598 2,
6599 1,
6600 IncompleteReviewPolicy::Block
6601 ));
6602 }
6603
6604 #[test]
6605 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
6606 // Every seat quota'd, nobody answered: there is no panel to decide
6607 // on, so this must fall through to the existing block-and-retry
6608 // fallback rather than call an unreviewed patch clean.
6609 assert!(!round_is_clean(
6610 0,
6611 true,
6612 0,
6613 2,
6614 2,
6615 IncompleteReviewPolicy::Block
6616 ));
6617 }
6618
6619 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
6620 CommandOutcome {
6621 command: "test".to_owned(),
6622 code,
6623 output_tail: String::new(),
6624 duration_ms: 0,
6625 resource_blocked,
6626 }
6627 }
6628
6629 #[test]
6630 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
6631 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
6632 assert!(
6633 !verify_inconclusive(&[outcome(Some(1), false)]),
6634 "an ordinary failure is still evidence about the patch"
6635 );
6636 assert!(verify_inconclusive(&[outcome(None, true)]));
6637 assert!(
6638 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
6639 "one inconclusive outcome taints the whole batch"
6640 );
6641 assert!(!verify_inconclusive(&[]));
6642 }
6643
6644 #[tokio::test]
6645 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
6646 // Alive for the first two checks, then dead - confirms the loop
6647 // actually re-polls rather than deciding once and sleeping out the
6648 // ceiling regardless.
6649 let calls = std::sync::atomic::AtomicUsize::new(0);
6650 let started = Instant::now();
6651 wait_for_pids_with(
6652 &[123],
6653 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
6654 Duration::from_millis(5),
6655 Duration::from_secs(5),
6656 )
6657 .await;
6658 assert!(
6659 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
6660 "must keep checking rather than deciding on the first answer"
6661 );
6662 assert!(
6663 started.elapsed() < Duration::from_secs(1),
6664 "must return the moment it is confirmed dead, not wait out the ceiling"
6665 );
6666 }
6667
6668 #[tokio::test]
6669 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
6670 let started = Instant::now();
6671 wait_for_pids_with(
6672 &[123],
6673 |_| true, // never reports dead
6674 Duration::from_millis(5),
6675 Duration::from_millis(30),
6676 )
6677 .await;
6678 let elapsed = started.elapsed();
6679 assert!(
6680 elapsed >= Duration::from_millis(30),
6681 "must not give up before its own ceiling: {elapsed:?}"
6682 );
6683 assert!(
6684 elapsed < Duration::from_secs(1),
6685 "must not wait past its own ceiling either: {elapsed:?}"
6686 );
6687 }
6688
6689 #[tokio::test]
6690 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
6691 let started = Instant::now();
6692 wait_for_pids_with(
6693 &[],
6694 |_| true,
6695 Duration::from_secs(5),
6696 Duration::from_secs(5),
6697 )
6698 .await;
6699 assert!(
6700 started.elapsed() < Duration::from_millis(200),
6701 "an empty pid list has nothing to confirm"
6702 );
6703 }
6704
6705 // `review_conclusion` is the exact decision the review hand-off task
6706 // fixed: a round budget spent (or a tree that stopped moving) must not
6707 // collapse into `Blocked` regardless of what verification actually
6708 // said. Deterministic and process-free for the same reason the
6709 // `round_is_clean` family above is.
6710 fn review_round(
6711 clean: bool,
6712 blocking: usize,
6713 answered: usize,
6714 expected: usize,
6715 progressed: bool,
6716 e2e_ok: bool,
6717 ) -> ReviewRound {
6718 ReviewRound {
6719 round: 1,
6720 head: "h".to_owned(),
6721 verified_head: None,
6722 verified_at: None,
6723 reviews: Vec::new(),
6724 e2e: vec![CommandOutcome {
6725 command: "test".to_owned(),
6726 code: Some(if e2e_ok { 0 } else { 1 }),
6727 output_tail: String::new(),
6728 duration_ms: 0,
6729 resource_blocked: false,
6730 }],
6731 verify_retried: false,
6732 e2e_deferred: false,
6733 e2e_defer_reason: None,
6734 fix: None,
6735 blocking,
6736 answered,
6737 expected,
6738 clean,
6739 progressed,
6740 vote_split: false,
6741 reconsideration: Vec::new(),
6742 verdict: None,
6743 }
6744 }
6745
6746 #[test]
6747 fn review_conclusion_is_none_when_nothing_has_run() {
6748 assert_eq!(review_conclusion(&[], 3), None);
6749 }
6750
6751 #[test]
6752 fn review_conclusion_is_none_while_rounds_remain() {
6753 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
6754 assert_eq!(review_conclusion(&rounds, 3), None);
6755 }
6756
6757 #[test]
6758 fn review_conclusion_is_gating_once_a_round_is_clean() {
6759 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
6760 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
6761 }
6762
6763 #[test]
6764 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
6765 let rounds = vec![
6766 review_round(false, 1, 2, 2, true, true),
6767 review_round(false, 1, 2, 2, true, true),
6768 ];
6769 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
6770 }
6771
6772 #[test]
6773 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
6774 let rounds = vec![
6775 review_round(false, 1, 2, 2, true, true),
6776 review_round(false, 1, 2, 2, true, false),
6777 ];
6778 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
6779 }
6780
6781 #[test]
6782 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
6783 // Magi never got a command to run against this round's own head — a
6784 // resource-blocked attempt, not a red one — so this must never
6785 // settle on `Blocked` the way a genuine e2e failure would. `None`
6786 // here is what tells `Runner::review_loop` to retry the check
6787 // itself rather than trust this cheap recomputation with a verdict
6788 // it cannot actually produce.
6789 let mut blocked = review_round(false, 1, 2, 2, true, false);
6790 blocked.e2e[0].resource_blocked = true;
6791 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
6792 assert_eq!(review_conclusion(&rounds, 2), None);
6793 }
6794
6795 #[test]
6796 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
6797 // Missing input, not a verified tree — never a hand-off candidate.
6798 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
6799 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
6800 }
6801
6802 #[test]
6803 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
6804 let rounds = vec![
6805 review_round(false, 1, 2, 2, false, true),
6806 review_round(false, 1, 2, 2, false, true),
6807 ];
6808 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
6809 }
6810
6811 fn secs(n: u64) -> Duration {
6812 Duration::from_secs(n)
6813 }
6814
6815 /// A throwaway repo with one commit on `main`, for tests that need `merge`
6816 /// to make real (and, if it runs at all, real*ly fail*) git calls.
6817 fn init_repo(dir: &Path) {
6818 let run = |args: &[&str]| {
6819 let out = std::process::Command::new("git")
6820 .args(args)
6821 .current_dir(dir)
6822 .quiet()
6823 .output()
6824 .expect("spawn git");
6825 assert!(
6826 out.status.success(),
6827 "git {args:?} failed: {}",
6828 String::from_utf8_lossy(&out.stderr)
6829 );
6830 };
6831 run(&["init", "-b", "main"]);
6832 run(&["config", "user.name", "magi test"]);
6833 run(&["config", "user.email", "magi@example.com"]);
6834 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
6835 run(&["add", "-A"]);
6836 run(&["commit", "-m", "init"]);
6837 }
6838
6839 // `settle_questions` is what closes the ghost the phone showed: a run's
6840 // seat asked something, the run then ended, and nothing was left to
6841 // abandon the question it left `open`. `HOME` is a process-wide
6842 // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
6843 // first time it runs in the binary — every test below still reaches the
6844 // same directory whichever call won, and each gets its own run id from
6845 // `RunState::new`, so they never collide there.
6846 fn ask_test_home() {
6847 crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
6848 }
6849
6850 /// A minimal, git-free `Runner` at a given status — `settle_questions`
6851 /// reads nothing else off it.
6852 fn runner_at(status: RunStatus) -> Runner {
6853 let mut state = RunState::new(
6854 PathBuf::from("/nonexistent/repo"),
6855 "main".to_owned(),
6856 "deadbeef".to_owned(),
6857 "task".to_owned(),
6858 Config::default(),
6859 );
6860 state.status = status;
6861 Runner {
6862 state,
6863 roles: ResolvedRoles {
6864 implementers: Vec::new(),
6865 judges: Vec::new(),
6866 reviewers: Vec::new(),
6867 fixer: None,
6868 conductor: conductor(),
6869 implementer_roster: Vec::new(),
6870 },
6871 sem: Arc::new(Semaphore::new(1)),
6872 pause: Pause::new(),
6873 interrupt: Pause::new(),
6874 }
6875 }
6876
6877 /// `park_here` folding in the reason `Pause::park_because` recorded -
6878 /// this is what lets an operator reading a run's events tell an
6879 /// interrupt-driven park from an ordinary shutdown park.
6880 #[test]
6881 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
6882 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6883 let mut runner = runner_at(RunStatus::Implementing);
6884 let interrupt = Pause::new();
6885 runner.watch_interrupt(interrupt.clone());
6886
6887 interrupt.park_because("task a1b2 asked to run first");
6888
6889 assert!(runner.park_here().expect("park_here"));
6890 assert!(runner.state.parked);
6891 let last = runner.state.events.last().expect("a park event");
6892 assert_eq!(last.node, "park");
6893 assert!(
6894 last.message.contains("task a1b2 asked to run first"),
6895 "expected the interrupt reason in {:?}",
6896 last.message
6897 );
6898 }
6899
6900 /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
6901 /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
6902 /// cleared) must not make a *different* run - one only watching its own,
6903 /// unshared interrupt `Pause` - see itself as parked. If a future change
6904 /// ever collapsed these back into one handle, the interrupt scheduler
6905 /// would park every run for the rest of the daemon's life, not just the
6906 /// one it meant to interrupt.
6907 #[test]
6908 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
6909 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6910 let mut runner = runner_at(RunStatus::Implementing);
6911 let shutdown = Pause::new();
6912 runner.on_pause(shutdown.clone());
6913 let interrupt = Pause::new();
6914 runner.watch_interrupt(interrupt.clone());
6915
6916 // Nobody has asked for anything yet.
6917 assert!(!runner.park_here().expect("park_here"));
6918 assert!(!runner.state.parked);
6919
6920 // Only the interrupt handle fires; the shutdown handle stays clear.
6921 interrupt.park_because("test");
6922 assert!(!shutdown.parked());
6923 assert!(runner.park_here().expect("park_here"));
6924 }
6925
6926 /// The property every prior attempt at this feature failed to pin down:
6927 /// asking a run to park while one of its nodes has a real, in-flight
6928 /// async operation running (an agent call, in production) must not cut
6929 /// that operation short. `park_here` is only ever consulted *between*
6930 /// `execute`'s node calls - see its own doc - so nothing inside a node
6931 /// can observe a park request until the node itself returns. This proves
6932 /// that structurally, with real `tokio` concurrency and a channel
6933 /// handshake (never a sleep, which would only prove "usually", not
6934 /// "cannot"): the "node" below reports that it has genuinely started,
6935 /// and only then is the park requested; the node still has to be told to
6936 /// finish before `park_here` is ever called, exactly mirroring every
6937 /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
6938 /// in `execute`.
6939 #[tokio::test]
6940 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
6941 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6942 let mut runner = runner_at(RunStatus::Implementing);
6943 let interrupt = Pause::new();
6944 runner.watch_interrupt(interrupt.clone());
6945
6946 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
6947 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
6948
6949 // Stands in for one node's in-flight agent call: it proves it has
6950 // genuinely started, then blocks - exactly as a spawned CLI process
6951 // does - until told to finish.
6952 let node = async move {
6953 started_tx.send(()).expect("send started");
6954 finish_rx.await.expect("recv finish");
6955 "node finished"
6956 };
6957
6958 let interrupter = async move {
6959 started_rx.await.expect("recv started");
6960 // The call is now genuinely in flight. Ask it to park.
6961 interrupt.park_because("higher-priority task waiting");
6962 // Nothing the node does can observe this yet - there is no
6963 // check inside it, by construction - so let the executor run
6964 // anything pending and then let the node finish on its own.
6965 tokio::task::yield_now().await;
6966 finish_tx.send(()).expect("send finish");
6967 };
6968
6969 let (node_result, ()) = tokio::join!(node, interrupter);
6970 assert_eq!(
6971 node_result, "node finished",
6972 "the in-flight call ran to completion"
6973 );
6974
6975 // Only now, at the boundary the real `execute` would check right
6976 // after this node, does the park take effect.
6977 assert!(runner.park_here().expect("park_here"));
6978 assert!(runner.state.parked);
6979 }
6980
6981 /// A run parked mid-competition carries every field it had accumulated
6982 /// through the exact same disk round-trip an ordinary resume uses -
6983 /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
6984 /// Nothing about parking for an interrupt is a special case of that path;
6985 /// this is what proves it rather than assuming it.
6986 #[test]
6987 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
6988 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
6989 let mut runner = runner_at(RunStatus::Judging);
6990 // `Runner::resume` re-resolves roles from the saved config, which
6991 // refuses an empty roster - give it the same minimal one `conductor`
6992 // itself uses.
6993 runner.state.config.agents = vec![conductor()];
6994 runner.state.candidates = vec![Candidate {
6995 index: 0,
6996 label: 'A',
6997 agent: "alpha".to_owned(),
6998 branch: "magi/x/A".to_owned(),
6999 worktree: PathBuf::from("/nonexistent/worktree"),
7000 summary: "did the thing".to_owned(),
7001 stat: "1 file changed".to_owned(),
7002 files: 1,
7003 commits: 1,
7004 empty: false,
7005 failed: None,
7006 verified_noop: None,
7007 duration_ms: 1234,
7008 folded: false,
7009 }];
7010 let run_id = runner.state.id.clone();
7011
7012 let interrupt = Pause::new();
7013 runner.watch_interrupt(interrupt.clone());
7014 interrupt.park_because("task c3d4 asked to run first");
7015 assert!(runner.park_here().expect("park_here"));
7016
7017 let resumed = Runner::resume(&run_id).expect("resume");
7018 assert_eq!(resumed.state.candidates.len(), 1);
7019 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
7020 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
7021 assert_eq!(resumed.state.status, runner.state.status);
7022 assert!(
7023 resumed.state.parked,
7024 "still parked until `execute` actually walks the graph again"
7025 );
7026 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
7027 }
7028
7029 /// A fresh open question on `run`, stored and handed back for assertions.
7030 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
7031 let mut q = ask::Question::new(
7032 run.to_owned(),
7033 "implement".to_owned(),
7034 "impl-A".to_owned(),
7035 "Which storage backend should the cache use?".to_owned(),
7036 String::new(),
7037 vec!["SQLite".to_owned(), "Redis".to_owned()],
7038 );
7039 store.put(&mut q).unwrap();
7040 q
7041 }
7042
7043 #[test]
7044 fn a_failed_runs_open_question_is_abandoned() {
7045 ask_test_home();
7046 let store = ask::Questions::open();
7047 let mut runner = runner_at(RunStatus::Failed);
7048 let run = runner.state.id.clone();
7049 let q = ask_open_question(&store, &run);
7050
7051 runner.settle_questions();
7052
7053 let back = store.get(&q.id).unwrap();
7054 assert!(
7055 !back.status.open(),
7056 "the seat that asked died with the run; nobody is left to read an answer"
7057 );
7058 assert!(
7059 back.detail.contains(&run) && back.detail.contains("failed"),
7060 "the reason names what the run became, not just that it is gone: {}",
7061 back.detail
7062 );
7063 }
7064
7065 #[test]
7066 fn a_merged_runs_open_question_is_abandoned_too() {
7067 ask_test_home();
7068 let store = ask::Questions::open();
7069 // A run that finishes cleanly still leaves nobody to read an answer -
7070 // this is not only a failure-path cleanup.
7071 for status in [RunStatus::Merged, RunStatus::Ready] {
7072 let mut runner = runner_at(status);
7073 let run = runner.state.id.clone();
7074 let q = ask_open_question(&store, &run);
7075
7076 runner.settle_questions();
7077
7078 let back = store.get(&q.id).unwrap();
7079 assert!(
7080 !back.status.open(),
7081 "{status:?} run's question must not outlive the run"
7082 );
7083 }
7084 }
7085
7086 #[test]
7087 fn a_still_resumable_runs_open_question_is_left_alone() {
7088 ask_test_home();
7089 let store = ask::Questions::open();
7090 // `Blocked` and `Stalled` can still be resumed — the candidates, the
7091 // review round and the seat sessions are all still on disk — so a
7092 // question asked mid-round may yet get a real answer from a real
7093 // resume. Sweeping it here would be exactly the failure mode this
7094 // whole feature exists to avoid on the other side.
7095 for status in [RunStatus::Blocked, RunStatus::Stalled] {
7096 let mut runner = runner_at(status);
7097 let run = runner.state.id.clone();
7098 let q = ask_open_question(&store, &run);
7099
7100 runner.settle_questions();
7101
7102 let back = store.get(&q.id).unwrap();
7103 assert!(
7104 back.status.open(),
7105 "{status:?} is still alive; the question must still be waiting"
7106 );
7107 }
7108 }
7109
7110 #[test]
7111 fn settle_questions_never_touches_an_already_answered_question() {
7112 ask_test_home();
7113 let store = ask::Questions::open();
7114 let mut runner = runner_at(RunStatus::Failed);
7115 let run = runner.state.id.clone();
7116 let mut q = ask_open_question(&store, &run);
7117 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
7118 .unwrap();
7119 store.put(&mut q).unwrap();
7120
7121 // Called twice, the way a crash-recovered daemon reclaim and the
7122 // graph's own cleanup both can for the same run — `abandon_for_run`
7123 // only ever touches what is still open, so this must be inert both
7124 // times, not merely the second.
7125 runner.settle_questions();
7126 runner.settle_questions();
7127
7128 let back = store.get(&q.id).unwrap();
7129 assert_eq!(
7130 back.status,
7131 ask::QuestionStatus::Answered,
7132 "a real answer is a decision on record, never overwritten by a sweep"
7133 );
7134 }
7135
7136 /// `fold_run(&mut state, drop_winner = false)` is exactly the call
7137 /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
7138 /// without merging, whose winner is still the operator's answer to read.
7139 /// Nothing previously called `fold_run` itself with a real `tally`, so
7140 /// this is the first test to pin down the one distinction the whole
7141 /// automatic-fold feature depends on: the winner's worktree and branch
7142 /// must survive, everything else sharing the run's worktree bay - a
7143 /// loser, standing in for a judge/review worktree too, since `fold_run`'s
7144 /// second sweep treats every non-winner directory under the bay alike -
7145 /// must not.
7146 #[tokio::test]
7147 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
7148 crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
7149 let tmp = tempfile::tempdir().expect("tempdir");
7150 let repo = tmp.path().join("repo");
7151 std::fs::create_dir_all(&repo).unwrap();
7152 init_repo(&repo);
7153
7154 let mut config = Config::default();
7155 config.graph.worktree_root = Some(tmp.path().join("wt"));
7156
7157 let mut state = RunState::new(
7158 repo.clone(),
7159 "main".to_owned(),
7160 "deadbeef".to_owned(),
7161 "task".to_owned(),
7162 config,
7163 );
7164 let root = state.worktree_root();
7165 let wt_a = root.join("cand-A");
7166 let wt_b = root.join("cand-B");
7167 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
7168 .await
7169 .expect("worktree A");
7170 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
7171 .await
7172 .expect("worktree B");
7173
7174 state.candidates = vec![
7175 Candidate {
7176 index: 0,
7177 label: 'A',
7178 agent: "alpha".to_owned(),
7179 branch: "magi/x/A".to_owned(),
7180 worktree: wt_a.clone(),
7181 summary: String::new(),
7182 stat: String::new(),
7183 files: 0,
7184 commits: 0,
7185 empty: false,
7186 failed: None,
7187 verified_noop: None,
7188 duration_ms: 0,
7189 folded: false,
7190 },
7191 Candidate {
7192 index: 1,
7193 label: 'B',
7194 agent: "beta".to_owned(),
7195 branch: "magi/x/B".to_owned(),
7196 worktree: wt_b.clone(),
7197 summary: String::new(),
7198 stat: String::new(),
7199 files: 0,
7200 commits: 0,
7201 empty: false,
7202 failed: None,
7203 verified_noop: None,
7204 duration_ms: 0,
7205 folded: false,
7206 },
7207 ];
7208 state.tally = Some(Tally {
7209 first_choice: BTreeMap::from([('A', 1)]),
7210 borda: BTreeMap::new(),
7211 winner: 'A',
7212 rankings: 1,
7213 unanimous_initial: true,
7214 deliberated: false,
7215 changed_votes: 0,
7216 unanimous_final: true,
7217 tie_break: None,
7218 judges: 1,
7219 present: 1,
7220 quorum: 1,
7221 met_quorum: true,
7222 uncontested: None,
7223 });
7224 state.status = RunStatus::Ready;
7225
7226 fold_run(&mut state, false, &crate::run::home())
7227 .await
7228 .expect("fold_run");
7229
7230 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
7231 assert!(
7232 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7233 "the unmerged winner's branch survives"
7234 );
7235 assert!(
7236 !state.candidates[0].folded,
7237 "the winner is not marked folded"
7238 );
7239
7240 assert!(!wt_b.exists(), "the loser's worktree is removed");
7241 assert!(
7242 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
7243 "the loser's branch is removed"
7244 );
7245 assert!(state.candidates[1].folded, "the loser is marked folded");
7246 }
7247
7248 /// `status == Ready` used to be read as "this is the harmless
7249 /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
7250 /// this test). But `land` sets the very same status when a `MergeMode::Pr`
7251 /// run's PR was closed without merging — and reentering `merge` with
7252 /// `mode` still `Pr` does not know the difference, so it pushed and
7253 /// opened a second pull request. `mode == Local` reproduces the same
7254 /// blind spot without a network call: reentry must not attempt another
7255 /// git merge once this node has already recorded an outcome.
7256 #[tokio::test]
7257 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
7258 let tmp = tempfile::tempdir().expect("tempdir");
7259 let repo = tmp.path().join("repo");
7260 std::fs::create_dir_all(&repo).unwrap();
7261 init_repo(&repo);
7262
7263 let mut config = Config::default();
7264 config.merge.mode = MergeMode::Local;
7265
7266 let mut state = RunState::new(
7267 repo.clone(),
7268 "main".to_owned(),
7269 "deadbeef".to_owned(),
7270 "task".to_owned(),
7271 config,
7272 );
7273 state.candidates = vec![Candidate {
7274 index: 0,
7275 label: 'A',
7276 agent: "alpha".to_owned(),
7277 branch: "does-not-exist".to_owned(),
7278 worktree: repo.clone(),
7279 summary: String::new(),
7280 stat: String::new(),
7281 files: 0,
7282 commits: 0,
7283 empty: false,
7284 failed: None,
7285 verified_noop: None,
7286 duration_ms: 0,
7287 folded: false,
7288 }];
7289 state.tally = Some(Tally {
7290 first_choice: BTreeMap::from([('A', 1)]),
7291 borda: BTreeMap::new(),
7292 winner: 'A',
7293 rankings: 1,
7294 unanimous_initial: true,
7295 deliberated: false,
7296 changed_votes: 0,
7297 unanimous_final: true,
7298 tie_break: None,
7299 judges: 0,
7300 present: 0,
7301 quorum: 0,
7302 met_quorum: true,
7303 uncontested: Some("only candidate A produced a change".to_owned()),
7304 });
7305 state.reviews = vec![ReviewRound {
7306 round: 1,
7307 head: "deadbeef".to_owned(),
7308 verified_head: None,
7309 verified_at: None,
7310 reviews: Vec::new(),
7311 e2e: Vec::new(),
7312 fix: None,
7313 blocking: 0,
7314 answered: 0,
7315 expected: 0,
7316 clean: true,
7317 verify_retried: false,
7318 e2e_deferred: false,
7319 e2e_defer_reason: None,
7320 progressed: false,
7321 vote_split: false,
7322 reconsideration: Vec::new(),
7323 verdict: None,
7324 }];
7325 state.gate = vec![CommandOutcome {
7326 command: "test".to_owned(),
7327 code: Some(0),
7328 output_tail: String::new(),
7329 duration_ms: 0,
7330 resource_blocked: false,
7331 }];
7332 state.gate_ran = true;
7333 // Reached its conclusion already — e.g. `land` closing the PR without
7334 // merging it, which (like the honest `MergeMode::None` path) leaves
7335 // `status` at `Ready`. The recorded outcome is what actually marks
7336 // this node done.
7337 state.status = RunStatus::Ready;
7338 state.merge = Some(MergeOutcome {
7339 mode: MergeMode::Local,
7340 ok: false,
7341 detail: "already concluded".to_owned(),
7342 });
7343
7344 let mut runner = Runner {
7345 state,
7346 roles: ResolvedRoles {
7347 implementers: Vec::new(),
7348 judges: Vec::new(),
7349 reviewers: Vec::new(),
7350 fixer: None,
7351 conductor: conductor(),
7352 implementer_roster: Vec::new(),
7353 },
7354 sem: Arc::new(Semaphore::new(1)),
7355 pause: Pause::new(),
7356 interrupt: Pause::new(),
7357 };
7358
7359 runner.merge().await.expect("merge");
7360
7361 assert_eq!(
7362 runner.state.status,
7363 RunStatus::Ready,
7364 "a concluded run's status must not change on reentry"
7365 );
7366 assert_eq!(
7367 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
7368 Some("already concluded"),
7369 "merge must not run again once the node already recorded an outcome"
7370 );
7371 }
7372
7373 /// `gate` leaves `state.gate_ran` false both before it has ever run and
7374 /// when its last attempt was resource-blocked (the shared build cache
7375 /// could not be acquired or confirmed fresh in time - see
7376 /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
7377 /// `Vec` this also leaves behind used to read as "nothing failed" and let
7378 /// a run merge a tree the gate never actually checked - exactly the case
7379 /// a contended cache produces on every retry until it clears. `merge`
7380 /// must refuse until `gate` has actually recorded an attempt.
7381 #[tokio::test]
7382 async fn merge_refuses_a_gate_that_has_not_actually_run() {
7383 let tmp = tempfile::tempdir().expect("tempdir");
7384 let repo = tmp.path().join("repo");
7385 std::fs::create_dir_all(&repo).unwrap();
7386 init_repo(&repo);
7387
7388 let mut config = Config::default();
7389 config.merge.mode = MergeMode::Local;
7390
7391 let mut state = RunState::new(
7392 repo.clone(),
7393 "main".to_owned(),
7394 "deadbeef".to_owned(),
7395 "task".to_owned(),
7396 config,
7397 );
7398 state.candidates = vec![Candidate {
7399 index: 0,
7400 label: 'A',
7401 agent: "alpha".to_owned(),
7402 branch: "does-not-exist".to_owned(),
7403 worktree: repo.clone(),
7404 summary: String::new(),
7405 stat: String::new(),
7406 files: 0,
7407 commits: 0,
7408 empty: false,
7409 failed: None,
7410 verified_noop: None,
7411 duration_ms: 0,
7412 folded: false,
7413 }];
7414 state.tally = Some(Tally {
7415 first_choice: BTreeMap::from([('A', 1)]),
7416 borda: BTreeMap::new(),
7417 winner: 'A',
7418 rankings: 1,
7419 unanimous_initial: true,
7420 deliberated: false,
7421 changed_votes: 0,
7422 unanimous_final: true,
7423 tie_break: None,
7424 judges: 0,
7425 present: 0,
7426 quorum: 0,
7427 met_quorum: true,
7428 uncontested: Some("only candidate A produced a change".to_owned()),
7429 });
7430 state.reviews = vec![ReviewRound {
7431 round: 1,
7432 head: "deadbeef".to_owned(),
7433 verified_head: None,
7434 verified_at: None,
7435 reviews: Vec::new(),
7436 e2e: Vec::new(),
7437 fix: None,
7438 blocking: 0,
7439 answered: 0,
7440 expected: 0,
7441 clean: true,
7442 verify_retried: false,
7443 e2e_deferred: false,
7444 e2e_defer_reason: None,
7445 progressed: false,
7446 vote_split: false,
7447 reconsideration: Vec::new(),
7448 verdict: None,
7449 }];
7450 // The point: `gate` has not recorded anything yet.
7451 state.gate = Vec::new();
7452 state.gate_ran = false;
7453 state.status = RunStatus::Gating;
7454
7455 let mut runner = Runner {
7456 state,
7457 roles: ResolvedRoles {
7458 implementers: Vec::new(),
7459 judges: Vec::new(),
7460 reviewers: Vec::new(),
7461 fixer: None,
7462 conductor: conductor(),
7463 implementer_roster: Vec::new(),
7464 },
7465 sem: Arc::new(Semaphore::new(1)),
7466 pause: Pause::new(),
7467 interrupt: Pause::new(),
7468 };
7469
7470 runner.merge().await.expect("merge");
7471
7472 assert!(
7473 runner.state.merge.is_none(),
7474 "an empty gate must never be read as a passing one: {:?}",
7475 runner.state.merge
7476 );
7477 }
7478
7479 /// The `shoka` repro this schema bump exists for: `verify.gate` has no
7480 /// commands configured and `merge.mode` is `none` (a review-only run).
7481 /// `gate` must still record a real attempt — zero commands, vacuously
7482 /// passed — rather than leaving `state.gate` empty in a way `merge`
7483 /// cannot tell apart from "never ran"; otherwise the run reaches
7484 /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
7485 #[tokio::test]
7486 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
7487 let tmp = tempfile::tempdir().expect("tempdir");
7488 let repo = tmp.path().join("repo");
7489 std::fs::create_dir_all(&repo).unwrap();
7490 init_repo(&repo);
7491
7492 // Default config: `verify.gate` empty, `merge.mode` is `none`.
7493 let config = Config::default();
7494
7495 let mut state = RunState::new(
7496 repo.clone(),
7497 "main".to_owned(),
7498 "deadbeef".to_owned(),
7499 "task".to_owned(),
7500 config,
7501 );
7502 state.candidates = vec![Candidate {
7503 index: 0,
7504 label: 'A',
7505 agent: "alpha".to_owned(),
7506 branch: "does-not-exist".to_owned(),
7507 worktree: repo.clone(),
7508 summary: String::new(),
7509 stat: String::new(),
7510 files: 0,
7511 commits: 0,
7512 empty: false,
7513 failed: None,
7514 verified_noop: None,
7515 duration_ms: 0,
7516 folded: false,
7517 }];
7518 state.tally = Some(Tally {
7519 first_choice: BTreeMap::from([('A', 1)]),
7520 borda: BTreeMap::new(),
7521 winner: 'A',
7522 rankings: 1,
7523 unanimous_initial: true,
7524 deliberated: false,
7525 changed_votes: 0,
7526 unanimous_final: true,
7527 tie_break: None,
7528 judges: 0,
7529 present: 0,
7530 quorum: 0,
7531 met_quorum: true,
7532 uncontested: Some("only candidate A produced a change".to_owned()),
7533 });
7534 state.reviews = vec![ReviewRound {
7535 round: 1,
7536 head: "deadbeef".to_owned(),
7537 verified_head: None,
7538 verified_at: None,
7539 reviews: Vec::new(),
7540 e2e: Vec::new(),
7541 fix: None,
7542 blocking: 0,
7543 answered: 0,
7544 expected: 0,
7545 clean: true,
7546 verify_retried: false,
7547 e2e_deferred: false,
7548 e2e_defer_reason: None,
7549 progressed: false,
7550 vote_split: false,
7551 reconsideration: Vec::new(),
7552 verdict: None,
7553 }];
7554
7555 let mut runner = Runner {
7556 state,
7557 roles: ResolvedRoles {
7558 implementers: Vec::new(),
7559 judges: Vec::new(),
7560 reviewers: Vec::new(),
7561 fixer: None,
7562 conductor: conductor(),
7563 implementer_roster: Vec::new(),
7564 },
7565 sem: Arc::new(Semaphore::new(1)),
7566 pause: Pause::new(),
7567 interrupt: Pause::new(),
7568 };
7569
7570 runner.gate().await.expect("gate");
7571 assert!(
7572 runner.state.gate_ran,
7573 "zero configured commands is still a real attempt, not an unrun gate"
7574 );
7575 assert!(runner.state.gate.is_empty());
7576 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
7577 assert_ne!(
7578 runner.state.status,
7579 RunStatus::Blocked,
7580 "a gate with nothing to check must not read as failed"
7581 );
7582
7583 runner.merge().await.expect("merge");
7584 assert_eq!(
7585 runner.state.status,
7586 RunStatus::Ready,
7587 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
7588 );
7589 }
7590
7591 /// `Config::cache_dir` is derived from `verify.e2e` as well as
7592 /// `verify.gate` (so the e2e leg and the final gate never build against
7593 /// different directories). With zero `verify.gate` commands but a
7594 /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
7595 /// that lease before discovering it had nothing to run - so a repo with
7596 /// no gate commands could come back `resource_blocked` (and therefore
7597 /// still `gate_ran == false`) on nothing but an unrelated run holding the
7598 /// cache, exactly the contention this run's own zero commands could
7599 /// never have touched. `gate` must recognise there is nothing to check
7600 /// before it ever asks for the lease.
7601 #[tokio::test]
7602 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
7603 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7604 let home = crate::run::home();
7605
7606 let tmp = tempfile::tempdir().expect("tempdir");
7607 let repo = tmp.path().join("repo");
7608 std::fs::create_dir_all(&repo).unwrap();
7609 init_repo(&repo);
7610 // Unique to this test, so holding its lease cannot collide with
7611 // another test sharing the same process-wide `home`.
7612 let cache_dir = tmp.path().join("target");
7613
7614 let mut config = Config::default();
7615 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
7616 // `verify.gate` stays empty (the default). Bounded so a regression
7617 // that does start waiting fails the test in seconds, not hangs it.
7618 config.graph.timeout_verify = Some(2);
7619
7620 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
7621 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
7622 .expect("no io error acquiring directly")
7623 {
7624 crate::cache::AcquireOutcome::Acquired(g) => g,
7625 crate::cache::AcquireOutcome::Busy(b) => {
7626 panic!("expected the direct acquire to win the lease first: {b:?}")
7627 }
7628 };
7629
7630 let mut state = RunState::new(
7631 repo.clone(),
7632 "main".to_owned(),
7633 "deadbeef".to_owned(),
7634 "task".to_owned(),
7635 config,
7636 );
7637 state.candidates = vec![Candidate {
7638 index: 0,
7639 label: 'A',
7640 agent: "alpha".to_owned(),
7641 branch: "does-not-exist".to_owned(),
7642 worktree: repo.clone(),
7643 summary: String::new(),
7644 stat: String::new(),
7645 files: 0,
7646 commits: 0,
7647 empty: false,
7648 failed: None,
7649 verified_noop: None,
7650 duration_ms: 0,
7651 folded: false,
7652 }];
7653 state.tally = Some(Tally {
7654 first_choice: BTreeMap::from([('A', 1)]),
7655 borda: BTreeMap::new(),
7656 winner: 'A',
7657 rankings: 1,
7658 unanimous_initial: true,
7659 deliberated: false,
7660 changed_votes: 0,
7661 unanimous_final: true,
7662 tie_break: None,
7663 judges: 0,
7664 present: 0,
7665 quorum: 0,
7666 met_quorum: true,
7667 uncontested: Some("only candidate A produced a change".to_owned()),
7668 });
7669 state.reviews = vec![ReviewRound {
7670 round: 1,
7671 head: "deadbeef".to_owned(),
7672 verified_head: None,
7673 verified_at: None,
7674 reviews: Vec::new(),
7675 e2e: Vec::new(),
7676 fix: None,
7677 blocking: 0,
7678 answered: 0,
7679 expected: 0,
7680 clean: true,
7681 verify_retried: false,
7682 e2e_deferred: false,
7683 e2e_defer_reason: None,
7684 progressed: false,
7685 vote_split: false,
7686 reconsideration: Vec::new(),
7687 verdict: None,
7688 }];
7689
7690 let mut runner = Runner {
7691 state,
7692 roles: ResolvedRoles {
7693 implementers: Vec::new(),
7694 judges: Vec::new(),
7695 reviewers: Vec::new(),
7696 fixer: None,
7697 conductor: conductor(),
7698 implementer_roster: Vec::new(),
7699 },
7700 sem: Arc::new(Semaphore::new(1)),
7701 pause: Pause::new(),
7702 interrupt: Pause::new(),
7703 };
7704
7705 let started = std::time::Instant::now();
7706 runner.gate().await.expect("gate");
7707 assert!(
7708 started.elapsed() < Duration::from_secs(1),
7709 "a gate with nothing to run must never wait on a lease it never needed"
7710 );
7711 assert!(
7712 runner.state.gate_ran,
7713 "zero commands is still a real, immediate attempt"
7714 );
7715 assert!(runner.state.gate.is_empty());
7716 assert_ne!(
7717 runner.state.status,
7718 RunStatus::Blocked,
7719 "must not read as resource-blocked on a lease it never asked for"
7720 );
7721 }
7722
7723 /// The addendum's second gap: a `verify.gate` command running for real
7724 /// wall-clock time had nothing at all to show for it in `active` before
7725 /// `run_commands` learned to record it — a run could sit in `Gating` for
7726 /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
7727 /// what was actually happening. Proven with a genuinely still-running
7728 /// command, not just a before/after check on the final state: a poller
7729 /// task reads the same `run.json` `gate()` is writing, the same way the
7730 /// phone or `magi show` would, while the shell command is still blocked
7731 /// on its own release marker.
7732 #[tokio::test]
7733 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
7734 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7735
7736 let tmp = tempfile::tempdir().expect("tempdir");
7737 let repo = tmp.path().join("repo");
7738 std::fs::create_dir_all(&repo).unwrap();
7739 init_repo(&repo);
7740
7741 let mut config = Config::default();
7742 config.verify.gate = vec![
7743 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
7744 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
7745 .to_owned(),
7746 ];
7747
7748 let mut state = RunState::new(
7749 repo.clone(),
7750 "main".to_owned(),
7751 "deadbeef".to_owned(),
7752 "task".to_owned(),
7753 config,
7754 );
7755 let run_id = state.id.clone();
7756 state.candidates = vec![Candidate {
7757 index: 0,
7758 label: 'A',
7759 agent: "alpha".to_owned(),
7760 branch: "does-not-exist".to_owned(),
7761 worktree: repo.clone(),
7762 summary: String::new(),
7763 stat: String::new(),
7764 files: 0,
7765 commits: 0,
7766 empty: false,
7767 failed: None,
7768 verified_noop: None,
7769 duration_ms: 0,
7770 folded: false,
7771 }];
7772 state.tally = Some(Tally {
7773 first_choice: BTreeMap::from([('A', 1)]),
7774 borda: BTreeMap::new(),
7775 winner: 'A',
7776 rankings: 1,
7777 unanimous_initial: true,
7778 deliberated: false,
7779 changed_votes: 0,
7780 unanimous_final: true,
7781 tie_break: None,
7782 judges: 0,
7783 present: 0,
7784 quorum: 0,
7785 met_quorum: true,
7786 uncontested: Some("only candidate A produced a change".to_owned()),
7787 });
7788 state.reviews = vec![ReviewRound {
7789 round: 1,
7790 head: "deadbeef".to_owned(),
7791 verified_head: None,
7792 verified_at: None,
7793 reviews: Vec::new(),
7794 e2e: Vec::new(),
7795 fix: None,
7796 blocking: 0,
7797 answered: 0,
7798 expected: 0,
7799 clean: true,
7800 verify_retried: false,
7801 e2e_deferred: false,
7802 e2e_defer_reason: None,
7803 progressed: false,
7804 vote_split: false,
7805 reconsideration: Vec::new(),
7806 verdict: None,
7807 }];
7808
7809 let mut runner = Runner {
7810 state,
7811 roles: ResolvedRoles {
7812 implementers: Vec::new(),
7813 judges: Vec::new(),
7814 reviewers: Vec::new(),
7815 fixer: None,
7816 conductor: conductor(),
7817 implementer_roster: Vec::new(),
7818 },
7819 sem: Arc::new(Semaphore::new(1)),
7820 pause: Pause::new(),
7821 interrupt: Pause::new(),
7822 };
7823
7824 let started_marker = repo.join("started.marker");
7825 let release_marker = repo.join("release.marker");
7826 let poller = tokio::spawn(async move {
7827 // Bounded so a regression that never records the task entry
7828 // fails this test in seconds instead of hanging the suite —
7829 // the same shape `a_park_requested_while_a_seat_is_mid_call_
7830 // does_not_cut_it_short` uses for the same reason.
7831 for _ in 0..100 {
7832 if started_marker.exists()
7833 && let Ok(s) = crate::run::RunState::load(&run_id)
7834 && let Some(a) = s.active.get("gate")
7835 {
7836 std::fs::write(&release_marker, b"go").expect("release marker");
7837 return Some(a.clone());
7838 }
7839 tokio::time::sleep(Duration::from_millis(50)).await;
7840 }
7841 None
7842 });
7843
7844 runner.gate().await.expect("gate");
7845 let captured = poller.await.expect("poller task");
7846 let captured = captured.expect(
7847 "the poller never saw a `gate` task entry in run.json while the command was \
7848 still blocked on its own release marker",
7849 );
7850
7851 assert_eq!(captured.task.as_deref(), Some("gate"));
7852 assert_eq!(captured.node, "gate");
7853 assert_eq!(captured.index, Some(1));
7854 assert_eq!(captured.total, Some(1));
7855 assert!(
7856 captured
7857 .command
7858 .as_deref()
7859 .is_some_and(|c| c.contains("started.marker")),
7860 "{captured:?}"
7861 );
7862
7863 assert!(
7864 runner.state.active.is_empty(),
7865 "the entry must be cleared once the command actually finished: {:?}",
7866 runner.state.active
7867 );
7868 assert!(runner.state.gate_ran);
7869 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
7870 }
7871
7872 /// The shape the incident this whole fix responds to actually had: the
7873 /// round budget spent, the last round's own e2e blocked on the shared
7874 /// build cache (held here by a live pid — this test process — exactly
7875 /// `cache`'s own unit tests' pattern for "another owner, still alive"
7876 /// without forking a process). `stop_reviewing` must retry it — not
7877 /// silently leave the round looking untouched (the catch-up-only half of
7878 /// the bug), and not read the contention as a red `e2e` and block the
7879 /// run on it (the other half). Called directly, the same way
7880 /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
7881 /// above exercises `gate`, so this never needs a real cargo build to
7882 /// reach: the lease is never released, so `with_cache_lease` never gets
7883 /// past acquiring it into anything that would need a real workspace.
7884 #[tokio::test]
7885 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
7886 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
7887 let home = crate::run::home();
7888
7889 let tmp = tempfile::tempdir().expect("tempdir");
7890 let repo = tmp.path().join("repo");
7891 std::fs::create_dir_all(&repo).unwrap();
7892 init_repo(&repo);
7893 let head = crate::git::rev_parse(&repo, "HEAD")
7894 .await
7895 .expect("rev-parse");
7896 // Unique to this test, so holding its lease cannot collide with
7897 // another test sharing the same process-wide `home`.
7898 let cache_dir = tmp.path().join("target");
7899
7900 let mut config = Config::default();
7901 config.verify.e2e = vec![format!(
7902 "CARGO_TARGET_DIR='{}' test -f README.md",
7903 cache_dir.display()
7904 )];
7905 config.graph.review_rounds = 1;
7906 // Bounded so a regression that does start waiting fails the test in
7907 // seconds, not hangs it.
7908 config.graph.timeout_verify = Some(2);
7909
7910 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
7911 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
7912 .expect("no io error acquiring directly")
7913 {
7914 crate::cache::AcquireOutcome::Acquired(g) => g,
7915 crate::cache::AcquireOutcome::Busy(b) => {
7916 panic!("expected the direct acquire to win the lease first: {b:?}")
7917 }
7918 };
7919
7920 let mut state = RunState::new(
7921 repo.clone(),
7922 "main".to_owned(),
7923 head.clone(),
7924 "task".to_owned(),
7925 config,
7926 );
7927 state.candidates = vec![Candidate {
7928 index: 0,
7929 label: 'A',
7930 agent: "alpha".to_owned(),
7931 branch: "does-not-exist".to_owned(),
7932 worktree: repo.clone(),
7933 summary: String::new(),
7934 stat: String::new(),
7935 files: 0,
7936 commits: 0,
7937 empty: false,
7938 failed: None,
7939 verified_noop: None,
7940 duration_ms: 0,
7941 folded: false,
7942 }];
7943 state.tally = Some(Tally {
7944 first_choice: BTreeMap::from([('A', 1)]),
7945 borda: BTreeMap::new(),
7946 winner: 'A',
7947 rankings: 1,
7948 unanimous_initial: true,
7949 deliberated: false,
7950 changed_votes: 0,
7951 unanimous_final: true,
7952 tie_break: None,
7953 judges: 0,
7954 present: 0,
7955 quorum: 0,
7956 met_quorum: true,
7957 uncontested: Some("only candidate A produced a change".to_owned()),
7958 });
7959 // The round budget's last round, deferred: `needs_catchup_run`'s
7960 // other trigger. `stop_reviewing`'s retry machinery must treat this
7961 // exactly like a resource-blocked attempt once it actually runs.
7962 state.reviews = vec![ReviewRound {
7963 round: 1,
7964 head: head.clone(),
7965 verified_head: None,
7966 verified_at: None,
7967 reviews: Vec::new(),
7968 e2e: Vec::new(),
7969 fix: None,
7970 blocking: 1,
7971 answered: 1,
7972 expected: 1,
7973 clean: false,
7974 verify_retried: false,
7975 e2e_deferred: true,
7976 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
7977 progressed: false,
7978 vote_split: false,
7979 reconsideration: Vec::new(),
7980 verdict: None,
7981 }];
7982
7983 let mut runner = Runner {
7984 state,
7985 roles: ResolvedRoles {
7986 implementers: Vec::new(),
7987 judges: Vec::new(),
7988 reviewers: Vec::new(),
7989 fixer: None,
7990 conductor: conductor(),
7991 implementer_roster: Vec::new(),
7992 },
7993 sem: Arc::new(Semaphore::new(1)),
7994 pause: Pause::new(),
7995 interrupt: Pause::new(),
7996 };
7997
7998 let shell = runner.state.config.shell();
7999 runner
8000 .stop_reviewing("round budget spent", &shell, &repo)
8001 .await
8002 .expect("stop_reviewing");
8003
8004 let last = runner.state.reviews.last().expect("round record");
8005 assert_eq!(
8006 last.e2e_status(),
8007 E2eStatus::ResourceBlocked,
8008 "the shared cache is still held; the attempt must read as blocked, not deferred or \
8009 failed: {last:?}"
8010 );
8011 assert_eq!(
8012 last.verified_head.as_deref(),
8013 Some(head.as_str()),
8014 "which commit this attempt targeted is known even though nothing finished checking \
8015 it"
8016 );
8017 let first_attempt_at = last
8018 .verified_at
8019 .expect("when this attempt ran is known too");
8020 assert_ne!(
8021 runner.state.status,
8022 RunStatus::Blocked,
8023 "contention is evidence about the machine, not the patch — it must not settle the \
8024 run as blocked: {:?}",
8025 runner.state.status
8026 );
8027 assert!(
8028 !runner
8029 .state
8030 .events
8031 .iter()
8032 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
8033 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
8034 runner.state.events
8035 );
8036
8037 // The cache is still held: a later reentry must retry the same
8038 // round's verification again — not leave it looking exactly as
8039 // untouched as the first blocked attempt, which is indistinguishable
8040 // from never having tried again at all.
8041 runner
8042 .stop_reviewing("round budget spent", &shell, &repo)
8043 .await
8044 .expect("stop_reviewing retry");
8045 assert_eq!(
8046 runner.state.reviews.len(),
8047 1,
8048 "no new round was started: {:?}",
8049 runner.state.reviews
8050 );
8051 let last = runner.state.reviews.last().expect("round record");
8052 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
8053 assert!(
8054 last.verified_at.expect("still known") > first_attempt_at,
8055 "a second reentry must be a fresh attempt, not a stale copy of the first"
8056 );
8057 assert_ne!(runner.state.status, RunStatus::Blocked);
8058
8059 held.release();
8060 }
8061
8062 /// A resumed run — a fresh `Runner`, `self.state.reviews` already
8063 /// holding the round `stop_reviewing` left `ResourceBlocked` from a
8064 /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
8065 /// own top-of-function fast path (`review_conclusion`) correctly reads
8066 /// this shape as `None` rather than guessing `Blocked`, and the loop's
8067 /// own `for` range is empty once the round budget is spent, so
8068 /// `review_loop` must retry the check itself rather than silently doing
8069 /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
8070 /// above exercises directly, but through `review_loop`'s own entry point
8071 /// this time, proving the wiring between the two rather than just the
8072 /// retry logic in isolation.
8073 #[tokio::test]
8074 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
8075 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8076 let home = crate::run::home();
8077
8078 let tmp = tempfile::tempdir().expect("tempdir");
8079 let repo = tmp.path().join("repo");
8080 std::fs::create_dir_all(&repo).unwrap();
8081 init_repo(&repo);
8082 let head = crate::git::rev_parse(&repo, "HEAD")
8083 .await
8084 .expect("rev-parse");
8085 let cache_dir = tmp.path().join("target");
8086
8087 let mut config = Config::default();
8088 config.verify.e2e = vec![format!(
8089 "CARGO_TARGET_DIR='{}' test -f README.md",
8090 cache_dir.display()
8091 )];
8092 config.graph.review_rounds = 1;
8093 config.graph.timeout_verify = Some(2);
8094
8095 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8096 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8097 .expect("no io error acquiring directly")
8098 {
8099 crate::cache::AcquireOutcome::Acquired(g) => g,
8100 crate::cache::AcquireOutcome::Busy(b) => {
8101 panic!("expected the direct acquire to win the lease first: {b:?}")
8102 }
8103 };
8104
8105 let mut state = RunState::new(
8106 repo.clone(),
8107 "main".to_owned(),
8108 head.clone(),
8109 "task".to_owned(),
8110 config,
8111 );
8112 state.candidates = vec![Candidate {
8113 index: 0,
8114 label: 'A',
8115 agent: "alpha".to_owned(),
8116 branch: "does-not-exist".to_owned(),
8117 worktree: repo.clone(),
8118 summary: String::new(),
8119 stat: String::new(),
8120 files: 0,
8121 commits: 0,
8122 empty: false,
8123 failed: None,
8124 verified_noop: None,
8125 duration_ms: 0,
8126 folded: false,
8127 }];
8128 state.tally = Some(Tally {
8129 first_choice: BTreeMap::from([('A', 1)]),
8130 borda: BTreeMap::new(),
8131 winner: 'A',
8132 rankings: 1,
8133 unanimous_initial: true,
8134 deliberated: false,
8135 changed_votes: 0,
8136 unanimous_final: true,
8137 tie_break: None,
8138 judges: 0,
8139 present: 0,
8140 quorum: 0,
8141 met_quorum: true,
8142 uncontested: Some("only candidate A produced a change".to_owned()),
8143 });
8144 // The exact shape a prior process's `stop_reviewing` would have left
8145 // on disk: the round budget's last round, a real attempt already
8146 // made and already resource-blocked.
8147 state.reviews = vec![ReviewRound {
8148 round: 1,
8149 head: head.clone(),
8150 verified_head: Some(head.clone()),
8151 verified_at: Some(jiff::Timestamp::now()),
8152 reviews: Vec::new(),
8153 e2e: vec![CommandOutcome {
8154 command: format!(
8155 "CARGO_TARGET_DIR='{}' test -f README.md",
8156 cache_dir.display()
8157 ),
8158 code: None,
8159 output_tail: "waiting for the shared build cache".to_owned(),
8160 duration_ms: 0,
8161 resource_blocked: true,
8162 }],
8163 fix: None,
8164 blocking: 1,
8165 answered: 1,
8166 expected: 1,
8167 clean: false,
8168 verify_retried: false,
8169 e2e_deferred: false,
8170 e2e_defer_reason: None,
8171 progressed: false,
8172 vote_split: false,
8173 reconsideration: Vec::new(),
8174 verdict: None,
8175 }];
8176
8177 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
8178 let mut runner = Runner {
8179 state,
8180 roles: ResolvedRoles {
8181 implementers: Vec::new(),
8182 judges: Vec::new(),
8183 reviewers: Vec::new(),
8184 fixer: None,
8185 conductor: conductor(),
8186 implementer_roster: Vec::new(),
8187 },
8188 sem: Arc::new(Semaphore::new(1)),
8189 pause: Pause::new(),
8190 interrupt: Pause::new(),
8191 };
8192
8193 // The lease is still held throughout, so this reentry's own retry is
8194 // also contended — proving `review_loop` actually tried again (not
8195 // that it happened to succeed) is what the timestamp comparison
8196 // below is for.
8197 runner.review_loop().await.expect("review_loop");
8198
8199 assert_eq!(
8200 runner.state.reviews.len(),
8201 1,
8202 "no new round was started on top of the unresolved one: {:?}",
8203 runner.state.reviews
8204 );
8205 let last = &runner.state.reviews[0];
8206 assert_eq!(
8207 last.e2e_status(),
8208 E2eStatus::ResourceBlocked,
8209 "still contended: {last:?}"
8210 );
8211 assert!(
8212 last.verified_at.expect("still known") > first_attempt_at,
8213 "review_loop must have actually retried the check, not left it exactly as found"
8214 );
8215 assert_ne!(
8216 runner.state.status,
8217 RunStatus::Blocked,
8218 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
8219 runner.state.status
8220 );
8221
8222 held.release();
8223 }
8224
8225 #[tokio::test]
8226 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
8227 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8228 let tmp = tempfile::tempdir().expect("tempdir");
8229 let repo = tmp.path().join("repo");
8230 std::fs::create_dir_all(&repo).unwrap();
8231 init_repo(&repo);
8232
8233 let mut config = Config::default();
8234 config.merge.mode = MergeMode::Pr;
8235 config.graph.land = true;
8236 config.graph.land_approval = false;
8237
8238 let mut state = RunState::new(
8239 repo.clone(),
8240 "main".to_owned(),
8241 "deadbeef".to_owned(),
8242 "task".to_owned(),
8243 config,
8244 );
8245 state.candidates = vec![Candidate {
8246 index: 0,
8247 label: 'A',
8248 agent: "alpha".to_owned(),
8249 branch: "does-not-exist".to_owned(),
8250 worktree: repo.clone(),
8251 summary: String::new(),
8252 stat: String::new(),
8253 files: 0,
8254 commits: 0,
8255 empty: false,
8256 failed: None,
8257 verified_noop: None,
8258 duration_ms: 0,
8259 folded: false,
8260 }];
8261 state.tally = Some(Tally {
8262 first_choice: BTreeMap::from([('A', 1)]),
8263 borda: BTreeMap::new(),
8264 winner: 'A',
8265 rankings: 1,
8266 unanimous_initial: true,
8267 deliberated: false,
8268 changed_votes: 0,
8269 unanimous_final: true,
8270 tie_break: None,
8271 judges: 0,
8272 present: 0,
8273 quorum: 0,
8274 met_quorum: true,
8275 uncontested: Some("only candidate A produced a change".to_owned()),
8276 });
8277 state.reviews = vec![ReviewRound {
8278 round: 1,
8279 head: "deadbeef".to_owned(),
8280 verified_head: None,
8281 verified_at: None,
8282 reviews: Vec::new(),
8283 e2e: Vec::new(),
8284 fix: None,
8285 blocking: 0,
8286 answered: 0,
8287 expected: 0,
8288 clean: true,
8289 verify_retried: false,
8290 e2e_deferred: false,
8291 e2e_defer_reason: None,
8292 progressed: false,
8293 vote_split: false,
8294 reconsideration: Vec::new(),
8295 verdict: None,
8296 }];
8297 state.gate = vec![CommandOutcome {
8298 command: "test".to_owned(),
8299 code: Some(0),
8300 output_tail: String::new(),
8301 duration_ms: 0,
8302 resource_blocked: false,
8303 }];
8304 state.gate_ran = true;
8305 // A first pass through `merge` already pushed and opened this pull
8306 // request; `status` is `Landing` because a previous call into `land`
8307 // parked or was interrupted before it reached a terminal outcome.
8308 state.status = RunStatus::Landing;
8309 state.merge = Some(MergeOutcome {
8310 mode: MergeMode::Pr,
8311 ok: true,
8312 detail: "https://example.invalid/x/y/pull/1".to_owned(),
8313 });
8314
8315 // The Landing-resume shortcut calls `run_land` directly rather than
8316 // through `merge`, which is exactly the call site that used to skip
8317 // `settle_questions` - see the fixture below.
8318 ask_test_home();
8319 let store = ask::Questions::open();
8320 let q = ask_open_question(&store, &state.id);
8321
8322 let mut runner = Runner {
8323 state,
8324 roles: ResolvedRoles {
8325 implementers: Vec::new(),
8326 judges: Vec::new(),
8327 reviewers: Vec::new(),
8328 fixer: None,
8329 conductor: conductor(),
8330 implementer_roster: Vec::new(),
8331 },
8332 sem: Arc::new(Semaphore::new(1)),
8333 pause: Pause::new(),
8334 interrupt: Pause::new(),
8335 };
8336
8337 // `execute`, not `merge` directly: the Landing-resume shortcut lives
8338 // at the top of `execute`, not inside `merge` (see `execute`'s doc)
8339 // exactly because `review_loop` would otherwise clobber the marker
8340 // first.
8341 runner.execute().await.expect("execute");
8342
8343 assert_eq!(
8344 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8345 Some("https://example.invalid/x/y/pull/1"),
8346 "reentry must not push again or open a second pull request over the \
8347 one `land` is already watching"
8348 );
8349 assert_ne!(
8350 runner.state.status,
8351 RunStatus::Landing,
8352 "land could not actually reach the fake pull request, so it must \
8353 have given up rather than left the run silently parked forever"
8354 );
8355 // `land` could not reach the fake pull request, so it gave up into
8356 // `Blocked` - still resumable, so the question must not have been
8357 // swept just because this branch now also calls `settle_questions`.
8358 assert_eq!(runner.state.status, RunStatus::Blocked);
8359 assert!(
8360 store.get(&q.id).unwrap().status.open(),
8361 "Blocked is still alive; settle_questions must have been a no-op here"
8362 );
8363 }
8364
8365 fn state_with_round(round: ReviewRound) -> RunState {
8366 let mut s = RunState::new(
8367 PathBuf::from("/repo"),
8368 "main".to_owned(),
8369 "abc1234".to_owned(),
8370 "add retries".to_owned(),
8371 Config::default(),
8372 );
8373 s.reviews = vec![round];
8374 s
8375 }
8376
8377 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
8378 crate::verdict::Finding {
8379 id: id.to_owned(),
8380 severity,
8381 file: None,
8382 line: None,
8383 title: title.to_owned(),
8384 detail: String::new(),
8385 }
8386 }
8387
8388 #[test]
8389 fn pr_body_names_open_findings_and_declined_ones() {
8390 let round = ReviewRound {
8391 round: 2,
8392 head: "deadbee".to_owned(),
8393 verified_head: None,
8394 verified_at: None,
8395 reviews: vec![ReviewRecord {
8396 attempts: 0,
8397 reviewer: 1,
8398 agent: "alpha".to_owned(),
8399 summary: String::new(),
8400 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
8401 vote: None,
8402 failed: None,
8403 duration_ms: 0,
8404 }],
8405 e2e: vec![CommandOutcome {
8406 command: "cargo test".to_owned(),
8407 code: Some(0),
8408 output_tail: String::new(),
8409 duration_ms: 0,
8410 resource_blocked: false,
8411 }],
8412 verify_retried: false,
8413 e2e_deferred: false,
8414 e2e_defer_reason: None,
8415 fix: Some(FixRecord {
8416 agent: "alpha".to_owned(),
8417 addressed: Vec::new(),
8418 rejected: vec![crate::verdict::Rejection {
8419 id: "R1-1-1".to_owned(),
8420 why: "not reachable from any caller".to_owned(),
8421 }],
8422 notes: String::new(),
8423 committed: true,
8424 failed: None,
8425 duration_ms: 0,
8426 continuation: None,
8427 }),
8428 blocking: 0,
8429 answered: 1,
8430 expected: 1,
8431 clean: false,
8432 progressed: true,
8433 vote_split: false,
8434 reconsideration: Vec::new(),
8435 verdict: None,
8436 };
8437 let state = state_with_round(round);
8438 let body = pr_message(&state, 'A').body;
8439
8440 assert!(body.contains("add retries"), "the task must still be there");
8441 assert!(body.contains("R2-1-1"), "{body}");
8442 assert!(body.contains("unused import"), "{body}");
8443 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
8444 assert!(
8445 body.contains("not reachable from any caller"),
8446 "the reason it was declined: {body}"
8447 );
8448 }
8449
8450 #[test]
8451 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
8452 let round = ReviewRound {
8453 round: 1,
8454 head: "deadbee".to_owned(),
8455 verified_head: None,
8456 verified_at: None,
8457 reviews: vec![ReviewRecord {
8458 attempts: 0,
8459 reviewer: 1,
8460 agent: "alpha".to_owned(),
8461 summary: String::new(),
8462 findings: Vec::new(),
8463 vote: None,
8464 failed: None,
8465 duration_ms: 0,
8466 }],
8467 e2e: Vec::new(),
8468 verify_retried: false,
8469 e2e_deferred: false,
8470 e2e_defer_reason: None,
8471 fix: None,
8472 blocking: 0,
8473 answered: 1,
8474 expected: 1,
8475 clean: true,
8476 progressed: false,
8477 vote_split: false,
8478 reconsideration: Vec::new(),
8479 verdict: None,
8480 };
8481 let state = state_with_round(round);
8482 let body = pr_message(&state, 'A').body;
8483 assert!(!body.contains("Open review findings"), "{body}");
8484 assert!(!body.contains("Declined"), "{body}");
8485 }
8486
8487 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
8488 let mut state = RunState::new(
8489 PathBuf::from("/repo"),
8490 "main".to_owned(),
8491 "abc1234".to_owned(),
8492 instruction.to_owned(),
8493 Config::default(),
8494 );
8495 state.candidates.push(Candidate {
8496 index: 0,
8497 label: 'A',
8498 agent: "alpha".to_owned(),
8499 branch: "magi/x/A".to_owned(),
8500 worktree: PathBuf::from("/wt"),
8501 summary: summary.to_owned(),
8502 stat: String::new(),
8503 files: 1,
8504 commits: 1,
8505 empty: false,
8506 failed: None,
8507 verified_noop: None,
8508 folded: false,
8509 duration_ms: 0,
8510 });
8511 state
8512 }
8513
8514 #[test]
8515 fn pr_message_describes_the_change_not_the_task() {
8516 let state = state_with_summary(
8517 "今回やってほしいこと: results projector を直す",
8518 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
8519 );
8520 let m = pr_message(&state, 'A');
8521 assert_eq!(m.title, "fix(web): batch the runs list reads");
8522 assert!(
8523 m.body.starts_with("## Summary\n\n- reads run.json once"),
8524 "{}",
8525 m.body
8526 );
8527 assert!(!m.body.contains("TITLE:"), "{}", m.body);
8528 let task_at = m.body.find("今回やってほしいこと").unwrap();
8529 let details_at = m.body.find("<details>").unwrap();
8530 assert!(
8531 details_at < task_at,
8532 "the task lives inside <details>: {}",
8533 m.body
8534 );
8535 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
8536 assert!(m.body.contains("magi:candidate-a"));
8537 }
8538
8539 #[test]
8540 fn pr_message_falls_back_to_the_task_without_a_title_line() {
8541 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
8542 let m = pr_message(&state, 'A');
8543 assert_eq!(m.title, "add retries");
8544 assert!(
8545 m.body.contains("## Summary\n\n- did some things"),
8546 "{}",
8547 m.body
8548 );
8549
8550 let none = RunState::new(
8551 PathBuf::from("/repo"),
8552 "main".to_owned(),
8553 "abc1234".to_owned(),
8554 "add retries".to_owned(),
8555 Config::default(),
8556 );
8557 let m = pr_message(&none, 'A');
8558 assert_eq!(m.title, "add retries");
8559 assert!(!m.body.contains("## Summary"), "{}", m.body);
8560 }
8561
8562 #[test]
8563 fn pr_message_refuses_the_candidate_commit_subject() {
8564 for bad in [
8565 "TITLE: magi: candidate A (uncommitted work)",
8566 "TITLE: chore: stuff (uncommitted work)",
8567 "TITLE: ",
8568 ] {
8569 let state = state_with_summary("add retries", bad);
8570 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
8571 }
8572 }
8573
8574 #[test]
8575 fn pr_message_bounds_a_very_long_task_and_title() {
8576 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
8577 let state = state_with_summary(&long, "- nothing");
8578 let m = pr_message(&state, 'A');
8579 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
8580 assert!(!m.title.contains('\n'));
8581
8582 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
8583 let m = pr_message(&state, 'A');
8584 assert!(m.title.starts_with("feat: "));
8585 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
8586 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
8587 }
8588
8589 #[test]
8590 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
8591 // What magi itself writes stays English under any configured language,
8592 // so a future localisation of these headings fails here. (The agents'
8593 // own text is held to English by the prompt only; magi cannot check it.)
8594 let mut state = state_with_summary(
8595 "add retries",
8596 "TITLE: fix(web): batch reads\n- reads run.json once",
8597 );
8598 state.config.graph.language = "ja".to_owned();
8599 let m = pr_message(&state, 'A');
8600 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
8601
8602 // The task is the operator's own text: it goes in untouched, and the
8603 // fallback title (no summary) may be in its language too.
8604 let task = "今回やってほしいこと: results projector を直す";
8605 let mut state = state_with_summary(task, "- no title line");
8606 state.config.graph.language = "ja".to_owned();
8607 let m = pr_message(&state, 'A');
8608 assert_eq!(m.title, task);
8609 assert!(
8610 m.body.contains(&format!(
8611 "<summary>Original task</summary>\n\n{task}\n\n</details>"
8612 )),
8613 "{}",
8614 m.body
8615 );
8616 }
8617
8618 #[test]
8619 fn pr_message_survives_a_task_that_closes_details() {
8620 let state = state_with_summary("a </details> b", "TITLE: fix: x");
8621 let m = pr_message(&state, 'A');
8622 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
8623 }
8624
8625 #[test]
8626 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
8627 let cmd = manual_merge_command(
8628 MergeStyle::Squash,
8629 Path::new("/repo"),
8630 "b",
8631 "fix: \"quoted\" $(x) `y`\n\nbody",
8632 );
8633 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
8634 }
8635
8636 #[test]
8637 fn manual_merge_command_matches_the_configured_style() {
8638 let repo = Path::new("/repo");
8639 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
8640
8641 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
8642 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
8643
8644 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
8645 assert_eq!(
8646 squash,
8647 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
8648 \"Merge magi run 0832 (candidate A)\""
8649 );
8650
8651 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
8652 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
8653 }
8654
8655 #[test]
8656 fn a_nudge_gets_a_quarter_of_the_budget() {
8657 // The judge and implement budgets magi ships with.
8658 assert_eq!(retry_budget(secs(1200), true), secs(300));
8659 assert_eq!(retry_budget(secs(3600), true), secs(900));
8660 }
8661
8662 #[test]
8663 fn a_resent_prompt_keeps_the_whole_budget() {
8664 // The seat kept no context, so the retry is the original job again and
8665 // shortening it would only guarantee a second failure.
8666 assert_eq!(retry_budget(secs(1200), false), secs(1200));
8667 assert_eq!(retry_budget(secs(60), false), secs(60));
8668 }
8669
8670 #[test]
8671 fn the_floor_never_exceeds_the_original_budget() {
8672 // A short configured timeout must not be *raised* by the floor: the
8673 // operator asked for a bound, and a retry may not outlast the attempt
8674 // it is retrying.
8675 assert_eq!(retry_budget(secs(60), true), secs(60));
8676 assert_eq!(retry_budget(secs(480), true), secs(120));
8677 assert_eq!(retry_budget(secs(0), true), secs(0));
8678 }
8679
8680 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
8681 agent::CommandEvidence {
8682 id: "item1".to_owned(),
8683 description: "cargo test".to_owned(),
8684 exit_code,
8685 result_summary: String::new(),
8686 source: "codex".to_owned(),
8687 }
8688 }
8689
8690 #[test]
8691 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
8692 // No evidence is not the same fact as unconfirmed evidence: a
8693 // backend with no adapter, or a reply that ran no commands at all,
8694 // must not be misread as carrying a dangling job.
8695 assert!(!has_unconfirmed_command(&[]));
8696 }
8697
8698 #[test]
8699 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
8700 // Deliberately not a check on the exit code's *value*: a fixer
8701 // legitimately runs something that fails mid-iteration before it
8702 // succeeds, and that must never by itself reopen a valid report.
8703 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
8704 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
8705 assert!(!has_unconfirmed_command(&[
8706 evidence(Some(0)),
8707 evidence(Some(101))
8708 ]));
8709 }
8710
8711 #[test]
8712 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
8713 assert!(has_unconfirmed_command(&[
8714 evidence(Some(0)),
8715 evidence(None)
8716 ]));
8717 }
8718
8719 #[test]
8720 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
8721 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
8722 assert_eq!(
8723 verified_noop_claim(true, &[], text).as_deref(),
8724 Some("already fixed by b32cfc4, on main.")
8725 );
8726 }
8727
8728 #[test]
8729 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
8730 // A timeout or a bad exit code reads as the ordinary loss it is,
8731 // whatever the reply's own prose claims.
8732 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
8733 assert!(verified_noop_claim(false, &[], text).is_none());
8734 }
8735
8736 #[test]
8737 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
8738 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
8739 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
8740 // A confirmed command alongside the marker is fine.
8741 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
8742 }
8743
8744 #[test]
8745 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
8746 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
8747 }
8748
8749 /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
8750 /// pair, in order, labelled A, B, C, ...
8751 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
8752 runner.state.candidates = shape
8753 .iter()
8754 .enumerate()
8755 .map(|(i, &(empty, verified))| Candidate {
8756 index: i,
8757 label: (b'A' + i as u8) as char,
8758 agent: "sonnet".to_owned(),
8759 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
8760 worktree: PathBuf::from(format!("/wt/{i}")),
8761 summary: String::new(),
8762 stat: String::new(),
8763 files: 0,
8764 commits: 0,
8765 empty,
8766 failed: None,
8767 verified_noop: verified.map(str::to_owned),
8768 duration_ms: 0,
8769 folded: false,
8770 })
8771 .collect();
8772 }
8773
8774 #[test]
8775 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
8776 ask_test_home();
8777 let mut runner = runner_at(RunStatus::Implementing);
8778 set_candidates(
8779 &mut runner,
8780 &[
8781 (true, Some("already on main at b32cfc4")),
8782 (true, Some("same fix, see the existing test")),
8783 ],
8784 );
8785
8786 runner
8787 .after_implement()
8788 .expect("a verified no-op is not an error");
8789
8790 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
8791 }
8792
8793 #[test]
8794 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
8795 ask_test_home();
8796 let mut runner = runner_at(RunStatus::Implementing);
8797 // Candidate A declares a verified no-op; candidate B simply wrote
8798 // nothing and said nothing about why. One candidate's claim is not
8799 // the whole run's agreement.
8800 set_candidates(
8801 &mut runner,
8802 &[(true, Some("already on main at b32cfc4")), (true, None)],
8803 );
8804
8805 let err = runner
8806 .after_implement()
8807 .expect_err("an unverified empty candidate must still fail the run");
8808
8809 assert!(
8810 err.to_string().contains("no candidate produced a change"),
8811 "{err}"
8812 );
8813 assert_eq!(runner.state.status, RunStatus::Failed);
8814 }
8815
8816 #[test]
8817 fn after_implement_still_fails_an_ordinary_all_empty_run() {
8818 ask_test_home();
8819 let mut runner = runner_at(RunStatus::Implementing);
8820 set_candidates(&mut runner, &[(true, None), (true, None)]);
8821
8822 let err = runner
8823 .after_implement()
8824 .expect_err("no candidate declared anything; this is an ordinary failure");
8825
8826 assert!(
8827 err.to_string().contains("no candidate produced a change"),
8828 "{err}"
8829 );
8830 assert_eq!(runner.state.status, RunStatus::Failed);
8831 }
8832}