//! The competition graph.
//!
//! ```text
//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
//! │ │
//! └──── unanimous ───────────┤
//! ▼
//! merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
//! ```
//!
//! Every node persists before the next one starts, so a run can be resumed
//! after a crash, a rate limit, or a reboot without re-spending the work that
//! already landed.
//!
//! The design decision that matters most is *where the facilitator lives*.
//! There is no moderator agent: magi assigns the labels, decides the
//! presentation order, relays the transcript, and collects the final votes
//! one-to-one. A moderator that never learns an author cannot leak one.
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use anyhow::{Context as _, Result, bail};
use jiff::Timestamp;
use tokio::sync::Semaphore;
use crate::advise;
use crate::agent::{self, AgentOutput, Invocation, SeatState};
use crate::ask;
use crate::blind;
use crate::bump;
use crate::config::{
AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
ResolvedRoles,
};
use crate::fixer;
use crate::git;
use crate::graft;
use crate::land;
use crate::proc::Quiet as _;
use crate::prompt::{
self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
};
use crate::queue;
use crate::refs;
use crate::run::{
BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
Tally, VoteRecord, tail, write_artifact,
};
use crate::verdict::{
self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
ReviewVote, Severity,
};
use crate::worktree_setup;
/// How much verification output is kept and fed back to the fixer.
const OUTPUT_TAIL: usize = 8_000;
/// Bytes of a failing command's output kept in an event, so the reason a run
/// stopped is readable from the report without opening `run.json`.
const EVENT_OUTPUT_TAIL: usize = 2_000;
/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
/// command's pid before releasing the build cache's lease.
const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
/// command's pid to actually exit before giving up and releasing anyway.
///
/// A timeout means the process was asked to die (`kill_on_drop`,
/// `start_kill`), not that it already has — on Windows in particular that can
/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
/// the instant the command returns would let the very next acquirer (this
/// run's own next round, another run's verification, the janitor's prune)
/// start touching the same directory while it might still be writing to it,
/// so this polls the actual pid — real confirmation, not a fixed guess —
/// until it is gone or this ceiling is reached. It is still not full
/// process-tree reaping: a grandchild the timed-out process spawned and that
/// outlives it independently is invisible to a pid check, and continuing to
/// observe and collect *that* stays a different piece of work with its own
/// owner. Set generously because the common case returns early the moment
/// the pid is confirmed gone, not because every timeout pays this in full.
const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
/// Consecutive review rounds with no tree progress (see
/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
/// instead of spending the rest of the round budget.
///
/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
/// legitimately finds nothing left to change (its previous round's fix already
/// covered it, and this round's reviewers re-raised only nits) looks the same
/// as one that is spinning, for exactly one round. Two in a row is where the
/// two stop being distinguishable, and a review round on this workload has
/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
/// third attempt at a tree that has not moved twice running is pure cost.
/// This does not touch `review_rounds` itself, which stays the operator's
/// call.
pub(crate) const STAGNANT_LIMIT: usize = 2;
/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
/// a base that moved before giving up and leaving the run `Blocked` for a
/// person.
///
/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
/// that keeps moving faster than a run can catch it is not something more
/// rebasing fixes, it is a person's call. Not the same *number as*
/// `land_rounds` - this budget is spent before a pull request exists, land's
/// after - but bounded for the identical reason, so it uses the same
/// default. Counted across both call sites in [`Runner::finish_after_tally`]
/// (once before review, once before the gate), because either one finding
/// the base still moving is the same signal.
const BASE_SYNC_ROUNDS: usize = 4;
/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
/// reply held no [`FixReport`].
///
/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
/// "I'll pause here until the `cargo make check` background run reports
/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
/// No `FixReport` was ever collected from that seat, and the run moved on to
/// the next review round regardless.
///
/// Bounded independently of `review_rounds` and `graph.retries`: this
/// recovers one seat's missing report mid-round, not a new round of review or
/// an ordinary parse retry, and must not itself become the unbounded wait the
/// rest of this module exists to avoid.
const MAX_FIX_CONTINUATIONS: usize = 2;
/// One queued agent invocation.
///
/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
/// CLI hung up on its own stream is asked again from the same job rather than
/// rebuilt from scratch. See [`Runner::resume_undelivered`].
#[derive(Clone)]
struct SeatJob {
spec: AgentSpec,
seat: SeatState,
cwd: PathBuf,
prompt: String,
timeout: Duration,
allow_write: bool,
sessions: bool,
artifacts: PathBuf,
stem: String,
/// The prompt for a seat that has been handed to another roster agent
/// (a fresh session): everything the original seat would have
/// remembered. `None` when `prompt` already carries it, as the first
/// ranking and the implement prompt do. Never a resume-style prompt.
handover: Option<String>,
}
/// How the graph reads one agent invocation.
///
/// Quota is split out from an ordinary failure on purpose: a rate-limited call
/// is known to fail again if retried now, so the retry loop must not spend an
/// attempt on it. `Dropped` is split out for the opposite reason: unlike
/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
/// error JSON, never the agent's answer — a caller that matched only
/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
/// left to read that JSON as if it were usable output. `resume_undelivered`
/// is the only caller that acts on it; everywhere else it is reported like an
/// ordinary failure.
enum AgentOutcome {
/// A usable output.
Ok(AgentOutput),
/// The CLI ran out of quota / rate limit. Retrying now is pointless.
Quota(AgentOutput),
/// The CLI hung up on its own stream after billed work. See
/// [`agent::AgentOutput::work_undelivered`].
Dropped(AgentOutput),
/// Any other failure: a timeout, a bad exit code, an empty reply.
Failed(String),
/// The agent was never spawned: a shared resource it needs (the build
/// cache's lease) stayed busy past `[graph] timeout_resource_wait`, or the
/// wait was abandoned because the run is parking. Not a failure of the
/// agent: it must not advance a fallback chain, enter seat history, cost
/// quota or spend an attempt. [`RunState::resource_block`] says why.
ResourceBlocked(String),
}
/// A request to park the run at its next node boundary.
///
/// Cloning is how the request travels: the loop keeps one handle and hands a
/// clone to each [`Runner`], and every clone points at the same flag. There
/// is no channel because there is nothing to send - the only message is
/// "park", it is idempotent, and a flag cannot be missed by a receiver that
/// was not listening yet.
///
/// The boundary is what makes this cheap. Every node writes the run's state
/// before the next one starts, and every node skips what is already recorded:
/// `prep` returns early once candidates exist, `implement` asks only the seats
/// with nothing on disk, `judge` returns early once judgements exist. So a
/// parked run resumes into exactly the node it stopped before, and no agent
/// work is thrown away. Killing the process mid-node, by contrast, loses
/// whatever the seats in flight had not yet written - which for an implement
/// wave is an hour of paid work.
///
/// A [`Runner`] watches two independent handles of this type - see
/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
/// clone covering the whole daemon's lifetime and is never asked to un-park,
/// which is correct exactly because nothing is dispatched after it fires.
/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
/// that parks for an interrupted task must go on to run other tasks
/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
/// reusing the daemon-wide one.
#[derive(Debug, Clone, Default)]
pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
impl Pause {
/// A pause nobody has asked for yet.
#[must_use]
pub fn new() -> Self {
Self::default()
}
/// Ask the run to park at its next node boundary. Idempotent.
pub fn park(&self) {
self.0.store(true, Ordering::SeqCst);
}
/// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
/// fold into the run's own `park` event - so an operator reading the run
/// later knows this was a deliberate interrupt rather than a shutdown or
/// a binary swap. The first reason recorded wins; a park already in
/// flight is not relabelled by a second, unrelated request.
pub fn park_because(&self, reason: impl Into<String>) {
self.park_for(ParkCause::Interrupt {
reason: reason.into(),
});
}
/// [`Pause::park`], recording the structured cause. The first cause
/// recorded wins, like [`Pause::park_because`].
pub fn park_for(&self, cause: ParkCause) {
let mut reason_guard = self
.1
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if reason_guard.is_none() {
*reason_guard = Some(cause);
}
drop(reason_guard);
self.park();
}
/// Has a park been asked for?
#[must_use]
pub fn parked(&self) -> bool {
self.0.load(Ordering::SeqCst)
}
/// Why the park was asked for, when the caller used [`Pause::park_because`].
#[must_use]
pub fn reason(&self) -> Option<String> {
self.cause().map(|c| c.label())
}
/// The structured cause, when one was recorded.
#[must_use]
pub fn cause(&self) -> Option<ParkCause> {
self.1
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
}
/// Drives one run.
pub struct Runner {
/// Run state; public so the CLI can report on it.
pub state: RunState,
roles: ResolvedRoles,
sem: Arc<Semaphore>,
/// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
/// run parked at its next node boundary. See [`Pause`]'s own doc for why
/// this is never the same handle as `interrupt`.
pause: Pause,
/// Set when `magi serve`'s interrupt scheduler wants this specific run
/// parked at its next node boundary, to let a task marked
/// [`crate::queue::Task::interrupt`] run alone before this one carries
/// on. Unlike `pause`, a fresh, unshared handle per run - see
/// [`Runner::watch_interrupt`].
interrupt: Pause,
}
/// Where the branch's own commits start: its merge base with the base branch
/// as the remote has it now, else with the recorded `base_commit`. A branch
/// rebased onto a base that moved past `base_commit` would otherwise count
/// the base's commits as its own under `base_commit..branch`.
async fn review_base(
repo: &Path,
remote: &str,
base_branch: &str,
base_commit: &str,
branch: &str,
) -> String {
review_base_checked(repo, remote, base_branch, base_commit, branch)
.await
.0
}
/// [`review_base`] plus whether the base was read from a freshly fetched
/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
/// never older than `base_commit`'s view of the base), but the answer is then
/// not trusted to rewrite a pull request's title.
async fn review_base_checked(
repo: &Path,
remote: &str,
base_branch: &str,
base_commit: &str,
branch: &str,
) -> (String, bool) {
let tracking = format!("{remote}/{base_branch}");
let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
if git::rev_exists(repo, &tracking).await
&& let Ok(mb) = git::merge_base(repo, &tracking, branch).await
&& !mb.is_empty()
{
return (mb, fresh);
}
let mb = git::merge_base(repo, base_commit, branch)
.await
.ok()
.filter(|mb| !mb.is_empty())
.unwrap_or_else(|| base_commit.to_owned());
(mb, false)
}
/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
/// when git cannot say or finds nothing: a stale list is better than a wrong
/// or empty one.
pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
if !is_review_run(state) {
return;
}
let base = review_base(
&state.repo,
&state.config.merge.remote,
&state.base_branch,
&state.base_commit,
branch,
)
.await;
if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
&& !subjects.is_empty()
&& state.reviewed_commits.as_ref() != Some(&subjects)
{
state.reviewed_commits = Some(subjects);
state.save().ok();
}
}
/// Subjects of the base's commits between the recorded start and the branch's
/// merge base: what a stale `base_commit..branch` would have mistaken for the
/// branch's own work.
async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
let (base, trusted) = review_base_checked(
&state.repo,
&state.config.merge.remote,
&state.base_branch,
&state.base_commit,
branch,
)
.await;
if !trusted {
return None;
}
git::subjects(&state.repo, &state.base_commit, &base)
.await
.ok()
}
/// May an adopted pull request's title be replaced with `computed`? Only when
/// it is empty, magi's own shape, or a base commit's subject that leaked in;
/// a title a person wrote stays. Never when `computed` is itself a leak.
pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
if is_leak(computed) {
return false;
}
let cur = current.trim();
cur.is_empty()
|| cur.starts_with(REVIEW_PROMPT_OPENING)
|| cur.starts_with("chore: land candidate")
|| cur.starts_with("magi: candidate")
|| is_leak(cur)
}
/// The commit a run branches from: the base branch as the remote has it.
///
/// Two failures this replaces. A run used to branch off `HEAD` and so refused
/// to start on a dirty tree, which made `magi serve` decline every task for as
/// long as the operator had work in progress - most of the time. Branching off
/// the *local* base branch fixed that and introduced a worse one: `land` merges
/// the winner on GitHub, nothing updates the local ref, and the next run
/// branches off a base missing everything the previous runs landed. Two tasks
/// in a row from a phone would have had the second silently re-implementing
/// against stale code and opening a pull request that reverted the first.
///
/// Only refs move here - no checkout, no local branch, no merge - so it is safe
/// with uncommitted work in the tree. A machine with no network still starts:
/// the fetch may fail and the local tip is used with a warning, because
/// refusing to run offline is a worse failure than running against a base the
/// operator can see for themselves.
///
/// One function, called by both entry points. Two answers to "where does a run
/// branch from" is the kind of drift nobody notices until a diff is wrong.
/// Bring the local `branch` in line with `<remote>/<branch>` before a review
/// checks it out.
///
/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
/// anything other than plain `git push` from this checkout (a jj colocated
/// workspace, another clone) moves only the remote-tracking ref - so the local
/// one can be a stale placeholder. It moves only when local is behind the remote or is an
/// empty placeholder that diverged from it; unpushed local work is kept, and a real
/// divergence is refused rather than guessed at.
async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
let tracking = format!("{remote}/{branch}");
let fetched = git::fetch(repo, remote, branch).await;
let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
let local_exists = git::branch_exists(repo, branch).await?;
if !fresh {
if !local_exists {
bail!("no branch `{branch}` in {} or on {remote}", repo.display());
}
tracing::warn!(
"could not read {tracking}; reviewing the local `{branch}`, which may be stale"
);
return Ok(());
}
let remote_sha = git::rev_parse(repo, &tracking).await?;
if !local_exists {
git::git(repo, &["branch", branch, &tracking]).await?;
return Ok(());
}
let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
return Ok(());
}
if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
// Diverged. `reconcile` settles it only when it can prove nothing is
// lost: a local tip that is the remote's change rebased is pushed over
// it (lease pinned to the tip read here), a tip whose every extra
// commit is empty is a placeholder the remote's work replaced, and
// anything else is two different changes - a question for a person.
match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
.await?
{
crate::reconcile::Reconciliation::Pushed => {
tracing::warn!(
"local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
short(&local_sha),
short(&remote_sha)
);
return Ok(());
}
crate::reconcile::Reconciliation::Placeholder => {}
crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
}
}
let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
if !out.ok() {
bail!(
"local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
short(&local_sha),
short(&remote_sha),
out.stderr
);
}
tracing::warn!(
"local `{branch}` was stale: fast-forwarded {} -> {}",
short(&local_sha),
short(&remote_sha)
);
Ok(())
}
async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
let tracking = format!("{remote}/{base_branch}");
let fetched = git::fetch(repo, remote, base_branch).await;
if let Ok(out) = &fetched
&& out.ok()
&& git::rev_exists(repo, &tracking).await
{
return git::rev_parse(repo, &tracking).await;
}
let why = match &fetched {
Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
Ok(_) => format!("{remote} has no {base_branch}"),
Err(e) => e.to_string(),
};
// No fallback to the local branch: it may be behind, and branching off an
// old commit is the stale-checkout bug this check exists to prevent.
bail!(
"cannot read {tracking} ({why}); refusing to branch off the local \
`{base_branch}`, which may be behind. Fix the remote, or set [merge] \
base / remote in magi.toml"
)
}
/// Exclusive claim on one run's `magi fix` step, released on drop — including
/// on an early return or a panic.
///
/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
/// manual `magi fix` invocations against the same run are otherwise
/// invisible to each other and would race to remove and recreate the same
/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
/// which is only ever reclaimed later, out of band, by
/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
/// `magi fix` invocation is not necessarily running under either of those, so
/// nothing would ever sweep a lock a killed or crashed process left behind.
/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
/// lease use: an unreadable or unparsable pid, or a liveness query the
/// platform cannot answer, reads as alive and the lock is left in place.
struct FixClaim {
path: PathBuf,
}
impl FixClaim {
fn acquire(dir: &Path) -> Result<Self> {
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
let path = dir.join("fix.lock");
match Self::create(&path) {
Ok(claim) => Ok(claim),
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
if Self::reclaim_if_dead(&path) {
Self::create(&path).with_context(|| format!("lock {}", path.display()))
} else {
bail!(
"another `magi fix` is already running for this run ({} exists)",
path.display()
)
}
}
Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
}
}
fn create(path: &Path) -> std::io::Result<Self> {
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(path)?;
use std::io::Write as _;
// Read back by `reclaim_if_dead` on a later, stuck invocation.
writeln!(f, "{}", std::process::id())?;
Ok(Self {
path: path.to_owned(),
})
}
/// True if the lock named a process confirmed dead, in which case it was
/// also removed. Never true on an unreadable file, an unparsable pid, or
/// a liveness query the platform cannot answer — see this type's own doc.
fn reclaim_if_dead(path: &Path) -> bool {
let dead = std::fs::read_to_string(path)
.ok()
.and_then(|body| body.trim().parse::<u32>().ok())
.is_some_and(|pid| !crate::proc::pid_alive(pid));
dead && std::fs::remove_file(path).is_ok()
}
}
impl Drop for FixClaim {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
impl Runner {
/// Start a fresh run against `repo`.
pub async fn start(
repo: &Path,
instruction: String,
config: Config,
origin: Origin,
) -> Result<Self> {
Self::start_naming(repo, instruction, "", config, origin).await
}
/// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
/// searched for branch and commit references along with the instruction,
/// since a task may name the work it is about only in its title.
pub async fn start_naming(
repo: &Path,
instruction: String,
also_scan: &str,
config: Config,
origin: Origin,
) -> Result<Self> {
let repo = git::toplevel(repo).await?;
let missing = agent::missing_programs(&config.agents);
if !missing.is_empty() {
bail!(
"these agent programs are not on PATH: {}. Fix the roster in \
magi.toml or install them.",
missing.join(", ")
);
}
let base_branch =
git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
.await?;
let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
let roles = config.resolve_roles()?;
let max_parallel = config.graph.max_parallel.max(1);
// A task that points at work already in the repository starts from
// it; what the repository says about each reference is recorded.
let seeds = refs::resolve(
&repo,
&base_commit,
&config.merge.remote,
&format!("{also_scan}\n{instruction}"),
)
.await;
refs::plan(&repo, &seeds).await?;
let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
// Recorded before the first save, so a crash right after minting
// cannot leave a run with no origin. Legibility only: nothing reads it
// to decide anything.
state.origin = Some(origin);
for seed in &seeds {
state.event(
"seed",
refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
);
}
state.seeds = seeds;
state.event("start", format!("run {} created", state.id));
state.save()?;
Ok(Self {
state,
roles,
sem: Arc::new(Semaphore::new(max_parallel)),
pause: Pause::new(),
interrupt: Pause::new(),
})
}
/// Open a review-only run against work that already exists on `branch`.
///
/// The expensive half of the graph is the implement wave — measured at
/// 111 and 134 internal tool-loop turns on this repository, against a
/// handful for a judge or a reviewer. The cheap half is worth running on
/// hand-written work too, and there was no way to reach it.
///
/// No new state and no schema change are needed: a run with **one** viable
/// candidate and a tally already decided degrades `execute` to exactly
/// review → gate → merge, because `judge` skips a single-candidate field,
/// `deliberate` has fewer than two first choices to reconcile, `vote`
/// returns early, `tally` is already present and `fold_losers` has no
/// losers. Resuming such a run therefore does the right thing as well.
pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
Self::review_taking_over(repo, branch, config, None, origin).await
}
/// [`Runner::review`] for a queued task's retry: when an earlier attempt
/// at the same task still has `branch` checked out, its worktree is
/// released first if that is safe (see [`crate::handover`]), and the
/// review refuses with the reason if it is not. `None` is a hand-run
/// review: it has no earlier attempts, so only a worktree of a dead run
/// magi recorded itself can be released.
pub async fn review_taking_over(
repo: &Path,
branch: &str,
config: Config,
takeover: Option<crate::handover::Takeover>,
origin: Origin,
) -> Result<Self> {
let repo = git::toplevel(repo).await?;
let missing = agent::missing_programs(&config.agents);
if !missing.is_empty() {
bail!(
"these agent programs are not on PATH: {}. Fix the roster in \
magi.toml or install them.",
missing.join(", ")
);
}
let base_branch =
git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
.await?;
if base_branch == branch {
bail!("`{branch}` is the base branch; there is nothing to review against");
}
let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
let roles = config.resolve_roles()?;
let max_parallel = config.graph.max_parallel.max(1);
let mut state = RunState::new(
repo.clone(),
base_branch,
base_commit.clone(),
String::new(),
config,
);
state.origin = Some(origin);
// Released before anything else touches the branch: a stale local
// branch is moved with `git branch -f`, which git refuses while an
// earlier attempt's worktree still has it checked out. Everything
// after this point that can fail puts the old run back.
// A hand-run review has no task, hence no earlier attempts, but a
// worktree of a dead run magi made may still be released.
let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
earlier: Vec::new(),
home: crate::run::home(),
choice: None,
task_instruction: None,
});
let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
if let Some(released) = &released {
state.event(
"release",
format!(
"took `{branch}` over from run {}: its worktree was released: {}",
crate::run::short_of(&released.old_id),
released.audit
),
);
}
// The owner's answer to an earlier divergence question is applied
// here: after the release (git will not move a checked-out branch)
// and before the sync that would otherwise ask again.
if let Some(choice) = takeover.choice.as_ref()
&& let Err(e) =
crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
.await
{
if let Some(released) = &released {
released.restore(&repo, branch).await;
}
return Err(e.context("applying the owner's answer about the diverged branch"));
}
// An earlier attempt of this task that published the task head, and
// whose work is the branch under review, hands that head over: the
// review run updates the same pull request. Without one (a legacy
// `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
// own head and its own open pull request is adopted as before.
for id in takeover.earlier.iter().rev() {
if let Ok(old) = RunState::load_under(id, &takeover.home)
&& let Some(head) = old.pr_head.clone()
&& (head == branch || old.winner().is_some_and(|w| w.branch == branch))
{
state.pr_head = Some(head);
break;
}
}
let opened = Self::open_review(
&repo,
branch,
state,
roles,
max_parallel,
base_commit,
&takeover,
)
.await;
if opened.is_err()
&& let Some(released) = &released
{
released.restore(&repo, branch).await;
}
opened
}
/// What the newest earlier attempt that reviewed `branch` leaves for this
/// run (see [`crate::recovery`]). `None` for a hand-run review, which has
/// no earlier attempts. Reads the earlier record only: ownership of its
/// worktree and branch is [`crate::handover`]'s business.
async fn inherit(
repo: &Path,
branch: &str,
base_commit: &str,
takeover: &crate::handover::Takeover,
) -> Option<crate::recovery::RecoveryRecord> {
use crate::recovery::{Relation, assess};
let newest = takeover.earlier.last()?;
let mut chosen = None;
for id in takeover.earlier.iter().rev() {
if let Ok(old) = RunState::load_under(id, &takeover.home)
&& !old.reviews.is_empty()
&& (old.winner().is_some_and(|w| w.branch == branch)
|| old.pr_head.as_deref() == Some(branch))
{
chosen = Some(old);
break;
}
}
let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
.await
.unwrap_or_default();
let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
Some(last) if last.head == tip => Relation::Same,
Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
_ => Relation::Diverged,
};
let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
Some(assess(from, chosen.as_ref(), base_commit, relation))
}
/// The half of [`Runner::review_taking_over`] that can fail after an
/// earlier attempt's worktree was released.
async fn open_review(
repo: &Path,
branch: &str,
mut state: RunState,
roles: ResolvedRoles,
max_parallel: usize,
base_commit: String,
takeover: &crate::handover::Takeover,
) -> Result<Self> {
sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
// The commit subjects are the closest thing to a task statement that
// existing work carries, and the reviewers are told as much.
let start = review_base(
repo,
&state.config.merge.remote,
&state.base_branch,
&base_commit,
branch,
)
.await;
let log = git::log_oneline(repo, &start, branch)
.await
.unwrap_or_default();
let log_text = if log.trim().is_empty() {
"(no commit messages)"
} else {
log.trim()
};
let mut instruction = format!(
"Review the work already on branch `{branch}`. There is no task \
statement: what the change claims to do is whatever its commits \
say.\n\n{log_text}"
);
// A retry of a task whose earlier attempt left reviews behind carries
// that attempt's evidence, and the task's own statement with it: the
// completion criteria are the task's, not whatever the commits claim.
if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
if let Some(task) = takeover
.task_instruction
.as_deref()
.filter(|t| !t.trim().is_empty())
&& rec.reviewed_head.is_some()
{
instruction = format!(
"{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
task.trim()
);
rec.criteria_inherited = true;
}
state.event("recovery", rec.summary());
state.recovery = Some(rec);
}
state.instruction = instruction;
state.reviewed_commits = Some(
git::subjects(repo, &start, branch)
.await
.unwrap_or_default(),
);
// An attached worktree, so the fixer's commits land on the branch under
// review rather than on a detached head nobody will look at again.
let worktree = state.worktree_root().join("under-review");
if let Some(parent) = worktree.parent() {
tokio::fs::create_dir_all(parent).await.ok();
}
let path = worktree.to_string_lossy().to_string();
git::git(repo, &["worktree", "add", &path, branch])
.await
.with_context(|| {
format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
})?;
let setup = match ensure_setup_config(&mut state, repo).await {
Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
Err(e) => Err(e),
};
if let Err(e) = setup {
git::worktree_remove(repo, &worktree).await.ok();
return Err(e);
}
let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
.await
.unwrap_or(0);
if commits == 0 {
git::worktree_remove(repo, &worktree).await.ok();
bail!("`{branch}` has no commits beyond {}", short(&base_commit));
}
let files = git::changed_files(&worktree, &base_commit, "HEAD")
.await
.map(|f| f.len())
.unwrap_or(0);
if files == 0
&& let (Ok(head_tree), Ok(base_tree)) = (
git::tree_of(&worktree, "HEAD").await,
git::tree_of(&worktree, &base_commit).await,
)
&& head_tree == base_tree
{
let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
git::worktree_remove(repo, &worktree).await.ok();
bail!(
"`{branch}` at {} has a tree identical to base {}; this usually means \
the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
against `{}/{branch}`) rather than an empty change",
short(&head),
short(&base_commit),
state.config.merge.remote
);
}
let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
.await
.unwrap_or_default();
state.candidates.push(Candidate {
index: 0,
label: 'A',
// Not an agent id on purpose: nothing in the roster wrote this, and
// the stats tables must not credit anyone with a win for it.
agent: EXISTING_BRANCH.to_owned(),
branch: branch.to_owned(),
worktree,
summary: String::new(),
stat,
files,
commits,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
});
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 0)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 0,
unanimous_initial: false,
deliberated: false,
changed_votes: 0,
unanimous_final: false,
tie_break: None,
// No panel sat, so no quorum applies. Zero judges is the correct
// number for work that never competed, and must not be reported as
// a collapsed panel.
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("review-only run: nothing competed".to_owned()),
});
state.status = RunStatus::Reviewing;
state.event(
"start",
format!(
"review-only run {} on `{branch}` ({files} files, {commits} commits)",
state.id
),
);
state.save()?;
Ok(Self {
state,
roles,
sem: Arc::new(Semaphore::new(max_parallel)),
pause: Pause::new(),
interrupt: Pause::new(),
})
}
/// Reopen an existing run.
pub fn resume(id: &str) -> Result<Self> {
let state = RunState::load(id)?;
if let Some(to) = &state.released_to {
bail!(
"run {} cannot be resumed: its worktree was released to run {}",
state.short(),
crate::run::short_of(to)
);
}
let roles = state.config.resolve_roles()?;
let max_parallel = state.config.graph.max_parallel.max(1);
Ok(Self {
state,
roles,
sem: Arc::new(Semaphore::new(max_parallel)),
pause: Pause::new(),
interrupt: Pause::new(),
})
}
/// Walk the graph to a terminal state, skipping nodes already recorded.
///
/// Every way a run is driven - the queue loop, `magi run`, a resume from
/// the phone - ends here, so this is the one place a run that ended
/// Blocked / Stalled / Failed, or died with an error, is announced to the
/// notification centre. Best-effort: see [`crate::notices::raise`].
pub async fn execute(&mut self) -> Result<()> {
let result = self.execute_graph().await;
self.mark_driver_exited();
let ended = if result.is_err() {
Some(crate::notices::run_stopped(&self.state.id, &self.state))
} else {
crate::notices::run_ended(&self.state)
};
if let Some(notice) = ended {
crate::notices::raise(notice);
}
result
}
/// Record that this process no longer drives the run, so its pid (a
/// daemon's outlives the run) is not read as a live driver.
///
/// Written onto the record as it is on disk, never this copy: another
/// process may have resumed the run (recording its own pid and clearing
/// the flag) or released its worktree since this copy was read, and
/// saving over that would mark a running driver dead. Only a record still
/// naming this process as the driver is touched.
fn mark_driver_exited(&mut self) {
self.state.driver_exited = true;
let pid = std::process::id();
let Ok(_lock) = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) else {
return;
};
let Ok(mut disk) = RunState::load(&self.state.id) else {
return;
};
if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
return;
}
disk.driver_exited = true;
if let Err(e) = disk.save() {
tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
}
}
/// Validate ownership and persist the driver claim in one critical section.
fn claim_driver(&mut self) -> Result<()> {
let _lock = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
let disk = RunState::load(&self.state.id)?;
crate::pr_head::ensure_owner(&disk)?;
// Moving again, so it is no longer parked. Set before the walk rather
// than in `resume`, so every way of re-entering the graph clears it
// and a card cannot claim a run is waiting to be resumed while the
// agents are already working.
self.state.parked = false;
self.state.resource_wait = None;
self.state.resource_block = None;
// Any seat this state still lists as answering belongs to whatever
// process last drove this run — this one included, if it crashed
// mid-wave. Cleared and flushed immediately, before anything else
// runs, so a resume can never show a seat as live when nothing is
// asking it anything yet; the node that actually dispatches the next
// wave repopulates it.
self.state.clear_active();
// Recorded in the same spot, and flushed together with the clear
// above: this is the pid a reader checks (`RunState::liveness`) when
// no daemon claim exists to answer "is a process still driving this
// run" — a plain `magi run` / `magi review` typed into a terminal
// claims nothing there. Always overwritten, never only-if-absent, so
// a resumed run's stale pid from a previous, possibly-dead process
// can never survive into this one's own report. Unlike
// `clear_active`, this changes on every single `execute()` call, so
// the save below is now unconditional rather than only-if-cleared.
//
// `driver_started_at` is recorded in the same breath, from this same
// pid, so `liveness` can tell a live pid that is genuinely still us
// apart from one the OS has since handed to an unrelated process —
// see that field's own doc for why the pid alone is not enough.
// A resume that raced a takeover: the record on disk says the worktree
// was handed to a later run after this copy was read. Saving over it
// would erase that and drive a run with nothing to run in.
if let Some(to) = &disk.released_to {
bail!(
"run {} cannot continue: its worktree was released to run {}",
self.state.short(),
crate::run::short_of(to)
);
}
let pid = std::process::id();
self.state.driver_pid = Some(pid);
self.state.driver_started_at = crate::proc::process_started_at(pid);
self.state.driver_exited = false;
self.state.begin_pass();
self.state.save()?;
Ok(())
}
async fn execute_graph(&mut self) -> Result<()> {
self.claim_driver()?;
// Publication already passed review and gate. Resume only that node:
// syncing to a newer base here could change the verified commit.
if self.state.pr_publication_pending {
self.merge().await?;
if self.resource_halt()? {
return Ok(());
}
self.state.save()?;
self.settle_questions();
return Ok(());
}
// A run that already lost its quorum never resumes into the verdict
// machinery: `deliberate` and `vote` would otherwise clobber the
// stalled marker back to Voting and the run would keep going past a
// verdict that is no longer trustworthy. Everything already recorded is
// kept, so the run stays resumable (or foldable) for a human to pick up.
//
// On --resume the run gets one chance to repair itself: the seats a
// rate limit took out are re-asked. If their quota has since reset and
// the quorum is restored, the run picks up and finishes; otherwise it
// stays stale and still-resumable for a later retry. If it does not
// recover, the returned status stays `Stalled` and nothing was
// clobbered (the recovery only mutates entries for the lost seats).
if self.state.status == RunStatus::Stalled {
if self.recover_stall().await? {
self.finish_after_tally().await?;
} else {
// Still below quorum: persist the marker and stay resumable.
self.state.save()?;
}
return Ok(());
}
// A run parked inside `land` - watching CI, mid fix-round, or
// waiting on the owner's merge approval - resumes directly into it,
// never back through `prep`. Everything before `merge` already
// concluded; that is the only way `status` reaches `Landing` in the
// first place. Re-walking `review_loop` first would also be actively
// wrong: its own status recomputation (see its doc) treats any
// clean round as reason to set `status` to `Gating`, which would
// clobber this marker before `merge` ever ran, and this run would
// never find its way back into `land` at all.
if self.state.status == RunStatus::Landing {
self.run_land().await?;
// `run_land` may have settled the run right here - CI came back
// green and the PR merged, say - without ever passing back
// through `merge`'s own trailing call. Whatever it left `status`
// as is what this has to read.
self.settle_questions();
return Ok(());
}
// A run parked at `merge` on the owner's word about a withheld PR text
// resumes straight into `merge`. Re-walking the tail would sync to the
// base again (possibly rebasing, possibly spending the fixer) after the
// gate had passed, and then push a tree nobody reviewed or gated.
if self.state.github_text.is_some()
&& self.state.merge.is_none()
&& self.state.status == RunStatus::Gating
{
self.merge().await?;
if self.resource_halt()? {
return Ok(());
}
self.state.save()?;
return Ok(());
}
self.prep().await?;
if self.park_here()? {
return Ok(());
}
self.advise().await?;
if self.park_here()? {
return Ok(());
}
self.implement().await?;
if self.park_here()? {
return Ok(());
}
// `after_implement` already saved the state and settled any open
// questions when it set this; nothing later in the graph has
// anything to judge.
if self.state.status == RunStatus::VerifiedNoop {
return Ok(());
}
self.judge().await?;
if self.park_here()? {
return Ok(());
}
self.deliberate().await?;
if self.park_here()? {
return Ok(());
}
self.vote().await?;
if self.park_here()? {
return Ok(());
}
self.tally()?;
// A verdict that lost its quorum is not trustworthy: do not review,
// gate, or merge on it. Everything already done is kept, so the run
// stays resumable (or foldable); the human can replace the agent that
// ran out of quota and pick it up.
if self.state.status == RunStatus::Stalled {
// Persist the stalled marker now — the normal end-of-execute save
// below is below this early return, and without it a resumed run
// would reload a pre-tally status and keep going.
self.state.save()?;
return Ok(());
}
self.finish_after_tally().await?;
Ok(())
}
/// A wave could not start its agents because a shared resource stayed busy
/// (see [`RunState::resource_block`]): end the node without recording any
/// result, leaving the run `Blocked` and resumable. No seat history,
/// handover, quota loss or candidate state was written for it.
fn stop_for_resource(&mut self) -> Result<()> {
// A wait abandoned because the run is parking stays a park.
if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
{
self.state.status = RunStatus::Blocked;
}
self.state.save()
}
/// Stop walking the graph if a node ended on a resource block.
fn resource_halt(&mut self) -> Result<bool> {
if self.state.resource_block.is_none() {
return Ok(false);
}
self.stop_for_resource()?;
Ok(true)
}
/// Park here if asked to, recording it in the run's own timeline.
///
/// Returns whether the caller should stop walking the graph. The state is
/// saved either way by the node that just finished; this adds the event so
/// the operator's card says why a run that is neither finished nor moving
/// is sitting where it is.
fn park_here(&mut self) -> Result<bool> {
let parking = self.pause.parked() || self.interrupt.parked();
if !parking && self.state.resource_block.is_some() {
self.stop_for_resource()?;
return Ok(true);
}
// Either handle asking is enough - see `Pause`'s own doc for why
// they are never the same one. `interrupt` is checked second so a
// reason it carries is preferred in the message below over a plain
// shutdown park racing it at the same boundary.
if !self.pause.parked() && !self.interrupt.parked() {
return Ok(false);
}
let cause = self.interrupt.cause().or_else(|| self.pause.cause());
let why = match cause.as_ref().map(ParkCause::label) {
Some(reason) => format!(
"parked after `{}` ({reason}) — resume to carry on from here",
self.state.status.as_str()
),
None => format!(
"parked after `{}` — resume to carry on from here",
self.state.status.as_str()
),
};
self.state.event("park", why);
self.state
.end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
self.state.parked = true;
self.state.save()?;
Ok(true)
}
/// Hand the runner the pause `magi serve`'s own shutdown watches.
pub fn on_pause(&mut self, pause: Pause) {
self.pause = pause;
}
/// Hand the runner a second, independent pause: `magi serve`'s interrupt
/// scheduler asking this one run - and no other - to park so a task
/// marked [`crate::queue::Task::interrupt`] can run alone. See
/// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
/// handle.
pub fn watch_interrupt(&mut self, pause: Pause) {
self.interrupt = pause;
}
/// Abandon this run's own open questions, once `status` has actually
/// settled rather than merely paused.
///
/// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
/// either back up with the candidates, the review round and the seat
/// sessions already on disk, so a question an implementer asked mid-round
/// may still get a real answer read by a real resume. Only the statuses
/// `resumable` excludes are actually final: the run merged, it reached
/// `Ready` with nothing left to do, it failed outright with no
/// established point to continue from, or every candidate agreed, with
/// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
/// every one of those the seat that asked is gone for good, exactly like
/// the run being deleted under `magi run rm` - so the same cleanup
/// applies, worded for what actually happened instead of "the run was
/// deleted".
///
/// Best-effort and silent on success: called from every place `status`
/// can land on one of those three, including ones a resumed run revisits,
/// so it must cost nothing when there was nothing open to begin with.
fn settle_questions(&mut self) {
if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
}
}
/// The tail of the graph after a trustworthy tally: fold losers, review,
/// gate, merge, and persist.
async fn finish_after_tally(&mut self) -> Result<()> {
// Before the losers are folded: their branches are what the graft
// reads from.
self.graft().await?;
if self.resource_halt()? {
return Ok(());
}
self.fold_losers().await?;
// Before review starts, and again right before the gate: a run's
// review rounds can themselves take long enough for the base to move
// a second time, and the gate is the one node whose "green" gets
// acted on.
self.sync_to_base().await?;
if self.state.status == RunStatus::AlreadyInBase {
return Ok(());
}
self.review_loop().await?;
if self.resource_halt()? {
return Ok(());
}
self.sync_to_base().await?;
if self.state.status == RunStatus::AlreadyInBase {
return Ok(());
}
self.gate().await?;
self.merge().await?;
if self.resource_halt()? {
return Ok(());
}
self.state.save()?;
Ok(())
}
// ---------------------------------------------------------------- prep
/// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
/// run with the step and its output in the event; it is never swallowed.
/// During `prep` (`in_prep`) the whole preparation is rolled back, so a
/// resume starts over instead of finding candidates and missing seats.
async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
let result = match ensure_setup_config(&mut self.state, repo).await {
Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
Err(e) => Err(e),
};
if let Err(e) = result {
git::worktree_remove(repo, wt).await.ok();
if in_prep {
self.rollback_prep(repo).await;
}
self.state.status = RunStatus::Blocked;
self.state.event(
"setup",
format!("worktree setup failed in {}: {e:#}", wt.display()),
);
self.state.save()?;
return Err(e);
}
Ok(())
}
/// Undo everything `prep` created, so a resume re-runs it whole.
async fn rollback_prep(&mut self, repo: &Path) {
let root = self.state.worktree_root();
for c in std::mem::take(&mut self.state.candidates) {
git::worktree_remove(repo, &c.worktree).await.ok();
git::branch_delete(repo, &c.branch).await.ok();
}
let seats = self.roles.judges.len().max(self.state.config.graph.judges);
for j in 1..=seats {
git::worktree_remove(repo, &root.join(format!("judge-{j}")))
.await
.ok();
}
for k in 1..=self.state.config.graph.advisors {
git::worktree_remove(repo, &root.join(format!("advisor-{k}")))
.await
.ok();
}
}
async fn prep(&mut self) -> Result<()> {
if !self.state.candidates.is_empty() {
return Ok(());
}
self.state.status = RunStatus::Prep;
let repo = self.state.repo.clone();
let base = self.state.base_commit.clone();
let plan = refs::plan(&repo, &self.state.seeds).await?;
let start = plan.start.clone().unwrap_or_else(|| base.clone());
let root = self.state.worktree_root();
let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
// The hook is the write-time half of the blindness contract; the
// presentation filter in `blind` is the half that cannot be bypassed.
let hooks_dir = self.state.dir().join("hooks");
if self.state.config.blind.commit_msg_hook {
std::fs::create_dir_all(&hooks_dir)
.with_context(|| format!("create {}", hooks_dir.display()))?;
let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
let path = hooks_dir.join("commit-msg");
std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
make_executable(&path)?;
// Ref-counted rather than a plain idempotent set: with more than
// one run able to be in flight in the same repository at once
// (see `Config::daemon.max_concurrent_runs`), a bare "already
// true?" check cannot tell "another run of mine still needs
// this" from "nobody does", and the run that happens to finish
// first would disable the hook out from under a sibling still
// relying on it.
git::acquire_worktree_config(&repo).await?;
self.state.enabled_worktree_config = true;
}
// `[worktree] setup` hides its products per worktree, which needs the
// same per-worktree config the hook does. Held until fold, like it.
ensure_setup_config(&mut self.state, &repo).await?;
for (index, (spec, label)) in self
.roles
.implementers
.clone()
.into_iter()
.zip(labels)
.enumerate()
{
let branch = self.state.branch_for(label);
let worktree = root.join(format!("cand-{label}"));
git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
if self.state.config.blind.commit_msg_hook {
git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
}
git::local_exclude(&worktree, "/.magi/").await?;
if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
// Nothing of this prep may survive: a resume must start over,
// not find half the candidates and skip the rest.
git::worktree_remove(&repo, &worktree).await.ok();
git::branch_delete(&repo, &branch).await.ok();
self.rollback_prep(&repo).await;
self.state.status = RunStatus::Blocked;
self.state
.event("prep", format!("worktree setup failed: {e:#}"));
self.state.save()?;
return Err(e);
}
for pick in &plan.picks {
if let Err(e) = git::cherry_pick(&worktree, pick).await {
self.state.status = RunStatus::Blocked;
self.state
.event("prep", format!("cannot apply referenced commit: {e}"));
self.state.save()?;
return Err(e);
}
}
self.state.candidates.push(Candidate {
index,
label,
agent: spec.id.clone(),
branch,
worktree,
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
});
}
for j in 1..=self.roles.judges.len() {
let wt = root.join(format!("judge-{j}"));
if !wt.exists() {
git::worktree_add_detached(&repo, &wt, &base).await?;
self.setup_seat_worktree(&repo, &wt, true).await?;
}
}
// Disposable, detached checkouts for the design-deliberation stage's
// advisor seats — the same shape as the judges' above, at the same
// base commit, since advisors also only ever read. Sized off the
// configured count directly rather than a resolved roster: unlike
// `implementers`/`judges`/`reviewers`, advisor seats are resolved
// lazily inside `advise` itself (see `Config::advisors`'s doc), so
// `prep` has no `ResolvedRoles` field to read a count from here.
if self.state.config.graph.advise {
for k in 1..=self.state.config.graph.advisors {
let wt = root.join(format!("advisor-{k}"));
if !wt.exists() {
git::worktree_add_detached(&repo, &wt, &base).await?;
self.setup_seat_worktree(&repo, &wt, true).await?;
}
}
}
// A judge cannot tell it is looking at its own patch — the seats keep
// separate conversations — but a panel that shares agents with the
// field is less independent than it looks, and that is worth saying out
// loud once per run rather than leaving it in the config.
let authors: Vec<&str> = self
.roles
.implementers
.iter()
.map(|a| a.id.as_str())
.collect();
let overlap: Vec<String> = self
.roles
.judges
.iter()
.enumerate()
.filter(|(_, j)| authors.contains(&j.id.as_str()))
.map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
.collect();
if !overlap.is_empty() {
let note = format!(
"{} also authored a candidate; blind, but the panel is less \
independent than {} distinct agents would be",
overlap.join(", "),
self.roles.judges.len()
);
self.state.event("prep", note);
}
self.state.event(
"prep",
format!(
"{} candidates, {} judges, base {} ({})",
self.state.candidates.len(),
self.roles.judges.len(),
&self.state.base_commit[..7.min(self.state.base_commit.len())],
self.state.base_branch
),
);
self.state.status = RunStatus::Implementing;
self.state.save()?;
Ok(())
}
// -------------------------------------------------------------- advise
/// The design-deliberation stage: independent, read-only advisor seats
/// each sketch a design before any implementer touches the repository,
/// and (when at least one produced a usable proposal) a synthesis seat
/// blends them into a brief `implement` carries in every candidate's
/// prompt.
///
/// `[graph] advise` is the on/off switch, on by default; `[graph]
/// advisors` is the proposal count. Everything here is best-effort and
/// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
/// that cannot reach quota, or a synthesis seat that produced nothing
/// usable all leave `implement` exactly as it was before this stage
/// existed — the task instruction alone — rather than failing the whole
/// competition over an enrichment stage. Every outcome is still recorded
/// as an event, so a run that got nothing from this stage says why.
///
/// [`RunState::advise_attempted`] is this node's idempotency marker, the
/// same role [`RunState::judge_skipped`] plays for `judge`: without it a
/// resumed run whose stage failed would re-run it, and re-spend the
/// agent calls, on every reentry before `implement`.
///
/// Also skipped once any candidate shows implementation progress — the
/// exact predicate `implement` itself uses to decide a candidate is no
/// longer "todo" (see its own `todo` filter). `advise_attempted` alone
/// is not enough: a run created by an older binary that predates this
/// field deserializes it as `false` (`#[serde(default)]`), so resuming
/// an already-`Implementing`-or-later run under this build would
/// otherwise walk straight back through `prep` (a no-op once candidates
/// exist) into this node and spawn every advisor seat against worktrees
/// `prep` never recreated — after implementation has already started,
/// which is exactly the invariant this stage exists to guarantee.
async fn advise(&mut self) -> Result<()> {
let implement_untouched = self
.state
.candidates
.iter()
.all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
if !self.state.config.graph.advise || self.state.advise_attempted {
return Ok(());
}
if !implement_untouched {
self.state.event(
"advise",
"skipping the design-deliberation stage: at least one \
candidate already shows implementation progress, so this \
run is past the point the stage exists to run before"
.to_owned(),
);
self.state.advise_attempted = true;
self.state.save()?;
return Ok(());
}
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
let instruction = self.state.instruction.clone();
let language = self.state.config.graph.language.clone();
let root = self.state.worktree_root();
let n = self.state.config.graph.advisors;
let where_recorded = self.state.dir().join("run.json");
let seats = match self.state.config.advisors() {
Ok(seats) if !seats.is_empty() => seats,
Ok(_) => {
self.state.event(
"advise",
format!(
"[graph] advisors is 0; skipping the design-deliberation \
stage and continuing without a synthesis brief (see {})",
where_recorded.display()
),
);
self.state.advise_attempted = true;
self.state.save()?;
return Ok(());
}
Err(e) => {
self.state.event(
"advise",
format!(
"could not resolve advisor seats ({e:#}); continuing \
without a design-deliberation brief (see {})",
where_recorded.display()
),
);
self.state.advise_attempted = true;
self.state.save()?;
return Ok(());
}
};
let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
let artifacts = agent::artifacts_dir(&self.state.dir());
let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
let mut jobs = Vec::new();
for (i, spec) in seats.iter().cloned().enumerate() {
let seat_key = format!("advisor-{}", i + 1);
let seat = self.seat(&seat_key, &spec.id);
jobs.push(SeatJob {
prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
spec,
seat,
cwd: worktrees[i % worktrees.len()].clone(),
timeout,
allow_write: false,
sessions: false,
artifacts: artifacts.clone(),
stem: seat_key,
handover: None,
});
}
self.state.event(
"advise",
format!(
"{} advisor seat(s) sketching a design in parallel",
jobs.len()
),
);
let mut quota_losses = Vec::new();
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "advise",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
};
let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
let results = ask_json_wave::<Proposal>(
jobs,
Arc::clone(&self.sem),
self.state.config.graph.retries,
&advisor_roster,
&ctx,
&mut quota_losses,
&mut self.state,
&|p: &Proposal| p.validate(),
)
.await;
if self.state.resource_block.is_some() {
return self.stop_for_resource();
}
self.state.quota.extend(quota_losses);
let mut records = Vec::with_capacity(results.len());
for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
match res {
Ok((proposal, out)) => {
self.state
.event("advise", format!("advisor-{} proposed a design", i + 1));
records.push(advise::AdvisorRecord::proposed(
i + 1,
agent_id,
proposal,
out.duration_ms,
));
}
Err(e) => {
self.state.event(
"advise",
format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
);
records.push(advise::AdvisorRecord::failed(
i + 1,
agent_id,
e.to_string(),
));
}
}
}
let mut advice = advise::Advice {
records,
synthesis: None,
};
if advice.proposals().is_empty() {
self.state.event(
"advise",
"no advisor produced a usable proposal; continuing without a \
synthesis brief"
.to_owned(),
);
} else {
match self
.synthesize_brief(
&advice,
&instruction,
&language,
&worktrees[0],
&artifacts,
&run_id,
&prompts,
cache.as_deref(),
)
.await
{
Ok(Some(text)) => {
self.state.event(
"advise",
"synthesized a design brief for the implementer".to_owned(),
);
advice.synthesis = Some(text);
}
Ok(None) => {
self.state.event(
"advise",
"the synthesis seat produced nothing usable; continuing \
without a design brief"
.to_owned(),
);
}
Err(e) => {
self.state.event(
"advise",
format!("could not synthesize a design brief: {e:#}"),
);
}
}
}
advise::apply_reflection(&mut advice);
self.state.advice = Some(advice);
self.state.advise_attempted = true;
self.state.save()?;
Ok(())
}
/// The synthesis seat: reads every advisor's proposal and blends them
/// into the design brief `advise` stores on [`RunState::advice`]. Split
/// out of [`Runner::advise`] only for readability — it is not called
/// anywhere else.
///
/// Picked the same way [`crate::talk`]'s standing conversation and
/// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
/// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
/// order (a claude seat, else the first runnable agent in roster order)
/// used when that field is unset — see `[roles] synthesizer`'s own doc
/// in [`crate::config`] for why a dedicated field exists here at all.
#[allow(clippy::too_many_arguments)]
async fn synthesize_brief(
&mut self,
advice: &advise::Advice,
instruction: &str,
language: &str,
cwd: &Path,
artifacts: &Path,
run_id: &str,
prompts: &Prompts,
cache: Option<&Path>,
) -> Result<Option<String>> {
let chain = agent::pick_chain(
&self.state.config.agents,
self.state.config.roles.synthesizer.as_ref(),
&agent::installed,
"synthesizer",
)?;
let proposals = advice.proposals();
let mut prompt = prompt::with_overlay(
prompt::synthesize_brief(instruction, &proposals, language),
prompts.overlay("advise"),
);
if cache.is_some() {
// This seat never writes, so it is never handed `CARGO_TARGET_DIR`
// below — see `prompt::build_cache_note`'s doc for why telling a
// read-only seat to build through the shared cache is exactly how
// a sandbox's write refusal gets misread as a defect.
prompt.push('\n');
prompt.push_str(&prompt::build_cache_note("advise", false));
}
let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
// Each id is tried once, in order; a quota hit, error or unusable
// answer moves to the next. The seat is single-turn (`sessions:
// false`) and the prompt is the whole context, so a fallback agent
// needs nothing carried over.
let mut last = None;
for (n, spec) in chain.iter().enumerate() {
if n > 0 {
self.state
.event("advise", format!("synthesis falling back to {}", spec.id));
}
let mut seat = self.seat("advise-synthesis", &spec.id);
let outcome = agent::invoke(
spec,
&mut seat,
&Invocation {
cwd,
prompt: &prompt,
timeout,
allow_write: false,
unsandboxed: false,
sessions: false,
artifacts,
stem: &if n == 0 {
"advise-synthesis".to_owned()
} else {
format!("advise-synthesis-{}", spec.id)
},
run: run_id,
node: "advise",
cache_dir: None,
attachments: &[],
writable: &[],
},
)
.await;
if outcome.is_ok() {
self.state.seats.insert(seat.key.clone(), seat);
}
let advance = agent::chain_advances(&outcome);
last = Some(outcome);
if !advance {
break;
}
}
// Exhausted: the last attempt's result is what a single failed seat
// would have produced.
let out = last.expect("a chain holds at least one agent")?;
if !out.usable() {
return Ok(None);
}
let text =
verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
Ok((!text.trim().is_empty()).then_some(text))
}
// ----------------------------------------------------------- implement
async fn implement(&mut self) -> Result<()> {
// Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
// agent files with `magi task add` name the run that paid for it. The
// prompt overlay is cloned alongside it because the waves borrow it
// while `self` is mutably borrowed by the node's own bookkeeping.
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
let todo: Vec<usize> = self
.state
.candidates
.iter()
.enumerate()
.filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
.map(|(i, _)| i)
.collect();
if todo.is_empty() {
return self.after_implement();
}
self.state.status = RunStatus::Implementing;
let language = self.state.config.graph.language.clone();
let github_language = self.state.config.graph.github_language.clone();
let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
// The design-deliberation stage's blended brief, when `advise` found
// one — carried into every implementer's prompt the same way
// regardless of which candidate it is.
let brief = self
.state
.advice
.as_ref()
.and_then(|a| a.synthesis.as_deref())
.map(str::to_owned);
let attachments = self.state.attachments.clone();
let mut jobs = Vec::new();
for &i in &todo {
let (index, label, worktree) = {
let c = &self.state.candidates[i];
(c.index, c.label, c.worktree.clone())
};
let spec = self.roles.implementers[index].clone();
let seat_key = format!("impl-{label}");
let seat = self.seat(&seat_key, &spec.id);
let instruction = seeded_instruction(&self.state);
jobs.push(SeatJob {
spec,
seat,
prompt: prompt::implement_in(
&instruction,
&worktree.to_string_lossy(),
&language,
&github_language,
brief.as_deref(),
&attachments,
),
cwd: worktree,
timeout,
allow_write: true,
sessions,
artifacts: artifacts.clone(),
stem: format!("impl-{label}"),
handover: None,
});
}
self.state.event(
"implement",
format!("{} candidates in parallel", jobs.len()),
);
// Kept so a seat whose CLI hung up can be asked again from the same
// job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
// can update a seat's own entry once a fallback agent takes it over —
// `resume_unconfirmed_commands`, which reads `sent` afterward, must see
// whichever agent actually answered, not the one that quota'd out.
let mut sent = jobs.clone();
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "implement",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
};
let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
if wave_blocked(&results) {
return self.stop_for_resource();
}
self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
.await;
self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
.await;
self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
.await;
if self.state.resource_block.is_some() {
return self.stop_for_resource();
}
for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
let seat_key = seat.key.clone();
// A quota fallback (`resume_seat_handovers`) may have handed this
// seat to a different agent than the one `prep` recorded on the
// candidate; the stats tables and any later fixer-defaults-to-
// winner's-author lookup must credit whoever actually answered —
// unless every fallback also quota'd out, in which case nobody
// actually answered and crediting the last agent tried would
// erase every earlier agent's own quota loss from the stats
// tables instead of just this one seat's.
let agent = seat.agent.clone();
let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
self.state.seats.insert(seat.key.clone(), seat);
let label = self.state.candidates[i].label;
let worktree = self.state.candidates[i].worktree.clone();
let base = self.state.base_commit.clone();
let (summary, duration, failed, verified_claim) = match out {
AgentOutcome::Ok(o) => {
let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
let failed = (!o.usable()).then(|| {
if o.timed_out {
"agent timed out".to_owned()
} else {
format!("agent exited with {:?}", o.exit_code)
}
});
let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
(text, o.duration_ms, failed, verified_claim)
}
// Left un-resumed by `resume_undelivered` (a dirty tree
// already rescues the work, or there was no session left to
// resume into) — reported like the ordinary failure it is,
// never as if `o.text` (the CLI's raw error JSON) were an
// answer.
AgentOutcome::Dropped(o) => {
let why = o
.dropped
.as_ref()
.map(|d| d.why.as_str())
.unwrap_or("the CLI ended the stream without delivering its answer");
(
String::new(),
o.duration_ms,
Some(format!("the CLI dropped the stream ({why})")),
None,
)
}
AgentOutcome::Quota(o) => {
self.state.quota.push(QuotaLoss {
seat: seat_key,
node: "implement".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
(
String::new(),
o.duration_ms,
Some("rate limited (quota); produced no change".to_owned()),
None,
)
}
AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
(String::new(), 0, Some(e), None)
}
};
// Rescue anything the agent edited but never committed: an
// uncommitted candidate would silently be an empty one.
let rescued = match git::rescue_commit(
&worktree,
&format!("magi: candidate {label} (uncommitted work)"),
)
.await
{
Ok(r) => {
self.state.note_withheld("implement", &r.withheld);
r.committed
}
Err(_) => false,
};
let commits = git::commits_ahead(&worktree, &base, "HEAD")
.await
.unwrap_or(0);
let patch = git::diff(&worktree, &base, "HEAD")
.await
.unwrap_or_default();
let stat = git::diff_stat(&worktree, &base, "HEAD")
.await
.unwrap_or_default();
let files = git::changed_files(&worktree, &base, "HEAD")
.await
.map(|f| f.len())
.unwrap_or(0);
write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
let c = &mut self.state.candidates[i];
if !exhausted_the_fallback_chain {
c.agent = agent;
}
c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
c.stat = stat;
c.files = files;
c.commits = commits;
c.duration_ms = duration;
c.empty = commits == 0 || patch.trim().is_empty();
// An agent that failed but still produced a committed change stays
// in the running: the patch is what gets judged, not the exit code.
c.failed = match failed {
Some(_) if c.empty => failed,
_ => None,
};
// Only an empty candidate can be a verified no-op: a claim next
// to a real patch is not what the marker is for, and `c.failed`
// being `Some` here already implies `verified_claim` was never
// set (see the guard above the match that produced it).
c.verified_noop = if c.empty { verified_claim } else { None };
let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
(Some(e), _, _, _) => format!("candidate {label}: {e}"),
(None, true, Some(_), _) => {
format!("candidate {label}: no change produced (agent-verified no-op)")
}
(None, true, None, _) => format!("candidate {label}: no change produced"),
(None, false, _, true) => {
format!(
"candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
)
}
(None, false, _, false) => {
format!("candidate {label}: {files} files, {commits} commits")
}
};
self.state.event("implement", note);
self.state.save()?;
}
self.after_implement()
}
/// Ask again, once, for work a CLI did and then failed to hand over.
///
/// [`agent::dropped_stream`] recognises the one shape observed: an error
/// status with an empty response and a usage report showing output tokens,
/// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
/// seven minutes and 14,267 output tokens that arrived as an empty
/// candidate, because `agy`'s own subscriber fell behind and hung up.
///
/// Two conditions, and both matter:
///
/// - **Only when the tree is untouched.** Often the agent has already
/// written its files and only the closing message was lost; the rescue
/// commit below picks that up and there is nothing to ask for. Re-asking
/// then would pay for a second implementation of work already on disk.
/// - **Once.** A CLI that drops one stream can drop the next, and this
/// node is the most expensive in the graph.
///
/// The re-ask is a resume, not a re-run: `has_context` is true because the
/// dropped reply still carried its `conversation_id`, so the seat is asked
/// to finish what it was doing rather than sent the whole task again. It
/// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
/// node's - for the same reason a re-ranked judge does: restating finished
/// work is not the work.
///
/// Unlike a quota this is worth retrying at all: a rate limit fails the
/// same way until it resets, while an abandoned conversation is still
/// there to be picked up.
async fn resume_undelivered(
&mut self,
results: &mut [(usize, SeatState, AgentOutcome)],
sent: &[SeatJob],
prompts: &Prompts,
run_id: &str,
) {
for (wi, seat, out) in results.iter_mut() {
let Some(dropped) = (match &*out {
AgentOutcome::Dropped(o) => o.dropped.clone(),
_ => None,
}) else {
continue;
};
let Some(job) = sent.get(*wi) else { continue };
// Already on disk? Then only the closing message was lost.
if !git::is_clean(&job.cwd).await.unwrap_or(true) {
self.state.event(
"implement",
format!(
"{}: the CLI dropped the stream after {} output tokens ({}), but the \
work is in the tree",
seat.key, dropped.output_tokens, dropped.why
),
);
continue;
}
// The re-ask only makes sense as a resume: `resume_after_drop`
// says nothing about the task, trusting the seat to still hold it.
// Without a session to resume — sessions disabled, or this CLI's
// drop shape happened not to carry a session id — that prompt
// would open a brand-new conversation with no context at all,
// which is worse than leaving this as the ordinary failure it
// already is.
if !has_context(&job.spec, seat, job.sessions) {
self.state.event(
"implement",
format!(
"{}: the CLI dropped the stream after {} output tokens ({}), but there \
is no session left to resume",
seat.key, dropped.output_tokens, dropped.why
),
);
continue;
}
self.state.event(
"implement",
format!(
"{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
conversation",
seat.key, dropped.output_tokens, dropped.why
),
);
let mut retry = job.clone();
retry.seat = seat.clone();
retry.prompt = prompt::resume_after_drop(&dropped.why);
retry.timeout = retry_budget(job.timeout, true);
retry.stem = format!("{}-resume", job.stem);
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: run_id,
node: "implement",
prompts,
cache: cache.as_deref(),
round: None,
};
let (resumed_seat, resumed) =
run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
*seat = resumed_seat;
*out = resumed;
}
}
/// Fall an implement seat through to the next untried agent in the
/// implementer roster when it lost to quota — or, since the handover was
/// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
/// [`should_hand_over`] for when a non-quota failure stops the chain), the
/// quota path itself being unchanged — instead of leaving the
/// seat's loss final the moment one agent's account runs dry.
///
/// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
/// are the motivating case: `Config::resolve_roles`'s `implementers`
/// truncates to the single slot rotation picked, so a solo task whose one
/// implementer hits quota mid-run used to have nothing else to try. This
/// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
/// unrotated roster — which is the only place the *other* candidates in
/// the machine's roster still exist once `implementers` has been cut down
/// to size.
///
/// Walks forward from just past the seat's own original position in the
/// roster, never wrapping back to the front: a later candidate slot (say
/// `beta`, the roster's second entry) must fall through to the *next*
/// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
/// almost certainly a different candidate's own agent already — and once
/// the roster's tail is exhausted there is nothing left to fall through
/// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
/// whole [`AgentSpec`]: a roster with the same id named twice must not
/// let this retry that id forever. The loop keeps falling through until
/// an attempt lands something other than `Quota` or the roster's tail
/// runs out of untried ids, at which point the seat is left exactly as
/// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
/// `QuotaLoss` recorded, the candidate failed/empty.
///
/// `sent` is taken mutably and updated with the fallback agent's spec:
/// `resume_unconfirmed_commands`, which runs after this and also reads
/// `sent`, must see whichever agent actually ended up answering the seat
/// — reading the stale, original spec there would check session
/// eligibility against the wrong CLI and could hand a fallback agent's
/// session id to the agent that just lost the seat to quota.
///
/// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
/// — `self.seat` only reuses state when the agent id is unchanged, so
/// handing it a different id already gets this for free. Reusing the old
/// seat would resume a different CLI's session as if it were a
/// continuation of this one.
///
/// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
/// a quota loss cuts an agent off mid-turn, so anything already in the
/// tree is unfinished work, not a completed candidate a re-ask would pay
/// for twice. A dirty tree is rescued into a commit first (the same
/// neutral-identity rescue `implement`'s own outcome loop gives every
/// candidate) so the next agent starts clean.
///
/// The new agent gets the implementer's full prompt and full
/// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
/// it has no session and no context, and is implementing the task from
/// nothing, unlike a resumed drop which is only restating work already
/// done.
///
/// Every intermediate `Quota` this loop absorbs is folded into a plain
/// `implement` event, never into `self.state.quota` — that is what
/// `daemon.rs`'s own backoff reads to decide a run's task attempt should
/// go unspent, and a seat that ultimately recovered on its second or
/// third agent is not the stalled panel that check exists to catch. Only
/// the final, unrecovered `Quota` (once the roster runs out) ever reaches
/// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
/// outcome loop already has — this helper never pushes to it itself.
async fn resume_seat_handovers(
&mut self,
results: &mut [(usize, SeatState, AgentOutcome)],
sent: &mut [SeatJob],
prompts: &Prompts,
run_id: &str,
) {
let instruction = seeded_instruction(&self.state);
let language = self.state.config.graph.language.clone();
let github_language = self.state.config.graph.github_language.clone();
let brief = self
.state
.advice
.as_ref()
.and_then(|a| a.synthesis.as_deref())
.map(str::to_owned);
let attachments = self.state.attachments.clone();
for (wi, seat, out) in results.iter_mut() {
// Who holds the other candidate seats of this wave right now
// (earlier handovers already written back to `sent`).
let others: BTreeSet<String> = sent
.iter()
.enumerate()
.filter(|(j, _)| j != wi)
.map(|(_, j)| j.spec.id.clone())
.collect();
let Some(job) = sent.get_mut(*wi) else {
continue;
};
// Where the seat's own original agent sits in the roster — the
// fallback walk starts just past here, never at the front, so a
// later candidate slot's quota loss does not fall back onto an
// earlier slot's own agent.
let start = self
.roles
.implementer_roster
.iter()
.position(|s| s.id == job.spec.id)
.unwrap_or(0);
let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
let mut fallback_attempt = 0usize;
let mut prev: Option<FailClass> = None;
while let Some(cur) = FailClass::of(&*out) {
if !should_hand_over(prev.as_ref(), &cur) {
break;
}
let Some(next) =
pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
.cloned()
else {
break;
};
tried.insert(next.id.clone());
fallback_attempt += 1;
if let Ok(r) = git::rescue_commit(
&job.cwd,
&format!(
"magi: candidate {} (uncommitted work before {} fallback)",
seat.key,
if cur == FailClass::Quota {
"quota"
} else {
"handover"
}
),
)
.await
{
self.state.note_withheld("implement", &r.withheld);
}
record_handover(
&mut self.state,
"implement",
&seat.key,
&seat.agent,
&next.id,
&cur,
&fail_reason(&*out),
);
prev = Some(cur.clone());
let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
self.state.seats.insert(seat.key.clone(), new_seat.clone());
// Kept in sync on `sent` itself, not just the local retry: a
// later helper (`resume_unconfirmed_commands`) reads `sent`
// after this one returns and must see whichever agent is now
// occupying the seat, not the one that just quota'd out —
// otherwise it would judge session/continuation eligibility
// by the wrong CLI and could resend a fallback's session id
// to the agent that lost it the seat in the first place.
job.spec = next.clone();
let mut retry = job.clone();
retry.seat = new_seat;
retry.prompt = prompt::implement_in(
&instruction,
&job.cwd.to_string_lossy(),
&language,
&github_language,
brief.as_deref(),
&attachments,
);
retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: run_id,
node: "implement",
prompts,
cache: cache.as_deref(),
round: None,
};
let (fallback_seat, fallback_out) = run_one(
retry,
Arc::clone(&self.sem),
&ctx,
&mut self.state,
fallback_attempt,
)
.await;
*seat = fallback_seat;
*out = fallback_out;
}
}
}
/// Ask an implement seat's own CLI to confirm what it started, once, when
/// its reply reported a command whose completion status it never
/// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
/// that does and does not mean.
///
/// The completion contract this task asks for, extended to `implement`
/// with the same signal `continue_fix_report` reads for the fixer,
/// rather than a keyword search over the reply or a hard requirement on
/// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
/// a candidate's CLI turn ended cleanly while a test run it had started
/// had not. A short, ordinary reply with no `## SUMMARY` and no commands
/// named in it at all is untouched by this: `commands` is empty, so
/// there is nothing to be unconfirmed.
///
/// Unlike `resume_undelivered`, not gated on the tree being untouched:
/// this is not about recovering edits that might already be on disk, it
/// is about a result the seat itself never vouched for, which resuming
/// asks for regardless of what the tree already holds. Bounded to one
/// attempt for the same reason `resume_undelivered` is — this is the
/// most expensive node in the graph — and a seat that still cannot
/// confirm on that attempt is left as whatever its (possibly still
/// unconfirmed) reply says; this does not invent a new "failed" reason
/// for a candidate that otherwise produced a real, committed change.
async fn resume_unconfirmed_commands(
&mut self,
results: &mut [(usize, SeatState, AgentOutcome)],
sent: &[SeatJob],
prompts: &Prompts,
run_id: &str,
) {
for (wi, seat, out) in results.iter_mut() {
let AgentOutcome::Ok(o) = &*out else {
continue;
};
if !has_unconfirmed_command(&o.commands) {
continue;
}
let Some(job) = sent.get(*wi) else { continue };
if !has_context(&job.spec, seat, job.sessions) {
self.state.event(
"implement",
format!(
"{}: the reply named a command whose own CLI never confirmed the exit \
status of, but there is no session left to resume",
seat.key
),
);
continue;
}
self.state.event(
"implement",
format!(
"{}: the reply named a command whose own CLI never confirmed the exit \
status of; resuming the conversation",
seat.key
),
);
let mut retry = job.clone();
retry.seat = seat.clone();
retry.prompt = prompt::resume_incomplete(
"a command in your last reply had no confirmed exit status",
);
retry.timeout = retry_budget(job.timeout, true);
retry.stem = format!("{}-confirm", job.stem);
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: run_id,
node: "implement",
prompts,
cache: cache.as_deref(),
round: None,
};
let (resumed_seat, resumed) =
run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
*seat = resumed_seat;
*out = resumed;
}
}
/// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
/// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
/// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
/// that motivated this.
///
/// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
/// own nudge loop already covers for judge/review/vote seats, and not a
/// dropped stream, which [`Runner::resume_undelivered`] covers for
/// implement seats: here the CLI turn genuinely finished while the node's
/// own work — the fixer's account of what it did — had not. Gated purely
/// on `extract_json::<FixReport>` having failed on an otherwise-usable
/// reply, never on any wording in it, so a fixer whose valid, first-try
/// `FixReport` happens to mention having waited on a background test is
/// never resumed — the `Ok(report)` branch at the call site returns
/// before this is ever invoked.
///
/// Same discipline as `resume_undelivered`: a nudge-sized timeout per
/// attempt ([`retry_budget`]), nothing attempted once the session is
/// gone, and a quota hit ends the loop immediately rather than retrying a
/// rate limit that fails the same way again.
async fn continue_fix_report(
&mut self,
mut seat: SeatState,
parse_err: String,
job: &SeatJob,
prompts: &Prompts,
run_id: &str,
round: usize,
) -> (
SeatState,
Option<FixReport>,
Option<String>,
ContinuationRecord,
) {
let mut last_err = parse_err;
let mut cumulative_wait_ms = 0u64;
let mut attempts = 0usize;
loop {
if !has_context(&job.spec, &seat, job.sessions) {
self.state.event(
"fix",
format!(
"round {round}: fixer's reply had no adoption report ({last_err}); no \
session left to resume into"
),
);
let outcome = if attempts == 0 {
ContinuationOutcome::NoSession
} else {
ContinuationOutcome::Exhausted
};
return (
seat,
None,
Some(format!("unparsable fix report: {last_err}")),
ContinuationRecord {
attempts,
cumulative_wait_ms,
outcome,
},
);
}
if attempts >= MAX_FIX_CONTINUATIONS {
self.state.event(
"fix",
format!(
"round {round}: fixer's reply still had no adoption report after \
{attempts} continuation(s) ({last_err}); giving up"
),
);
return (
seat,
None,
Some(format!(
"unparsable fix report after {attempts} continuation(s): {last_err}"
)),
ContinuationRecord {
attempts,
cumulative_wait_ms,
outcome: ContinuationOutcome::Exhausted,
},
);
}
attempts += 1;
self.state.event(
"fix",
format!(
"round {round}: fixer's reply had no adoption report ({last_err}); resuming \
the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
),
);
let mut retry = job.clone();
retry.seat = seat.clone();
retry.prompt = prompt::resume_incomplete(&last_err);
retry.timeout = retry_budget(job.timeout, true);
retry.stem = format!("{}-continue{attempts}", job.stem);
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: run_id,
node: "fix",
prompts,
cache: cache.as_deref(),
round: Some(round),
};
let (resumed_seat, resumed_out) = run_one(
retry,
Arc::clone(&self.sem),
&ctx,
&mut self.state,
attempts,
)
.await;
seat = resumed_seat;
match resumed_out {
AgentOutcome::Ok(o) => {
cumulative_wait_ms += o.duration_ms;
match verdict::extract_json::<FixReport>(&o.text) {
Ok(report) if !has_unconfirmed_command(&o.commands) => {
self.state.event(
"fix",
format!(
"round {round}: fixer's adoption report recovered after \
{attempts} continuation(s)"
),
);
return (
seat,
Some(report),
None,
ContinuationRecord {
attempts,
cumulative_wait_ms,
outcome: ContinuationOutcome::Resumed,
},
);
}
// The report parsed, but this same reply's own
// CommandEvidence — the identical record `state.jobs`
// renders — names a command whose CLI never
// confirmed an exit status. Read together, that is
// not a resolved answer: keep nudging rather than
// accept a report standing next to a command the
// seat's own CLI cannot vouch for.
Ok(_) => {
last_err = "the reply parsed, but it reported a command whose own CLI \
never confirmed an exit status"
.to_owned();
}
Err(e) => last_err = e.to_string(),
}
}
AgentOutcome::Quota(o) => {
cumulative_wait_ms += o.duration_ms;
self.state.quota.push(QuotaLoss {
seat: seat.key.clone(),
node: "fix".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
self.state.event(
"fix",
format!(
"round {round}: continuation rate limited (quota); not retrying now"
),
);
return (
seat,
None,
Some("rate limited (quota) while recovering the fix report".to_owned()),
ContinuationRecord {
attempts,
cumulative_wait_ms,
outcome: ContinuationOutcome::QuotaLost,
},
);
}
AgentOutcome::Dropped(o) => {
cumulative_wait_ms += o.duration_ms;
let why = o
.dropped
.as_ref()
.map(|d| d.why.as_str())
.unwrap_or("the CLI ended the stream without delivering its answer");
last_err = format!("the CLI dropped the stream ({why})");
}
AgentOutcome::Failed(e) => last_err = e,
// The continuation was never started; the report stays lost
// and the caller records that, without any handover.
AgentOutcome::ResourceBlocked(e) => {
return (
seat,
None,
Some(format!("the continuation was not started: {e}")),
ContinuationRecord {
attempts: attempts.saturating_sub(1),
cumulative_wait_ms,
outcome: ContinuationOutcome::NoSession,
},
);
}
}
}
}
fn after_implement(&mut self) -> Result<()> {
// Scan every candidate patch once the set is complete.
if self.state.leaks.is_empty() {
let cfg = self.state.config.blind.clone();
let mut leaks = Vec::new();
for c in &self.state.candidates {
let Some(patch) =
crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
else {
continue;
};
leaks.extend(blind::scan(
&format!("candidate {} patch", c.label),
&patch,
&cfg.vendor_tokens,
));
}
if !leaks.is_empty() {
let summary = leaks
.iter()
.map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
.collect::<Vec<_>>()
.join(", ");
match cfg.on_leak {
LeakPolicy::Fail => {
self.state.status = RunStatus::Failed;
self.state
.event("blind", format!("vendor text in a patch: {summary}"));
self.state.leaks = leaks;
self.state.save()?;
self.settle_questions();
bail!(
"blind.on_leak = \"fail\" and vendor text reached a \
judged patch: {summary}"
);
}
LeakPolicy::Redact => self.state.event(
"blind",
format!("redacting vendor text for judging: {summary}"),
),
LeakPolicy::Warn => self.state.event(
"blind",
format!("vendor text present in a judged patch (shown as-is): {summary}"),
),
}
self.state.leaks = leaks;
}
}
if self.state.viable().is_empty() {
if self.state.all_candidates_verified_noop() {
// Every candidate agreed, with evidence the adoption guard
// accepted, that nothing belongs in this worktree. That is
// not the same fact as a candidate that simply failed to
// write anything, and settling it as an ordinary `Failed`
// (see `SCHEMA`'s doc for schema 10) is what let two of
// task 391f's attempts burn a retry each re-discovering the
// same already-landed fix. Terminal either way, so `judge`
// must never run over an empty candidate set — unlike the
// `Failed` branch below this returns `Ok`, not an error:
// nothing here failed.
self.state.status = RunStatus::VerifiedNoop;
self.state.save()?;
self.settle_questions();
return Ok(());
}
self.state.status = RunStatus::Failed;
self.state.save()?;
self.settle_questions();
bail!("no candidate produced a change; nothing to judge");
}
self.state.status = RunStatus::Judging;
self.state.save()?;
Ok(())
}
// --------------------------------------------------------------- judge
async fn judge(&mut self) -> Result<()> {
// Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
// agent files with `magi task add` name the run that paid for it. The
// prompt overlay is cloned alongside it because the waves borrow it
// while `self` is mutably borrowed by the node's own bookkeeping.
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
if !self.state.judgements.is_empty() || self.state.judge_skipped {
return Ok(());
}
let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
if viable.len() == 1 {
// Recorded so this is a one-time event: `judgements` stays empty
// either way, which without this flag is indistinguishable from
// "not yet judged" on the next reentry — and status is left
// untouched, so a later node's conclusion (e.g. `Blocked` after
// the review budget ran out) survives a resume instead of being
// clobbered back to `Judging` by this node running again.
self.state.judge_skipped = true;
self.state.event(
"judge",
format!(
"only candidate {} produced a change; judging skipped",
viable[0].label
),
);
self.state.save()?;
return Ok(());
}
self.state.status = RunStatus::Judging;
let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
let language = self.state.config.graph.language.clone();
let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let root = self.state.worktree_root();
let base_short = short(&self.state.base_commit);
let mut jobs = Vec::new();
let mut orders = Vec::new();
for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
let order = blind::presentation_order(viable.len(), j, self.state.seed);
let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
let seat_key = format!("judge-{}", j + 1);
let seat = self.seat(&seat_key, &spec.id);
jobs.push(SeatJob {
prompt: prompt::judge(
&self.state.instruction,
&views,
self.roles.judges.len(),
&base_short,
&language,
),
spec,
seat,
cwd: root.join(format!("judge-{}", j + 1)),
timeout,
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("judge-{}", j + 1),
handover: None,
});
}
self.state.event(
"judge",
format!(
"{} judges ranking {} candidates blind",
jobs.len(),
viable.len()
),
);
let labels_for_check = labels.clone();
let mut quota_losses = Vec::new();
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "judge",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
};
let results = ask_json_wave::<Ranking>(
jobs,
Arc::clone(&self.sem),
self.state.config.graph.retries,
&self.roles.judge_roster,
&ctx,
&mut quota_losses,
&mut self.state,
&move |r: &Ranking| r.validate(&labels_for_check),
)
.await;
if self.state.resource_block.is_some() {
return self.stop_for_resource();
}
self.state.quota.extend(quota_losses);
for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let mut record = Judgement {
judge: j + 1,
seat: format!("judge-{}", j + 1),
agent: agent_id,
ranking: Vec::new(),
reasons: BTreeMap::new(),
confidence: None,
strengths: BTreeMap::new(),
order: orders[j].clone(),
failed: None,
duration_ms: 0,
};
match res {
Ok((ranking, out)) => {
record.ranking = ranking.normalized();
record.reasons = ranking.reasons;
record.strengths = ranking.strengths;
record.confidence = ranking.confidence;
record.duration_ms = out.duration_ms;
self.state.event(
"judge",
format!(
"judge {} ranked {}",
j + 1,
record.ranking.iter().collect::<String>()
),
);
}
Err(e) => {
record.failed = Some(e.to_string());
self.state
.event("judge", format!("judge {} produced no ranking: {e}", j + 1));
}
}
self.state.judgements.push(record);
self.state.save()?;
}
Ok(())
}
// ---------------------------------------------------------- deliberate
async fn deliberate(&mut self) -> Result<()> {
// Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
// agent files with `magi task add` name the run that paid for it. The
// prompt overlay is cloned alongside it because the waves borrow it
// while `self` is mutably borrowed by the node's own bookkeeping.
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
if !self.state.deliberation.is_empty() {
return Ok(());
}
let tops: Vec<char> = self
.state
.judgements
.iter()
.filter_map(|j| j.ranking.first().copied())
.collect();
let rounds = self.state.config.graph.deliberate_rounds;
if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
self.state.event(
"deliberate",
format!("judges agreed on {} outright; no deliberation", tops[0]),
);
}
self.state.status = RunStatus::Voting;
self.state.save()?;
return Ok(());
}
self.state.status = RunStatus::Deliberating;
self.state.event(
"deliberate",
format!(
"split: first choices were {} — opening {rounds} round(s)",
tops.iter().collect::<String>()
),
);
let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
let language = self.state.config.graph.language.clone();
let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let root = self.state.worktree_root();
let base_short = short(&self.state.base_commit);
// Judges argue in sequence so that a turn can answer the one before it;
// that is the difference between deliberation and three parallel
// monologues.
for round in 1..=rounds {
let mut turns: Vec<DeliberationTurn> = Vec::new();
for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
if self.state.judgements[j].failed.is_some() {
continue;
}
let seat_key = format!("judge-{}", j + 1);
let spec = self.occupant(&seat_key, spec);
let mut seat = self.seat(&seat_key, &spec.id);
let transcript = self.transcript(&turns, j);
let build = |context: Option<&str>| {
prompt::deliberate(
&self.state.instruction,
context,
&transcript,
round,
rounds,
&language,
)
};
let block = self.candidate_block(&viable, &base_short);
let full = build(Some(&block));
let text = if has_context(&spec, &seat, sessions) {
build(None)
} else {
full.clone()
};
let job = SeatJob {
spec,
seat: seat.clone(),
prompt: text,
cwd: root.join(format!("judge-{}", j + 1)),
timeout,
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("delib-{round}-judge-{}", j + 1),
handover: Some(full),
};
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "deliberate",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
};
// A turn is never nudged (`retries` 0): a failed seat is
// handed to the next roster agent, which gets the full
// context. An empty answer is a turn, not a failure.
let mut losses = Vec::new();
let mut results = ask_wave_with::<String>(
vec![job],
Arc::clone(&self.sem),
0,
&self.roles.judge_roster,
&ctx,
&mut losses,
&mut self.state,
&|text: &str| {
Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
},
)
.await;
self.state.quota.extend(losses);
let (updated, res, _) = results.pop().expect("one job in, one result out");
seat = updated;
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let body = match res {
Ok((body, _)) => body,
// Skip the seat; a CLI's raw error JSON is never read as
// this judge's position.
Err(e) => {
self.state
.event("deliberate", format!("judge {} skipped: {e}", j + 1));
continue;
}
};
let tentative = verdict::extract_json::<Position>(&body)
.ok()
.and_then(|p| p.tentative)
.and_then(|s| s.trim().chars().next())
.map(|c| c.to_ascii_uppercase());
self.state.event(
"deliberate",
format!(
"round {round}: judge {} now favours {}",
j + 1,
tentative.map_or("—".to_owned(), |c| c.to_string())
),
);
turns.push(DeliberationTurn {
judge: j + 1,
agent: agent_id,
body: blind::sanitize_prose(&body, &self.state.config.blind),
tentative,
});
}
self.state
.deliberation
.push(DeliberationRound { round, turns });
self.state.save()?;
}
self.state.status = RunStatus::Voting;
self.state.save()?;
Ok(())
}
// ---------------------------------------------------------------- vote
async fn vote(&mut self) -> Result<()> {
// Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
// agent files with `magi task add` name the run that paid for it. The
// prompt overlay is cloned alongside it because the waves borrow it
// while `self` is mutably borrowed by the node's own bookkeeping.
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
if !self.state.votes.is_empty() {
return Ok(());
}
let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
if viable.len() == 1 {
return Ok(());
}
self.state.status = RunStatus::Voting;
let language = self.state.config.graph.language.clone();
let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let root = self.state.worktree_root();
let base_short = short(&self.state.base_commit);
let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
let mut jobs = Vec::new();
let mut seats_at = Vec::new();
for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
if self
.state
.judgements
.get(j)
.is_some_and(|r| r.failed.is_some())
{
continue;
}
let seat_key = format!("judge-{}", j + 1);
let spec = self.occupant(&seat_key, spec);
let seat = self.seat(&seat_key, &spec.id);
let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
let text = if has_context(&spec, &seat, sessions) {
prompt::final_vote(&viable, &language)
} else {
full.clone()
};
jobs.push(SeatJob {
spec,
seat,
prompt: text,
cwd: root.join(format!("judge-{}", j + 1)),
timeout,
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("vote-judge-{}", j + 1),
handover: Some(full),
});
seats_at.push(j);
}
self.state.event(
"vote",
format!(
"collecting {} final votes one by one, privately",
jobs.len()
),
);
let allowed = viable.clone();
let mut quota_losses = Vec::new();
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "vote",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
};
let results = ask_json_wave::<FinalVote>(
jobs,
Arc::clone(&self.sem),
self.state.config.graph.retries,
&self.roles.judge_roster,
&ctx,
&mut quota_losses,
&mut self.state,
&move |v: &FinalVote| match v.label() {
Some(c) if allowed.contains(&c) => Ok(()),
other => bail!("vote {other:?} is not one of {allowed:?}"),
},
)
.await;
if self.state.resource_block.is_some() {
return self.stop_for_resource();
}
self.state.quota.extend(quota_losses);
for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let initial = self
.state
.judgements
.get(j)
.and_then(|r| r.ranking.first().copied());
let mut record = VoteRecord {
judge: j + 1,
agent: agent_id,
vote: None,
reason: String::new(),
changed: false,
};
match res {
Ok((v, _)) => {
record.vote = v.label();
record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
self.state.event(
"vote",
format!(
"judge {} voted {}{}",
j + 1,
record.vote.unwrap_or('?'),
if record.changed { " (changed)" } else { "" }
),
);
}
Err(e) => {
self.state
.event("vote", format!("judge {} cast no vote: {e}", j + 1));
}
}
self.state.votes.push(record);
self.state.save()?;
}
Ok(())
}
// --------------------------------------------------------------- tally
fn tally(&mut self) -> Result<()> {
if self.state.tally.is_some() {
return Ok(());
}
let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
let tops: Vec<char> = self
.state
.judgements
.iter()
.filter_map(|j| j.ranking.first().copied())
.collect();
let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
// A judge whose private vote failed still counted once, in the initial
// ranking; using it beats discarding a whole seat.
let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
let mut cast: Vec<char> = Vec::new();
for (i, j) in self.state.judgements.iter().enumerate() {
let vote = self
.state
.votes
.iter()
.find(|v| v.judge == i + 1)
.and_then(|v| v.vote)
.or_else(|| j.ranking.first().copied());
if let Some(v) = vote {
*first_choice.entry(v).or_insert(0) += 1;
cast.push(v);
}
}
let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
for j in &self.state.judgements {
let n = j.ranking.len();
for (pos, label) in j.ranking.iter().enumerate() {
*borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
}
}
let best = first_choice.values().copied().max().unwrap_or(0);
let mut leaders: Vec<char> = first_choice
.iter()
.filter(|(_, v)| **v == best)
.map(|(k, _)| *k)
.collect();
let mut tie_break = None;
if leaders.len() > 1 {
let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
let borda_leaders: Vec<char> = leaders
.iter()
.copied()
.filter(|l| borda[l] == top_borda)
.collect();
tie_break = Some(if borda_leaders.len() == 1 {
format!(
"{} way tie on first-choice votes, broken by Borda points from the initial rankings",
leaders.len()
)
} else {
format!(
"{} way tie on both first-choice votes and Borda points, broken by label order",
leaders.len()
)
});
leaders = borda_leaders;
leaders.sort_unstable();
}
let winner = *leaders
.first()
.or(viable.first())
.context("no candidate to declare a winner from")?;
let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
let deliberated = !self.state.deliberation.is_empty();
// Whose verdict is this? A rate-limited seat is absent even if it
// ranked before the limit hit, so presence is measured against the
// recorded losses, not just "did a ranking ever appear".
let quota_seats: std::collections::BTreeSet<&str> =
self.state.quota.iter().map(|q| q.seat.as_str()).collect();
let mut present = 0usize;
for (i, j) in self.state.judgements.iter().enumerate() {
if quota_seats.contains(j.seat.as_str()) {
continue;
}
let ranked = !j.ranking.is_empty() && j.failed.is_none();
let voted = self
.state
.votes
.iter()
.any(|v| v.judge == i + 1 && v.vote.is_some());
if ranked || voted {
present += 1;
}
}
// Strict majority of the configured panel. A bare majority is real
// signal we can act on, while a minority verdict must never stand in
// for a healthy one. A one-candidate run needs no panel at all, and
// `judges` stays `0` rather than the roster size a panel that never
// sat would otherwise be credited with.
let needs_quorum = viable.len() > 1;
let judges_total = if needs_quorum {
self.roles.judges.len()
} else {
0
};
let quorum = if needs_quorum {
judges_total / 2 + 1
} else {
0
};
let met_quorum = !needs_quorum || present >= quorum;
let uncontested = (!needs_quorum).then(|| {
format!("only one candidate ({winner}) produced a usable change; no panel was asked")
});
self.state.event(
"tally",
match &uncontested {
Some(reason) => format!("winner {winner} — {reason}"),
None => format!(
"winner {winner} — votes {} | initial {} | {} changed | \
{present}/{judges_total} judges{}",
first_choice
.iter()
.map(|(k, v)| format!("{k}:{v}"))
.collect::<Vec<_>>()
.join(" "),
if unanimous_initial {
"unanimous"
} else {
"split"
},
changed_votes,
if met_quorum {
String::new()
} else {
format!(" — below quorum ({quorum} required)")
},
),
},
);
if !met_quorum {
self.state.event(
"stall",
format!(
"verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
the run stops here, resumable"
),
);
}
self.state.tally = Some(Tally {
first_choice,
borda,
winner,
rankings: tops.len(),
unanimous_initial,
deliberated,
changed_votes,
unanimous_final,
tie_break,
judges: judges_total,
present,
quorum,
met_quorum,
uncontested,
});
self.state.status = if met_quorum {
RunStatus::Reviewing
} else {
RunStatus::Stalled
};
self.state.save()?;
Ok(())
}
// ------------------------------------------------------------- recover
/// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
/// resumed toward completion once the transient cause clears.
///
/// A seat is absent — and therefore re-asked — when `tally` refuses to count
/// it toward the quorum, which is exactly the set of seats whose absence
/// collapsed the panel: struck by a rate limit at *any* node (the quorum must
/// not depend on which node happened to hit the limit), or an ordinary
/// failure (`failed = Some`) that never produced a usable ranking. A healthy
/// seat is never disturbed.
///
/// A seat that now answers with a usable ranking is "recovered": its
/// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
/// `tally` counts it present again), and its vote re-collected. A seat that
/// still fails keeps its loss and stays absent.
///
/// Returns `true` when the re-tally restores the quorum (the run may proceed
/// to review/gate/merge), `false` when it is still below quorum (the run
/// stays `Stalled`, still resumable for a later retry).
#[allow(clippy::too_many_lines)]
async fn recover_stall(&mut self) -> Result<bool> {
// Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
// agent files with `magi task add` name the run that paid for it. The
// prompt overlay is cloned alongside it because the waves borrow it
// while `self` is mutably borrowed by the node's own bookkeeping.
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
// Absent seats = quota-lost at any node, or failed outright. Mirroring
// `tally`'s presence test (rather than the old quota-judge/vote filter)
// is what keeps a non-quota collapse — or a quota loss recorded at the
// deliberate node — from being a permanent dead-end on `--resume`.
let quota_seats: BTreeSet<&str> =
self.state.quota.iter().map(|q| q.seat.as_str()).collect();
let absent: Vec<String> = self
.state
.judgements
.iter()
.filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
.map(|j| j.seat.clone())
.collect();
if absent.is_empty() {
return Ok(false);
}
let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
if viable.len() <= 1 {
return Ok(false);
}
let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
let language = self.state.config.graph.language.clone();
let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let root = self.state.worktree_root();
let base_short = short(&self.state.base_commit);
let candidates: Vec<Candidate> = viable.clone();
// Map each absent seat key to its 0-based position in `roles.judges`.
let mut positions: Vec<usize> = absent
.iter()
.filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
.collect();
if positions.is_empty() {
return Ok(false);
}
positions.sort_unstable();
positions.dedup();
// Re-rank the lost seats, one blind prompt each.
let mut judge_jobs = Vec::new();
for &j in &positions {
let order = blind::presentation_order(viable.len(), j, self.state.seed);
let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
let seat_key = format!("judge-{}", j + 1);
let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
let seat = self.seat(&seat_key, &spec.id);
judge_jobs.push(SeatJob {
spec,
seat,
prompt: prompt::judge(
&self.state.instruction,
&views,
self.roles.judges.len(),
&base_short,
&language,
),
cwd: root.join(seat_key),
timeout,
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("judge-{}-recover", j + 1),
handover: None,
});
}
let labels_for_check = labels.clone();
let mut judge_losses = Vec::new();
let retries = self.state.config.graph.retries;
let cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "judge",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
};
let results = ask_json_wave::<Ranking>(
judge_jobs,
Arc::clone(&self.sem),
retries,
&self.roles.judge_roster,
&ctx,
&mut judge_losses,
&mut self.state,
&move |r: &Ranking| r.validate(&labels_for_check),
)
.await;
if self.state.resource_block.is_some() {
self.stop_for_resource()?;
return Ok(false);
}
// Refresh the judgement of every seat that ranked again.
let mut recovered: BTreeSet<usize> = BTreeSet::new();
for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let record = &mut self.state.judgements[j];
match res {
Ok((ranking, out)) => {
record.agent = agent_id;
record.ranking = ranking.normalized();
record.reasons = ranking.reasons;
record.strengths = ranking.strengths;
record.confidence = ranking.confidence;
record.failed = None;
record.duration_ms = out.duration_ms;
recovered.insert(j);
self.state.event(
"recover",
format!("judge {} ranked again after the limit", j + 1),
);
}
Err(e) => {
self.state
.event("recover", format!("judge {} still cannot rank: {e}", j + 1));
}
}
}
// Re-ask the votes of the seats that recovered a ranking.
let mut vote_jobs = Vec::new();
let mut vote_pos: Vec<usize> = Vec::new();
for &j in &recovered {
let seat_key = format!("judge-{}", j + 1);
let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
let seat = self.seat(&seat_key, &spec.id);
let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
let text = if has_context(&spec, &seat, sessions) {
prompt::final_vote(&labels, &language)
} else {
full.clone()
};
vote_jobs.push(SeatJob {
spec,
seat,
prompt: text,
cwd: root.join(seat_key),
timeout,
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("vote-judge-{}-recover", j + 1),
handover: Some(full),
});
vote_pos.push(j);
}
let allowed = labels.clone();
let mut vote_losses = Vec::new();
let vote_retries = self.state.config.graph.retries;
let vote_cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "vote",
prompts: &prompts,
cache: vote_cache.as_deref(),
round: None,
};
let votes = ask_json_wave::<FinalVote>(
vote_jobs,
Arc::clone(&self.sem),
vote_retries,
&self.roles.judge_roster,
&ctx,
&mut vote_losses,
&mut self.state,
&move |v: &FinalVote| match v.label() {
Some(c) if allowed.contains(&c) => Ok(()),
other => bail!("vote {other:?} is not one of {allowed:?}"),
},
)
.await;
if self.state.resource_block.is_some() {
self.stop_for_resource()?;
return Ok(false);
}
for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
match res {
Ok((v, _)) => {
if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
rec.vote = v.label();
rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
} else {
self.state.votes.push(VoteRecord {
judge: j + 1,
agent: agent_id,
vote: v.label(),
reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
changed: false,
});
}
self.state.event(
"recover",
format!("judge {} voted again after the limit", j + 1),
);
}
Err(e) => {
self.state
.event("recover", format!("judge {} still cannot vote: {e}", j + 1));
}
}
}
// A seat that ranked again is present even if its re-vote failed —
// `tally` falls back to the initial ranking's first choice — so clear
// its quota loss. Seats that still fail keep theirs and stay absent.
let recovered_keys: BTreeSet<String> = recovered
.iter()
.map(|&j| format!("judge-{}", j + 1))
.collect();
self.state
.quota
.retain(|q| !recovered_keys.contains(&q.seat));
// A seat that hit the limit again is a fresh loss, not the old one:
// replace the stale entry so the history stays one-per-seat and the
// daemon can tell this attempt's loss from a previous session's.
for loss in judge_losses.into_iter().chain(vote_losses) {
if recovered_keys.contains(&loss.seat) {
continue;
}
self.state.quota.retain(|q| q.seat != loss.seat);
self.state.quota.push(loss);
}
// Recompute the verdict from the refreshed panel.
self.state.tally = None;
self.tally()?;
Ok(self
.state
.tally
.as_ref()
.map(|t| t.met_quorum)
.unwrap_or(false))
}
// ---------------------------------------------------------------- graft
/// Port the strengths the judges attributed to the losing candidates onto
/// the winner, once, between the tally and the review loop.
///
/// Best-effort by construction: it returns nothing and fails nothing. A
/// skip, a fixer that failed or hit quota, a round that changed nothing
/// and a result over the size limit ([`graft::limit`]) all leave the
/// winner exactly at the commit it had before (`GraftRecord::pre`) and the
/// run carries on as if the phase were off. See [`graft`] for the
/// decisions and `AGENTS.md` for the invariants.
///
/// The one exception is a rollback that could not be confirmed: the
/// rejected commit may still be the winner's tip, so the run is blocked
/// (resumable; the record stays `Running`, which makes a resume retry the
/// restore) instead of reviewing, gating and merging it.
async fn graft(&mut self) -> Result<()> {
let Err(e) = self.graft_inner().await else {
return Ok(());
};
if e.downcast_ref::<GraftRestoreFailed>().is_some() {
self.state.status = RunStatus::Blocked;
self.state.event("graft", format!("blocked: {e:#}"));
self.state.save()?;
return Err(e);
}
// An error after the `Running` record was saved may have left the
// fixer's work in the tree: the record must not stay unsettled
// (a later resume would reset past the review fixes), so confirm the
// restore now, or block.
if let Some(rec) = self.state.graft.clone()
&& !rec.outcome.settled()
{
if let Some(winner) = self.state.winner().cloned() {
if let Err(re) = Self::graft_restore(&winner.worktree, &rec.pre).await {
self.state.status = RunStatus::Blocked;
self.state.event("graft", format!("blocked: {re:#}"));
self.state.save()?;
return Err(re);
}
}
if let Some(g) = self.state.graft.as_mut() {
g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
}
}
self.state.event("graft", format!("skipped: {e:#}"));
if let Err(e) = self.state.save() {
tracing::warn!("save after graft: {e:#}");
}
Ok(())
}
/// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
/// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
async fn graft_restore(wt: &Path, pre: &str) -> Result<()> {
let restored = git::restore_to(wt, pre).await;
let tip = git::rev_parse(wt, "HEAD").await;
match (restored, tip) {
(Ok(()), Ok(tip)) if tip == pre => Ok(()),
(Err(e), _) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
(Ok(()), Ok(tip)) => {
Err(GraftRestoreFailed(format!("the winner is at {tip}, not {pre}")).into())
}
(Ok(()), Err(e)) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
}
}
/// A skip is a run event, recorded once however often the node is re-entered.
fn graft_skipped(&mut self, why: &str) {
let message = format!("skipped: {why}");
if !self
.state
.events
.iter()
.any(|e| e.node == "graft" && e.message == message)
{
self.state.event("graft", message);
}
}
async fn graft_inner(&mut self) -> Result<()> {
// A round that was started and never finished (crash, kill): the
// fixer may have left anything in the tree, so put the winner back.
if let Some(rec) = self.state.graft.clone()
&& !rec.outcome.settled()
{
// Reviews exist only if the graft ended without its record being
// settled (a swallowed error): the tree has moved on since, and
// a reset to `pre` would discard the review fixes.
if self.state.reviews.is_empty()
&& let Some(winner) = self.state.winner().cloned()
{
Self::graft_restore(&winner.worktree, &rec.pre).await?;
}
if let Some(g) = self.state.graft.as_mut() {
g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
}
self.state.event(
"graft",
"rolled back: the previous round was interrupted; the winner is untouched",
);
self.state.save()?;
return Ok(());
}
let plan = match graft::plan(&self.state) {
Ok(plan) => plan,
Err(skip) => {
if skip != graft::Skip::AlreadyDone {
self.graft_skipped(skip.reason());
self.state.save()?;
}
return Ok(());
}
};
let Some(winner) = self.state.winner().cloned() else {
return Ok(());
};
let wt = winner.worktree.clone();
if !git::is_clean(&wt).await? {
self.graft_skipped("the winner's worktree has uncommitted changes");
self.state.save()?;
return Ok(());
}
let pre = git::rev_parse(&wt, "HEAD").await?;
let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
else {
self.graft_skipped("the winner's diff could not be measured");
self.state.save()?;
return Ok(());
};
let limit = graft::limit(winner_lines);
// Saved before the fixer is called: a stop from here on is repaired
// by the branch above on the next entry.
self.state.graft = Some(GraftRecord {
pre: pre.clone(),
post: None,
lines: None,
limit: Some(limit),
agent: None,
outcome: GraftOutcome::Running,
requested: graft::requested(&plan),
adoption: None,
adoption_error: None,
});
self.state.event(
"graft",
format!(
"porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
plan.strengths
.iter()
.map(|(l, _, _)| l.to_string())
.collect::<Vec<_>>()
.join(", "),
plan.winner
),
);
self.state.save()?;
let fix_prompt = prompt::graft(
&self.state.instruction,
plan.winner,
&plan.strengths,
limit,
&self.state.config.graph.language,
);
let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
let (_, seat, out) = self
.ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
SeatJob {
prompt: fix_prompt.clone(),
spec,
seat,
cwd: wt.clone(),
timeout,
allow_write: true,
sessions,
artifacts: artifacts.clone(),
stem: "graft".to_owned(),
handover: None,
}
})
.await;
// Never spawned: nothing changed (`HEAD == pre` is confirmed), so the
// record goes back to "not started" - not `RolledBack`, which would
// settle it - and the run stops resumably. The seat, the chain and
// the quota are untouched.
if matches!(out, AgentOutcome::ResourceBlocked(_)) {
Self::graft_restore(&wt, &pre).await?;
self.state.graft = None;
self.state.save()?;
return self.stop_for_resource();
}
let mut why: Option<String> = None;
match &out {
AgentOutcome::Ok(o) if agent::output_advances(o) => {
why = Some("the fixer produced nothing usable".to_owned());
}
AgentOutcome::Ok(_) => {}
AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
AgentOutcome::Quota(o) => {
self.state.quota.push(QuotaLoss {
seat: seat.key.clone(),
node: "graft".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
why = Some("rate limited (quota)".to_owned());
}
AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
why = Some(format!(
"the fixer failed: {}",
e.lines()
.next()
.unwrap_or("")
.chars()
.take(160)
.collect::<String>()
));
}
}
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let mut post = None;
let mut lines = None;
if why.is_none() {
let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
if let Ok(r) = &rescued {
self.state.note_withheld("graft", &r.withheld);
}
let head = git::rev_parse(&wt, "HEAD").await?;
// What is measured must be what gets reviewed and gated: a failed
// rescue, or anything still uncommitted (withheld lockfiles
// aside, which are never part of the tree), rejects the graft.
let dirty = match &rescued {
Ok(r) => git::status_porcelain(&wt)
.await
.map(|st| {
st.lines().any(|l| {
!r.withheld
.iter()
.any(|w| l.trim_end().ends_with(w.path.as_str()))
})
})
.unwrap_or(true),
Err(_) => true,
};
if dirty {
why = Some("the fixer left changes that could not be committed".to_owned());
} else if head == pre {
why = Some("the fixer changed nothing".to_owned());
} else {
lines = git::changed_lines(&wt, &pre, &head).await;
if graft::within_budget(lines, limit) {
post = Some(head);
} else {
why = Some(format!(
"{} changed lines against a limit of {limit}",
lines.map_or_else(
|| "an unmeasurable number of".to_owned(),
|l| l.to_string()
)
));
}
}
}
let (adoption, adoption_error) = match &out {
AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
_ => (None, None),
};
let requested = graft::requested(&plan);
let outcome = match why {
None => {
self.state.event(
"graft",
format!(
"applied: {} changed lines (limit {limit})",
lines.unwrap_or(0)
),
);
GraftOutcome::Applied
}
Some(why) => {
// Not settled until the restore is confirmed: the record
// stays `Running`, so a resume retries it.
Self::graft_restore(&wt, &pre).await?;
self.state.event(
"graft",
format!("rolled back: {why}; the winner is untouched"),
);
GraftOutcome::RolledBack(why)
}
};
self.state.graft = Some(GraftRecord {
pre,
post,
lines,
limit: Some(limit),
agent: Some(agent_id),
outcome,
requested,
adoption,
adoption_error,
});
self.state.save()?;
Ok(())
}
// ----------------------------------------------------------------- fold
async fn fold_losers(&mut self) -> Result<()> {
let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
return Ok(());
};
let repo = self.state.repo.clone();
let mut folded = Vec::new();
for i in 0..self.state.candidates.len() {
let c = &self.state.candidates[i];
if c.label == winner || c.folded {
continue;
}
let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
git::worktree_remove(&repo, &wt).await.ok();
git::branch_delete(&repo, &branch).await.ok();
self.state.candidates[i].folded = true;
folded.push(label.to_string());
}
// The judges are finished; their checkouts are pure cost from here.
let root = self.state.worktree_root();
for j in 1..=self.roles.judges.len() {
let wt = root.join(format!("judge-{j}"));
if wt.exists() {
git::worktree_remove(&repo, &wt).await.ok();
}
}
// The design-deliberation stage is finished by the time a tally
// exists — same reasoning as the judges above.
if self.state.config.graph.advise {
for k in 1..=self.state.config.graph.advisors {
let wt = root.join(format!("advisor-{k}"));
if wt.exists() {
git::worktree_remove(&repo, &wt).await.ok();
}
}
}
if !folded.is_empty() {
self.state
.event("fold", format!("folded candidates {}", folded.join(", ")));
self.state.save()?;
}
Ok(())
}
// ------------------------------------------------------------ base sync
/// Land the winner's tree on the current tip of `<remote>/<base>` before
/// anything verifies it.
///
/// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
/// read whatever is checked out in the winner's worktree. Left alone that
/// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
/// it when the run *branched* - and a run takes long enough that the base
/// has usually moved by the time it gets here. A gate that ran there
/// answers "green on the commit this run started from", not "green on
/// what is about to land", and the difference showed up three times in
/// one day as a green run whose merge would have reverted a file another
/// pull request had already landed.
///
/// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
/// `land::Step::Rebase` calls, rather than a second implementation of the
/// same idea: a throwaway worktree, nothing runs in the primary tree, and
/// a second rebase path is exactly the kind of drift `resolve_base`'s own
/// doc warns about ("two answers to a question nobody notices until a
/// diff is wrong").
///
/// A conflict is not the end of the road: the standing rebase is handed
/// to the fixer seat, at most `graph.review_rounds` times, counted in
/// `state.rebase_fixes` (so it survives a park/resume and is shared with
/// land). Once it finishes, review and the gate run as usual on the
/// rebased tree, which is where a breakage the new base caused is caught
/// by the ordinary gate-fix round. magi resolves nothing itself.
///
/// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
/// `state.base_sync.attempts`, is how many times the base is *rebased
/// onto* (a base that keeps moving); `rebase_fixes` is how many times a
/// *conflict* was given to a fixer. When the fixer cannot finish the
/// rebase the branch is restored, `state.base_sync.conflict` is set with
/// what was tried (rounds spent, paths still conflicted) and the branch
/// and worktree stay exactly as they were - untouched, for a person to
/// look at - which is also what makes re-entering this function
/// afterwards a no-op instead of a second attempt at the same wall. A
/// push failure ends the same way.
async fn sync_to_base(&mut self) -> Result<()> {
if self.state.status == RunStatus::AlreadyInBase {
return Ok(());
}
let conflicted = self
.state
.base_sync
.as_ref()
.is_some_and(|s| s.conflict.is_some());
let Some(winner) = self.state.winner().cloned() else {
return Ok(());
};
let repo = self.state.repo.clone();
let remote = self.state.config.merge.remote.clone();
let base_branch = self.state.base_branch.clone();
let tracking = format!("{remote}/{base_branch}");
git::fetch(&repo, &remote, &base_branch).await.ok();
// No network, or the remote never had this branch: the run already
// started from a fetched `<remote>/<base>` (`resolve_base` refuses
// otherwise), and one that got this far is not blocked by a fetch
// that fails now. It just has no newer tip to compare against.
let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
return Ok(());
};
let head = git::rev_parse(&winner.worktree, "HEAD").await?;
let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
// Before any rebase, and before a recorded conflict is honoured: a
// branch whose change reached the base under other commit ids has
// nothing to rebase and nothing to conflict with, and a run that
// already stopped on that phantom conflict recovers here on resume.
// `behind == 0` with head == tip is a branch the base has since taken
// in whole, whether or not a conflict was ever recorded: the ancestry
// proof must still run (`classify` ignores a head still on the start
// commit).
if (behind > 0 || conflicted || head == tip)
&& self
.settle_already_in(&winner.branch, &tip, &head, attempts, behind)
.await?
{
return Ok(());
}
if conflicted {
return Ok(());
}
if behind == 0 {
// A fixer-finished rebase moves the branch ref before the
// winner's worktree is told (`sync_to_head` below). A run that
// died in between resumes here with `behind == 0` and a tree still
// holding the pre-rebase files, which review and the gate would
// then read. That state is exactly: HEAD moved off the tip the
// rebase started from, yet the tree is still identical to that
// tip. A tree with edits of its own differs from it, so nothing
// is thrown away.
if let Some(from) = self
.state
.rebase_fixes
.iter()
.rev()
.find_map(|r| r.from.clone())
&& from != head
&& git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
.await
.is_ok_and(|o| o.ok())
{
git::sync_to_head(&winner.worktree).await?;
}
// An earlier attempt may have rebased the branch locally and died
// before pushing it (only the fresh-rebase arm below pushes).
// Publish it now, so the plain push at PR time is not refused as
// a non-fast-forward. A run already holding a recorded conflict
// never reaches here; that case is out of scope.
let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
if let Some(why) = &conflict {
self.state.status = RunStatus::Blocked;
self.state.event("land", why.clone());
}
self.state.base_sync = Some(BaseSync {
tip,
behind: 0,
attempts,
conflict,
already_in: None,
});
self.state.save()?;
return Ok(());
}
if attempts >= BASE_SYNC_ROUNDS {
let why = format!(
"{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
rebase(s); rebasing again would only race it",
winner.branch
);
self.state.status = RunStatus::Blocked;
self.state.base_sync = Some(BaseSync {
tip,
behind,
attempts,
conflict: Some(why.clone()),
already_in: None,
});
self.state.event("land", why);
self.state.save()?;
return Ok(());
}
self.state.event(
"land",
format!(
"{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
winner.branch
),
);
self.state.save()?;
// The remote's copy of the branch, read now and only if the fetch
// really succeeded (a stale tracking ref must never pin a lease). It is
// pushed over after a rebase only when it is a commit this branch
// already contains, by ancestry or by patch (an earlier rebase of ours
// that never reached the remote): anything else is somebody else's work.
let branch_tracking = format!("{remote}/{}", winner.branch);
let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
let remote_tip = if self.state.task_pr_branch().is_none()
&& matches!(&fetched_branch, Ok(o) if o.ok())
{
git::rev_parse(&repo, &branch_tracking).await.ok()
} else {
None
};
// A remote tip this branch does not contain is somebody else's work:
// rebasing would leave a local tip that can never be pushed. Stop
// before touching anything and say so.
if let Some(theirs) = &remote_tip
&& !git::is_ancestor(&repo, theirs, &head).await
&& !crate::reconcile::origin_missing(&repo, &head, theirs)
.await
.is_ok_and(|missing| missing.is_empty())
{
let why = format!(
"{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
them",
short(theirs),
winner.branch
);
self.state.status = RunStatus::Blocked;
self.state.base_sync = Some(BaseSync {
tip,
behind,
attempts,
conflict: Some(why.clone()),
already_in: None,
});
self.state.event("land", why);
self.state.save()?;
return Ok(());
}
let scratch = self.state.dir().join("base-sync");
let rebased = match crate::rebase::rebase_with_fixer(
&mut self.state,
&scratch,
&winner.branch,
&tracking,
)
.await
{
Ok(crate::rebase::Rebased::Applied) => Ok(None),
Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
Err(e) => Err(e),
};
let attempts = attempts + 1;
match rebased {
Ok(None) => {
// The branch ref moved, but a worktree that already had it
// checked out (the winner's) was not told; sync its index and
// files before anything reads them.
git::sync_to_head(&winner.worktree).await?;
refresh_reviewed_commits(&mut self.state, &winner.branch).await;
let mut conflict = None;
if let Some(pinned) = &remote_tip {
let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
match pushed {
Ok(o) if o.ok() => self.state.event(
"land",
format!("pushed rebased {} to {remote}", winner.branch),
),
Ok(o) => {
conflict = Some(format!(
"rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
winner.branch,
short(pinned),
o.stderr.chars().take(600).collect::<String>()
));
}
Err(e) => {
conflict = Some(format!(
"rebased {} locally but could not push it: {e:#}",
winner.branch
));
}
}
}
if let Some(why) = &conflict {
self.state.status = RunStatus::Blocked;
self.state.event("land", why.clone());
}
self.state.base_sync = Some(BaseSync {
tip: tip.clone(),
behind: 0,
attempts,
conflict,
already_in: None,
});
self.state
.event("land", format!("rebased {} onto {tracking}", winner.branch));
}
Ok(Some(conflict)) => {
let why = format!(
"{} conflicts with {tracking} and did not rebase: {}",
winner.branch,
conflict.chars().take(600).collect::<String>()
);
self.state.status = RunStatus::Blocked;
self.state.base_sync = Some(BaseSync {
tip,
behind,
attempts,
conflict: Some(why.clone()),
already_in: None,
});
self.state.event("land", why);
}
Err(e) => {
let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
self.state.status = RunStatus::Blocked;
self.state.base_sync = Some(BaseSync {
tip,
behind,
attempts,
conflict: Some(why.clone()),
already_in: None,
});
self.state.event("land", why);
}
}
self.state.save()?;
Ok(())
}
/// Push a branch an earlier attempt rebased locally but never published,
/// pinned to the remote tip read right after a successful fetch. Returns
/// the reason when the run must stop; `None` when there was nothing to do
/// (no remote copy, the same tip, or a remote copy this branch already
/// contains, which the PR-time push fast-forwards) or the push succeeded.
async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
// Task competitions publish only after the PR text gate, to their
// public head; never resurrect a candidate branch on the remote.
if self.state.task_pr_branch().is_some() {
return None;
}
let repo = self.state.repo.clone();
let remote = self.state.config.merge.remote.clone();
let fetched = git::fetch(&repo, &remote, branch).await;
if !matches!(&fetched, Ok(o) if o.ok()) {
return None;
}
let branch_tracking = format!("{remote}/{branch}");
let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
return None;
}
if !crate::reconcile::origin_missing(&repo, head, &theirs)
.await
.is_ok_and(|missing| missing.is_empty())
{
return Some(format!(
"{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
them",
short(&theirs)
));
}
match git::push_pinned(&repo, &remote, branch, &theirs).await {
Ok(o) if o.ok() => {
self.state
.event("land", format!("pushed rebased {branch} to {remote}"));
None
}
Ok(o) => Some(format!(
"{branch} is rebased locally but {remote} refused the push (it moved since {}; \
someone may have pushed): {}",
short(&theirs),
o.stderr.chars().take(600).collect::<String>()
)),
Err(e) => Some(format!(
"{branch} is rebased locally but could not be pushed: {e:#}"
)),
}
}
/// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
/// is already on `tip` under other commit ids ([`crate::already`]); returns
/// whether it did.
///
/// Checked only when the base is ahead of the branch. A failing check is
/// "not proven" - the ordinary rebase path then decides - never a reason to
/// stop the run.
///
/// The remote copy of the branch is held to the same standard as the local
/// one: if it carries a tip this worktree does not, that tip must itself be
/// proven in the base, or nothing is settled (a pull request would
/// otherwise be closed over commits nobody checked). The pull request is
/// closed *before* the terminal status is saved; if that fails for a
/// reason other than a refusal (no network, a `gh` error) the run is left
/// `Blocked` with the reason as its conflict, which a resume retries -
/// the same recovery a phantom conflict gets.
async fn settle_already_in(
&mut self,
branch: &str,
tip: &str,
head: &str,
attempts: usize,
behind: usize,
) -> Result<bool> {
let repo = self.state.repo.clone();
let remote = self.state.config.merge.remote.clone();
let start = self.state.base_commit.clone();
let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
Ok(Some(e)) => e,
Ok(None) => return Ok(false),
Err(e) => {
tracing::warn!("already-in-base check for {branch}: {e:#}");
return Ok(false);
}
};
let mut verified = vec![head.to_owned()];
let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
let fetched = git::fetch(&repo, &remote, &public_branch).await;
if matches!(&fetched, Ok(o) if o.ok())
&& let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
&& theirs != head
{
match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
Ok(Some(_)) => verified.push(theirs),
_ => return Ok(false),
}
}
let base_branch = self.state.base_branch.clone();
let message = format!(
"{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
land",
evidence.names(),
evidence.proof.as_str()
);
let closed =
crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
.await;
match closed {
Ok(Ok(url)) => self
.state
.event("land", format!("closed {url}: superseded on {base_branch}")),
Ok(Err(why)) => self
.state
.event("land", format!("did not close a pull request: {why}")),
Err(e) => {
let why = format!(
"{branch} is already in {remote}/{base_branch}, but its pull request could \
not be closed ({e:#}); resume to retry"
);
self.state.status = RunStatus::Blocked;
self.state.base_sync = Some(BaseSync {
tip: tip.to_owned(),
behind,
attempts,
conflict: Some(why.clone()),
already_in: None,
});
self.state.event("land", why);
self.state.save()?;
return Ok(true);
}
}
self.state.status = RunStatus::AlreadyInBase;
self.state.base_sync = Some(BaseSync {
tip: tip.to_owned(),
behind,
attempts,
conflict: None,
already_in: Some(evidence),
});
self.state.event("land", message);
self.state.save()?;
self.settle_questions();
Ok(true)
}
/// The commit review and gate diff against: the tip [`Self::sync_to_base`]
/// last landed the winner on, once it has run, else the commit the run
/// branched from.
///
/// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
/// and `vote` all happen before there is a winner to rebase, so they
/// compare every candidate against the branch point on purpose, and a
/// base that moves after they are already done cannot change an answer
/// they already gave.
fn landing_base(&self) -> String {
self.state
.base_sync
.as_ref()
.map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
}
// ------------------------------------------------------- operator fix
/// Route specific, already-recorded review findings to a fixer for a
/// targeted, out-of-band fix on the winning branch — `magi fix`'s own
/// entry point.
///
/// Distinct from `review_loop`'s own fix step in three ways: it never
/// runs a reviewer wave, it never spends review-round budget, and what
/// happened is recorded as an [`OperatorFixRequest`] appended to
/// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
/// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
/// and vote must never be rewritten to look like a manufactured blocking
/// verdict.
///
/// Only meaningful once review has actually concluded: `Ready` (handed
/// off with findings still open, or simply concluded clean while minor
/// findings sat unaddressed) or `Blocked` (round budget spent, or the
/// gate failed). Everything else is refused: a run still in progress
/// should simply be resumed, and a `Merged` run's branch has already
/// landed — reopening *this* run's own record cannot change that, so the
/// answer there is a fresh `magi review <branch>`.
///
/// A real commit here re-verifies through a fresh, ordinary review-only
/// run on the same branch ([`Self::review`]) rather than reopening this
/// run's own `review_loop`: once any round in this run's history went
/// clean, `review_conclusion` treats that as permanent by design (the
/// same purity `gate`/`merge` rely on for safe reentry), so there is no
/// way to force one more genuine reviewer wave out of *this* run without
/// either rewriting history or weakening that guarantee for every other
/// caller. A review-only run costs nothing extra — no implementation, no
/// judging, no vote — and exercises the exact same review → verify →
/// gate → (human) merge path, unmodified.
pub async fn fix_selected(
&mut self,
ids: &[String],
reason: &str,
allow_stale: bool,
) -> Result<()> {
let reason = reason.trim();
if reason.is_empty() {
bail!("a fix request needs a reason — that is the operator's own record of why");
}
if ids.is_empty() {
bail!("no finding id given");
}
if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
bail!(
"run {} is `{}`; only a `ready` or `blocked` run — one whose review \
has already concluded — can be given a targeted fix. A run still \
in progress should simply be resumed; a `merged` run's branch has \
already landed, so its answer is a fresh `magi review <branch>`, \
not reopening this run's own record",
self.state.id,
self.state.status.as_str()
);
}
let Some(winner) = self.state.winner().cloned() else {
bail!("run {} has no winning candidate to fix", self.state.id);
};
if !git::branch_exists(&self.state.repo, &winner.branch).await? {
bail!(
"branch `{}` no longer exists; this run cannot be extended",
winner.branch
);
}
let home = crate::run::home();
if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
bail!(
"run {} is currently being worked on by another magi process",
self.state.id
);
}
// Held for the rest of this call, including the follow-up review
// below: two `magi fix` invocations against the same run must not
// both reach the worktree manipulation further down, which would
// otherwise race to remove and recreate the same directory — see
// [`FixClaim`]'s own doc.
let _claim = FixClaim::acquire(&self.state.dir())?;
// Resolve every id before spending anything — an unknown id refuses
// the whole request rather than silently dropping it — and dedup
// while keeping the operator's own order.
let mut seen = BTreeSet::new();
let mut findings = Vec::new();
let mut missing = Vec::new();
for id in ids {
if !seen.insert(id.clone()) {
continue;
}
match self.state.finding(id) {
Some((round, rec, f)) => findings.push(OperatorFixFinding {
id: f.id.clone(),
severity: f.severity,
reviewer_vote: rec.vote,
round: round.round,
round_head: round.head.clone(),
reviewer: rec.reviewer,
agent: rec.agent.clone(),
file: f.file.clone(),
line: f.line,
title: f.title.clone(),
detail: f.detail.clone(),
outcome: OperatorFixOutcome::Pending,
}),
None => missing.push(id.clone()),
}
}
if !missing.is_empty() {
bail!(
"unknown finding id(s): {}; nothing was changed",
missing.join(", ")
);
}
let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
let stale_details: Vec<(String, String)> = findings
.iter()
.filter(|f| f.round_head != head_at_request)
.map(|f| (f.id.clone(), f.round_head.clone()))
.collect();
let stale = !stale_details.is_empty();
if stale && !allow_stale {
bail!(
"the branch has moved since some finding(s) were raised — {} — now \
at {}; pass --allow-stale to fix anyway, or re-run review first",
stale_details
.iter()
.map(|(id, head)| format!("{id} (raised against {})", short(head)))
.collect::<Vec<_>>()
.join(", "),
short(&head_at_request)
);
}
let request = OperatorFixRequest {
requested_at: Timestamp::now(),
reason: reason.to_owned(),
findings,
head_at_request: head_at_request.clone(),
allow_stale,
stale,
fix: None,
result_head: None,
follow_up_review_run: None,
};
self.state.event(
"fix",
format!(
"operator requested a targeted fix on {} finding(s) ({}): {reason}",
request.findings.len(),
request
.findings
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", "),
),
);
// Recorded now, before any worktree work or the fixer call itself —
// and re-saved at each checkpoint below: a crash at any point after
// this (mid fixer call, mid follow-up review) must not lose the fact
// that this was requested, for which findings, and why. Everything
// past this point reads and writes through `request_index` rather
// than a local variable, since `request` itself is moved here.
self.state.operator_fixes.push(request);
self.state.save()?;
let request_index = self.state.operator_fixes.len() - 1;
// A fresh, dedicated worktree for this one call, never the winner's
// own worktree in place: that one may already be gone (folded away),
// and reusing it in place would leave the branch checked out there
// when the follow-up review below tries to check it out again. Freed
// immediately after, either way — but only once confirmed clean:
// `worktree_remove` is a `git worktree remove --force`, which would
// otherwise discard uncommitted work left there by the operator or
// another process before this had a chance to even look at it.
if winner.worktree.exists() {
// Lockfiles a rescue commit withheld stay untracked on purpose and
// are already recorded; they are not the operator's work to protect.
let dirty = git::git(
&winner.worktree,
&["status", "--porcelain", "--untracked-files=all"],
)
.await?;
let only_withheld = dirty.lines().all(|l| {
l.strip_prefix("?? ")
.is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
});
if !only_withheld {
bail!(
"`{}` has uncommitted changes; refusing to touch it — commit or \
discard them first",
winner.worktree.display()
);
}
git::worktree_remove(&self.state.repo, &winner.worktree)
.await
.ok();
}
let fix_worktree = self.state.worktree_root().join("operator-fix");
let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
git::git(
&self.state.repo,
&["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
)
.await
.with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
if !git::is_clean(&fix_worktree).await? {
git::worktree_remove(&self.state.repo, &fix_worktree)
.await
.ok();
bail!(
"`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
winner.branch
);
}
let repo_for_setup = self.state.repo.clone();
let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
Ok(()) => {
worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
}
Err(e) => Err(e),
};
if let Err(e) = setup {
git::worktree_remove(&self.state.repo, &fix_worktree)
.await
.ok();
return Err(e);
}
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
let language = self.state.config.graph.language.clone();
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
.findings
.iter()
.map(|f| Finding {
id: f.id.clone(),
severity: f.severity,
file: f.file.clone(),
line: f.line,
title: f.title.clone(),
detail: f.detail.clone(),
})
.collect();
let fix_prompt = prompt::operator_fix_in(
&self.state.instruction,
&finding_list,
reason,
&stale_details,
&head_at_request,
&language,
&self.state.config.graph.github_language,
);
let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
let (job, seat, out) = self
.ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
prompt: fix_prompt.clone(),
spec,
seat,
cwd: fix_worktree.clone(),
timeout,
allow_write: true,
sessions,
artifacts: artifacts.clone(),
stem: "operator-fix".to_owned(),
handover: None,
})
.await;
let agent_id = seat.agent.clone();
let mut fix = FixRecord {
agent: agent_id,
addressed: Vec::new(),
rejected: Vec::new(),
notes: String::new(),
committed: false,
failed: None,
duration_ms: 0,
continuation: None,
};
let mut final_seat = seat.clone();
match out {
AgentOutcome::Ok(o) => {
fix.duration_ms = o.duration_ms;
let parsed = verdict::extract_json::<FixReport>(&o.text);
let incomplete_reason = match &parsed {
Ok(_) if has_unconfirmed_command(&o.commands) => Some(
"the reply parsed, but it reported a command whose own CLI \
never confirmed an exit status"
.to_owned(),
),
Ok(_) => None,
Err(e) => Some(e.to_string()),
};
match incomplete_reason {
None => {
let report = parsed.expect("checked Ok above");
fix.addressed = report.addressed;
fix.rejected = report.rejected;
fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
}
Some(reason) => {
let (resumed_seat, resolved, failure, cont) = self
.continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
.await;
fix.duration_ms += cont.cumulative_wait_ms;
fix.continuation = Some(cont);
final_seat = resumed_seat;
match resolved {
Some(report) => {
fix.addressed = report.addressed;
fix.rejected = report.rejected;
fix.notes =
blind::sanitize_prose(&report.notes, &self.state.config.blind);
}
None => fix.failed = failure,
}
}
}
}
AgentOutcome::Dropped(o) => {
fix.duration_ms = o.duration_ms;
let why = o
.dropped
.as_ref()
.map(|d| d.why.as_str())
.unwrap_or("the CLI ended the stream without delivering its answer");
fix.failed = Some(format!("the CLI dropped the stream ({why})"));
}
AgentOutcome::Quota(o) => {
self.state.quota.push(QuotaLoss {
seat: final_seat.key.clone(),
node: "fix".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
}
AgentOutcome::Failed(e) => fix.failed = Some(e),
AgentOutcome::ResourceBlocked(e) => {
fix.failed = Some(format!("the fixer was not started: {e}"));
}
}
if fix.continuation.is_none() {
fix.continuation = Some(ContinuationRecord::not_needed());
}
self.state.seats.insert(final_seat.key.clone(), final_seat);
let rescue_message = format!(
"magi: operator-selected fix ({}) (uncommitted work)",
self.state.operator_fixes[request_index]
.findings
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", ")
);
if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
self.state.note_withheld("fix", &r.withheld);
}
let after = git::rev_parse(&fix_worktree, "HEAD").await?;
fix.committed = after != head_at_request;
git::worktree_remove(&self.state.repo, &fix_worktree)
.await
.ok();
self.state.event(
"fix",
match &fix.failed {
Some(reason) => format!(
"operator fix: adoption report was lost ({reason}); {}",
if fix.committed {
"committed"
} else {
"NO new commit"
}
),
None => format!(
"operator fix: {} addressed, {} rejected, {}",
fix.addressed.len(),
fix.rejected.len(),
if fix.committed {
"committed"
} else {
"NO new commit"
}
),
},
);
// Every selected finding gets an outcome — never left `Pending` once
// the fixer's own turn is over. A report that never came back at all
// marks every one of them `Unreported`, not silently "not addressed":
// quota, a dropped stream, or an exhausted continuation are gaps in
// the report, not evidence about the finding itself (see [`SCHEMA`]'s
// doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
for f in &mut self.state.operator_fixes[request_index].findings {
f.outcome = if fix.failed.is_some() {
OperatorFixOutcome::Unreported
} else if fix.addressed.contains(&f.id) {
OperatorFixOutcome::Addressed
} else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
OperatorFixOutcome::Rejected { why: r.why.clone() }
} else {
OperatorFixOutcome::Unreported
};
}
let committed = fix.committed;
if committed {
self.state.operator_fixes[request_index].result_head = Some(after.clone());
}
self.state.operator_fixes[request_index].fix = Some(fix);
// Saved again now that the fixer's own outcome is final, on top of
// the save right after the request was first pushed above.
self.state.save()?;
if committed {
self.state.event(
"fix",
format!(
"operator fix committed {}; opening a follow-up review-only run",
short(&after)
),
);
// The operator asked for the fix, and the follow-up serves whatever
// task the run it follows served.
let origin =
Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
match Self::review(
&self.state.repo,
&winner.branch,
self.state.config.clone(),
origin,
)
.await
{
Ok(mut follow_up) => {
follow_up.state.event(
"start",
format!(
"requested by an operator fix on run {} for finding(s) {}",
self.state.id,
self.state.operator_fixes[request_index]
.findings
.iter()
.map(|f| f.id.as_str())
.collect::<Vec<_>>()
.join(", "),
),
);
follow_up.state.save()?;
let follow_up_id = follow_up.state.id.clone();
// The follow-up is a run like any other: it belongs to the
// task of the run it follows, or to one filed for it.
let adopted = match crate::direct::adopt(
&follow_up.state,
self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
) {
Ok(a) => a,
Err(e) => {
// An ownerless run must not spend agent calls; it
// stays saved, and `magi run --resume` adopts it.
self.state.event(
"fix",
format!(
"follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
),
);
self.state.save()?;
return Ok(());
}
};
let executed = follow_up.execute().await;
let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
if let Some(a) = adopted {
a.finish(&follow_up.state, executed);
}
if let Some(e) = failure {
self.state.event(
"fix",
format!(
"follow-up review {follow_up_id} did not complete cleanly: {e:#}"
),
);
}
self.state.operator_fixes[request_index].follow_up_review_run =
Some(follow_up_id);
}
Err(e) => {
self.state.event(
"fix",
format!("committed the fix but could not open a follow-up review: {e:#}"),
);
}
}
self.state.save()?;
}
Ok(())
}
// --------------------------------------------------------------- review
/// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
/// names, in order, each at most once, moving on only when the call
/// advances (an error, a quota hit or nothing usable - the same decision
/// point `agent::chain_advances` is for the other chained roles).
///
/// Each agent gets its own seat from `Runner::seat`: the same agent
/// continues its conversation (the winner's own implementer seat when it
/// is the winner's author, now that the competition is over), another
/// takes a fresh one so the full prompt is sent again. A handover is
/// recorded under `node`, which is what makes the fallback stick for the
/// rest of the run (see `crate::fixer`). Only the last attempt's outcome
/// is returned, so an exhausted chain reads exactly like a single failed
/// fixer: one quota loss, the same wording, the same refund. An earlier
/// attempt's edits are left in the tree and judged, with the final
/// attempt's, by what git says afterwards.
///
/// Returns the job that produced the outcome, for `continue_fix_report`.
async fn ask_fixer(
&mut self,
winner: &Candidate,
node: &'static str,
round: Option<usize>,
build: impl Fn(AgentSpec, SeatState) -> SeatJob,
) -> (SeatJob, SeatState, AgentOutcome) {
let attempts = fixer::attempts(&self.state, &self.roles, winner);
self.ask_fixer_among(attempts, node, round, None, build)
.await
}
/// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
/// worktree and a commit: before the next agent is asked after one that
/// did not satisfy the call, the worktree is put back on that commit, so
/// a failed agent's partial edits are not carried into the next one.
async fn ask_fixer_among(
&mut self,
attempts: Vec<(AgentSpec, String)>,
node: &'static str,
round: Option<usize>,
restore: Option<(&Path, &str)>,
build: impl Fn(AgentSpec, SeatState) -> SeatJob,
) -> (SeatJob, SeatState, AgentOutcome) {
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
let cache = self.state.config.cache_dir();
let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
let mut last = None;
for (i, (spec, key)) in attempts.into_iter().enumerate() {
let seat = self.seat(&key, &spec.id);
let mut job = build(spec.clone(), seat);
if i > 0 {
job.stem = format!("{}-{}", job.stem, spec.id);
}
let ctx = WaveCtx {
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node,
prompts: &prompts,
cache: cache.as_deref(),
round,
};
let (seat, out) =
run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
// A resource block never advances the chain: the chosen agent
// was not asked, so it must be the one asked again on resume.
let advances = match &out {
AgentOutcome::Ok(o) => agent::output_advances(o),
AgentOutcome::ResourceBlocked(_) => false,
_ => true,
};
if let Some(next) = ids.get(i + 1)
&& advances
{
self.state.seats.insert(seat.key.clone(), seat.clone());
let class =
FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
let reason = match &out {
AgentOutcome::Ok(_) => "nothing usable".to_owned(),
other => fail_reason(other),
};
record_handover(
&mut self.state,
node,
&seat.key,
&spec.id,
next,
&class,
&reason,
);
// A failed agent's edits must not reach the next one, and a
// restore that did not work ends the chain: the call fails
// and the caller's own (confirmed) restore decides.
if let Some((wt, rev)) = restore
&& let Err(e) = git::restore_to(wt, rev).await
{
let why =
format!("could not restore the worktree before the next fixer: {e:#}");
tracing::warn!("{why}");
last = Some((job, seat, AgentOutcome::Failed(why)));
break;
}
continue;
}
last = Some((job, seat, out));
break;
}
last.expect("the fixer chain always has an entry")
}
/// The one repair a verification-only failure earns (see
/// [`crate::recovery`]), before the first review round of a run that took
/// a branch over.
///
/// Saved as started *before* the fixer is called, so a resume never
/// repairs twice; a start found on re-entry is rolled back to the commit
/// it began from. A failed or unusable fixer leaves the tree as it was and
/// the review goes on over it, so the first round's own verification
/// reports the failure again. The failure is never carried into a round as
/// a result: verification is re-run on whatever commit this leaves.
async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
use crate::recovery::{Mode, Repair};
if !self.state.reviews.is_empty() {
return Ok(());
}
let Some(mut rec) = self.state.recovery.clone() else {
return Ok(());
};
if rec.mode != Mode::VerifyRepair {
return Ok(());
}
let wt = winner.worktree.clone();
let head = git::rev_parse(&wt, "HEAD").await?;
match rec.repair.clone() {
Repair::Done { .. } => return Ok(()),
Repair::Started { before } => {
// Interrupted: the unfinished edits are not trusted.
// An unconfirmed rollback blocks the run with `Started` kept,
// so a resume retries it; the tree is never reviewed as is.
if let Err(e) = Self::graft_restore(&wt, &before).await {
return self
.stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
.await;
}
rec.repair = Repair::Done {
before: before.clone(),
after: before,
committed: false,
failed: Some("interrupted; rolled back".to_owned()),
};
self.state
.event("recovery", "the repair was interrupted and rolled back");
self.state.recovery = Some(rec);
self.state.save()?;
return Ok(());
}
Repair::NotStarted => {}
}
if crate::recovery::recheck(&mut rec, &head) {
self.state.event("recovery", rec.summary());
self.state.recovery = Some(rec);
self.state.save()?;
return Ok(());
}
rec.repair = Repair::Started {
before: head.clone(),
};
self.state.recovery = Some(rec.clone());
self.state.save()?;
let language = self.state.config.graph.language.clone();
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let fix_prompt = prompt::recovery_fix(
&self.state.instruction,
&rec,
self.state.config.graph.review_rounds,
&language,
);
let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
self.state.event(
"recovery",
format!(
"repair: the fixer starts from the failed verification on {}",
short(&head)
),
);
let (_job, seat, out) = self
.ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
prompt: fix_prompt.clone(),
spec,
seat,
cwd: wt.clone(),
timeout,
allow_write: true,
sessions,
artifacts: artifacts.clone(),
stem: "fix-recovery".to_owned(),
handover: None,
})
.await;
// Never spawned: nothing changed, so the repair goes back to "not
// started" and the run stops resumably, as for a graft.
if matches!(out, AgentOutcome::ResourceBlocked(_)) {
if let Err(e) = Self::graft_restore(&wt, &head).await {
return self
.stop_recovery(rec, &head, format!("resource wait; restore failed: {e:#}"))
.await;
}
rec.repair = Repair::NotStarted;
self.state.recovery = Some(rec);
self.state.save()?;
return self.stop_for_resource();
}
let failed = match &out {
AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
AgentOutcome::Quota(o) => {
self.state.quota.push(QuotaLoss {
seat: seat.key.clone(),
node: "fix".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
Some("rate limited (quota); the fixer could not run".to_owned())
}
AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
AgentOutcome::Failed(e) => Some(e.clone()),
};
self.state.seats.insert(seat.key.clone(), seat);
let failed = match failed {
// The agent's partial edits are not kept.
Some(why) => {
if let Err(e) = git::restore_to(&wt, &head).await {
return self
.stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
.await;
}
Some(why)
}
None => {
if let Ok(r) = git::rescue_commit(
&wt,
"magi: recovery repair of the failed verification (uncommitted work)",
)
.await
{
self.state.note_withheld("fix", &r.withheld);
}
None
}
};
let after = git::rev_parse(&wt, "HEAD").await?;
self.state.event(
"recovery",
match &failed {
Some(why) => format!("repair: nothing kept ({why})"),
None if after != head => {
format!("repair: committed {}", short(&after))
}
None => "repair: the fixer made no commit".to_owned(),
},
);
rec.repair = Repair::Done {
before: head.clone(),
committed: after != head,
after,
failed,
};
self.state.recovery = Some(rec);
self.state.save()?;
Ok(())
}
/// A repair whose rollback could not be confirmed: the run is blocked and
/// resumable, and the unconfirmed state is never reviewed.
async fn stop_recovery(
&mut self,
mut rec: crate::recovery::RecoveryRecord,
head: &str,
why: String,
) -> Result<()> {
// `Started` stays recorded: a resume retries the restore.
rec.repair = crate::recovery::Repair::Started {
before: head.to_owned(),
};
self.state.recovery = Some(rec);
self.state
.event("recovery", format!("repair could not be undone: {why}"));
self.state.status = RunStatus::Blocked;
self.state.save()?;
bail!("recovery repair could not be rolled back: {why}")
}
async fn review_loop(&mut self) -> Result<()> {
// A base that would not rebase is a person's decision, not a review
// round: nothing here would change the answer, and reviewers and a
// fixer would be spending real budget on a tree that cannot land
// regardless of what they find.
if self
.state
.base_sync
.as_ref()
.is_some_and(|s| s.conflict.is_some())
{
return Ok(());
}
// Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
// agent files with `magi task add` name the run that paid for it. The
// prompt overlay is cloned alongside it because the waves borrow it
// while `self` is mutably borrowed by the node's own bookkeeping.
let run_id = self.state.id.clone();
let prompts = self.state.config.prompts.clone();
let Some(winner) = self.state.winner().cloned() else {
return Ok(());
};
let max_rounds = self.state.config.graph.review_rounds;
// A clean round, an exhausted round budget, or a stalled tree (see
// `STAGNANT_LIMIT`) are all already-decided conclusions the moment
// they are recorded — recomputed here, not read off `status`, so a
// reentry into a run that already stopped restates the identical
// verdict instead of silently handing back whatever an earlier node
// in this same walk clobbered `status` to (a solo-candidate
// `judge`/`deliberate` skip rewrites it on every reentry). The loop
// below runs an empty range once the budget is spent, and would
// otherwise fall through without touching `status` at all.
if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
// A reentry after a crash between the last round's save and
// `stop_reviewing` reaches the hand-off here, not there.
if status == RunStatus::Gating {
self.record_contested_handoff();
}
self.state.status = status;
self.state.save()?;
return Ok(());
}
self.state.status = RunStatus::Reviewing;
// A last recorded round whose own verification never resolved
// (`ResourceBlocked` — the shared build cache, not the patch) is
// never a concluded round, whatever the round budget says: starting
// a fresh round on top of it would spend a whole new reviewer wave
// re-reading an unchanged patch instead of just retrying the one
// check that actually needs it, and once the budget is spent the
// loop below has nothing left to do at all (its range is empty).
// Retry that check directly instead, exactly the same retry
// `stop_reviewing` already does for its own catch-up case.
if self
.state
.reviews
.last()
.is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
{
let shell = self.state.config.shell();
return self
.stop_reviewing(
"the last round's own verification never resolved",
&shell,
&winner.worktree,
)
.await;
}
self.recovery_repair(&winner).await?;
if self.resource_halt()? {
return Ok(());
}
let repo = self.state.repo.clone();
let root = self.state.worktree_root();
let language = self.state.config.graph.language.clone();
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
let base = self.landing_base();
let base_short = short(&base);
let reviewers = self.roles.reviewers.clone();
let shell = self.state.config.shell();
for round in (self.state.reviews.len() + 1)..=max_rounds {
let head = git::rev_parse(&winner.worktree, "HEAD").await?;
let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
// The prior round's own record, already persisted — never a
// hand-carried variable of just its failing output: that is
// exactly what let a round's e2e result drift out of sync with
// which commit it was actually about (see `SCHEMA`'s doc for
// schema 8). Judged against `head`, the commit reviewers are
// about to look at now, so the summary always reads as "an
// earlier head" here — this round's own patch has not been
// checked yet.
let prev_verification = self
.state
.reviews
.last()
.and_then(|r| r.verification_summary(&head));
// Only the first round of a run that took a branch over: later
// rounds have this run's own history.
let prior_section = match (&self.state.recovery, round) {
(Some(rec), 1) if self.state.reviews.is_empty() => {
let since = match (rec.mode, rec.reviewed_head.as_deref()) {
(crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
Some(crate::recovery::head_of(
&git::diff(&winner.worktree, reviewed, "HEAD")
.await
.unwrap_or_default(),
crate::recovery::SINCE_DIFF_MAX,
))
}
_ => None,
};
Some(prompt::recovery_review(rec, since.as_deref()))
}
_ => None,
};
// Each reviewer gets its own detached checkout of exactly this
// commit: nobody can perturb the winner's tree, and the fixer can
// keep working without racing a reviewer.
let mut jobs = Vec::new();
for (r, spec) in reviewers.iter().cloned().enumerate() {
let wt = root.join(format!("review-{}", r + 1));
// `clean -fdx` in a reset wipes setup's products, so every
// round sets the seat up again.
if wt.exists() {
git::reset_detached(&wt, &head).await?;
} else {
git::worktree_add_detached(&repo, &wt, &head).await?;
}
self.setup_seat_worktree(&repo, &wt, false).await?;
let seat_key = format!("review-{}", r + 1);
// The seat starts the round on whoever answered it last, not
// on the agent the spec names, so a failure is not re-paid.
let spec = pick_start_spec(
&self.roles.reviewer_roster,
spec,
self.state.seat_history.get(&seat_key),
);
let seat = self.seat(&seat_key, &spec.id);
jobs.push(SeatJob {
prompt: prompt::review(&prompt::ReviewCtx {
instruction: &self.state.instruction,
branch: &winner.branch,
base_short: &base_short,
stat: &stat,
patch: &patch,
verification: prev_verification.as_ref(),
prior: prior_section.as_deref(),
reviewers: reviewers.len(),
round,
rounds: max_rounds,
// A review-only run has no rankings, so nothing
// competed for this patch and the reviewer is told so.
competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
lens: Lens::for_seat(r),
language: &language,
github_language: &self.state.config.graph.github_language,
}),
spec,
seat,
cwd: wt,
timeout: Duration::from_secs(self.state.config.graph.timeout_review),
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("review-{round}-{}", r + 1),
handover: None,
});
}
self.state.event(
"review",
format!(
"round {round}: {} reviewers on {}",
jobs.len(),
short(&head)
),
);
let mut quota_losses = Vec::new();
let review_retries = self.state.config.graph.retries;
let review_cache = self.state.config.cache_dir();
let ctx = WaveCtx {
carry_seats: true,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "review",
prompts: &prompts,
cache: review_cache.as_deref(),
round: Some(round),
};
let results = ask_json_wave::<Review>(
jobs,
Arc::clone(&self.sem),
review_retries,
&self.roles.reviewer_roster,
&ctx,
&mut quota_losses,
&mut self.state,
&|_: &Review| Ok(()),
)
.await;
if self.state.resource_block.is_some() {
return self.stop_for_resource();
}
// Counted before the move below: how many of *this* round's
// reviewer seats were lost to their own rate limit, as opposed to
// a crash, a timeout, or unparsable output — see `round_is_clean`.
let round_quota_missing = quota_losses.len();
self.state.quota.extend(quota_losses);
let mut records = Vec::new();
let mut all_findings = Vec::new();
for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let mut record = ReviewRecord {
reviewer: r + 1,
agent: agent_id,
summary: String::new(),
findings: Vec::new(),
vote: None,
failed: None,
duration_ms: 0,
// Set for both outcomes: `failed: Some(_)` with
// `attempts > 0` is a seat every retry still lost, not a
// recovered one — only `failed: None` with `attempts > 0`
// reads as "answered after a nudge" (see this field's own
// doc).
attempts,
};
match res {
Ok((review, out)) => {
// Sanitized here, at the point every other piece of
// agent prose in this file is (candidate summaries,
// deliberation turns, vote reasons): a reviewer's own
// words are the one thing about it that could name
// it, and reconsideration below broadcasts this same
// summary and these same findings to every other
// seat on the panel.
record.summary =
blind::sanitize_prose(&review.summary, &self.state.config.blind);
record.vote = Some(review.vote);
record.duration_ms = out.duration_ms;
for (n, mut f) in review.findings.into_iter().enumerate() {
// ids are magi's, never the agent's: the fixer's
// adoption report is keyed by them.
f.id = format!("R{round}-{}-{}", r + 1, n + 1);
f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
// `file` is agent-supplied prose too, never
// checked against the real tree — the same
// exposure `title`/`detail` above have, just in
// a field easy to forget because it looks like a
// path rather than free text.
f.file = f
.file
.map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
all_findings.push(f.clone());
record.findings.push(f);
}
self.state.event(
"review",
format!(
"round {round}: reviewer {} voted {} with {} finding(s)",
r + 1,
review.vote.label(),
record.findings.len()
),
);
}
Err(e) => {
record.failed = Some(e.to_string());
self.state.event(
"review",
format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
);
}
}
records.push(record);
}
// Tally the round's votes and, if they split, spend the one
// round of reconsideration the split -> deliberate -> revote
// shape `judge`/`vote` use for the panel, sized down to what a
// read-only review round can afford: one round, and a revote
// rather than an argument, because the panel already wrote its
// reasoning down as findings the first time around.
let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
let vote_split =
initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
if vote_split {
self.state.event(
"review",
format!(
"round {round}: votes split ({}) — one round of reconsideration",
initial_votes
.iter()
.map(|v| v.label())
.collect::<Vec<_>>()
.join(", ")
),
);
// Seats read every seat's findings and votes, still numbered
// and never named — the same anonymity `review` itself keeps.
let panel: Vec<ReviewSeatReport<'_>> = records
.iter()
.filter_map(|r| {
r.vote.map(|vote| ReviewSeatReport {
reviewer: r.reviewer,
vote,
summary: &r.summary,
findings: &r.findings,
})
})
.collect();
let mut jobs = Vec::new();
let mut seats_at = Vec::new();
for (r, spec) in reviewers.iter().cloned().enumerate() {
// A seat with no initial vote has nothing to reconsider
// from and stays absent, the same as it stayed absent
// from `panel` above.
if records[r].vote.is_none() {
continue;
}
let wt = root.join(format!("review-{}", r + 1));
let seat_key = format!("review-{}", r + 1);
let spec = self.occupant(&seat_key, spec);
let seat = self.seat(&seat_key, &spec.id);
// A seat with no live session has already forgotten the
// initial review's prompt — restate the patch it is
// voting on, the same as `deliberate`/`vote` do for a
// judge in the same position.
// The panel already carries this seat's own review and
// vote, so restating the patch makes the prompt whole for
// a seat handed to another agent.
let build = |with_patch: bool| {
prompt::review_reconsider(&ReviewReconsiderCtx {
instruction: &self.state.instruction,
reviewer: r + 1,
lens: Lens::for_seat(r),
panel: &panel,
patch: with_patch.then_some(ReviewPatch {
branch: &winner.branch,
base_short: &base_short,
stat: &stat,
patch: &patch,
}),
round,
rounds: max_rounds,
language: &language,
})
};
let full = build(true);
let prompt = if has_context(&spec, &seat, sessions) {
build(false)
} else {
full.clone()
};
jobs.push(SeatJob {
prompt,
spec,
seat,
cwd: wt,
timeout: Duration::from_secs(self.state.config.graph.timeout_review),
allow_write: false,
sessions,
artifacts: artifacts.clone(),
stem: format!("review-{round}-reconsider-{}", r + 1),
handover: Some(full),
});
seats_at.push(r);
}
let mut recon_quota_losses = Vec::new();
let recon_cache = self.state.config.cache_dir();
let recon_ctx = WaveCtx {
carry_seats: true,
pauses: Some((&self.pause, &self.interrupt)),
run: &run_id,
node: "review",
prompts: &prompts,
cache: recon_cache.as_deref(),
round: Some(round),
};
let recon_results = ask_json_wave::<ReviewRevote>(
jobs,
Arc::clone(&self.sem),
review_retries,
&self.roles.reviewer_roster,
&recon_ctx,
&mut recon_quota_losses,
&mut self.state,
&|_: &ReviewRevote| Ok(()),
)
.await;
if self.state.resource_block.is_some() {
return self.stop_for_resource();
}
self.state.quota.extend(recon_quota_losses);
for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
let agent_id = seat.agent.clone();
self.state.seats.insert(seat.key.clone(), seat);
let mut rec = ReviewRevoteRecord {
reviewer: r + 1,
agent: agent_id,
vote: None,
reason: String::new(),
failed: None,
};
match res {
Ok((rv, _)) => {
rec.vote = Some(rv.vote);
rec.reason =
blind::sanitize_prose(&rv.reason, &self.state.config.blind);
self.state.event(
"review",
format!(
"round {round}: reviewer {} revoted {}",
r + 1,
rv.vote.label()
),
);
}
Err(e) => {
rec.failed = Some(e.to_string());
self.state.event(
"review",
format!("round {round}: reviewer {} did not revote: {e}", r + 1),
);
}
}
reconsideration.push(rec);
}
} else if initial_votes.len() > 1 {
self.state.event(
"review",
format!(
"round {round}: votes agreed ({}) — no reconsideration",
initial_votes[0].label()
),
);
}
let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
// A round that already has a blocking finding and a round left to
// try is going back to the fixer no matter what `verify.e2e`
// says, so running it first only spends the loop's slowest step
// (minutes, for a Rust repo's full test suite) on a head about
// to be rewritten. Deferred, never skipped: `verify.e2e` still
// runs once a round has no blocking findings left (see
// `round_is_clean`, which a deferred — empty — `e2e` can never
// satisfy since `blocking` is nonzero whenever this branch is
// taken), and `stop_reviewing` forces a real run before it will
// ever read a deferred round as green.
let defer_e2e =
blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
let reason =
format!("{blocking} blocking finding(s) already required a fix this round");
self.state.event(
"verify",
format!(
"round {round}: {reason} — e2e deferred to the fixer (reviewed head \
{}); it will run once a round has none left",
short(&head)
),
);
(Vec::new(), false, true, Some(reason))
} else {
let e2e_commands = self.state.config.verify.e2e.clone();
let cache_dir = self.state.config.cache_dir();
let context = format!("round {round}");
let (e2e, verify_retried) = with_cache_lease(
&mut self.state,
cache_dir.as_deref(),
"e2e",
"e2e",
&winner.worktree,
&head,
verify_timeout,
&context,
|state, budget| {
let shell = shell.clone();
let e2e_commands = e2e_commands.clone();
let worktree = winner.worktree.clone();
let context = context.clone();
async move {
run_e2e_with_retry(
state,
&shell,
&e2e_commands,
&worktree,
budget,
&context,
)
.await
}
},
)
.await;
(e2e, verify_retried, false, None)
};
let expected = records.len();
let answered = records.iter().filter(|r| r.failed.is_none()).count();
let incomplete = answered < expected;
let e2e_ok = e2e.iter().all(CommandOutcome::ok);
let policy = self.state.config.graph.incomplete_review;
let clean = round_is_clean(
blocking,
e2e_ok,
answered,
expected,
round_quota_missing,
policy,
);
let mut round_record = ReviewRound {
round,
head: head.clone(),
verified_head: None,
verified_at: None,
reviews: records,
e2e,
verify_retried,
e2e_deferred,
e2e_defer_reason,
fix: None,
blocking,
answered,
expected,
clean,
progressed: false,
vote_split,
reconsideration,
verdict: None,
};
// The final vote per seat is its revote where reconsideration
// ran and answered, its initial vote otherwise — the same
// fallback `tally` uses for a judge whose private vote failed.
round_record.verdict = ReviewVote::worst(
round_record
.final_votes()
.into_iter()
.map(|(_, _, vote)| vote),
);
// Which commit and when magi actually attempted to check —
// known the moment a command was dispatched against `head`,
// whether or not it finished: a resource-blocked attempt still
// targeted a specific commit at a specific time, and leaving
// that unrecorded is exactly what made `verification_summary`
// report a fresh attempt as "commit unknown ... recorded before
// this was tracked", indistinguishable from a genuinely old,
// untracked record. Only a deferred or unconfigured round never
// ran at all and has nothing to record — see
// `ReviewRound::verified_head`'s own doc.
if !matches!(
round_record.e2e_status(),
E2eStatus::Deferred | E2eStatus::NotConfigured
) {
round_record.verified_head = Some(head.clone());
round_record.verified_at = Some(Timestamp::now());
}
let this_round_verification = round_record.verification_summary(&head);
if incomplete {
let missing: Vec<String> = round_record
.reviews
.iter()
.filter(|r| r.failed.is_some())
.map(|r| format!("review-{}", r.reviewer))
.collect();
self.state.event(
"review",
format!(
"round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
missing.join(", ")
),
);
}
if clean {
self.state.event(
"review",
if incomplete && policy == IncompleteReviewPolicy::Warn {
format!(
"round {round}: clean (warn policy, incomplete panel) — no \
blocking findings from the seats that answered, verification green"
)
} else if incomplete {
format!(
"round {round}: clean ({} rate-limited reviewer(s) excluded from \
quorum) — no blocking findings from the seats that answered, \
verification green",
expected - answered
)
} else {
format!("round {round}: clean — no blocking findings, verification green")
},
);
self.state.reviews.push(round_record);
self.state.status = RunStatus::Gating;
self.state.save()?;
return Ok(());
}
// Nothing was raised and verification passed, but not every seat
// answered and `round_is_clean` still refused to call it clean —
// either a seat is missing for a reason other than its own quota
// (a crash, a timeout, unparsable output — worth another try), or
// every seat that could have answered lost its quota and nobody
// is left to decide on: re-review rather than send the fixer
// after a round with nothing to fix.
if incomplete && blocking == 0 && e2e_ok {
self.state.reviews.push(round_record);
self.state.save()?;
if round == max_rounds {
self.state.status = RunStatus::Blocked;
self.state.event(
"review",
format!(
"{} reviewer seat(s) never answered after {max_rounds} rounds; \
refusing to call it clean",
expected - answered
),
);
return Ok(());
}
continue;
}
// Nothing for the fixer to act on (`blocking == 0`) and the only
// reason this round is not clean is that magi itself never got
// a command to run — the shared build cache, not the patch (see
// `CommandOutcome::resource_blocked`'s own doc). Sending that to
// the fixer would invite a change to appease contention that has
// nothing to do with the diff, and would leave this attempt
// sitting in the next round's prompt as if it were about an
// earlier, superseded commit rather than what it actually is:
// the same head, still waiting to be checked. Wait for it the
// same way the final round's own contention is already handled,
// whatever round this happens to be.
if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
self.state.reviews.push(round_record);
return self
.stop_reviewing(
"the round's own verification could not run",
&shell,
&winner.worktree,
)
.await;
}
if round == max_rounds {
self.state.reviews.push(round_record);
return self
.stop_reviewing(
&format!(
"{blocking} blocking finding(s) still open after {max_rounds} round(s)"
),
&shell,
&winner.worktree,
)
.await;
}
// Fix. The winner's own implementer seat continues its conversation:
// the competition is over, so context is pure benefit now.
let blocking_findings: Vec<_> = all_findings
.iter()
.filter(|f| f.severity.blocks())
.cloned()
.collect();
let fix_prompt = prompt::fix_in(
&self.state.instruction,
&blocking_findings,
this_round_verification.as_ref(),
round,
max_rounds,
&language,
&self.state.config.graph.github_language,
);
let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
let before = git::rev_parse(&winner.worktree, "HEAD").await?;
let (job, seat, out) = self
.ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
prompt: fix_prompt.clone(),
spec,
seat,
cwd: winner.worktree.clone(),
timeout,
allow_write: true,
sessions,
artifacts: artifacts.clone(),
stem: format!("fix-{round}"),
handover: None,
})
.await;
// Not asked: the round is not recorded (so a resume repeats it),
// no fixer was handed over to, and the seat is left as it was.
if matches!(out, AgentOutcome::ResourceBlocked(_)) {
return self.stop_for_resource();
}
let agent_id = seat.agent.clone();
let mut fix = FixRecord {
agent: agent_id,
addressed: Vec::new(),
rejected: Vec::new(),
notes: String::new(),
committed: false,
failed: None,
duration_ms: 0,
continuation: None,
};
let mut continuation = ContinuationRecord::not_needed();
let mut final_seat = seat.clone();
match out {
AgentOutcome::Ok(o) => {
fix.duration_ms = o.duration_ms;
let parsed = verdict::extract_json::<FixReport>(&o.text);
// A parsed report standing next to a command this same
// reply's own CLI never confirmed the exit status of is
// not a resolved answer — the identical `CommandEvidence`
// `state.jobs` renders, read here instead of only on
// display, per the completion judgment and the shown
// record needing to agree.
let incomplete_reason = match &parsed {
Ok(_) if has_unconfirmed_command(&o.commands) => Some(
"the reply parsed, but it reported a command whose own CLI never \
confirmed an exit status"
.to_owned(),
),
Ok(_) => None,
Err(e) => Some(e.to_string()),
};
match incomplete_reason {
None => {
let report = parsed.expect("checked Ok above");
fix.addressed = report.addressed;
fix.rejected = report.rejected;
fix.notes =
blind::sanitize_prose(&report.notes, &self.state.config.blind);
}
Some(reason) => {
let (resumed_seat, resolved, failure, cont) = self
.continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
.await;
fix.duration_ms += cont.cumulative_wait_ms;
continuation = cont;
final_seat = resumed_seat;
match resolved {
Some(report) => {
fix.addressed = report.addressed;
fix.rejected = report.rejected;
fix.notes = blind::sanitize_prose(
&report.notes,
&self.state.config.blind,
);
}
None => fix.failed = failure,
}
}
}
}
// The CLI's raw error JSON is not a fix report to parse.
AgentOutcome::Dropped(o) => {
fix.duration_ms = o.duration_ms;
let why = o
.dropped
.as_ref()
.map(|d| d.why.as_str())
.unwrap_or("the CLI ended the stream without delivering its answer");
fix.failed = Some(format!("the CLI dropped the stream ({why})"));
}
AgentOutcome::Quota(o) => {
self.state.quota.push(QuotaLoss {
seat: final_seat.key.clone(),
node: "fix".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
}
AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
}
fix.continuation = Some(continuation);
self.state.seats.insert(final_seat.key.clone(), final_seat);
if let Ok(r) = git::rescue_commit(
&winner.worktree,
&format!("magi: review round {round} fixes (uncommitted work)"),
)
.await
{
self.state.note_withheld("fix", &r.withheld);
}
let after = git::rev_parse(&winner.worktree, "HEAD").await?;
fix.committed = after != before;
// Judged by what `git` says moved against base, never by the
// fixer's own `addressed`/`rejected` count — see
// `ReviewRound::progressed`. Propagated with `?`, the same as the
// `patch` snapshot above: swallowing this error would default
// `diff_after` to empty, which almost always differs from a
// non-empty `patch` and reads as "progressed" — exactly backwards
// for a `git` failure the stagnation check cannot see through.
let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
let progressed = diff_after != patch;
let commit_note = if fix.committed {
"committed"
} else {
"NO new commit"
};
let tree_note = if progressed {
"changed vs base"
} else {
"unchanged vs base"
};
self.state.event(
"fix",
match &fix.failed {
// Distinct on purpose from "0 addressed, 0 rejected": the
// fixer's own diff still landed (blocking counts do keep
// falling round over round), only its adoption report did
// not come back, so this must never read like every
// finding was reviewed and declined.
Some(reason) => {
format!(
"round {round}: fixer's adoption report was lost ({reason}); \
{commit_note}, tree {tree_note}"
)
}
None => format!(
"round {round}: {} addressed, {} rejected, {commit_note}, tree \
{tree_note}{}",
fix.addressed.len(),
fix.rejected.len(),
if continuation.outcome == ContinuationOutcome::Resumed {
format!(
" (adoption report recovered after {} continuation(s))",
continuation.attempts
)
} else {
String::new()
},
),
},
);
round_record.fix = Some(fix);
round_record.progressed = progressed;
self.state.reviews.push(round_record);
self.state.save()?;
// The fixer's own report never came back this round, even after
// `continue_fix_report`'s own budget was spent on it — not an
// ordinary "no report" (dropped stream, quota, plain failure),
// which already reads that way and is left to the existing round
// budget. Stopping here, rather than opening another round, is
// what keeps a next reviewer/fixer wave from ever being
// dispatched onto `winner.worktree` while whatever the seat's
// last call may still have running there is unaccounted for: no
// process liveness check exists (and none is being added — see
// AGENTS.md/this task's own scope), so the only way to honour
// "nothing starts before a valid report returns" is to not start
// anything further on this worktree from this run at all.
if matches!(
continuation.outcome,
ContinuationOutcome::Exhausted
| ContinuationOutcome::QuotaLost
| ContinuationOutcome::NoSession
) {
return self
.stop_reviewing(
"the fixer's adoption report never came back, even after resuming its \
own seat; refusing to start another round against the same worktree \
while that is unresolved",
&shell,
&winner.worktree,
)
.await;
}
let streak = self
.state
.reviews
.iter()
.rev()
.take_while(|r| !r.progressed)
.count();
if streak >= STAGNANT_LIMIT {
return self
.stop_reviewing(
&format!(
"the tree has not moved against base for {streak} round(s) in a row"
),
&shell,
&winner.worktree,
)
.await;
}
}
Ok(())
}
/// Decide, from the last recorded round's own verification, whether
/// stopping the review loop is a hand-off or a genuine block.
///
/// Called once the loop has given up trying — the round budget is spent,
/// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
/// findings still open, never while a round is still clean or the
/// incomplete-panel case handled inline above. Gate and e2e are facts
/// about the tree; a lingering review finding is an opinion, and this
/// workload's own `magi stats` puts reviewer precision low enough
/// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
/// must not by itself stand between a green, verified change and the
/// human who decides what to do with it. A red e2e is not an opinion, so
/// that case still blocks, with the failing command and a tail of its
/// output recorded here rather than left in `run.json` for someone to go
/// find.
///
/// A round that deferred its own e2e (see [`Config::graph`]'s
/// `e2e_every_round`) is never read as that green: its `e2e` is empty
/// only because nothing ran, and treating an empty list as a passing one
/// here is exactly the "deferred painted green" bug this function exists
/// to not have. When the last round's own verification never resolved —
/// deferred on purpose, or a real attempt the shared build cache blocked
/// — this makes (or retries) the real run, on the actual worktree this
/// loop is about to stop touching, before deciding anything. A
/// resource-blocked attempt is likewise never read as either green or
/// red: it is evidence about the machine, not the patch (see
/// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
/// blocked cache leaves this call without deciding rather than guessing
/// — the caller retries on a later reentry.
/// Record, once, that the review loop handed off over a blocking finding
/// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
/// `land` asks the owner even with `land_approval` off. Called from every
/// path that concludes `Gating`; a reentry keeps the first record.
fn record_contested_handoff(&mut self) {
if self.state.contested_handoff.is_some() {
return;
}
let Some(contested) = self
.state
.reviews
.last()
.and_then(ReviewRound::contested_handoff)
else {
return;
};
self.state.event(
"review",
format!(
"{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
wait for the owner's approval",
contested.findings.len(),
contested.rejecters.len()
),
);
self.state.contested_handoff = Some(contested);
}
async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
let round_idx = self.state.reviews.len() - 1;
// A deferred round and a resource-blocked one are the same shape
// here: neither has a real result yet, and both get one more
// attempt. Read off `e2e_status` — the single source for this —
// rather than `e2e.is_empty()` alone, so a resource-blocked attempt
// (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
// still retries instead of being read as a settled result the
// instant it stops being empty.
let needs_catchup_run = matches!(
self.state.reviews[round_idx].e2e_status(),
E2eStatus::Deferred | E2eStatus::ResourceBlocked
);
if needs_catchup_run {
let round = self.state.reviews[round_idx].round;
let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
let commands = self.state.config.verify.e2e.clone();
let attempted_head = git::rev_parse(worktree, "HEAD").await?;
let cache_dir = self.state.config.cache_dir();
let context = format!(
"round {round}: verification unresolved, catching up before the final decision"
);
let (outcomes, verify_retried) = with_cache_lease(
&mut self.state,
cache_dir.as_deref(),
"e2e",
"e2e",
worktree,
&attempted_head,
timeout,
&context,
|state, budget| {
let shell = shell.to_vec();
let commands = commands.clone();
let context = context.clone();
async move {
run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
.await
}
},
)
.await;
let last = &mut self.state.reviews[round_idx];
last.e2e = outcomes;
last.verify_retried = verify_retried;
// Always the commit and time this attempt actually targeted,
// whether or not it happens to equal the reviewed `head` and
// whether or not a command finished — see
// `ReviewRound::verified_head`'s own doc. A still-inconclusive
// attempt is recorded too, so a later reader sees "attempted
// again at T2" rather than silence.
last.verified_head = Some(attempted_head);
last.verified_at = Some(Timestamp::now());
if verify_inconclusive(&last.e2e) {
// Still not a real result: `e2e_deferred` is left exactly
// as it was, so `needs_catchup_run` above reads
// `ResourceBlocked` (via `e2e_status`, which checks
// `resource_blocked` before `e2e_deferred`) and retries
// again on the next reentry, rather than recording
// contention as a red e2e and blocking the run on it.
self.state.save()?;
return Ok(());
}
last.e2e_deferred = false;
}
let last = &self.state.reviews[round_idx];
let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
match last.e2e_status() {
E2eStatus::Failed => {
let red: Vec<String> = last
.e2e
.iter()
.filter(|o| !o.ok())
.map(|o| {
format!(
"`{}` -> {:?}\n{}",
o.command,
o.code,
tail(&o.output_tail, EVENT_OUTPUT_TAIL)
)
})
.collect();
self.state
.event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
self.state.status = RunStatus::Blocked;
}
// `needs_catchup_run` above already retried once this call; if
// it is still blocked, this is magi's own admission it could
// not get a command to run, never a verdict on the patch — the
// run is left exactly where a later reentry can retry again.
E2eStatus::ResourceBlocked => {
self.state.event(
"review",
format!(
"{why}; e2e could not run (shared build cache unavailable); not \
deciding yet"
),
);
}
E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
self.state.event(
"review",
format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
);
self.record_contested_handoff();
self.state.status = RunStatus::Gating;
}
}
self.state.save()?;
Ok(())
}
// ----------------------------------------------------------------- gate
async fn gate(&mut self) -> Result<()> {
// Judged by the review record itself, not by `status`: a solo
// candidate's `judge`/`deliberate` skip rewrites `status` on every
// reentry (see `judge`), and trusting it here is exactly how a run
// that exhausted its review budget got gated and merged a second
// time around. `review_conclusion` recomputes the review loop's own
// verdict from the round records themselves — `Gating` for a clean
// round or a hand-off (see `stop_reviewing`), anything else means the
// loop is still going or genuinely blocked.
// A base the winner could not be replayed onto is a decision, not a
// round: there is no landing tree to gate. Read as its own record for
// the same reason the review verdict is.
if self.state.status == RunStatus::Failed
|| self
.state
.base_sync
.as_ref()
.is_some_and(|s| s.conflict.is_some())
|| review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
!= Some(RunStatus::Gating)
{
return Ok(());
}
if self.state.gate_ran {
// `review_loop` derives its conclusion from the clean review
// record on every reentry and therefore puts a completed run back
// in `Gating`. A recorded gate is a stronger, terminal fact:
// retain its original command output (or lack of any, for a repo
// with no `verify.gate` commands — see `RunState::gate_ran`'s own
// doc) and restore `Blocked` on a real failure rather than
// pretending the command is still running or running it a second
// time. `gate_ran == false` remains the only shape — unattempted,
// or a resource-blocked retry — that may still need to execute a
// command.
if self.state.gate.iter().any(|outcome| !outcome.ok()) {
self.state.status = RunStatus::Blocked;
self.state.save()?;
}
return Ok(());
}
let Some(winner) = self.state.winner().cloned() else {
return Ok(());
};
self.state.status = RunStatus::Gating;
let mut outcomes = self.run_gate(&winner).await?;
loop {
// A resource-blocked outcome means the gate command never actually
// ran - the shared build cache could not be acquired or confirmed
// fresh in time - which is evidence about the machine, not about
// the tree (see `CommandOutcome::resource_blocked`'s own doc).
// Recording it as a red gate would mark a run `Blocked` on nothing
// but contention magi has already logged; leaving `self.state.gate`
// empty and `self.state.gate_ran` false instead keeps the shape
// this function already treats as "still needs to run" (see the
// early-return above), so the next call retries the command
// rather than concluding anything.
if verify_inconclusive(&outcomes) {
self.state.save()?;
return Ok(());
}
if outcomes.iter().all(CommandOutcome::ok) {
break;
}
match self.gate_fix_round(&winner, &outcomes).await? {
GateFix::Retry => outcomes = self.run_gate(&winner).await?,
GateFix::Stop => break,
GateFix::Defer => {
self.state.save()?;
return Ok(());
}
}
}
let passed = outcomes.iter().all(CommandOutcome::ok);
self.state.gate = outcomes;
self.state.gate_ran = true;
self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
if !passed {
self.state.status = RunStatus::Blocked;
let spent = self.state.gate_fixes.len();
self.state.event(
"gate",
if spent == 0 {
"gate failed; not merging".to_owned()
} else {
format!("gate failed after {spent} gate-fix round(s); not merging")
},
);
}
self.state.save()?;
Ok(())
}
/// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
/// changed into one commit. Reached only from [`Self::run_gate`], i.e.
/// after review is clean and never on a candidate awaiting judging.
///
/// Never fails the run: a non-zero exit or timeout is a warning and a
/// recorded outcome, and the gate remains the single arbiter. Nothing
/// configured means nothing happens - no event, no commit. `commit_all`
/// commits any leftover change under the neutral identity and returns
/// `false` when the tree is clean, so no empty commit is ever made.
async fn run_pre_gate(&mut self, winner: &Candidate) {
let commands = self.state.config.verify.pre_gate.clone();
if commands.is_empty() {
return;
}
let shell = self.state.config.shell();
let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
let (outcomes, _) = run_commands(
&mut self.state,
"pre_gate",
"pre_gate",
0,
&shell,
&commands,
&winner.worktree,
timeout,
)
.await;
for o in &outcomes {
if !o.ok() {
tracing::warn!(
"pre_gate `{}` failed ({:?}); the gate decides",
o.command,
o.code
);
}
self.state.event(
"pre_gate",
format!(
"`{}` -> {}",
o.command,
if o.ok() {
"pass".to_owned()
} else {
format!(
"FAIL ({:?})\n{}",
o.code,
tail(&o.output_tail, EVENT_OUTPUT_TAIL)
)
}
),
);
}
self.state.pre_gate = outcomes;
match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
Ok(head) => {
self.state
.event("pre_gate", format!("committed mechanical fixes ({head})"));
self.state.pre_gate_commit = Some(head);
}
Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
},
Ok(false) => {}
Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
}
if let Err(e) = self.state.save() {
tracing::warn!("could not persist the pre_gate record: {e:#}");
}
}
/// Run `verify.gate` once against the winner's current tree, logging one
/// event per command. Empty when nothing is configured.
async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
self.run_pre_gate(winner).await;
let shell = self.state.config.shell();
let gate_commands = self.state.config.verify.gate.clone();
// Zero commands has nothing to run and nothing that could touch the
// shared build cache, so it never needs a lease: `Config::cache_dir`
// is derived from `verify.e2e` too, so a repo with no `verify.gate`
// commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
// otherwise queue behind an unrelated run's lease and come back
// resource-blocked - `gate_ran` would stay false on nothing but
// cache contention, for a step that had nothing to check in the
// first place.
let outcomes = if gate_commands.is_empty() {
Vec::new()
} else {
let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
let cache_dir = self.state.config.cache_dir();
let head = git::rev_parse(&winner.worktree, "HEAD").await?;
let (outcomes, _) = with_cache_lease(
&mut self.state,
cache_dir.as_deref(),
"gate",
"gate",
&winner.worktree,
&head,
timeout,
"final gate",
|state, budget| {
let shell = shell.clone();
let gate_commands = gate_commands.clone();
let worktree = winner.worktree.clone();
async move {
let (outcomes, timed_out_pids) = run_commands(
state,
"gate",
"gate",
0,
&shell,
&gate_commands,
&worktree,
budget,
)
.await;
(outcomes, false, timed_out_pids)
}
},
)
.await;
outcomes
};
if outcomes.is_empty() {
// Nothing configured to check — distinct from every other
// silence in this run's event log, since an empty `gate` alone
// no longer says whether the gate ran at all (see
// `RunState::gate_ran`'s own doc).
self.state.event(
"gate",
"no gate commands configured; nothing to check, passing",
);
}
for o in &outcomes {
self.state.event(
"gate",
format!(
"`{}` -> {}",
o.command,
if o.ok() {
"pass".to_owned()
} else {
format!(
"FAIL ({:?})\n{}",
o.code,
tail(&o.output_tail, EVENT_OUTPUT_TAIL)
)
}
),
);
}
Ok(outcomes)
}
/// One bounded fix round for a failing gate.
///
/// The fixer is told the failure came from the gate itself, not from a
/// reviewer, and is shown the failed commands, their exit codes and a tail
/// of their output - whatever `[verify].gate` holds, nothing here knows
/// what those commands run. Only a normal non-zero exit that printed
/// something earns a round (see [`gate_fixable`]): a timeout, a missing
/// command or a full disk says nothing about the code, and a fixer sent
/// after it can only appease the machine. The round is judged by what git
/// says moved, never by the fixer's own report, and `verify.e2e` runs
/// again before the gate does, so a fix cannot trade a green gate for a
/// red e2e unnoticed.
async fn gate_fix_round(
&mut self,
winner: &Candidate,
outcomes: &[CommandOutcome],
) -> Result<GateFix> {
let cap = self.state.config.graph.gate_fix_rounds;
let spent = self.state.gate_fixes.len();
if spent >= cap {
if cap > 0 {
self.state.event(
"gate",
format!("{spent} gate-fix round(s) spent and the gate still fails"),
);
}
return Ok(GateFix::Stop);
}
if !gate_fixable(outcomes) {
self.state.event(
"gate",
"gate failure is not an ordinary non-zero exit with output (timeout, missing \
command or similar); not spending a fix round on it",
);
return Ok(GateFix::Stop);
}
let min_free = self.state.config.disk.min_free_bytes;
if min_free > 0 {
match crate::disk::free_bytes(&winner.worktree) {
Ok(free) if crate::disk::enough_space(free, min_free) => {}
Ok(free) => {
self.state.event(
"gate",
format!(
"only {free} bytes free ({min_free} required by `[disk] \
min_free_bytes`); not spending a fix round on a failure the disk \
may explain"
),
);
return Ok(GateFix::Stop);
}
Err(e) => {
self.state.event(
"gate",
format!("free disk space could not be measured ({e:#}); no fix round"),
);
return Ok(GateFix::Stop);
}
}
}
let attempt = spent + 1;
let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
let base = self.landing_base();
let fix_prompt = prompt::gate_fix_in(
&self.state.instruction,
&failed,
attempt,
cap,
&self.state.config.graph.language,
&self.state.config.graph.github_language,
);
let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
let sessions = self.state.config.graph.sessions;
let artifacts = agent::artifacts_dir(&self.state.dir());
self.state.event(
"gate",
format!("gate failed; gate-fix round {attempt} of {cap}"),
);
let before = git::rev_parse(&winner.worktree, "HEAD").await?;
let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
let (_, seat, out) = self
.ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
prompt: fix_prompt.clone(),
spec,
seat,
cwd: winner.worktree.clone(),
timeout,
allow_write: true,
sessions,
artifacts: artifacts.clone(),
stem: format!("gate-fix-{attempt}"),
handover: None,
})
.await;
// The fixer was never asked: no round is spent (nothing is recorded)
// and a later reentry asks the same agent again.
if matches!(out, AgentOutcome::ResourceBlocked(_)) {
return Ok(GateFix::Defer);
}
let mut record = GateFixRecord {
agent: seat.agent.clone(),
failed,
notes: String::new(),
committed: false,
error: None,
};
match out {
AgentOutcome::Ok(o) => {
// A missing report is not a failed fix: the round is judged
// by the tree below, and the report only carries prose.
if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
}
}
AgentOutcome::Dropped(_) => {
record.error = Some("the CLI dropped the stream".to_owned());
}
AgentOutcome::Quota(o) => {
self.state.quota.push(QuotaLoss {
seat: seat.key.clone(),
node: "gate-fix".to_owned(),
at: Timestamp::now(),
reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
});
record.error = Some("rate limited (quota); fixer could not run".to_owned());
}
AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
}
self.state.seats.insert(seat.key.clone(), seat);
if let Ok(r) = git::rescue_commit(
&winner.worktree,
&format!("magi: gate fix {attempt} (uncommitted work)"),
)
.await
{
self.state.note_withheld("gate-fix", &r.withheld);
}
let after = git::rev_parse(&winner.worktree, "HEAD").await?;
record.committed = after != before;
let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
let note = record.error.clone();
self.state.gate_fixes.push(record);
self.state.save()?;
if !changed {
self.state.event(
"gate",
match note {
Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
None => format!("gate-fix round {attempt}: the tree did not change"),
},
);
return Ok(GateFix::Stop);
}
self.state.event(
"gate",
format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
);
let commands = self.state.config.verify.e2e.clone();
if !commands.is_empty() {
let shell = self.state.config.shell();
let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
let cache_dir = self.state.config.cache_dir();
let context = format!("gate-fix round {attempt}");
let (e2e, _) = with_cache_lease(
&mut self.state,
cache_dir.as_deref(),
"e2e",
"e2e",
&winner.worktree,
&after,
timeout,
&context,
|state, budget| {
let shell = shell.clone();
let commands = commands.clone();
let context = context.clone();
let worktree = winner.worktree.clone();
async move {
run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
.await
}
},
)
.await;
if verify_inconclusive(&e2e) {
return Ok(GateFix::Defer);
}
if e2e.iter().any(|o| !o.ok()) {
self.state.event(
"gate",
format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
);
return Ok(GateFix::Stop);
}
}
Ok(GateFix::Retry)
}
// ---------------------------------------------------------------- merge
/// One read-only rewrite by the summary's author; a text the gate still
/// withholds goes to the owner (see `github_text`) and the fixed fallback
/// applies on their word or on silence. `None` means the run is parked on
/// that question and nothing may be pushed or opened yet.
async fn guarded_pr_message(
&mut self,
winner: &Candidate,
facts: Option<&BranchFacts>,
posting: bool,
) -> Result<Option<PrMessage>> {
let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
let head = match final_head.as_deref() {
Some(h) => crate::pr_outcome::FinalHead::At(h),
None => crate::pr_outcome::FinalHead::Unknown,
};
// Only a pull request that is about to be posted pays for a summary.
let summarized = if posting {
Some(self.summarize_change(winner, final_head.as_deref()).await)
} else {
None
};
// A busy shared resource is not a permanent cause: publishing an
// "unavailable" description would stick (a later head move never
// edits the body). Stop before anything is pushed or opened; the
// run is `Blocked` and a resume asks the summary again.
if self.state.resource_block.is_some() {
self.state.event(
"pr-summary",
"a shared resource was busy; nothing was published, resume to summarize again",
);
return Ok(None);
}
if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
anyhow::bail!(
"the final commit moved during public description generation; retry before publishing"
);
}
let note = match (&summarized, &self.state.change_summary) {
(Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
(Some(Err(why)), _) => ChangeNote::Unavailable(why),
(Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
(None, _) => ChangeNote::NotAsked,
};
let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
if !posting {
let identity = crate::scrub::Identity::current();
return Ok(Some(PrMessage {
title: crate::scrub::scrub(&pr.title, &identity),
body: crate::scrub::scrub(&pr.body, &identity),
}));
}
// This very text was already rejected and asked about: no second
// judge call, rewrite or question, just carry the record on.
let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
if let Some(known) = self
.state
.github_text
.clone()
.filter(|g| g.fingerprint == raw_fp)
{
return self.settle_withheld(winner, &pr, known).await;
}
let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
let mut violations =
crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
let identity = crate::scrub::Identity::current();
if (crate::scrub::scrub(&pr.title, &identity) != pr.title
|| crate::scrub::scrub(&pr.body, &identity) != pr.body)
&& !violations.contains(&crate::github_text::Violation::SensitiveData)
{
violations.push(crate::github_text::Violation::SensitiveData);
}
// A rewrite cannot fix sensitive data (e.g. a quoted original task), so
// only a language violation starts one; sensitive data alone goes
// straight to the owner, with the redacted text as the fallback.
let has_language = violations
.iter()
.any(|v| *v != crate::github_text::Violation::SensitiveData);
if self.state.config.graph.github_text_guard && !violations.is_empty() {
self.state.event(
"github-text",
if has_language {
format!("description rejected: {violations:?}; requesting one rewrite")
} else {
format!("description rejected: {violations:?}; asking the owner")
},
);
let mut rewritten = false;
let mut rejected_rewrite = None;
if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
{
let output = text;
if let Some(title) = summary_title(&output) {
// Judge the redacted candidate that will actually be posted.
// Sensitive data alone must not discard a useful rewrite.
let raw_body = summary_without_title(&output);
let clean_title = crate::scrub::scrub(&title, &identity);
let mut body = crate::scrub::scrub(&raw_body, &identity);
let redacted = clean_title != title || body != raw_body;
let title = clean_title;
let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
if !complete {
self.state.event(
"github-text",
"description rewrite omitted required findings",
);
}
if !body.trim().is_empty() {
body.push_str("\n\n");
body.push_str(&public_pr_evidence(&self.state, head));
}
let marker = format!("magi:run/{}", self.state.id);
if !body.trim().is_empty() && !body.contains(&marker) {
body.push_str(&format!("\n\n{marker}"));
}
let rewrite = PrMessage {
title: title.clone(),
body: body.clone(),
};
let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
let mut failed = crate::github_text::check_for(
&self.state.config.graph,
&title,
&body,
redecision,
);
if (!crate::github_text::shareable(&title)
|| !crate::github_text::shareable(&body))
&& !failed.contains(&crate::github_text::Violation::SensitiveData)
{
failed.push(crate::github_text::Violation::SensitiveData);
}
if complete
&& !title.trim().is_empty()
&& !body.trim().is_empty()
&& !failed.is_empty()
{
rejected_rewrite = Some(crate::run::GithubTextRewrite {
title: title.clone(),
body: body.clone(),
failed: failed
.iter()
.map(|v| crate::github_text::category(*v).to_owned())
.collect(),
});
}
if complete
&& !title.trim().is_empty()
&& !raw_body.trim().is_empty()
&& failed.is_empty()
{
decision = redecision;
pr = PrMessage { title, body };
rewritten = true;
self.state
.event("github-text", "description rewrite passed");
if redacted {
self.state.event(
"github-text",
"sensitive data removed from description rewrite",
);
}
}
}
}
if !rewritten {
self.state.event(
"github-text",
"description rewrite unavailable or rejected; asking the owner",
);
if let Some(w) =
crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
{
// A different text than the one asked about before: that
// question no longer describes anything.
self.retire_github_text_question("the withheld text changed");
let record = crate::run::GithubTextAsk {
rewrite: rejected_rewrite,
fingerprint: raw_fp,
title: w.title,
body: w.body,
categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
question: None,
asks: 0,
resolved: false,
chosen_title: None,
choice: None,
};
return self.settle_withheld(winner, &pr, record).await;
}
}
}
let (title, body) =
crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
Ok(Some(PrMessage { title, body }))
}
/// Describe the whole change from its final diff in either run mode.
///
/// The summary is tied to the final commit and to the merge base of the
/// resolved remote base; a record for any other pair is dropped and the
/// agent asked again, so a description never outlives the diff it was
/// written from. The seat is read-only and the prompt tells it to read the
/// diff and the code around it. `[roles] synthesizer` selects the agents
/// (unset: the default pick), each asked at most once, advancing on an
/// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
/// carries the reason it is unavailable; nothing is invented then.
async fn summarize_change(
&mut self,
winner: &Candidate,
head: Option<&str>,
) -> std::result::Result<(), String> {
let Some(head) = head else {
return Err("the final commit could not be read".to_owned());
};
let remote = self.state.config.merge.remote.clone();
let tracking = format!("{remote}/{}", self.state.base_branch);
if !git::rev_exists(&winner.worktree, &tracking).await {
return Err(format!("the remote base `{tracking}` could not be read"));
}
let base = match git::merge_base(&winner.worktree, &tracking, head).await {
Ok(b) if !b.is_empty() => b,
_ => return Err("the merge base with the remote base could not be read".to_owned()),
};
let evidence = crate::pr_outcome::summary_evidence(
&self.state,
crate::pr_outcome::FinalHead::At(head),
);
let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
if self
.state
.change_summary
.as_ref()
.is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
{
return Ok(());
}
if self.state.change_summary.take().is_some() {
self.state.event(
"pr-summary",
"the final diff or evidence changed; the earlier change summary was discarded",
);
}
// Save the source snapshot even if no generation agent is available.
if let Err(e) = write_artifact(
&self.state,
&format!("public-description-{fingerprint}.md"),
&format!("Final diff: {base}..{head}\n\n{evidence}"),
) {
tracing::warn!("could not save public description evidence: {e:#}");
}
let chain = match agent::pick_chain(
&self.state.config.agents,
self.state.config.roles.synthesizer.as_ref(),
&agent::installed,
"synthesizer",
) {
Ok(chain) => chain,
Err(e) => {
tracing::warn!("no agent for the change summary: {e:#}");
self.state.event("pr-summary", "no summary agent available");
return Err("no agent was available to summarize it".to_owned());
}
};
let subjects = git::subjects(&winner.worktree, &base, head)
.await
.unwrap_or_default();
let stat = git::diff_stat(&winner.worktree, &base, head)
.await
.unwrap_or_default();
let stat: String = stat.chars().take(6_000).collect();
let mut prompt = crate::prompt::change_summary_in(
&base,
head,
&subjects,
stat.trim(),
&self.state.config.graph.github_language,
);
prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
let sessions = self.state.config.graph.sessions;
let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
let prompts = self.state.config.prompts.clone();
let cache = self.state.config.cache_dir();
let run = self.state.id.clone();
for (i, spec) in chain.into_iter().enumerate() {
let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
let job = SeatJob {
spec: spec.clone(),
seat,
cwd: winner.worktree.clone(),
prompt: prompt.clone(),
timeout,
allow_write: false,
sessions,
artifacts: agent::artifacts_dir(&self.state.dir()),
stem: if i == 0 {
"change-summary".to_owned()
} else {
format!("change-summary-{}", spec.id)
},
handover: None,
};
let ctx = WaveCtx {
run: &run,
node: "pr-summary",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
};
let (seat, outcome) =
run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
self.state.seats.insert(seat.key.clone(), seat);
let text = match outcome {
AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
AgentOutcome::ResourceBlocked(why) => {
// No agent was asked: the run stops resumably before
// anything is published (see `guarded_pr_message`).
if self.state.resource_block.is_none() {
self.state.resource_block = Some(crate::run::ResourceBlock {
node: "pr-summary".to_owned(),
kind: "shared resource".to_owned(),
detail: why,
at: Timestamp::now(),
});
}
return Err("a shared resource was busy".to_owned());
}
_ => continue,
};
let blind = &self.state.config.blind;
let Some(title) = summary_title(&text) else {
continue;
};
let body: String = summary_without_title(&text)
.lines()
.filter(|l| !l.contains("magi:run/"))
.collect::<Vec<_>>()
.join("\n");
let body = blind::sanitize_prose(body.trim(), blind);
let title = blind::sanitize_prose(&title, blind);
if body.trim().is_empty() || title.trim().is_empty() {
continue;
}
if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
continue;
}
// The branch must still be the one the agent was shown.
if git::rev_parse(&winner.worktree, "HEAD")
.await
.ok()
.as_deref()
!= Some(head)
{
return Err("the branch moved while the summary was written".to_owned());
}
if git::merge_base(&winner.worktree, &tracking, head)
.await
.ok()
.as_deref()
!= Some(base.as_str())
|| crate::pr_outcome::summary_evidence(
&self.state,
crate::pr_outcome::FinalHead::At(head),
) != evidence
{
return Err("the summary evidence changed while it was written".to_owned());
}
self.state.change_summary = Some(crate::run::ChangeSummary {
head: head.to_owned(),
base,
evidence: fingerprint,
title,
body,
});
self.state.event(
"pr-summary",
format!("change summary written by {} for the final diff", spec.id),
);
if let Err(e) = self.state.save() {
tracing::warn!("could not save the change summary: {e:#}");
}
return Ok(());
}
self.state
.event("pr-summary", "no agent produced a change summary");
Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
}
/// One read-only rewrite request for a withheld description. The
/// implementer's own conversation is reused only when it exists and can be
/// resumed; otherwise the `[roles] language_judge` chain (or the default
/// pick) writes it in a separate conversation. The prompt carries the whole
/// text, so no agent needs any memory. Each agent is asked at most once and
/// the chain advances on an error, quota, unusable answer or a reply with
/// no `TITLE:` line. Nothing is recorded as a handover.
async fn rewrite_pr_text(
&mut self,
winner: &Candidate,
pr: &PrMessage,
violations: &[crate::github_text::Violation],
) -> Option<String> {
let prompt = format!(
"Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
pr.title, pr.body
);
let sessions = self.state.config.graph.sessions;
let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
// (spec, seat, resumed)
let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
let key = format!("impl-{}", winner.label);
if let Some(spec) = self
.state
.config
.agents
.iter()
.find(|a| a.id == winner.agent)
.cloned()
&& let Some(existing) = self.state.seats.get(&key)
&& existing.agent == spec.id
&& agent::has_session(spec.kind, existing, sessions)
{
attempts.push((spec, existing.clone(), true));
} else {
let chain = agent::pick_chain(
&self.state.config.agents,
self.state.config.roles.language_judge.as_ref(),
&agent::installed,
"language_judge",
);
match chain {
Ok(chain) => {
for spec in chain {
let seat = crate::fixer::seat_for(
&mut self.state,
"github-text-rewrite",
&spec.id,
);
attempts.push((spec, seat, false));
}
}
Err(e) => {
tracing::warn!("no agent for the description rewrite: {e:#}");
self.state
.event("github-text", "no rewrite agent available");
return None;
}
}
}
let prompts = self.state.config.prompts.clone();
let cache = self.state.config.cache_dir();
let run = self.state.id.clone();
for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
let job = SeatJob {
spec: spec.clone(),
seat,
cwd: winner.worktree.clone(),
prompt: prompt.clone(),
timeout: retry_budget(timeout_full, resumed),
allow_write: false,
sessions,
artifacts: agent::artifacts_dir(&self.state.dir()),
stem: if i == 0 {
"github-text-rewrite".to_owned()
} else {
format!("github-text-rewrite-{}", spec.id)
},
handover: None,
};
let ctx = WaveCtx {
run: &run,
node: "github-text",
prompts: &prompts,
cache: cache.as_deref(),
round: None,
carry_seats: false,
pauses: Some((&self.pause, &self.interrupt)),
};
let (seat, outcome) =
run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
self.state.seats.insert(seat.key.clone(), seat);
match outcome {
AgentOutcome::Ok(o)
if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
{
return Some(o.text);
}
AgentOutcome::ResourceBlocked(_) => return None,
_ => {}
}
}
None
}
/// Abandon the standing posting-gate question, if any.
fn retire_github_text_question(&mut self, why: &str) {
let Some(id) = self
.state
.github_text
.as_ref()
.and_then(|g| g.question.clone())
else {
return;
};
let _ = ask::Questions::open().update(&id, |q| {
q.abandon(why);
Ok(())
});
}
/// Carry a withheld text to its end: file the question, park while it is
/// open, and on an answer, silence or timeout pick the text to post. The
/// decision is saved before it is used, so it applies once.
async fn settle_withheld(
&mut self,
winner: &Candidate,
pr: &PrMessage,
mut g: crate::run::GithubTextAsk,
) -> Result<Option<PrMessage>> {
use crate::github_text as gt;
let store = ask::Questions::open();
let timeout = self.state.config.graph.answer_timeout;
if !g.resolved {
let mut standing = g
.question
.as_ref()
.and_then(|id| store.list().into_iter().find(|q| &q.id == id));
if let Some(q) = standing.as_ref()
&& q.status.open()
&& gt::expired(q, timeout)
{
let why = format!("no answer within {}s of asking", timeout.max(1));
standing = store
.update(&q.id, |q| {
q.abandon(&why);
Ok(())
})
.ok()
.map(|(q, ())| q);
}
let mut ask_again: Option<Option<Vec<String>>> = None;
match standing {
// Recorded but never written (a stop between the two saves),
// or removed: ask again without spending the retry.
None if g.asks <= 1 => {
g.asks = 0;
ask_again = Some(None);
}
None => {
g.resolved = true;
g.chosen_title = None;
g.choice = Some(GithubTextChoice::Fallback);
}
Some(q) if q.status.open() => {
self.park_on_github_text(&g)?;
return Ok(None);
}
Some(q) => match gt::read_reply(&q) {
gt::Reply::Fallback => {
g.resolved = true;
g.chosen_title = None;
g.choice = Some(GithubTextChoice::Fallback);
}
gt::Reply::Original => {
g.resolved = true;
g.chosen_title = None;
g.choice = Some(GithubTextChoice::Original);
}
gt::Reply::Rewrite => {
g.resolved = true;
g.chosen_title = None;
g.choice = Some(if g.rewrite.is_some() {
GithubTextChoice::Rewrite
} else {
GithubTextChoice::Fallback
});
}
gt::Reply::Title(text) => {
let first = text
.lines()
.map(str::trim)
.find(|l| !l.is_empty())
.unwrap_or("");
let decision = if first.is_empty() {
None
} else {
let candidate = PrMessage {
title: first.to_owned(),
body: String::new(),
};
self.judge_pr_language(&winner.worktree, &candidate).await
};
match gt::vet_title(
&text,
decision,
self.state.config.graph.github_english(),
) {
Ok(title) => {
g.resolved = true;
g.chosen_title = Some(title);
g.choice = Some(GithubTextChoice::MyTitle);
}
Err(cats) if g.asks < 2 => {
ask_again = Some(Some(
cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
))
}
Err(_) => {
g.resolved = true;
g.chosen_title = None;
g.choice = Some(GithubTextChoice::Fallback);
}
}
}
},
}
if let Some(rejected) = ask_again {
// A retry (after a rejected replacement title) is still about
// the original text: the same ORIGINAL / FALLBACK comparison.
let w = gt::Withheld {
title: g.title,
body: g.body,
categories: g.categories.clone(),
};
// Named by the original text's fingerprint alone, so a resume
// rewrites the same file and the detail stays identical.
let artifact = match crate::run::write_artifact(
&self.state,
>::artifact_name(>::fingerprint(&pr.title, &pr.body)),
>::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
) {
Ok(p) => Some(p),
Err(e) => {
tracing::warn!("could not write the withheld-text artifact: {e:#}");
None
}
};
let (detail, comparison) = gt::question_content(
&w,
&pr.title,
&pr.body,
&self.state.id,
rejected.as_deref(),
artifact.as_deref(),
g.rewrite.as_ref(),
);
let mut q = ask::Question::new(
self.state.id.clone(),
gt::ASK_NODE.to_owned(),
gt::ASK_SEAT.to_owned(),
gt::question_summary(&self.state.config.graph.language, &w),
detail,
gt::question_choices(&w, g.rewrite.as_ref()),
);
q.posting_comparison = Some(comparison);
q.answer_timeout = timeout;
// Recorded before it is written: a stop in between leaves a
// record naming a question the resume then files again.
g.question = Some(q.id.clone());
g.asks += 1;
self.state.github_text = Some(g.clone());
self.state.event(
"github-text",
format!("asking the owner about the withheld text ({})", q.short()),
);
self.state.save()?;
store
.put(&mut q)
.context("file the posting-gate question")?;
if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
tracing::warn!(
"could not notify about posting-gate question {}: {e:#}",
q.short()
);
}
self.park_on_github_text(&g)?;
return Ok(None);
}
}
let resolved = g.chosen_title.clone();
let original = g.choice == Some(GithubTextChoice::Original);
// Field names only: the event log never records the text.
self.state.event(
"github-text",
if original {
format!(
"posting the original text for: {}",
match (g.title, g.body) {
(true, true) => "title, description",
(true, false) => "title",
_ => "description",
}
)
} else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
"posting the saved rewrite title and description".to_owned()
} else if resolved.is_some() {
"posting the owner's replacement title".to_owned()
} else {
"posting the neutral text for the withheld fields".to_owned()
},
);
self.state.github_text = Some(g.clone());
self.state.save()?;
if g.choice == Some(GithubTextChoice::Rewrite)
&& let Some(r) = g.rewrite.as_ref()
{
return Ok(Some(PrMessage {
title: r.title.clone(),
body: r.body.clone(),
}));
}
let w = gt::Withheld {
title: g.title,
body: g.body,
categories: g.categories.clone(),
};
let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
if let Some(t) = resolved {
title = t;
}
// Every withheld field is now neutral or vetted English.
let english = gt::LanguageDecision {
title_english: true,
body_english: true,
body_complete: true,
};
let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
if original {
// The owner's explicit choice overrides the gate for the fields it
// was made for; the others keep what the gate produced.
if g.title {
title = pr.title.clone();
}
if g.body {
body = pr.body.clone();
}
}
Ok(Some(PrMessage { title, body }))
}
fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
self.state.github_text = Some(g.clone());
self.state.end_pass_parked(ParkCause::Question);
self.state.parked = true;
self.state.event(
"github-text",
"parked awaiting the owner's word on the withheld pull request text - resumes once answered",
);
self.state.save()
}
/// Ask `[roles] language_judge` about this text and record which source
/// decided; the text itself is never recorded.
async fn judge_pr_language(
&mut self,
cwd: &Path,
pr: &PrMessage,
) -> Option<crate::github_text::LanguageDecision> {
self.state.config.roles.language_judge.as_ref()?;
if !self.state.config.graph.language_gate() {
return None;
}
let decision =
crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
self.state.event(
"github-text",
if decision.is_some() {
"language decided by the language judge"
} else {
"language judge unavailable; using built-in heuristics"
},
);
decision
}
async fn merge(&mut self) -> Result<()> {
// Same reasoning as `gate`: ask the review and gate records directly
// rather than `status`, which a solo-candidate `judge`/`deliberate`
// skip can rewrite on reentry to something that no longer says
// `Blocked`. `review_conclusion` is the same derivation `gate` uses,
// so a hand-off (open findings, green verification) reaches merge
// exactly like a genuinely clean round does.
//
// A run resumed mid-`land` never reaches here at all: `execute`
// recognises `RunStatus::Landing` before it even calls `prep`, and
// routes straight to `run_land` instead. That has to happen a level
// up from this function, not with a check in here, because
// `review_loop`'s own status recomputation (see its doc) runs
// *before* `merge` on every reentry and would otherwise overwrite
// the `Landing` marker with `Gating` before this node ever saw it.
if self
.state
.base_sync
.as_ref()
.is_some_and(|s| s.conflict.is_some())
|| review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
!= Some(RunStatus::Gating)
// `gate_ran == false` is not "passed" - `gate` leaves it false
// both before it has ever run and when its last attempt was
// resource-blocked (see `Runner::gate`'s own doc), and neither is
// permission to merge on nothing but the review record. Only a
// gate that actually ran - zero commands configured and
// vacuously passed, or one or more that all exited 0 - may
// proceed; `RunState::gate_status` is the single place that
// reading is computed.
|| !self.state.gate_status().ok()
{
return Ok(());
}
// This node's own record, not `status`: `status == Ready` is not
// unique to the harmless `MergeMode::None` path this line was
// written for. `land` (below) sets it too, when a `MergeMode::Pr`
// run's PR was closed without merging — and on that run `mode` is
// still `Pr`, so a reentry that fell through here would push and
// open a second pull request. `self.state.merge` is set exactly once
// this node (or `land`) has already produced a verdict, under every
// mode, which is what "already done" actually means here.
if self.state.merge.is_some() && !self.state.pr_publication_pending {
return Ok(());
}
let Some(winner) = self.state.winner().cloned() else {
return Ok(());
};
let repo = self.state.repo.clone();
let base = self.state.base_branch.clone();
let mode = self.state.config.merge.mode;
let style = self.state.config.merge.style;
let facts = if is_review_run(&self.state) {
refresh_reviewed_commits(&mut self.state, &winner.branch).await;
let start = review_base(
&repo,
&self.state.config.merge.remote,
&base,
&self.state.base_commit,
&winner.branch,
)
.await;
branch_facts(&repo, &start, &winner.branch).await
} else {
None
};
// `None` when the base could not be freshly read: then an adopted
// pull request's title is left alone.
let leaked = if is_review_run(&self.state) {
leaked_subjects(&self.state, &winner.branch).await
} else {
Some(Vec::new())
};
// `None`: parked on the owner's word about a withheld title or body.
// Nothing has been pushed or opened, and `merge` stays unrecorded so
// the resume arrives here again.
let Some(pr) = self
.guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
.await?
else {
return Ok(());
};
let message = pr.commit_message();
let outcome = match mode {
MergeMode::None => MergeOutcome {
mode,
ok: true,
detail: manual_merge_command(style, &repo, &winner.branch, &message),
empty: false,
},
MergeMode::Pr | MergeMode::Local
if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
{
MergeOutcome {
mode,
ok: false,
detail: empty_candidate_detail(&self.state, &base),
empty: true,
}
}
MergeMode::Local => {
let on = git::current_branch(&repo).await?;
if on.as_deref() != Some(base.as_str()) {
MergeOutcome {
mode,
ok: false,
detail: format!(
"{} has {} checked out, not the base branch {base}",
repo.display(),
on.unwrap_or_else(|| "a detached HEAD".to_owned())
),
empty: false,
}
} else if !git::is_clean(&repo).await? {
MergeOutcome {
mode,
ok: false,
detail: format!("{} is dirty; refusing to merge", repo.display()),
empty: false,
}
} else {
let out = match style {
MergeStyle::Merge => {
git::merge_no_ff(&repo, &winner.branch, &message).await?
}
MergeStyle::Squash => {
git::merge_squash(&repo, &winner.branch, &message).await?
}
MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
};
MergeOutcome {
mode,
ok: out.ok(),
detail: if out.ok() { out.stdout } else { out.stderr },
empty: false,
}
}
}
MergeMode::Pr if self.state.task_pr_branch().is_some() => {
self.state.pr_publication_pending = true;
self.state.save()?;
let out =
crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
self.state.pr_publication_pending = out.is_err();
MergeOutcome {
mode,
ok: out.is_ok(),
detail: match out {
Ok(url) => url,
Err(e) => format!("{e:#}"),
},
empty: false,
}
}
MergeMode::Pr => {
let remote = self.state.config.merge.remote.clone();
let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
if !pushed.ok() {
MergeOutcome {
mode,
ok: false,
detail: pushed.stderr,
empty: false,
}
} else {
// A retry or resume of a run whose branch already has an
// open pull request adopts it rather than failing on a
// duplicate. Only this winner branch into this base:
// `branch_for` derives the name from the run id, so a
// different run's pull request never matches.
let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
let out = match pr_merge_plan(found) {
PrPlan::Create => {
gh_pr_create(
&winner.worktree,
&base,
&winner.branch,
&pr.title,
&pr.body,
)
.await
}
PrPlan::Adopt { url, title } => {
self.state
.event("merge", format!("Pr: adopted open pull request {url}"));
if self.state.github_text.as_ref().is_some_and(|g| {
g.resolved
&& g.choice == Some(GithubTextChoice::Rewrite)
&& g.rewrite.is_some()
}) {
// The owner chose the entire pair, even if the
// existing title already matches or was edited.
land::set_pr_text_verbatim(
&winner.worktree,
&url,
&pr.title,
&pr.body,
)
.await
.map(|()| url)
} else {
if title != pr.title
&& (!is_review_run(&self.state)
|| leaked
.as_deref()
.is_some_and(|l| should_retitle(&title, &pr.title, l)))
&& let Err(e) =
if self.state.github_text.as_ref().is_some_and(|g| {
g.resolved
&& g.title
&& g.choice == Some(GithubTextChoice::Original)
}) {
// The owner chose to post this title unmodified.
land::set_pr_title_verbatim(
&winner.worktree,
&url,
&pr.title,
)
.await
} else {
land::set_pr_title(
&mut self.state,
&winner.worktree,
&url,
&pr.title,
)
.await
}
{
tracing::warn!("could not refresh title of {url}: {e:#}");
self.state
.event("merge", format!("Pr: title refresh failed: {e:#}"));
}
Ok(url)
}
}
PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
};
match out {
Ok(url) => MergeOutcome {
mode,
ok: true,
detail: url,
empty: false,
},
Err(e) => MergeOutcome {
mode,
ok: false,
detail: e.to_string(),
empty: false,
},
}
}
}
};
self.state.status = match (mode, outcome.ok) {
(MergeMode::None, _) => RunStatus::Ready,
(_, true) => RunStatus::Merged,
(_, false) => RunStatus::Blocked,
};
self.state.event(
"merge",
format!(
"{:?}: {}",
mode,
outcome.detail.lines().next().unwrap_or("")
),
);
self.state.merge = Some(outcome);
self.state.save()?;
// The PR is open and the run would historically stop here, leaving the
// operator to watch checks, feed review comments back to a fixer, and
// merge. That was done by hand six times in one session before this
// existed. Opt-in, because merging is the one irreversible thing magi
// can do to a repository.
if self.state.config.graph.land
&& mode == MergeMode::Pr
&& self.state.status == RunStatus::Merged
{
self.run_land().await?;
}
// `run_land` may have left `status` at `Landing` - still waiting on
// CI or the owner's approval, not actually settled - so this has to
// read whatever `status` ended up as here, not the `Merged` this
// function set a few lines up.
self.settle_questions();
Ok(())
}
/// Enter `land`.
///
/// Shared between a fresh run's first pass through [`Runner::merge`] and
/// a resumed run's re-entry. `land::land` itself is what serialises the
/// two git-mutating moments inside the loop — the rebase push and
/// `gh pr merge` — per repository (see its own doc); nothing here needs
/// to hold a lock across the whole call, and doing so would serialise
/// this run's CI wait against a *different* run's land-approval resume
/// in the same repository, which is exactly the "must not wait on
/// another task" property the daemon's slot-freeing exists to give.
async fn run_land(&mut self) -> Result<()> {
crate::pr_head::ensure_owner(&self.state)?;
let url = self
.state
.merge
.as_ref()
.map(|m| m.detail.clone())
.unwrap_or_default();
let url = url.lines().next().unwrap_or("").trim().to_owned();
if !url.starts_with("http") {
return Ok(());
}
// A land failure is not a lost run: the work is on a branch and the
// pull request is open, which is exactly where a human takes over.
match land::land(&mut self.state, &url).await {
Ok(pr) if self.state.parked => {
// `land` already saved the parked marker; nothing here
// overrides `status` back to a terminal value while an
// approval is still outstanding.
let _ = pr;
}
Ok(pr) => {
self.state.status = match pr.state {
land::PrLifecycle::Merged => RunStatus::Merged,
_ => RunStatus::Blocked,
};
// Downstream of a confirmed merge only - see
// `bump::should_release_bump`'s own doc for why this one
// check covers all three of `land`'s success paths.
// Best-effort: the run already landed, so a failure here
// (the decision call, `gh`, `cargo`) is recorded and never
// turns a landed run into a failed one.
if bump::should_release_bump(self.state.status)
&& let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
{
// The event is the run's own record. Not-eligible cases
// (disabled, no `Cargo.toml`, ...) return `Ok`, so an
// `Err` is a bump that was tried and failed:
// `after_merge` itself raises the operator notice for
// that, whether or not a release PR exists yet.
self.state
.event("bump", format!("release bump skipped: {e:#}"));
}
// Independent of the bump, and best-effort in the same way:
// findings the merge left open become follow-up tasks.
if self.state.status == RunStatus::Merged {
crate::followup::after_merge(&mut self.state, &pr.url).await;
}
self.state.save()?;
}
Err(e) => {
self.state.status = RunStatus::Blocked;
self.state.event("land", format!("gave up: {e}"));
self.state.save()?;
}
}
Ok(())
}
// -------------------------------------------------------------- helpers
/// Fetch or create a seat, keeping its conversation across nodes.
fn seat(&mut self, key: &str, agent: &str) -> SeatState {
if let Some(existing) = self.state.seats.get(key)
&& existing.agent == agent
{
return existing.clone();
}
// A seat that changes agent mints its session id from the agent too,
// like a handover: the old agent's uuid is already taken by the CLI.
let fresh = if self.state.seats.contains_key(key) {
handover_seat(key, agent, self.state.next_seat_seed())
} else {
SeatState::new(key, agent, self.state.seed)
};
self.state.seats.insert(key.to_owned(), fresh.clone());
fresh
}
/// The agent now holding seat `key`: `spec`, unless a handover moved the
/// seat to another roster agent, in which case that agent. Nodes that
/// continue a seat's conversation (deliberation, the votes, a reviewer's
/// reconsideration) must keep talking to whoever answered it, not slip
/// back to the agent that failed it.
fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
match self.state.seats.get(key) {
Some(s) if s.agent != spec.id => {
self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
}
_ => spec,
}
}
/// A candidate rendered for judging, with the leak policy applied.
fn view(&self, c: &Candidate) -> CandidateView {
let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
.unwrap_or_default();
let (patch, _) = blind::sanitize_patch(
&format!("candidate {} patch", c.label),
&raw,
&self.state.config.blind,
);
CandidateView {
label: c.label,
branch: c.branch.clone(),
summary: c.summary.clone(),
stat: c.stat.clone(),
patch,
}
}
/// The full candidate set as prompt text, for seats with no live session.
fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
prompt::judge(
"(see above)",
&views,
self.roles.judges.len(),
base_short,
"en",
)
}
/// The final-vote prompt with everything a seat that has no session of its
/// own needs: the candidates, the seat's own ranking and reasons, and the
/// anonymised deliberation it took part in (only when there was one, so a
/// handed-over seat never sees more than the seat it replaces did). The
/// `Final vote` heading stays first.
fn vote_prompt_full(
&self,
j: usize,
labels: &[char],
language: &str,
candidates: &[Candidate],
base_short: &str,
) -> String {
let mut text = format!(
"{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
prompt::final_vote(labels, language),
self.state.instruction,
self.candidate_block(candidates, base_short)
);
if let Some(own) = self
.state
.judgements
.get(j)
.filter(|r| !r.ranking.is_empty())
{
let reasons = own
.reasons
.iter()
.map(|(k, v)| format!("- {k}: {v}"))
.collect::<Vec<_>>()
.join("\n");
text.push_str(&format!(
"\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
own.ranking.iter().collect::<String>(),
if reasons.is_empty() {
""
} else {
", because:\n"
}
));
}
if !self.state.deliberation.is_empty() {
text.push_str("\n# What was argued before this vote\n");
for t in self.transcript(&[], j) {
text.push_str(&format!(
"\n## {}{}\n\n{}\n",
t.who,
if t.is_self { " (you)" } else { "" },
t.body.trim()
));
}
}
text
}
/// Anonymised transcript for judge `self_idx`.
///
/// The initial rankings are always the opening statements. Seeding them
/// only when no turn had been taken yet meant every judge after the first
/// argued against a single voice instead of against the actual split — the
/// disagreement is the information, so it is always on the table.
fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
let mut turns = Vec::new();
for j in &self.state.judgements {
if j.ranking.is_empty() {
continue;
}
let reasons = j
.reasons
.iter()
.map(|(k, v)| format!("- {k}: {v}"))
.collect::<Vec<_>>()
.join("\n");
turns.push(Turn {
who: format!("Judge {} (opening ranking)", j.judge),
is_self: j.judge == self_idx + 1,
body: format!(
"Ranked {}{}{reasons}",
j.ranking.iter().collect::<String>(),
if reasons.is_empty() {
""
} else {
", because:\n"
}
),
});
}
for t in self
.state
.deliberation
.iter()
.flat_map(|r| r.turns.iter())
.chain(current)
{
turns.push(Turn {
who: format!("Judge {}", t.judge),
is_self: t.judge == self_idx + 1,
body: t.body.clone(),
});
}
turns
}
}
/// Does this seat still hold the context a follow-up prompt would rely on?
fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
agent::has_session(spec.kind, seat, sessions)
}
/// The next entry in `roster` after `start`, never wrapping back to the
/// front, whose id is not in `tried` yet.
///
/// Starts one past `start` rather than at the front of `roster`: `start` is
/// the seat's own original position, and a seat whose candidate slot already
/// sits on the roster's second entry must fall through to the third next, not
/// restart at the first — which is very likely a different candidate's own
/// agent already. Never wraps back past `start`, for the same reason: an
/// entry earlier in the roster than the seat's own position is almost
/// certainly some *other* candidate slot's own agent, and once the tail of
/// the roster is exhausted there are no more untried agents for *this* seat
/// to fall through to — the caller's fallback chain ends there, exactly as
/// "no further untried agents remain in the list for that seat" asks for.
///
/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
/// the same id twice (an operator's `roles.implementers` typo, or a
/// `[[agents]]` list reused across roles) must not let
/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
/// over `roster` either finds an untried id or runs out, so this always
/// terminates regardless of duplicates.
fn next_untried_in_roster<'a>(
roster: &'a [AgentSpec],
start: usize,
tried: &BTreeSet<String>,
) -> Option<&'a AgentSpec> {
roster
.get(start + 1..)?
.iter()
.find(|s| !tried.contains(&s.id))
}
/// The successor for a seat, shared by all four seat kinds (implement, judge,
/// review, advise). `others` holds the ids that *currently* occupy the other
/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
/// sees them), so roster entries beyond the seat count act as spares: a failed
/// seat goes to an agent no other seat holds whenever the roster permits.
///
/// `carried` is `Some` only for the review loop's carried failure history.
/// Order: (1) forward from `start`, never wrapping, untried, not a carried
/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
/// whole roster: untried and not another seat's occupant; (3) the plain walk
/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
/// a better outcome than an empty seat, and it keeps the answer to "is there
/// a successor at all" exactly what it was before occupants were considered,
/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
/// carried failure is a cheaper bet than doubling an agent on the panel, and
/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
/// the same round are handled one at a time, so a seat later in the batch
/// sees an earlier one's new occupant but not yet its own freed agent.
fn pick_successor<'a>(
roster: &'a [AgentSpec],
start: usize,
tried: &BTreeSet<String>,
carried: Option<&BTreeSet<String>>,
others: &BTreeSet<String>,
) -> Option<&'a AgentSpec> {
let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
roster
.get(start + 1..)
.and_then(|tail| {
tail.iter()
.filter(free)
.find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
})
.or_else(|| {
carried
.is_some()
.then(|| roster.iter().find(free))
.flatten()
})
.or_else(|| match carried {
Some(c) => next_for_seat(roster, start, tried, c),
None => next_untried_in_roster(roster, start, tried),
})
}
/// The next agent for a seat that carries its failure history across rounds
/// (the review loop). `round_tried` is this round's own bound and starts
/// empty every round; `carried_failed` only decides priority.
///
/// First: an id walking forward from `start`, never wrapping, that is neither
/// tried this round nor failed in an earlier one. Only when that is exhausted
/// does it rescue: the first roster id (in roster order, so this one *does*
/// look before `start`) not yet tried this round, which is by then a carried
/// failure. Each id is rescued at most once per round, so it cannot loop.
fn next_for_seat<'a>(
roster: &'a [AgentSpec],
start: usize,
round_tried: &BTreeSet<String>,
carried_failed: &BTreeSet<String>,
) -> Option<&'a AgentSpec> {
roster
.get(start + 1..)?
.iter()
.find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
.or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
}
/// Where a reviewer seat starts a round: the agent that last answered it when
/// it is still on the roster and not marked failed, else the spec's own agent
/// unless it failed, else the next roster agent that has not failed, else the
/// spec's own agent again (the whole roster failed). Ids no longer on the
/// roster are ignored. An empty roster has no handover, so the spec stands.
fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
let Some(h) = hist.filter(|_| !roster.is_empty()) else {
return spec;
};
let ok = |id: &str| !h.failed.contains(id);
if let Some(last) = h.last_ok.as_deref()
&& ok(last)
&& let Some(s) = roster.iter().find(|s| s.id == last)
{
return s.clone();
}
if ok(&spec.id) {
return spec;
}
let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
roster
.get(from + 1..)
.into_iter()
.flatten()
.chain(roster.iter())
.find(|s| ok(&s.id))
.cloned()
.unwrap_or(spec)
}
/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
/// seed so a CLI that mints its session id up front (`--session-id`) never
/// reuses the uuid the previous agent already opened under the same seat key.
pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
}
/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
/// so the classifier below and the code that builds the message cannot drift.
const TIMED_OUT: &str = "timed out";
impl FailClass {
/// `None` for an answer; otherwise how the turn failed.
fn of(out: &AgentOutcome) -> Option<Self> {
match out {
AgentOutcome::Ok(_) => None,
AgentOutcome::Quota(_) => Some(Self::Quota),
AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
// Not a failure of the agent. Callers check for it first; `None`
// here means "no class to hand over on".
AgentOutcome::ResourceBlocked(_) => None,
}
}
/// The word in a handover's artifact stem (`impl-A-quota-beta`).
fn stem_word(&self) -> &'static str {
match self {
Self::Quota => "quota",
_ => "handover",
}
}
}
/// The message's first line with its variable parts removed — digit runs and
/// path-like tokens — so "exited with Some(2)" and "exited with Some(7)" read
/// as one kind of failure.
fn failure_signature(msg: &str) -> String {
let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
let mut out = Vec::new();
for word in line.split_whitespace() {
if word.contains('/') || word.contains('\\') {
out.push("<path>".to_owned());
continue;
}
let mut w = String::new();
let mut in_digits = false;
for c in word.chars() {
if c.is_ascii_digit() {
if !in_digits {
w.push('#');
}
in_digits = true;
} else {
in_digits = false;
w.push(c);
}
}
out.push(w);
}
out.join(" ").chars().take(120).collect()
}
/// Whether a seat that just failed with `cur` may go to the next roster agent.
/// A quota or a timeout always may. Any other failure may not when the agent
/// before it failed the same way: an error the prompt causes would otherwise
/// walk the whole roster. `prev` is the class of the immediately preceding
/// agent's failure, so a quota or timeout in between breaks the run of
/// identical failures by itself.
fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
match cur {
FailClass::Quota | FailClass::Timeout => true,
FailClass::Other(_) => prev != Some(cur),
}
}
/// A short human reason for a failed outcome, for the handover record.
fn fail_reason(out: &AgentOutcome) -> String {
match out {
AgentOutcome::Ok(_) => String::new(),
AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
AgentOutcome::Dropped(o) => format!(
"the CLI dropped the stream ({})",
o.dropped
.as_ref()
.map(|d| d.why.as_str())
.unwrap_or("it ended without delivering its answer")
),
AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
e.lines().next().unwrap_or("").chars().take(160).collect()
}
}
}
/// Note one handover in the run: the structured record and, in the timeline,
/// the sentence a person reads. A quota keeps the wording it always had.
pub(crate) fn record_handover(
state: &mut RunState,
node: &str,
seat: &str,
from: &str,
to: &str,
class: &FailClass,
reason: &str,
) {
let message = if *class == FailClass::Quota {
format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
} else {
format!("{seat}: handed over {from} -> {to} ({reason})")
};
state.event(node, message);
state.handovers.push(Handover {
at: Timestamp::now(),
node: node.to_owned(),
seat: seat.to_owned(),
from: from.to_owned(),
to: to.to_owned(),
reason: reason.to_owned(),
});
}
/// Did this reply report running a command whose own CLI never confirmed an
/// exit status?
///
/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
/// command *finished* (see that type's own doc), so this can only be `true`
/// for a command whose completion event carried no readable exit code — not
/// for one that simply is not mentioned at all. That is the one signal this
/// crate can read, from the same record `state.jobs` renders, about a reply
/// standing next to work its own CLI cannot vouch for finishing; it is
/// deliberately not a check on the exit code's *value* (a fixer legitimately
/// runs a command that fails mid-iteration before it succeeds) and not a
/// guess at a command still running in the background (which emits no event
/// at all, and so leaves no evidence here to find).
fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
commands.iter().any(|c| c.exit_code.is_none())
}
/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
/// trusted as a verified no-op — the adoption guard's own text-level half.
///
/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
/// timed out): a marker only earns the benefit of the doubt from a turn the
/// CLI itself vouches for finishing properly, the same house style
/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
/// *fix* report to for `commands`. A candidate that timed out, exited
/// non-zero, or left a command unconfirmed is read as the ordinary loss it
/// is, whatever prose it wrote — this returns `None` before it ever looks at
/// `text`. The remaining guards (the tree really is empty, the evidence is
/// non-empty) are the caller's: this only reads what the reply *claimed*.
fn verified_noop_claim(
usable: bool,
commands: &[agent::CommandEvidence],
text: &str,
) -> Option<String> {
(usable && !has_unconfirmed_command(commands))
.then(|| verdict::verified_noop(text))
.flatten()
}
fn short(commit: &str) -> String {
commit.chars().take(7).collect()
}
fn make_executable(path: &Path) -> Result<()> {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt as _;
let mut perms = std::fs::metadata(path)?.permissions();
perms.set_mode(0o755);
std::fs::set_permissions(path, perms)?;
}
#[cfg(not(unix))]
{
let _ = path;
}
Ok(())
}
/// What every seat in one batch shares: where the answers are attributed, the
/// prompt overlay they inherit, and the build cache they are told to use.
///
/// A struct rather than four more parameters: `wave` also needs the run's
/// state (to record who is answering right now) and the attempt number, and
/// eight positional arguments is both unreadable and a clippy error.
struct WaveCtx<'a> {
/// Exported as `MAGI_RUN`, so a task an agent files names the run that
/// paid for it.
run: &'a str,
/// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
node: &'a str,
prompts: &'a Prompts,
/// The shared `CARGO_TARGET_DIR`, when the config declares one.
cache: Option<&'a Path>,
/// The review round this wave belongs to, for `"review"`/`"fix"` — see
/// `JobRecord::round`. `None` for every other node.
round: Option<usize>,
/// Carry each seat's failed-agent history across waves (the review loop
/// only): start-of-round priority and handover choice read
/// [`RunState::seat_history`], and every answer or failure writes it.
carry_seats: bool,
/// The run's park / interrupt requests, so a wait for a shared resource
/// can be abandoned the moment the operator asks to park.
pauses: Option<(&'a Pause, &'a Pause)>,
}
/// Run one job, honouring the parallelism budget.
async fn run_one(
job: SeatJob,
sem: Arc<Semaphore>,
ctx: &WaveCtx<'_>,
state: &mut RunState,
attempt: usize,
) -> (SeatState, AgentOutcome) {
let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
.await
.pop()
.expect("one job in, one result out");
(seat, out)
}
/// Run every job concurrently, capped by the semaphore, preserving order.
///
/// Every seat in the batch is recorded into [`RunState::active`] before the
/// wave starts and cleared as each answer lands, so the run's own record says
/// who is still being waited on rather than only who finished.
async fn wave(
jobs: Vec<SeatJob>,
sem: Arc<Semaphore>,
ctx: &WaveCtx<'_>,
state: &mut RunState,
attempt: usize,
) -> Vec<(usize, SeatState, AgentOutcome)> {
let WaveCtx {
run,
node,
prompts,
cache,
round,
carry_seats: _,
pauses,
} = *ctx;
for job in &jobs {
state.seat_started(node, &job.seat.key, job.timeout, attempt);
}
if let Err(e) = state.save() {
// A failed persist of "who is answering right now" must not abort the
// wave: the seats are already being asked, and the alternative is
// losing the answers to save a status line nobody may even be
// watching.
tracing::warn!("could not persist in-progress seats: {e:#}");
}
// Hold the shared build cache's lease for the whole batch, not per job:
// several candidates (an implement wave) or a fixer legitimately share
// one cache concurrently within this run, and that stays untouched — a
// single lease taken once for the whole wave and released once it is
// done is what stops a *different* borrower (another run's own wave, its
// e2e/gate, a human's `magi review`) from interleaving a build into the
// same directory while this one is in flight. Best-effort, not
// all-or-nothing: a wave that cannot get the lease within its own
// longest job's budget still runs — an hour of paid implementer calls is
// not thrown away over cache contention — but every write-allowed seat
// then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
// below), the same fallback a read-only seat always gets, rather than
// building into a directory this run was never granted. The identity
// record is still invalidated below either way, so the next tracked
// caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
// The wait for the lease has its own bound, `timeout_resource_wait`; it is
// never borrowed from a job's agent budget (and never charged to it either,
// see below). The agent's clock starts in `agent::invoke`, at spawn.
let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
let cache_guard = if let Some(cache_dir) = cache {
if jobs_had_a_writer {
let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
{
Ok(Some(g)) => Some(g),
// Parking: leave without spawning anything. The run parks at
// the next boundary and the seats are asked afresh on resume.
Ok(None) => {
state.resource_block = Some(crate::run::ResourceBlock {
node: node.to_owned(),
kind: "shared build cache".to_owned(),
detail: "the run is parking".to_owned(),
at: Timestamp::now(),
});
return block_wave(
jobs,
state,
node,
attempt,
"parked while waiting for the shared build cache",
);
}
// An unreadable lease is not contention: keep the old
// fallback (no shared cache for this wave's seats).
Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
Err(e) => {
// Writing seats may not build into a cache this run does
// not hold, and an agent that cannot be given one is not
// asked at all: the run stops resumably instead.
let why = format!("{e:#}");
state.resource_block = Some(crate::run::ResourceBlock {
node: node.to_owned(),
kind: "shared build cache".to_owned(),
detail: why.clone(),
at: Timestamp::now(),
});
state.event(
"resource",
format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
);
return block_wave(jobs, state, node, attempt, &why);
}
}
} else {
None
}
} else {
None
};
// Every job keeps its whole budget: the wait above was not the agent's.
// Restart the seats' clocks so "started" means "being spawned now".
state.seats_spawning(node, attempt);
if let Err(e) = state.save() {
tracing::warn!("could not persist the start of a wave: {e:#}");
}
// Queueing behind this wave's own seats for a parallelism slot is normal
// (each of them is bounded by its own agent timeout); only a wait beyond
// all of them plus the resource limit means the slots never came back.
let slot_limit = resource_limit
+ jobs
.iter()
.map(|j| j.timeout)
.max()
.unwrap_or_default()
.saturating_mul(jobs.len() as u32);
let mut set = tokio::task::JoinSet::new();
// Seats announce themselves here the moment they hold a parallelism
// permit, i.e. just before their CLI is spawned.
let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
let overlay = prompts.overlay(node);
for (i, mut job) in jobs.into_iter().enumerate() {
job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
if cache.is_some() {
job.prompt.push('\n');
job.prompt
.push_str(&prompt::build_cache_note(node, job.allow_write));
}
let sem = Arc::clone(&sem);
let run = run.to_owned();
let node = node.to_owned();
// Only implementers were told about the task's attachments, so only
// their seats get the directory widened for reading.
let attachments = if node == "implement" {
state.attachments.clone()
} else {
Vec::new()
};
// A read-only seat is never handed `CARGO_TARGET_DIR` — see
// `prompt::build_cache_note`'s doc for why setting it anyway is
// exactly how a sandboxed reviewer's write refusal got reported as a
// defect in the patch, not a property of its own seat. And a
// write-allowed one is handed it only when the lease above was
// actually acquired: a wave that could not get it (`cache_guard` is
// `None`, see its own comment) must not send seats to build into a
// directory this run does not hold - that is the exact concurrent,
// unmanaged-write race this module exists to prevent, not something
// "proceeding anyway" is allowed to reintroduce.
let cache = cache
.filter(|_| job.allow_write && cache_guard.is_some())
.map(Path::to_path_buf);
let spawned_tx = spawned_tx.clone();
set.spawn(async move {
// A wait for a parallelism slot is scheduling, not the agent: it
// is bounded by its own limit and ends as a resource block, never
// as a timeout of the agent that was not started.
let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
Ok(p) => p,
Err(_) => {
return (
i,
job.seat,
AgentOutcome::ResourceBlocked(format!(
"no parallel slot became free within {}s",
slot_limit.as_secs()
)),
);
}
};
spawned_tx.send(job.seat.key.clone()).ok();
let mut seat = job.seat;
let out = agent::invoke(
&job.spec,
&mut seat,
&Invocation {
cwd: &job.cwd,
prompt: &job.prompt,
timeout: job.timeout,
allow_write: job.allow_write,
unsandboxed: false,
sessions: job.sessions,
artifacts: &job.artifacts,
stem: &job.stem,
run: &run,
node: &node,
cache_dir: cache.as_deref(),
attachments: &attachments,
writable: &[],
},
)
.await;
let out = match out {
Ok(o) if o.usable() => AgentOutcome::Ok(o),
Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
// Billed work the CLI failed to hand over is not an ordinary
// failure, but its text is the CLI's raw error JSON, not an
// answer — `Dropped` keeps it out of `Ok` so a caller cannot
// read it as one by forgetting to check. `usable()` is always
// false here (dropped implies an empty response), so this has
// to be checked before the catch-all `Failed` below or the
// one shape this exists for is lost with the rest.
Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
Ok(o) => AgentOutcome::Failed(format!(
"exited with {:?} and no usable output",
o.exit_code
)),
Err(e) => AgentOutcome::Failed(e.to_string()),
};
(i, seat, out)
});
}
drop(spawned_tx);
let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
loop {
let joined = tokio::select! {
Some(key) = spawned_rx.recv() => {
// The clock of this seat starts now, not when it queued.
state.seat_spawned(&key);
if let Err(e) = state.save() {
tracing::warn!("could not persist a seat's start: {e:#}");
}
continue;
}
joined = set.join_next() => joined,
};
let Some(joined) = joined else {
break;
};
let (i, seat, out) = match joined {
Ok(v) => v,
// No seat to clear: a panicked task never reported which one it
// was. The defensive sweep below this loop is what stops that
// seat's `active` entry from surviving forever.
Err(e) => {
tracing::error!("agent task panicked: {e}");
continue;
}
};
state.seat_finished(&seat.key);
if let AgentOutcome::ResourceBlocked(why) = &out
&& state.resource_block.is_none()
{
state.resource_block = Some(crate::run::ResourceBlock {
node: node.to_owned(),
kind: "parallel slot".to_owned(),
detail: why.clone(),
at: Timestamp::now(),
});
state.event("resource", format!("{node}: no agent was started - {why}"));
}
record_jobs(state, node, round, &seat.key, &out);
if let Err(e) = state.save() {
tracing::warn!("could not persist a seat's completion: {e:#}");
}
if collected.len() <= i {
collected.resize_with(i + 1, || None);
}
collected[i] = Some((i, seat, out));
}
// Belt-and-braces for the panic branch above: every seat this exact batch
// started shares this `(node, attempt)` pair, and every seat that finished
// normally already cleared itself, so anything left tagged with it here
// can only be a panicked task's leftover. Cleared unconditionally rather
// than left to read as still answering forever.
if state
.active
.values()
.any(|a| a.node == node && a.attempt == attempt)
{
state
.active
.retain(|_, a| !(a.node == node && a.attempt == attempt));
if let Err(e) = state.save() {
tracing::warn!("could not persist the end of a wave: {e:#}");
}
}
// Whether or not the lease above was actually held, several worktrees
// may just have built into the cache with nothing here able to name one
// coherent (worktree, head) for it - see `cache::invalidate_identity`'s
// own doc. Forgetting the old record costs the next `e2e`/`gate` one
// clean it might not have strictly needed; trusting a stale match would
// cost it a wrong answer.
if let Some(cache_dir) = cache
&& jobs_had_a_writer
{
crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
}
if let Some(guard) = cache_guard {
guard.release();
}
collected.into_iter().flatten().collect()
}
/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
/// this is data collection, not the fix-specific completion contract in
/// [`Runner::continue_fix_report`], and applies regardless of which node
/// asked.
///
/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
/// evidence from; `Failed` does not, and correctly contributes nothing — a
/// timeout or crash is not itself evidence about a command the seat may have
/// started.
fn record_jobs(
state: &mut RunState,
node: &str,
round: Option<usize>,
seat: &str,
out: &AgentOutcome,
) {
let commands: &[agent::CommandEvidence] = match out {
AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
};
let checked_at = Timestamp::now();
for c in commands {
state.jobs.push(JobRecord {
node: node.to_owned(),
round,
seat: seat.to_owned(),
id: c.id.clone(),
description: c.description.clone(),
checked_at,
status: match c.exit_code {
Some(0) => JobStatus::Completed,
Some(_) => JobStatus::Failed,
None => JobStatus::Unknown,
},
exit_code: c.exit_code,
result_summary: c.result_summary.clone(),
source: c.source.clone(),
});
}
}
/// Is a review round clean, given how many reviewer seats answered against
/// how many the round expected?
///
/// A seat that never answered (timeout, crash, unparsable output) is not a
/// seat that read the patch and found nothing — treating it as such is
/// exactly the bug this function exists to close. Under the default `block`
/// policy a missing seat can never be clean; `warn` still requires the seats
/// that *did* answer to have found nothing blocking and verification to be
/// green.
///
/// `quota_missing` narrows that `block` default for exactly one cause of
/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
/// a session limit lifts by the very next round buys nothing — the seat is
/// asked again with the same quota — so once every missing seat is accounted
/// for by a quota loss (and at least one seat *did* answer, so a decision has
/// something to rest on) the round is decided on the panel that could answer,
/// same as `warn` would. A panel that lost every seat to quota is not
/// decided here: `answered == 0` falls through to the existing `block`
/// fallback so a fully collapsed panel still waits rather than landing on no
/// review at all.
fn round_is_clean(
blocking: usize,
e2e_ok: bool,
answered: usize,
expected: usize,
quota_missing: usize,
policy: IncompleteReviewPolicy,
) -> bool {
if blocking != 0 || !e2e_ok {
return false;
}
if answered == expected || policy == IncompleteReviewPolicy::Warn {
return true;
}
answered > 0 && expected - answered <= quota_missing
}
/// The review loop's own conclusion, derived entirely from its persisted
/// round records and the round budget that produced them — never from
/// `status`, so a reentry (or `gate`/`merge` reading it independently)
/// recomputes the identical answer regardless of what an earlier node in the
/// same walk, or a previous walk, did to `status`.
///
/// `None` while more rounds remain to try, including when review never ran
/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
/// gone clean, or the budget is spent, or the tree has stopped moving (see
/// [`STAGNANT_LIMIT`]), the answer is one of two things:
///
/// - An incomplete panel that raised nothing is missing input, not a
/// verified tree — never a hand-off candidate, whatever verification said
/// (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
/// - Otherwise, green e2e on the last round hands off (see
/// [`Runner::stop_reviewing`]); red e2e blocks.
///
/// A last round whose own verification is still `ResourceBlocked` — magi
/// itself never got a command to run, not evidence the patch is broken —
/// is neither: this returns `None` for it too, the same as "more rounds
/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
/// handling of that shape) instead of this cheap recomputation guessing a
/// verdict a real attempt never produced.
fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
return Some(RunStatus::Gating);
}
let last = reviews.last()?;
let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
if reviews.len() < max_rounds && !stagnant {
return None;
}
if last.incomplete() && last.blocking == 0 {
return Some(RunStatus::Blocked);
}
if last.e2e_status() == E2eStatus::ResourceBlocked {
return None;
}
Some(if last.e2e.iter().all(CommandOutcome::ok) {
RunStatus::Gating
} else {
RunStatus::Blocked
})
}
/// How long a re-ask may take, given the budget the first attempt had.
///
/// A `nudged` retry is a request to restate an answer the seat has already
/// worked out: it carries no new work, so it does not deserve the original
/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
/// 133 seconds while a third sat for over ten minutes on a resumed session
/// holding 410 KB of prior output - and because the retry had inherited the
/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
/// judging round whose other seats were long finished.
///
/// A quarter of the budget, with a floor so that a deliberately short timeout
/// does not collapse to nothing. A retry that re-sends the whole prompt
/// (because the seat kept no context) is the original job again, and keeps the
/// original budget.
pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
if nudged {
(full / 4).max(Duration::from_secs(120)).min(full)
} else {
full
}
}
/// Run a wave and parse each reply, re-asking the seats whose reply was
/// unusable.
///
/// The re-ask is a nudge rather than the whole prompt again when the seat still
/// holds its conversation, which is the difference between a cheap retry and
/// paying for the entire candidate set twice.
///
/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
/// successor in `roster` is handed to it instead of being re-asked: the
/// handover is the retry. A seat with no successor left (a single-agent
/// roster, the roster's tail) is nudged as before, up to `retries` times. So
/// the asks to one seat in one node number at most
/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
/// is asked once (a dropped stream is nudged first), and only the last agent
/// of the chain gets the `retries` same-agent nudges. Each roster agent is
/// tried at most once per seat, walking forward from the seat's own position and never wrapping
/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
/// other failure stops the chain when the previous agent failed the same way
/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
/// [`has_context`] is false and the job's own full prompt and full budget are
/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
/// intermediate ones are not losses) and is returned as a failure like any
/// other absent seat — the caller decides whether the panel still has a
/// quorum. An empty `roster` disables handover: failures are nudged as they
/// always were, and a quota is simply lost. A reply that fails to parse or
/// validate is the prompt's doing and is only ever nudged, never handed over.
///
/// The returned [`SeatState`] names the agent that answered (or tried last).
#[allow(clippy::too_many_arguments)]
async fn ask_json_wave<T>(
jobs: Vec<SeatJob>,
sem: Arc<Semaphore>,
retries: usize,
roster: &[AgentSpec],
ctx: &WaveCtx<'_>,
losses: &mut Vec<QuotaLoss>,
state: &mut RunState,
validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
where
T: serde::de::DeserializeOwned + Send + 'static,
{
ask_wave_with(
jobs,
sem,
retries,
roster,
ctx,
losses,
state,
&|text: &str| {
let v = verdict::extract_json::<T>(text)?;
validate(&v)?;
Ok(v)
},
)
.await
}
/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
/// a node whose answer is prose (deliberation) shares the same handover,
/// failure classification, quota bookkeeping and bounds instead of a copy.
///
/// A seat handed to another roster agent is sent the job's `handover` prompt
/// (when it has one) rather than `prompt`: the new agent has no session, so a
/// resume-style prompt would be incomplete. That holds for the handover ask
/// and for every nudge to that agent whose `has_context` is false.
#[allow(clippy::too_many_arguments)]
async fn ask_wave_with<T>(
jobs: Vec<SeatJob>,
sem: Arc<Semaphore>,
retries: usize,
roster: &[AgentSpec],
ctx: &WaveCtx<'_>,
losses: &mut Vec<QuotaLoss>,
state: &mut RunState,
parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
where
T: Send + 'static,
{
let n = jobs.len();
let originals: Vec<SeatJob> = jobs;
let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
// Nudges each seat's *current* agent has taken — 0 for a first-ask
// answer, N once it has gone through N nudges. Read back once this
// returns, so a caller building a history record (`ReviewRecord`) can
// tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
// "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
// field's own doc.
let mut nudges: Vec<usize> = vec![0; n];
// The agent now occupying each seat, the ids it has already been through,
// where in the roster the walk began, the class of the last failure, and
// the stem word of a handover not yet asked (full prompt, full budget).
let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
let mut tried: Vec<BTreeSet<String>> = specs
.iter()
.map(|s| BTreeSet::from([s.id.clone()]))
.collect();
let starts: Vec<usize> = specs
.iter()
.map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
.collect();
let carry = ctx.carry_seats && !roster.is_empty();
// Carried across rounds: ids that failed the seat earlier, and how the
// last failure went (so a repeat of it is not handed over again).
let carried: Vec<BTreeSet<String>> = originals
.iter()
.map(|j| {
state
.seat_history
.get(&j.seat.key)
.filter(|_| carry)
.map(|h| h.failed.clone())
.unwrap_or_default()
})
.collect();
let mut prev: Vec<Option<FailClass>> = originals
.iter()
.map(|j| {
state
.seat_history
.get(&j.seat.key)
.filter(|_| carry)
.and_then(|h| h.last_fail.clone())
})
.collect();
let next_agent =
|i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
let others: BTreeSet<String> = specs
.iter()
.enumerate()
.filter(|(j, _)| *j != i)
.map(|(_, s)| s.id.clone())
.collect();
pick_successor(
roster,
starts[i],
tried,
carry.then(|| &carried[i]),
&others,
)
.cloned()
};
let mut fresh: Vec<Option<String>> = vec![None; n];
let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
let mut pending: Vec<usize> = (0..n).collect();
// Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
// asks: an agent with a successor is asked once and handed over, and only
// a seat with no successor spends `retries` nudges on the same agent. This
// only guarantees the loop's own termination whatever those say.
let max_rounds = (retries + 1) * roster.len().max(1) + 1;
for round in 0..max_rounds {
if pending.is_empty() {
break;
}
let mut batch = Vec::with_capacity(pending.len());
let mut renudged: Vec<&str> = Vec::new();
for &i in &pending {
let src = &originals[i];
// A seat now held by another agent than the job named has no
// session of its own: it gets the full-context prompt whenever
// it is asked in full (the handover ask, a nudge it cannot
// resume).
let full: &str = match &src.handover {
Some(h) if specs[i].id != src.spec.id => h,
_ => &src.prompt,
};
// The prompt and the budget are one decision: a nudge restates
// finished work, a re-sent prompt redoes it.
let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
(
full.to_owned(),
src.timeout,
format!("{}-{word}-{}", src.stem, specs[i].id),
)
} else if nudges[i] == 0 {
(full.to_owned(), src.timeout, src.stem.clone())
} else {
renudged.push(src.seat.key.as_str());
let why = done[i]
.as_ref()
.and_then(|r| r.as_ref().err().map(ToString::to_string))
.unwrap_or_else(|| "no parsable answer".to_owned());
let nudge = prompt::nudge(&why);
let nudged = has_context(&specs[i], &seats[i], src.sessions);
let prompt = if nudged {
nudge
} else {
format!("{full}\n\n---\n\n{nudge}")
};
(
prompt,
retry_budget(src.timeout, nudged),
format!("{}-retry{}", src.stem, nudges[i]),
)
};
batch.push(SeatJob {
spec: specs[i].clone(),
seat: seats[i].clone(),
cwd: src.cwd.clone(),
prompt,
timeout,
allow_write: src.allow_write,
sessions: src.sessions,
artifacts: src.artifacts.clone(),
stem,
handover: None,
});
}
if !renudged.is_empty() {
state.event(
ctx.node,
format!("retry {round}: re-asking {}", renudged.join(", ")),
);
}
let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
let mut still = Vec::new();
for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
seats[i] = seat;
// No agent was asked: nothing to hand over, nudge, remember or
// charge. The caller sees `resource_block` and stops the node.
if let AgentOutcome::ResourceBlocked(why) = &out {
done[i] = Some(Err(anyhow::anyhow!("{why}")));
continue;
}
let class = FailClass::of(&out);
// A dropped stream is nudged first (the conversation is still
// there to pick up); only a seat whose nudges are spent hands over.
let nudge_first = matches!(out, AgentOutcome::Dropped(_))
&& nudges[i] < retries
&& !roster.is_empty();
if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
let next = should_hand_over(prev[i].as_ref(), &cur)
.then(|| next_agent(i, &tried[i], &specs))
.flatten();
if carry {
let h = state
.seat_history
.entry(originals[i].seat.key.clone())
.or_default();
h.failed.insert(specs[i].id.clone());
h.last_fail = Some(cur.clone());
if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
h.last_ok = None;
}
// Saved before the next agent is asked, so a restart in
// between does not forget who failed.
if let Err(e) = state.save() {
tracing::warn!("could not persist a seat's failure history: {e:#}");
}
}
if let Some(next) = next {
record_handover(
state,
ctx.node,
&originals[i].seat.key,
&specs[i].id,
&next.id,
&cur,
&fail_reason(&out),
);
tried[i].insert(next.id.clone());
prev[i] = Some(cur.clone());
seats[i] =
handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
specs[i] = next;
fresh[i] = Some(cur.stem_word().to_owned());
nudges[i] = 0;
done[i] = Some(Err(anyhow::anyhow!(
"handed over after: {}",
fail_reason(&out)
)));
still.push(i);
continue;
}
}
if carry
&& !nudge_first
&& let Some(cur) = class.clone()
{
// The chain ended here (no successor, or a repeated failure).
let h = state
.seat_history
.entry(originals[i].seat.key.clone())
.or_default();
h.failed.insert(specs[i].id.clone());
h.last_fail = Some(cur);
}
let parsed = match out {
AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
AgentOutcome::Quota(o) => {
last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
}
// Not a parseable answer: the nudge loop re-asks it, which is
// exactly what a dropped stream needs. Just don't hand its raw
// error JSON to `extract_json`.
AgentOutcome::Dropped(o) => {
let why = o
.dropped
.as_ref()
.map(|d| d.why.as_str())
.unwrap_or("the CLI ended the stream without delivering its answer");
Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
}
AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
Err(anyhow::anyhow!(e))
}
};
let quota = class == Some(FailClass::Quota);
let failed = parsed.is_err();
done[i] = Some(parsed);
if carry && !failed {
let h = state
.seat_history
.entry(originals[i].seat.key.clone())
.or_default();
h.failed.remove(&specs[i].id);
h.last_ok = Some(specs[i].id.clone());
h.last_fail = None;
}
// Do not re-ask a rate-limited seat (quota) — a retry is known to
// fail the same way; and never re-ask a seat that already parsed.
// A failed agent that still has a successor is not re-asked
// either: the handover was its remedy and has just been refused
// (the chain stops on a repeated failure class). A seat with no
// successor left (a single-agent roster, the roster's tail, or an
// empty roster) keeps the same-agent nudge, bounded by `retries`.
let agent_failure = class.is_some()
&& !nudge_first
&& !roster.is_empty()
&& next_agent(i, &tried[i], &specs).is_some();
if failed && !quota && !agent_failure && nudges[i] < retries {
nudges[i] += 1;
still.push(i);
}
}
pending = still;
}
// One loss per seat whose chain ended on a quota: the intermediate ones
// were absorbed by a handover and are not losses.
for (i, q) in last_quota.into_iter().enumerate() {
if let Some(reset) = q {
losses.push(QuotaLoss {
seat: originals[i].seat.key.clone(),
node: ctx.node.to_owned(),
at: Timestamp::now(),
reset,
});
}
}
seats
.into_iter()
.zip(done)
.zip(nudges)
.map(|((seat, res), attempts)| {
(
seat,
res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
attempts,
)
})
.collect()
}
/// Did the wave end without asking any agent (see [`block_wave`])?
fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
results
.iter()
.any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
}
/// Settle a wave that never spawned anything: every job answers
/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
/// else about the seat (session, history, quota) is touched.
fn block_wave(
jobs: Vec<SeatJob>,
state: &mut RunState,
node: &str,
attempt: usize,
why: &str,
) -> Vec<(usize, SeatState, AgentOutcome)> {
state.resource_wait = None;
state
.active
.retain(|_, a| !(a.node == node && a.attempt == attempt));
if let Err(e) = state.save() {
tracing::warn!("could not persist a resource-blocked wave: {e:#}");
}
jobs.into_iter()
.enumerate()
.map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
.collect()
}
/// Acquire the shared build cache's lease, waiting out contention within
/// `budget` (never past it — see AGENTS.md's build-cache section on why an
/// unbounded wait is never acceptable).
///
/// A first, non-blocking check happens before ever waiting; if it finds the
/// lease busy, that fact is logged as a `verify` event *and* flushed with
/// [`RunState::save`] immediately — not only once the wait finally succeeds
/// or gives up — so a `magi show` run by a different process while this one
/// is still waiting reads a `run.json` that says so, rather than whatever it
/// looked like before the wait started. The same applies to the terminal
/// failure: logged and saved before this returns `Err`, so a caller that
/// could not get the lease at all still leaves a legible record of why.
async fn acquire_cache_lease(
state: &mut RunState,
cache_dir: &Path,
owner: &crate::cache::Owner,
budget: Duration,
context: &str,
cancel: &(dyn Fn() -> bool + Sync),
) -> Result<Option<crate::cache::Guard>> {
let home = crate::run::home();
let started = Instant::now();
let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
Err(e) => {
state.event(
"verify",
format!("{context}: could not check the shared build cache: {e:#}"),
);
if let Err(e2) = state.save() {
tracing::warn!("could not persist a cache-check failure: {e2:#}");
}
return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
}
};
state.event(
"verify",
format!(
"{context}: waiting for the shared build cache at {} ({})",
cache_dir.display(),
busy.describe()
),
);
// Seats of this node are waiting, not running: say so, so the countdown a
// reader sees only ever measures an agent that has been spawned.
state.resource_wait = Some(crate::run::ResourceWait {
node: context.to_owned(),
kind: "shared build cache".to_owned(),
holder: busy.describe(),
since: Timestamp::now(),
limit_secs: budget.as_secs(),
});
if let Err(e) = state.save() {
tracing::warn!("could not persist a cache wait: {e:#}");
}
let remaining = budget.saturating_sub(started.elapsed());
let waited = crate::cache::wait_for_unless(
&home,
cache_dir,
owner,
remaining,
Duration::from_secs(5),
cancel,
)
.await;
state.resource_wait = None;
match waited {
Ok(g) => Ok(g),
Err(e) => {
state.event("verify", format!("{context}: {e:#}"));
if let Err(e2) = state.save() {
tracing::warn!("could not persist a cache wait timeout: {e2:#}");
}
Err(e)
}
}
}
/// The lease state could not be read at all (unreadable lease file, a
/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
/// known to be contended, so a wave degrades to running without a shared cache
/// instead of stopping on a hold that [`crate::triage`] could never release.
#[derive(Debug)]
struct CacheCheckError(String);
impl std::fmt::Display for CacheCheckError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for CacheCheckError {}
/// Run `body` — a verify command batch — while holding the shared build
/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
/// never interleave with another borrower's build against the same
/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
/// timeout, or a human's own `magi review`. See the `cache` module doc for
/// why this matters more than Cargo's own per-target locking covers — two
/// *different* worktrees building the same package name/version into one
/// cache directory is a staleness bug, not a lock contention one.
///
/// The wait for the lease is carved out of `budget`, never on top of it —
/// `body` is handed whatever is left, so a caller's own node timeout is the
/// only clock involved, exactly what AGENTS.md's build-cache section asks
/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
/// cache configured at all — this is a pass-through: `body` runs with the
/// full budget and nothing is leased.
///
/// A lease that cannot be acquired within `budget` is reported as a single
/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
/// verification — the same shape a spawn failure already takes in
/// [`run_commands`], so a caller need not special-case it.
#[allow(clippy::too_many_arguments)]
async fn with_cache_lease<'s, F, Fut>(
state: &'s mut RunState,
cache_dir: Option<&Path>,
node: &str,
seat: &str,
worktree: &Path,
head: &str,
budget: Duration,
context: &str,
body: F,
) -> (Vec<CommandOutcome>, bool)
where
F: FnOnce(&'s mut RunState, Duration) -> Fut,
Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
{
let Some(cache_dir) = cache_dir else {
let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
return (outcomes, retried);
};
let home = crate::run::home();
let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
let started = Instant::now();
let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context, &|| false)
.await
.and_then(|g| {
g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
}) {
Ok(g) => g,
Err(e) => {
return (
vec![CommandOutcome {
command: "(waiting for the shared build cache)".to_owned(),
code: None,
output_tail: e.to_string(),
duration_ms: started.elapsed().as_millis() as u64,
resource_blocked: true,
}],
false,
);
}
};
let identity = crate::cache::Identity::new(worktree, head);
if let Err(e) = crate::cache::ensure_fresh(
&home,
cache_dir,
&identity,
&state.config.verify.cargo_manifests(),
) {
// A failed freshness check means this process cannot vouch for what
// is sitting in the cache right now - on Windows this is exactly the
// "a stale test executable is still locked, `cargo clean -p` cannot
// remove it" case the evidence log records. Running verify anyway
// and reporting whatever it says would let a result nobody can trust
// stand for the tree it claims to have checked; fail the step
// instead of the patch.
state.event(
"verify",
format!(
"{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
worktree.display(),
short(head)
),
);
guard.release();
return (
vec![CommandOutcome {
command: "(confirming the shared build cache is fresh)".to_owned(),
code: None,
output_tail: e.to_string(),
duration_ms: started.elapsed().as_millis() as u64,
resource_blocked: true,
}],
false,
);
}
let remaining = budget.saturating_sub(started.elapsed());
let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
// A timed-out command's process was only *asked* to die (`kill_on_drop`,
// `start_kill`); confirm it actually has before handing the directory to
// the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
// for what this can and cannot see.
if !timed_out_pids.is_empty() {
wait_for_timed_out_children_to_die(&timed_out_pids).await;
}
guard.release();
(outcomes, retried)
}
/// Poll `pids` — commands [`run_commands`] reports as still running when its
/// own timeout elapsed — until every one is confirmed gone, or
/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
///
/// Real confirmation where confirmation is possible, not a substitute for
/// full process-tree observation: a grandchild the timed-out process spawned
/// and that survives independently of it is invisible to a pid check the
/// same way it always was, and continuing to observe and collect *that*
/// stays a different piece of work with its own owner. This only narrows a
/// fixed blind wait into an actual check of the pids this process does know
/// about.
async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
wait_for_pids_with(
pids,
crate::proc::pid_alive,
LEASE_RELEASE_POLL,
LEASE_RELEASE_MAX_WAIT,
)
.await;
}
/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
/// interval and ceiling supplied by the caller, so the polling *logic* -
/// returns as soon as every pid reports dead, gives up at the ceiling
/// otherwise - is testable on millisecond durations without asking the real
/// OS about a pid at all.
async fn wait_for_pids_with<F: Fn(u32) -> bool>(
pids: &[u32],
alive: F,
poll: Duration,
max_wait: Duration,
) {
let deadline = Instant::now() + max_wait;
loop {
if pids.iter().all(|&pid| !alive(pid)) {
return;
}
if Instant::now() >= deadline {
return;
}
tokio::time::sleep(poll).await;
}
}
/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
/// admission that it could not even get a verify command to run, as opposed
/// to evidence the command actually produced? A caller that would otherwise
/// read a resource-blocked outcome as a red command must check this first:
/// see [`Runner::gate`], which retries rather than records `Blocked` when
/// this is true.
fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
outcomes.iter().any(|o| o.resource_blocked)
}
/// What [`Runner::gate_fix_round`] decided.
enum GateFix {
/// The tree changed and `verify.e2e` is still green: run the gate again.
Retry,
/// No more rounds, nothing to fix, or the fix did not hold: the gate's
/// last failure stands and the run ends blocked.
Stop,
/// `verify.e2e` could not run after the fix (magi's own contention):
/// decide nothing now, a later reentry retries.
Defer,
}
/// Is every red command in `outcomes` an ordinary failure the code could
/// explain: it ran, exited non-zero, and said something?
///
/// A timeout, a spawn failure and a killed process all leave `code` `None`;
/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
/// what the command is stays the gate's business.
fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
red.peek().is_some()
&& red.all(|o| {
!o.resource_blocked
&& matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
&& !o.output_tail.trim().is_empty()
})
}
/// Describe one verify command's outcome for the event log, distinguishing a
/// build/link failure — the toolchain never produced a binary to run — from
/// an actual test failure, since only the latter is a verdict on the patch.
fn e2e_outcome_label(o: &CommandOutcome) -> String {
if o.ok() {
return "pass".to_owned();
}
let reason = if o.build_failed() {
format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
} else {
format!("FAIL ({:?})", o.code)
};
format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
}
/// Run `verify.e2e`, retrying once if the first attempt could not build or
/// link — a build/link failure is frequently a race against a shared
/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
/// `verify` event per command, tagged with `context` (normally `"round N"`)
/// so the two call sites that need this — the ordinary per-round leg in
/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
/// it will ever call a round green — read identically in the event log.
async fn run_e2e_with_retry(
state: &mut RunState,
shell: &[String],
commands: &[String],
worktree: &Path,
timeout: Duration,
context: &str,
) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
let (mut e2e, mut timed_out_pids) = run_commands(
state, "verify", "e2e", 0, shell, commands, worktree, timeout,
)
.await;
for o in &e2e {
state.event(
"verify",
format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
);
}
// A build/link failure is not a verdict on the patch — it is frequently a
// race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
// one retry before letting a red like that decide the round.
let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
if verify_retried {
state.event(
"verify",
format!(
"{context}: verify could not build/link, not a test result — retrying once \
before concluding"
),
);
let retried = run_commands(
state, "verify", "e2e", 1, shell, commands, worktree, timeout,
)
.await;
e2e = retried.0;
// Both attempts' timeouts matter, not just the last one: the first
// attempt's descendants may still be alive alongside the retry's.
timed_out_pids.extend(retried.1);
for o in &e2e {
state.event(
"verify",
format!(
"{context}: retry `{}` -> {}",
o.command,
e2e_outcome_label(o)
),
);
}
}
(e2e, verify_retried, timed_out_pids)
}
/// Run configured shell commands in `cwd`, in order. The second element is
/// the pid of every command that hit `timeout` and was still running when
/// this stopped waiting on it (best-effort: `None` when the platform did not
/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
/// that releases a shared resource afterward needs to know.
///
/// Records `task` into [`RunState::active`] at every command boundary
/// (`RunState::task_command`) and clears it once the whole list has run
/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
/// for minutes with no seat and no output of its own to show for it (see
/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
/// apart from "not yet run" without this), and this is the only place that
/// knows which command is running right now and how many are left. Three
/// saves per command — start, not per second — matching the same "only at a
/// boundary" rule [`wave`] already follows for seats.
#[allow(clippy::too_many_arguments)]
async fn run_commands(
state: &mut RunState,
node: &str,
task: &str,
attempt: usize,
shell: &[String],
commands: &[String],
cwd: &Path,
timeout: Duration,
) -> (Vec<CommandOutcome>, Vec<u32>) {
if commands.is_empty() {
// Nothing to mark as running and nothing to clear — an empty list
// means "not configured", and touching `active` (or the disk) over
// that would be a write for every round of a repo with no
// `verify.e2e` / `verify.gate` commands at all.
return (Vec::new(), Vec::new());
}
let mut out = Vec::new();
let mut timed_out_pids = Vec::new();
let total = commands.len();
for (idx, command) in commands.iter().enumerate() {
state.task_command(task, node, attempt, command, idx + 1, total, timeout);
if let Err(e) = state.save() {
tracing::warn!("could not persist an in-progress {task} command: {e:#}");
}
let started = Instant::now();
let mut cmd = tokio::process::Command::new(&shell[0]);
cmd.quiet();
cmd.args(&shell[1..])
.arg(command)
.current_dir(cwd)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true);
let spawned = cmd.spawn();
let (code, body) = match spawned {
Ok(child) => {
// Captured before the child is consumed below: `kill_on_drop`
// only *asks* the process to die when the timeout branch
// drops it, and the pid is the only way anyone downstream can
// later check whether that request actually took.
let pid = child.id();
match tokio::time::timeout(timeout, child.wait_with_output()).await {
Ok(Ok(o)) => {
let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
body.push_str(&String::from_utf8_lossy(&o.stderr));
(o.status.code(), body)
}
Ok(Err(e)) => (None, format!("failed to run: {e}")),
Err(_) => {
if let Some(pid) = pid {
timed_out_pids.push(pid);
}
(None, format!("timed out after {}s", timeout.as_secs()))
}
}
}
Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
};
out.push(CommandOutcome {
command: command.clone(),
code,
output_tail: tail(&body, OUTPUT_TAIL),
duration_ms: started.elapsed().as_millis() as u64,
resource_blocked: false,
});
}
state.task_finished(task);
if let Err(e) = state.save() {
tracing::warn!("could not persist the end of {task}: {e:#}");
}
(out, timed_out_pids)
}
/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
/// section (`report::run`) and in the `merge` event this node records — for
/// the operator to run by hand.
///
/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
/// branch whose ruleset forbids merge commits (GitHub's "must not contain
/// merge commits", or "require linear history") rejects the push a `--no-ff`
/// merge would produce, which is exactly the guidance this function replaces.
/// `message`'s first line becomes the squash commit's subject, matching the
/// note `report::run` prints alongside this command — see that function for
/// why an explicit subject is not optional there.
fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
let repo = repo.display();
match style {
MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
MergeStyle::Squash => {
// The subject sits inside double quotes, and a title an agent
// wrote may carry the characters that break out of them.
let subject = message
.lines()
.next()
.unwrap_or(branch)
.replace(['\\', '"', '$', '`'], "");
format!(
"git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
)
}
MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
}
}
/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
/// under the hood, rejects a `title` over 256 characters and the whole
/// command fails — no PR at all, for a run whose body was otherwise fine
/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
/// this bound. It is a margin, not a guarantee — a title packed
/// with multi-unit characters could still in principle land close to the
/// edge, but a real task title's occasional emoji or accented letter fits
/// comfortably inside it.
const PR_TITLE_MAX: usize = 240;
/// A pull request title from the opening line of `text`, or `None` when that
/// line is not English (GitHub text is) or has no letters.
///
/// A line within `max` is kept as is. A longer one is cut at the end of its
/// first sentence when that falls inside `max`, else at a word boundary with a
/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
/// make every merely-truncated title look non-English). The language check
/// runs on the kept text before any mark is added, so only what GitHub will
/// show is judged: an English opening followed by non-ASCII far past the cut
/// still passes.
fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
let line = queue::first_line(text)?;
let chars: Vec<char> = line.chars().collect();
let (kept, mark) = if chars.len() <= max {
(line.to_owned(), "")
} else if let Some(end) = sentence_end(&chars, max) {
(chars[..end].iter().collect::<String>(), "")
} else {
let room = max.saturating_sub(3);
// Cut at the last space inside the room; when the char just past the
// room is a space the room already ends on a word.
let cut = if chars[room].is_whitespace() {
room
} else {
chars[..room]
.iter()
.rposition(|c| c.is_whitespace())
.unwrap_or(room)
};
let head: String = chars[..cut].iter().collect();
let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
(head.to_owned(), "...")
};
if kept.is_empty() {
return None;
}
if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
return None;
}
if !ascii_only && !kept.chars().any(char::is_alphabetic) {
return None;
}
Some(format!("{kept}{mark}"))
}
/// The char length of the first sentence of `chars` when it ends within `max`
/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
/// abbreviations so `e.g. foo` does not end a title early.
fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
const MIN: usize = 20;
for i in MIN..max.min(chars.len()) {
if !matches!(chars[i], '.' | '!' | '?') {
continue;
}
let Some(&next) = chars.get(i + 1) else {
continue;
};
if !next.is_whitespace() {
continue;
}
let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
if after.is_some_and(|c| c.is_ascii_lowercase()) {
continue;
}
let word: String = chars[..i]
.iter()
.rev()
.take_while(|c| !c.is_whitespace())
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
let word = word.to_ascii_lowercase();
if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
continue;
}
let end = chars[..i]
.iter()
.rposition(|c| !c.is_whitespace())
.map_or(i, |p| p + 1);
return Some(end);
}
None
}
/// What `merge = "pr"` (and the merge commit of the other modes) says about a
/// change: a title and a body describing what was *implemented*, not the task
/// that asked for it. A task reads as a request; a reader of the merged
/// history wants the change.
struct PrMessage {
title: String,
body: String,
}
impl PrMessage {
/// Title, blank line, body. The first line is the squash/merge commit
/// subject (`manual_merge_command` takes it via `lines().next()`), so it
/// has to stay one sensible line.
fn commit_message(&self) -> String {
format!("{}\n\n{}", self.title, self.body)
}
}
/// The text after a leading `TITLE:` (any case) on `line`.
fn title_marker(line: &str) -> Option<&str> {
let line = line.trim();
let head = line.get(..6)?;
head.eq_ignore_ascii_case("title:")
.then(|| line[6..].trim())
}
/// The implementer's own one-line title: the `TITLE:` line the implement
/// prompt asks for at the top of its SUMMARY. Candidate commits are all
/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
/// source, and a title that says as much is refused here too.
fn summary_title(summary: &str) -> Option<String> {
let first = summary.lines().find(|l| !l.trim().is_empty())?;
let raw = title_marker(first)?;
if raw.is_empty() {
return None;
}
let title = queue::title_from(raw, PR_TITLE_MAX);
let lower = title.to_ascii_lowercase();
if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
return None;
}
Some(title)
}
/// How `open_review`'s instruction begins; see [`landing_title`].
const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
/// Marker `open_review` gives a candidate that nothing in the roster wrote.
const EXISTING_BRANCH: &str = "(existing branch)";
/// Does this run review work that already existed, rather than implement a
/// task? Runs recorded before `reviewed_commits` existed carry only the
/// candidate marker.
fn is_review_run(state: &RunState) -> bool {
state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
}
/// The title of a review-only run: the subject of the oldest commit under
/// review. Later commits are usually fixups, and `instruction` is the review
/// prompt, which says nothing about the change. GitHub text is English, so a
/// non-ASCII or blank subject yields `None` and the caller's neutral title.
fn review_title(state: &RunState) -> Option<String> {
english_subject(
state.reviewed_commits.as_ref()?.first()?,
state.config.graph.github_english(),
)
}
/// `raw` as a pull request title, or `None` when it is blank, not English
/// (GitHub text is), or one of magi's own candidate commit subjects.
fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
let raw = raw.trim();
if raw.is_empty() {
return None;
}
if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
return None;
}
if !ascii_only && !raw.chars().any(char::is_alphabetic) {
return None;
}
let title = queue::title_from(raw, PR_TITLE_MAX);
let lower = title.to_ascii_lowercase();
if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
return None;
}
Some(title)
}
/// What a review-only run's branch says about itself, read at the moment the
/// pull request is opened.
#[derive(Debug, Clone, PartialEq, Eq)]
struct BranchFacts {
/// `(subject, body)` of each commit, oldest first.
commits: Vec<(String, String)>,
/// Trimmed `git diff --stat`.
stat: String,
}
/// Longest diff stat shown: this many file lines plus the summary line.
const STAT_FILE_LINES: usize = 25;
/// Cap on the commit list, well inside GitHub's 65536-character body limit.
const COMMITS_MAX_CHARS: usize = 20_000;
/// Read the commits and diff stat of `base..branch`. `None` when git cannot
/// say or finds nothing, so the caller falls back to what the run recorded.
async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
let commits = git::commit_log(repo, base, branch).await.ok()?;
if commits.is_empty() {
return None;
}
let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
let lines: Vec<&str> = stat.lines().collect();
let stat = if lines.len() > STAT_FILE_LINES + 1 {
let omitted = lines.len() - 1 - STAT_FILE_LINES;
let more = format!(" ... {omitted} more file(s)");
let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
kept.push(&more);
kept.push(lines[lines.len() - 1]);
kept.join("\n")
} else {
lines.join("\n")
};
Some(BranchFacts { commits, stat })
}
/// Defang what would break the surrounding markdown: a closing `</details>`
/// and a code fence.
fn markdown_safe(text: &str) -> String {
text.replace("</details>", "</details>")
.replace("\x60\x60\x60", "~~~")
}
fn neutral_title(state: &RunState, winner: char) -> String {
format!(
"chore: land candidate {} of run {}",
winner.to_ascii_uppercase(),
state.id
)
}
/// The pull request title to hand to `land::merge_subject`. A review-only run
/// opened by an earlier build titled its pull request with the review prompt;
/// that title is dropped (empty, so the fallback applies) rather than landed.
/// Any other title, including an operator's rename, passes through untouched,
/// and so does every title of a run that implements a task.
pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
""
} else {
pr_title
}
}
/// What the squash subject falls back to when the pull request title is empty
/// or candidate-shaped: for a review-only run the derived title, never the
/// review prompt held in `instruction`.
pub fn landing_subject_source(state: &RunState) -> String {
if is_review_run(state) {
let winner = state.candidates.first().map_or('A', |c| c.label);
return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
}
state.instruction.clone()
}
/// `summary` without its `TITLE:` line, which the pull request title already
/// carries.
fn summary_without_title(summary: &str) -> String {
let mut lines = summary.trim().lines().peekable();
if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
lines.next();
}
lines.collect::<Vec<_>>().join("\n").trim().to_owned()
}
/// The pull request title and body for the winning candidate.
///
/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
/// the task's own opening line via [`queue::title_from`] when there is none.
/// `state.instruction` can open with blank lines (`task_text` only rejects a
/// body that is blank *entirely*), which `title_from` skips.
///
/// Publishing uses an independent description generated from the final diff
/// and structured evidence. Internal summaries and reports remain local.
/// Non-publishing previews may display the original task and candidate summary.
/// The footer retains the run and candidate provenance.
#[cfg(test)]
fn pr_message(state: &RunState, winner: char) -> PrMessage {
pr_message_with(state, winner, None)
}
/// [`pr_message`] with what the branch of a review-only run says about itself.
/// `facts` is ignored for a run that implements a task.
#[cfg(test)]
fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
let raw = pr_message_raw(
state,
winner,
facts,
crate::pr_outcome::FinalHead::Unreported,
ChangeNote::NotAsked,
);
let id = crate::scrub::Identity::current();
PrMessage {
title: crate::scrub::scrub(&raw.title, &id),
body: crate::scrub::scrub(&raw.body, &id),
}
}
/// What the public description may say about the change itself.
#[derive(Debug, Clone, Copy)]
enum ChangeNote<'a> {
/// A local preview or non-publishing merge mode: no public generation needed.
NotAsked,
/// A summary written from the final diff.
Summary(&'a crate::run::ChangeSummary),
/// A summary was asked for and could not be produced; the text says why.
Unavailable(&'a str),
}
fn pr_message_raw(
state: &RunState,
winner: char,
facts: Option<&BranchFacts>,
head: crate::pr_outcome::FinalHead<'_>,
change: ChangeNote<'_>,
) -> PrMessage {
if !matches!(change, ChangeNote::NotAsked) {
let (title, mut body) = match change {
ChangeNote::Summary(c) => (
c.title.clone(),
format!("## Summary\n\n{}\n\n", c.body.trim()),
),
ChangeNote::Unavailable(why) => (
neutral_title(state, winner),
format!(
"## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
),
),
ChangeNote::NotAsked => unreachable!(),
};
body.push_str(&public_pr_evidence(state, head));
body.push_str(&format!(
"\n---\nmagi:run/{} magi:candidate-{}\n",
state.id,
winner.to_ascii_lowercase()
));
return PrMessage { title, body };
}
pr_preview(state, winner, facts, head)
}
/// Append public facts from the run records after every prose generation,
/// including posting-gate rewrites. The agent cannot drop this provenance.
fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
let mut body = crate::pr_outcome::review_rounds(state);
body.push_str(&crate::pr_outcome::final_verification(state, head));
let open = state.last_round_findings();
if !open.is_empty() {
body.push_str("## Open review findings\n\n");
for f in open {
body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
}
body.push('\n');
}
if let Some(c) = state.change_summary.as_ref()
&& head == crate::pr_outcome::FinalHead::At(&c.head)
{
body.push_str(&format!(
"Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
c.base, c.head
));
}
body
}
/// Local-only preview of the operator's task and candidate summary.
fn pr_preview(
state: &RunState,
winner: char,
facts: Option<&BranchFacts>,
head: crate::pr_outcome::FinalHead<'_>,
) -> PrMessage {
let summary = state
.candidates
.iter()
.find(|c| c.label == winner)
.map(|c| c.summary.as_str())
.unwrap_or_default();
// The fallback is the operator's own words and may not be English; GitHub
// text always is, so a non-English task gets a neutral title instead.
let review = is_review_run(state);
let title = if review {
facts
.and_then(|f| {
english_subject(&f.commits.first()?.0, state.config.graph.github_english())
})
.or_else(|| review_title(state))
.or_else(|| {
state
.candidates
.iter()
.find(|c| c.label == winner)
.filter(|c| !c.branch.starts_with("magi/"))
.and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
})
.unwrap_or_else(|| neutral_title(state, winner))
} else {
summary_title(summary).unwrap_or_else(|| {
english_title(
&state.instruction,
PR_TITLE_MAX,
state.config.graph.github_english(),
)
.unwrap_or_else(|| neutral_title(state, winner))
})
};
let mut body = String::new();
let what = summary_without_title(summary);
if !what.is_empty() {
body.push_str("## Summary\n\n");
body.push_str(&what);
body.push_str("\n\n");
}
// Review history uses structured facts across all rounds, never reports.
body.push_str(&crate::pr_outcome::review_rounds(state));
body.push_str(&crate::pr_outcome::final_verification(state, head));
let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
let open = state.open_findings();
if !open.is_empty() {
body.push_str("## Open review findings\n\n");
for f in &open {
body.push_str(&format!(
"- `{}` [{:?}] {}\n",
f.id, f.severity, "Context retained in local review records."
));
}
body.push('\n');
}
if let Some(fix) = fix
&& !fix.rejected.is_empty()
{
body.push_str("## Declined by the fixer\n\n");
for r in &fix.rejected {
body.push_str(&format!(
"- `{}`: {}\n",
r.id, "Reason retained in local review records."
));
}
body.push('\n');
}
if review {
// The review prompt is not the task; list what the branch carries.
body.push_str("## Commits under review\n\n");
if let Some(facts) = facts {
let mut left = COMMITS_MAX_CHARS;
for (i, (subject, text)) in facts.commits.iter().enumerate() {
let mut entry = format!("- {}\n", markdown_safe(subject));
for l in markdown_safe(text).lines() {
entry.push_str(format!(" {l}\n").trim_end_matches(' '));
}
if left == 0 {
body.push_str(&format!(
"- ... {} more commit(s)\n",
facts.commits.len() - i
));
break;
}
if entry.len() > left {
// Even the first commit is cut: one huge body must not
// push the whole description past GitHub's limit.
let mut end = left;
while !entry.is_char_boundary(end) {
end -= 1;
}
entry.truncate(end);
entry.push_str("\n ... (truncated)\n");
left = 0;
} else {
left -= entry.len();
}
body.push_str(&entry);
}
if !facts.stat.trim().is_empty() {
body.push_str(&format!(
"\n## Diff stat\n\n```\n{}\n```\n",
markdown_safe(facts.stat.trim())
));
}
} else {
match &state.reviewed_commits {
Some(subjects) => {
for s in subjects {
body.push_str(&format!("- {}\n", s.trim()));
}
}
None => {
// An older run kept only the prompt, with the commit list
// after its first paragraph.
let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
body.push_str(rest.trim());
body.push('\n');
}
}
}
} else {
let task = state.instruction.trim();
let task = if task.is_empty() {
"(empty task)"
} else {
task
};
body.push_str(&format!(
"<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
task.replace("</details>", "</details>")
));
}
body.push_str(&format!(
"\n---\nmagi:run/{} magi:candidate-{}\n",
state.id,
winner.to_ascii_lowercase()
));
PrMessage { title, body }
}
/// The task with what the repository says about the existing work it names
/// appended, so an implementer knows what it started from and what it must
/// not redo. Unchanged when the task names nothing.
fn seeded_instruction(state: &RunState) -> String {
match refs::describe(&state.seeds) {
Some(facts) => format!(
"{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
Candidates start from the unmerged branch named above, when there \
is one, and carry any unmerged commit named by sha as a \
cherry-pick. Check that this is what the task meant before \
building on it.",
state.instruction
),
None => state.instruction.clone(),
}
}
/// Does the winner have no commits ahead of the base it would land on?
/// Any failure to find out reads as "not empty": the merge then behaves as it
/// always did rather than refusing on a guess.
async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
let base = &state.base_branch;
let mut against = base.clone();
if mode == MergeMode::Pr {
// A pull request lands on the remote's base, never the local branch
// of the same name: if that cannot be read, "not empty" is the safe
// answer.
let remote = &state.config.merge.remote;
let tracking = format!("{remote}/{base}");
let fetched = git::fetch(repo, remote, base).await;
if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
against = tracking;
} else {
return false;
}
}
matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
}
/// Why nothing was opened for an empty winner, with what the task's own
/// references resolved to.
fn empty_candidate_detail(state: &RunState, base: &str) -> String {
let mut detail = format!(
"empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
nothing to open a pull request for"
);
match refs::describe(&state.seeds) {
Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
None => detail.push_str(
"\nThe task names no existing branch or commit; if it means to land work \
that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
),
}
detail
}
/// What the `Pr` merge does once it knows whether the branch already has an
/// open pull request.
#[derive(Debug, PartialEq, Eq)]
enum PrPlan {
Create,
Adopt { url: String, title: String },
Stop(String),
}
/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
/// a failed lookup -> stop with the real reason. Never guesses.
fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
match found {
Ok(land::OpenPr::None) => PrPlan::Create,
Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
"several open pull requests exist for this branch, not picking one: {}",
urls.join(" ")
)),
Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
}
}
/// `gh pr create`, returning the PR url.
pub(crate) async fn gh_pr_create(
cwd: &Path,
base: &str,
head: &str,
title: &str,
body: &str,
) -> Result<String> {
let out = tokio::process::Command::new("gh")
.args([
"pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
])
.current_dir(cwd)
.quiet()
.stdin(std::process::Stdio::null())
.output()
.await
.context("spawn gh")?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
} else {
bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
}
}
/// Tear a run's worktrees and branches down.
///
/// `home` is where the updated `run.json` is saved (via
/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
/// a housekeeping pass already has its own honest `home` handed to it, and
/// falling through to the global here would write back through whichever
/// directory some other process or test pinned into that `OnceLock` first,
/// not the one the caller actually resolved its `runs` and `state` from.
pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
let repo = state.repo.clone();
let root = state.worktree_root();
let winner = state.tally.as_ref().map(|t| t.winner);
let mut removed = Vec::new();
for i in 0..state.candidates.len() {
let c = state.candidates[i].clone();
let is_winner = Some(c.label) == winner;
if is_winner && !drop_winner {
continue;
}
if c.worktree.exists() {
git::worktree_remove(&repo, &c.worktree).await.ok();
removed.push(c.worktree.to_string_lossy().into_owned());
}
// A branch handed to a later run (and its pull request) is not this
// run's to delete.
let handed_over = state.released_branches.contains(&c.branch);
if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
git::branch_delete(&repo, &c.branch).await.ok();
removed.push(c.branch.clone());
}
state.candidates[i].folded = true;
}
for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
let path = name.path();
let keep = !drop_winner
&& winner.is_some_and(|w| {
path.file_name()
.is_some_and(|n| n == format!("cand-{w}").as_str())
});
if keep {
continue;
}
git::worktree_remove(&repo, &path).await.ok();
removed.push(path.to_string_lossy().into_owned());
}
// `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
// judge worktrees, so once the loop above has cleared all of them out,
// the parent is a bare directory nobody else was ever going to remove -
// git only ever managed what was inside it. Left alone, one of these
// accumulates per fully-folded run; the operator's own machine had 74.
// `remove_if_empty` re-checks rather than assuming: a run whose winner
// was kept (`!drop_winner`) leaves its directory behind on purpose, and
// so does anything a run never claimed that happens to share the bay.
remove_if_empty(&root);
if state.enabled_worktree_config && drop_winner {
// A release, not a raw disable: some sibling run in this repository
// may still hold its own reference (see `git::acquire_worktree_config`),
// and only the last release actually turns the setting back off.
git::release_worktree_config(&repo).await.ok();
state.enabled_worktree_config = false;
}
state.save_under(home)?;
Ok(removed)
}
/// Remove `dir` if it exists and has nothing in it.
///
/// Best-effort and silent by design: a directory that is not empty (a run
/// whose winner is still parked there, a stray file some other process left)
/// is exactly the case this must refuse, and a directory that is already gone
/// is not a failure worth reporting either. `std::fs::remove_dir` itself
/// already refuses a non-empty directory, so the emptiness check below is
/// belt, not suspenders - it is what keeps this from ever attempting the
/// removal in the case that matters, rather than trusting `remove_dir`'s
/// error path to have no side effects if it ever changed.
fn remove_if_empty(dir: &Path) {
if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
std::fs::remove_dir(dir).ok();
}
}
/// Severity of the worst open finding in the last review round, for reporting.
pub fn worst_open(state: &RunState) -> Option<Severity> {
state
.reviews
.last()?
.reviews
.iter()
.flat_map(|r| r.findings.iter())
.map(|f| f.severity)
.max()
}
/// `[worktree] setup` hides its products per worktree, which needs
/// `extensions.worktreeConfig`. Taken once per run and released with the run's
/// worktrees, like the hook's; every place that runs setup calls this first.
async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
git::acquire_worktree_config(repo).await?;
state.enabled_worktree_config = true;
}
Ok(())
}
#[cfg(test)]
mod tests {
#[test]
fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
let leaked = vec!["chore(deps): update a crate".to_owned()];
let own = "fix(daemon): apply a chosen action";
assert!(should_retitle("", own, &leaked));
assert!(should_retitle(
&format!("{REVIEW_PROMPT_OPENING} `x`"),
own,
&leaked
));
assert!(should_retitle(
"chore: land candidate A of run 1",
own,
&leaked
));
assert!(should_retitle(
"magi: candidate A (uncommitted work)",
own,
&leaked
));
assert!(should_retitle("chore(deps): update a crate", own, &leaked));
assert!(!should_retitle("feat: renamed by hand", own, &leaked));
assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
}
#[test]
fn pr_merge_plan_creates_adopts_or_stops() {
assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
assert_eq!(
pr_merge_plan(Ok(land::OpenPr::One {
url: "u".into(),
title: "t".into()
})),
PrPlan::Adopt {
url: "u".into(),
title: "t".into()
}
);
let PrPlan::Stop(many) =
pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
else {
panic!("many must stop");
};
assert!(many.contains('a') && many.contains('b'));
let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
panic!("a failed lookup must stop");
};
assert!(err.contains("bad token"));
}
use super::*;
use crate::run::GateStatus;
use std::collections::BTreeMap;
use std::time::Duration;
fn conductor() -> AgentSpec {
AgentSpec {
id: "conductor".to_owned(),
kind: crate::config::AgentKind::Command,
model: None,
command: vec!["true".to_owned()],
extra_args: Vec::new(),
env: BTreeMap::new(),
prompt_delivery: None,
}
}
fn spec(id: &str) -> AgentSpec {
AgentSpec {
id: id.to_owned(),
kind: crate::config::AgentKind::Command,
model: None,
command: vec!["true".to_owned()],
extra_args: Vec::new(),
env: BTreeMap::new(),
prompt_delivery: None,
}
}
fn ids(xs: &[&str]) -> BTreeSet<String> {
xs.iter().map(|s| (*s).to_owned()).collect()
}
#[test]
fn pick_successor_skips_an_agent_another_seat_holds() {
// Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
let roster = [spec("a"), spec("b"), spec("c")];
let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
}
#[test]
fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
// The other seat started on c but was handed to d; c is free again.
let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
}
#[test]
fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
let roster = [spec("a"), spec("b")];
let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
let carried = ids(&["b"]);
let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
}
#[test]
fn pick_successor_returns_none_without_any_untried_successor() {
let roster = [spec("a"), spec("b")];
assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
let carried = ids(&["a"]);
assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
}
#[test]
fn pick_successor_rescue_avoids_another_seats_occupant() {
// b is a carried failure and free; a is held by the other seat.
let roster = [spec("a"), spec("b"), spec("c")];
let carried = ids(&["b", "c"]);
let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
}
#[test]
fn next_for_seat_prefers_an_agent_that_has_not_failed() {
let roster = [spec("a"), spec("b"), spec("c")];
let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
}
#[test]
fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
let roster = [spec("a"), spec("b"), spec("c")];
let failed = ids(&["a", "b", "c"]);
// Rescue looks at the whole roster, once per id, then runs out.
let mut tried = ids(&["b"]);
let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
assert_eq!(first.id, "a");
tried.insert(first.id.clone());
let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
assert_eq!(second.id, "c");
tried.insert(second.id.clone());
assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
}
#[test]
fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
let roster = [spec("a"), spec("b")];
let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
}
#[test]
fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
let roster = [spec("a"), spec("b"), spec("c")];
let h = SeatHistory {
failed: ids(&["a"]),
last_ok: Some("b".to_owned()),
last_fail: None,
};
assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
// A last answerer that left the roster, or later failed, is ignored.
let gone = SeatHistory {
last_ok: Some("zzz".to_owned()),
..h.clone()
};
assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
let failed = SeatHistory {
failed: ids(&["a", "b"]),
last_ok: Some("b".to_owned()),
last_fail: None,
};
assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
}
#[test]
fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
let first = SeatState::new("review-1", "alpha", 7);
let next = handover_seat("review-1", "gamma", 7);
assert_ne!(first.claude_session, next.claude_session);
}
#[test]
fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
let mut state = state_with_summary("x", "y");
let a = handover_seat("review-1", "beta", state.next_seat_seed());
let b = handover_seat("review-1", "beta", state.next_seat_seed());
assert_ne!(a.claude_session, b.claude_session);
}
#[test]
fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
let roster = [spec("a"), spec("b")];
let h = SeatHistory {
failed: ids(&["a", "b"]),
..SeatHistory::default()
};
assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
}
// `next_untried_in_roster` is the property `resume_seat_handovers`'s own
// fallback loop depends on to terminate: it must walk forward from the
// seat's own position, never restart at the front of the roster, and it
// must never hand back an id already tried, however many times that id
// happens to appear.
#[test]
fn failure_signature_ignores_numbers_and_paths() {
assert_eq!(
failure_signature("exited with Some(2) and no usable output"),
failure_signature("exited with Some(137) and no usable output")
);
assert_eq!(
failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
failure_signature("cannot open /var/x.txt: denied")
);
assert_ne!(failure_signature("boom"), failure_signature("bang"));
}
#[test]
fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
let other = FailClass::Other("x".into());
assert!(should_hand_over(None, &FailClass::Quota));
assert!(should_hand_over(Some(&other), &FailClass::Quota));
assert!(should_hand_over(
Some(&FailClass::Timeout),
&FailClass::Timeout
));
assert!(should_hand_over(None, &other));
assert!(!should_hand_over(Some(&other), &other));
assert!(should_hand_over(
Some(&other),
&FailClass::Other("y".into())
));
// A quota or timeout in between ends the run of identical failures.
assert!(should_hand_over(Some(&FailClass::Timeout), &other));
assert!(should_hand_over(Some(&FailClass::Quota), &other));
}
#[test]
fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
let a = SeatState::new("judge-1", "alpha", 7);
let b = handover_seat("judge-1", "beta", 7);
assert_ne!(a.claude_session, b.claude_session);
assert_eq!(b.turns, 0);
}
#[test]
fn a_timeout_is_classified_apart_from_other_failures() {
assert_eq!(
FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
Some(FailClass::Timeout)
);
assert!(matches!(
FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
Some(FailClass::Other(_))
));
}
#[test]
fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
let tried = BTreeSet::from(["beta".to_owned()]);
// beta sits at index 1; the next candidate is gamma, never alpha —
// which is very likely a different candidate slot's own agent.
let next = next_untried_in_roster(&roster, 1, &tried);
assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
}
#[test]
fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
let roster = vec![spec("alpha"), spec("beta")];
let tried = BTreeSet::from(["beta".to_owned()]);
// beta is the roster's last entry: nothing follows it, and alpha —
// earlier in the roster, almost certainly a different candidate
// slot's own agent — must not be reached by wrapping back to it.
assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
}
#[test]
fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
// beta (index 1) and gamma (index 2, the only entry after it) have
// both been tried; alpha (index 0) never has, but it comes before
// beta's own position, so there is nothing further for this seat.
assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
}
#[test]
fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
let roster = vec![spec("a"), spec("a"), spec("b")];
let tried = BTreeSet::from(["a".to_owned()]);
let next = next_untried_in_roster(&roster, 0, &tried);
assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
}
#[test]
fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
let roster = vec![spec("a"), spec("b")];
let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
}
#[test]
fn remove_if_empty_only_ever_takes_a_bare_directory() {
let dir = tempfile::tempdir().unwrap();
let bay = dir.path().join("ffff");
// Not there yet: nothing to do, nothing to panic on.
remove_if_empty(&bay);
assert!(!bay.exists());
// Something still inside - the winner's worktree, or a stray file -
// keeps the directory standing.
std::fs::create_dir_all(bay.join("cand-A")).unwrap();
remove_if_empty(&bay);
assert!(bay.exists(), "non-empty directory must survive");
// Once the last entry is gone, so is the directory itself.
std::fs::remove_dir(bay.join("cand-A")).unwrap();
remove_if_empty(&bay);
assert!(!bay.exists(), "an empty bay is a leftover, not a record");
}
// `round_is_clean` is the exact decision this task fixed: a round with a
// seat that never answered must not read the same as a round every seat
// actually reviewed. These are deterministic and process-free by design —
// the equivalent end-to-end check (a real reviewer timing out under a
// live graph run) is a genuine race against wall-clock contention, and a
// spawn slow enough to blow even a generous budget under a loaded test
// run must not turn this specific regression check flaky.
#[test]
fn a_full_panel_that_found_nothing_is_clean() {
assert!(round_is_clean(
0,
true,
2,
2,
0,
IncompleteReviewPolicy::Block
));
}
#[test]
fn a_missing_seat_is_never_clean_under_the_default_policy() {
assert!(!round_is_clean(
0,
true,
1,
2,
0,
IncompleteReviewPolicy::Block
));
}
#[test]
fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
assert!(!round_is_clean(
1,
true,
1,
2,
0,
IncompleteReviewPolicy::Warn
));
}
#[test]
fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
assert!(round_is_clean(
0,
true,
1,
2,
0,
IncompleteReviewPolicy::Warn
));
}
#[test]
fn a_full_panel_with_an_open_finding_is_not_clean() {
assert!(!round_is_clean(
1,
true,
2,
2,
0,
IncompleteReviewPolicy::Block
));
}
#[test]
fn a_full_panel_with_a_red_e2e_is_not_clean() {
assert!(!round_is_clean(
0,
false,
2,
2,
0,
IncompleteReviewPolicy::Block
));
}
// The stall this task closes: under the default `block` policy, a seat
// missing only because it was rate limited must not force a wait for a
// session limit that will not lift by the next round. `round_is_clean`
// is where that quorum carve-out lives; the review loop around it never
// changes what a reviewer's vote or a finding's severity means.
#[test]
fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
// 1 of 2 answered, and the one missing was quota'd — the exact
// "review-2 rate limited (quota)" shape from the field report.
assert!(round_is_clean(
0,
true,
1,
2,
1,
IncompleteReviewPolicy::Block
));
}
#[test]
fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
// 1 of 2 answered, but the miss was a crash/timeout/parse failure,
// not a quota loss (`quota_missing` stays 0) — worth another try.
assert!(!round_is_clean(
0,
true,
1,
2,
0,
IncompleteReviewPolicy::Block
));
}
#[test]
fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
assert!(!round_is_clean(
1,
true,
1,
2,
1,
IncompleteReviewPolicy::Block
));
assert!(!round_is_clean(
0,
false,
1,
2,
1,
IncompleteReviewPolicy::Block
));
}
#[test]
fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
// Every seat quota'd, nobody answered: there is no panel to decide
// on, so this must fall through to the existing block-and-retry
// fallback rather than call an unreviewed patch clean.
assert!(!round_is_clean(
0,
true,
0,
2,
2,
IncompleteReviewPolicy::Block
));
}
fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
CommandOutcome {
command: "test".to_owned(),
code,
output_tail: String::new(),
duration_ms: 0,
resource_blocked,
}
}
#[test]
fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
assert!(
!verify_inconclusive(&[outcome(Some(1), false)]),
"an ordinary failure is still evidence about the patch"
);
assert!(verify_inconclusive(&[outcome(None, true)]));
assert!(
verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
"one inconclusive outcome taints the whole batch"
);
assert!(!verify_inconclusive(&[]));
}
#[tokio::test]
async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
// Alive for the first two checks, then dead - confirms the loop
// actually re-polls rather than deciding once and sleeping out the
// ceiling regardless.
let calls = std::sync::atomic::AtomicUsize::new(0);
let started = Instant::now();
wait_for_pids_with(
&[123],
|_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
Duration::from_millis(5),
Duration::from_secs(5),
)
.await;
assert!(
calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
"must keep checking rather than deciding on the first answer"
);
assert!(
started.elapsed() < Duration::from_secs(1),
"must return the moment it is confirmed dead, not wait out the ceiling"
);
}
#[tokio::test]
async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
let started = Instant::now();
wait_for_pids_with(
&[123],
|_| true, // never reports dead
Duration::from_millis(5),
Duration::from_millis(30),
)
.await;
let elapsed = started.elapsed();
assert!(
elapsed >= Duration::from_millis(30),
"must not give up before its own ceiling: {elapsed:?}"
);
assert!(
elapsed < Duration::from_secs(1),
"must not wait past its own ceiling either: {elapsed:?}"
);
}
#[tokio::test]
async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
let started = Instant::now();
wait_for_pids_with(
&[],
|_| true,
Duration::from_secs(5),
Duration::from_secs(5),
)
.await;
assert!(
started.elapsed() < Duration::from_millis(200),
"an empty pid list has nothing to confirm"
);
}
// `review_conclusion` is the exact decision the review hand-off task
// fixed: a round budget spent (or a tree that stopped moving) must not
// collapse into `Blocked` regardless of what verification actually
// said. Deterministic and process-free for the same reason the
// `round_is_clean` family above is.
fn review_round(
clean: bool,
blocking: usize,
answered: usize,
expected: usize,
progressed: bool,
e2e_ok: bool,
) -> ReviewRound {
ReviewRound {
round: 1,
head: "h".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: vec![CommandOutcome {
command: "test".to_owned(),
code: Some(if e2e_ok { 0 } else { 1 }),
output_tail: String::new(),
duration_ms: 0,
resource_blocked: false,
}],
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
fix: None,
blocking,
answered,
expected,
clean,
progressed,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}
}
#[test]
fn review_conclusion_is_none_when_nothing_has_run() {
assert_eq!(review_conclusion(&[], 3), None);
}
#[test]
fn review_conclusion_is_none_while_rounds_remain() {
let rounds = vec![review_round(false, 1, 2, 2, true, true)];
assert_eq!(review_conclusion(&rounds, 3), None);
}
#[test]
fn review_conclusion_is_gating_once_a_round_is_clean() {
let rounds = vec![review_round(true, 0, 2, 2, false, true)];
assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
}
#[test]
fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
let rounds = vec![
review_round(false, 1, 2, 2, true, true),
review_round(false, 1, 2, 2, true, true),
];
assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
}
#[test]
fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
let rounds = vec![
review_round(false, 1, 2, 2, true, true),
review_round(false, 1, 2, 2, true, false),
];
assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
}
#[test]
fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
// Magi never got a command to run against this round's own head — a
// resource-blocked attempt, not a red one — so this must never
// settle on `Blocked` the way a genuine e2e failure would. `None`
// here is what tells `Runner::review_loop` to retry the check
// itself rather than trust this cheap recomputation with a verdict
// it cannot actually produce.
let mut blocked = review_round(false, 1, 2, 2, true, false);
blocked.e2e[0].resource_blocked = true;
let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
assert_eq!(review_conclusion(&rounds, 2), None);
}
#[test]
fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
// Missing input, not a verified tree — never a hand-off candidate.
let rounds = vec![review_round(false, 0, 1, 2, false, true)];
assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
}
#[test]
fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
let rounds = vec![
review_round(false, 1, 2, 2, false, true),
review_round(false, 1, 2, 2, false, true),
];
assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
}
fn secs(n: u64) -> Duration {
Duration::from_secs(n)
}
/// A throwaway repo with one commit on `main`, for tests that need `merge`
/// to make real (and, if it runs at all, real*ly fail*) git calls.
fn init_repo(dir: &Path) {
let run = |args: &[&str]| {
let out = std::process::Command::new("git")
.args(args)
.current_dir(dir)
.quiet()
.output()
.expect("spawn git");
assert!(
out.status.success(),
"git {args:?} failed: {}",
String::from_utf8_lossy(&out.stderr)
);
};
run(&["init", "-b", "main"]);
run(&["config", "user.name", "magi test"]);
run(&["config", "user.email", "magi@example.com"]);
std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
run(&["add", "-A"]);
run(&["commit", "-m", "init"]);
}
// `settle_questions` is what closes the ghost the phone showed: a run's
// seat asked something, the run then ended, and nothing was left to
// abandon the question it left `open`. `HOME` is a process-wide
// `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
// first time it runs in the binary — every test below still reaches the
// same directory whichever call won, and each gets its own run id from
// `RunState::new`, so they never collide there.
fn ask_test_home() {
crate::run::pin_test_home();
}
/// A minimal, git-free `Runner` at a given status — `settle_questions`
/// reads nothing else off it.
fn runner_at(status: RunStatus) -> Runner {
let mut state = RunState::new(
PathBuf::from("/nonexistent/repo"),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
Config::default(),
);
state.status = status;
Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
}
}
/// `park_here` folding in the reason `Pause::park_because` recorded -
/// this is what lets an operator reading a run's events tell an
/// interrupt-driven park from an ordinary shutdown park.
#[test]
fn park_here_folds_the_interrupt_reason_into_the_park_event() {
crate::run::pin_test_home();
let mut runner = runner_at(RunStatus::Implementing);
let interrupt = Pause::new();
runner.watch_interrupt(interrupt.clone());
interrupt.park_because("task a1b2 asked to run first");
assert!(runner.park_here().expect("park_here"));
assert!(runner.state.parked);
let last = runner.state.events.last().expect("a park event");
assert_eq!(last.node, "park");
assert!(
last.message.contains("task a1b2 asked to run first"),
"expected the interrupt reason in {:?}",
last.message
);
}
/// The cause is stored on the pass when it parks, and the first cause a
/// shared pause was given is the one that stays.
#[test]
fn park_here_records_the_structured_cause_and_the_first_one_wins() {
crate::run::pin_test_home();
let mut runner = runner_at(RunStatus::Implementing);
runner.state.begin_pass();
let shutdown = Pause::new();
runner.on_pause(shutdown.clone());
let upgrade = ParkCause::Upgrade {
from: "0.121.0".to_owned(),
to: Some("0.121.1".to_owned()),
};
shutdown.park_for(upgrade.clone());
shutdown.park_for(ParkCause::Operator);
assert!(runner.park_here().expect("park_here"));
assert_eq!(
runner.state.passes[0].end,
Some(crate::run::PassEnd::Parked { cause: upgrade })
);
let last = runner.state.events.last().expect("a park event");
assert!(last.message.contains("0.121.0"), "{}", last.message);
}
/// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
/// shutdown `Pause` (what `Stop::park` hands every run, shared and never
/// cleared) must not make a *different* run - one only watching its own,
/// unshared interrupt `Pause` - see itself as parked. If a future change
/// ever collapsed these back into one handle, the interrupt scheduler
/// would park every run for the rest of the daemon's life, not just the
/// one it meant to interrupt.
#[test]
fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
crate::run::pin_test_home();
let mut runner = runner_at(RunStatus::Implementing);
let shutdown = Pause::new();
runner.on_pause(shutdown.clone());
let interrupt = Pause::new();
runner.watch_interrupt(interrupt.clone());
// Nobody has asked for anything yet.
assert!(!runner.park_here().expect("park_here"));
assert!(!runner.state.parked);
// Only the interrupt handle fires; the shutdown handle stays clear.
interrupt.park_because("test");
assert!(!shutdown.parked());
assert!(runner.park_here().expect("park_here"));
}
/// The property every prior attempt at this feature failed to pin down:
/// asking a run to park while one of its nodes has a real, in-flight
/// async operation running (an agent call, in production) must not cut
/// that operation short. `park_here` is only ever consulted *between*
/// `execute`'s node calls - see its own doc - so nothing inside a node
/// can observe a park request until the node itself returns. This proves
/// that structurally, with real `tokio` concurrency and a channel
/// handshake (never a sleep, which would only prove "usually", not
/// "cannot"): the "node" below reports that it has genuinely started,
/// and only then is the park requested; the node still has to be told to
/// finish before `park_here` is ever called, exactly mirroring every
/// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
/// in `execute`.
#[tokio::test]
async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
crate::run::pin_test_home();
let mut runner = runner_at(RunStatus::Implementing);
let interrupt = Pause::new();
runner.watch_interrupt(interrupt.clone());
let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
// Stands in for one node's in-flight agent call: it proves it has
// genuinely started, then blocks - exactly as a spawned CLI process
// does - until told to finish.
let node = async move {
started_tx.send(()).expect("send started");
finish_rx.await.expect("recv finish");
"node finished"
};
let interrupter = async move {
started_rx.await.expect("recv started");
// The call is now genuinely in flight. Ask it to park.
interrupt.park_because("higher-priority task waiting");
// Nothing the node does can observe this yet - there is no
// check inside it, by construction - so let the executor run
// anything pending and then let the node finish on its own.
tokio::task::yield_now().await;
finish_tx.send(()).expect("send finish");
};
let (node_result, ()) = tokio::join!(node, interrupter);
assert_eq!(
node_result, "node finished",
"the in-flight call ran to completion"
);
// Only now, at the boundary the real `execute` would check right
// after this node, does the park take effect.
assert!(runner.park_here().expect("park_here"));
assert!(runner.state.parked);
}
/// A run parked mid-competition carries every field it had accumulated
/// through the exact same disk round-trip an ordinary resume uses -
/// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
/// Nothing about parking for an interrupt is a special case of that path;
/// this is what proves it rather than assuming it.
#[test]
fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
crate::run::pin_test_home();
let mut runner = runner_at(RunStatus::Judging);
// `Runner::resume` re-resolves roles from the saved config, which
// refuses an empty roster - give it the same minimal one `conductor`
// itself uses.
runner.state.config.agents = vec![conductor()];
runner.state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "magi/x/A".to_owned(),
worktree: PathBuf::from("/nonexistent/worktree"),
summary: "did the thing".to_owned(),
stat: "1 file changed".to_owned(),
files: 1,
commits: 1,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 1234,
folded: false,
}];
let run_id = runner.state.id.clone();
let interrupt = Pause::new();
runner.watch_interrupt(interrupt.clone());
interrupt.park_because("task c3d4 asked to run first");
assert!(runner.park_here().expect("park_here"));
let resumed = Runner::resume(&run_id).expect("resume");
assert_eq!(resumed.state.candidates.len(), 1);
assert_eq!(resumed.state.candidates[0].summary, "did the thing");
assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
assert_eq!(resumed.state.status, runner.state.status);
assert!(
resumed.state.parked,
"still parked until `execute` actually walks the graph again"
);
assert!(resumed.state.events.iter().any(|e| e.node == "park"));
}
/// A fresh open question on `run`, stored and handed back for assertions.
fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
let mut q = ask::Question::new(
run.to_owned(),
"implement".to_owned(),
"impl-A".to_owned(),
"Which storage backend should the cache use?".to_owned(),
String::new(),
vec!["SQLite".to_owned(), "Redis".to_owned()],
);
store.put(&mut q).unwrap();
q
}
#[test]
fn a_failed_runs_open_question_is_abandoned() {
ask_test_home();
let store = ask::Questions::open();
let mut runner = runner_at(RunStatus::Failed);
let run = runner.state.id.clone();
let q = ask_open_question(&store, &run);
runner.settle_questions();
let back = store.get(&q.id).unwrap();
assert!(
!back.status.open(),
"the seat that asked died with the run; nobody is left to read an answer"
);
assert!(
back.detail.contains(&run) && back.detail.contains("failed"),
"the reason names what the run became, not just that it is gone: {}",
back.detail
);
}
#[test]
fn a_merged_runs_open_question_is_abandoned_too() {
ask_test_home();
let store = ask::Questions::open();
// A run that finishes cleanly still leaves nobody to read an answer -
// this is not only a failure-path cleanup.
for status in [RunStatus::Merged, RunStatus::Ready] {
let mut runner = runner_at(status);
let run = runner.state.id.clone();
let q = ask_open_question(&store, &run);
runner.settle_questions();
let back = store.get(&q.id).unwrap();
assert!(
!back.status.open(),
"{status:?} run's question must not outlive the run"
);
}
}
#[test]
fn a_still_resumable_runs_open_question_is_left_alone() {
ask_test_home();
let store = ask::Questions::open();
// `Blocked` and `Stalled` can still be resumed — the candidates, the
// review round and the seat sessions are all still on disk — so a
// question asked mid-round may yet get a real answer from a real
// resume. Sweeping it here would be exactly the failure mode this
// whole feature exists to avoid on the other side.
for status in [RunStatus::Blocked, RunStatus::Stalled] {
let mut runner = runner_at(status);
let run = runner.state.id.clone();
let q = ask_open_question(&store, &run);
runner.settle_questions();
let back = store.get(&q.id).unwrap();
assert!(
back.status.open(),
"{status:?} is still alive; the question must still be waiting"
);
}
}
#[test]
fn settle_questions_never_touches_an_already_answered_question() {
ask_test_home();
let store = ask::Questions::open();
let mut runner = runner_at(RunStatus::Failed);
let run = runner.state.id.clone();
let mut q = ask_open_question(&store, &run);
q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
.unwrap();
store.put(&mut q).unwrap();
// Called twice, the way a crash-recovered daemon reclaim and the
// graph's own cleanup both can for the same run — `abandon_for_run`
// only ever touches what is still open, so this must be inert both
// times, not merely the second.
runner.settle_questions();
runner.settle_questions();
let back = store.get(&q.id).unwrap();
assert_eq!(
back.status,
ask::QuestionStatus::Answered,
"a real answer is a decision on record, never overwritten by a sweep"
);
}
/// `fold_run(&mut state, drop_winner = false)` is exactly the call
/// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
/// without merging, whose winner is still the operator's answer to read.
/// Nothing previously called `fold_run` itself with a real `tally`, so
/// this is the first test to pin down the one distinction the whole
/// automatic-fold feature depends on: the winner's worktree and branch
/// must survive, everything else sharing the run's worktree bay - a
/// loser, standing in for a judge/review worktree too, since `fold_run`'s
/// second sweep treats every non-winner directory under the bay alike -
/// must not.
#[tokio::test]
async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let mut config = Config::default();
config.graph.worktree_root = Some(tmp.path().join("wt"));
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
let root = state.worktree_root();
let wt_a = root.join("cand-A");
let wt_b = root.join("cand-B");
git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
.await
.expect("worktree A");
git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
.await
.expect("worktree B");
state.candidates = vec![
Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "magi/x/A".to_owned(),
worktree: wt_a.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
},
Candidate {
index: 1,
label: 'B',
agent: "beta".to_owned(),
branch: "magi/x/B".to_owned(),
worktree: wt_b.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
},
];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 1,
present: 1,
quorum: 1,
met_quorum: true,
uncontested: None,
});
state.status = RunStatus::Ready;
fold_run(&mut state, false, &crate::run::home())
.await
.expect("fold_run");
assert!(wt_a.exists(), "the unmerged winner's worktree survives");
assert!(
git::branch_exists(&repo, "magi/x/A").await.unwrap(),
"the unmerged winner's branch survives"
);
assert!(
!state.candidates[0].folded,
"the winner is not marked folded"
);
assert!(!wt_b.exists(), "the loser's worktree is removed");
assert!(
!git::branch_exists(&repo, "magi/x/B").await.unwrap(),
"the loser's branch is removed"
);
assert!(state.candidates[1].folded, "the loser is marked folded");
}
/// A branch handed to a later run is that run's (and its pull request's):
/// folding the run that released it must not delete it.
#[tokio::test]
async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let home = tmp.path().join("home");
let mut config = Config::default();
config.graph.worktree_root = Some(tmp.path().join("wt"));
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
// The worktree is already gone (released); the branch survives.
git::git(&repo, &["branch", "magi/x/A", "main"])
.await
.expect("branch");
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "magi/x/A".to_owned(),
worktree: state.worktree_root().join("cand-A"),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: true,
}];
state.released_to = Some("20260901-000000-new1".to_owned());
state.released_branches = vec!["magi/x/A".to_owned()];
fold_run(&mut state, true, &home).await.expect("fold_run");
assert!(
git::branch_exists(&repo, "magi/x/A").await.unwrap(),
"the handed-over branch survives a fold"
);
}
/// A winner with nothing ahead of the base is caught before `gh` is ever
/// asked for a pull request, and the message carries what the task's
/// references resolved to.
#[tokio::test]
async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let run = |args: &[&str]| {
let out = std::process::Command::new("git")
.quiet()
.args(args)
.current_dir(&repo)
.output()
.expect("spawn git");
assert!(out.status.success(), "git {args:?}");
};
run(&["branch", "magi/x/A"]);
run(&["checkout", "-q", "-b", "magi/x/B"]);
std::fs::write(repo.join("f.txt"), "x\n").unwrap();
run(&["add", "-A"]);
run(&["commit", "-q", "-m", "work"]);
run(&["checkout", "-q", "main"]);
// A pull request is compared against the remote's base, so the
// fixture needs one. Before it exists nothing can be read, and the
// answer must be "not empty".
let probe = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
Config::default(),
);
assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
let bare = tmp.path().join("origin.git");
let out = std::process::Command::new("git")
.quiet()
.args(["init", "-q", "--bare"])
.arg(&bare)
.output()
.expect("spawn git");
assert!(out.status.success(), "git init --bare");
run(&["remote", "add", "origin", bare.to_str().unwrap()]);
run(&["push", "-q", "origin", "main"]);
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
Config::default(),
);
state.seeds = vec![refs::Seed {
token: "magi/27b2/A".to_owned(),
kind: refs::SeedKind::Unresolved,
sha: String::new(),
branch: true,
detail: "no branch or commit named magi/27b2/A".to_owned(),
}];
assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
let detail = empty_candidate_detail(&state, "main");
assert!(detail.starts_with("empty candidate"), "{detail}");
assert!(detail.contains("magi/27b2/A"), "{detail}");
}
/// `status == Ready` used to be read as "this is the harmless
/// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
/// this test). But `land` sets the very same status when a `MergeMode::Pr`
/// run's PR was closed without merging — and reentering `merge` with
/// `mode` still `Pr` does not know the difference, so it pushed and
/// opened a second pull request. `mode == Local` reproduces the same
/// blind spot without a network call: reentry must not attempt another
/// git merge once this node has already recorded an outcome.
#[tokio::test]
async fn merge_does_not_reattempt_once_a_run_has_concluded() {
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let mut config = Config::default();
config.merge.mode = MergeMode::Local;
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
state.reviews = vec![ReviewRound {
round: 1,
head: "deadbeef".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 0,
answered: 0,
expected: 0,
clean: true,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
state.gate = vec![CommandOutcome {
command: "test".to_owned(),
code: Some(0),
output_tail: String::new(),
duration_ms: 0,
resource_blocked: false,
}];
state.gate_ran = true;
// Reached its conclusion already — e.g. `land` closing the PR without
// merging it, which (like the honest `MergeMode::None` path) leaves
// `status` at `Ready`. The recorded outcome is what actually marks
// this node done.
state.status = RunStatus::Ready;
state.merge = Some(MergeOutcome {
mode: MergeMode::Local,
ok: false,
detail: "already concluded".to_owned(),
empty: false,
});
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
runner.merge().await.expect("merge");
assert_eq!(
runner.state.status,
RunStatus::Ready,
"a concluded run's status must not change on reentry"
);
assert_eq!(
runner.state.merge.as_ref().map(|m| m.detail.as_str()),
Some("already concluded"),
"merge must not run again once the node already recorded an outcome"
);
}
/// `gate` leaves `state.gate_ran` false both before it has ever run and
/// when its last attempt was resource-blocked (the shared build cache
/// could not be acquired or confirmed fresh in time - see
/// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
/// `Vec` this also leaves behind used to read as "nothing failed" and let
/// a run merge a tree the gate never actually checked - exactly the case
/// a contended cache produces on every retry until it clears. `merge`
/// must refuse until `gate` has actually recorded an attempt.
#[tokio::test]
async fn merge_refuses_a_gate_that_has_not_actually_run() {
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let mut config = Config::default();
config.merge.mode = MergeMode::Local;
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
state.reviews = vec![ReviewRound {
round: 1,
head: "deadbeef".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 0,
answered: 0,
expected: 0,
clean: true,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
// The point: `gate` has not recorded anything yet.
state.gate = Vec::new();
state.gate_ran = false;
state.status = RunStatus::Gating;
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
runner.merge().await.expect("merge");
assert!(
runner.state.merge.is_none(),
"an empty gate must never be read as a passing one: {:?}",
runner.state.merge
);
}
/// The `shoka` repro this schema bump exists for: `verify.gate` has no
/// commands configured and `merge.mode` is `none` (a review-only run).
/// `gate` must still record a real attempt — zero commands, vacuously
/// passed — rather than leaving `state.gate` empty in a way `merge`
/// cannot tell apart from "never ran"; otherwise the run reaches
/// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
#[tokio::test]
async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
// Default config: `verify.gate` empty, `merge.mode` is `none`.
let config = Config::default();
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
state.reviews = vec![ReviewRound {
round: 1,
head: "deadbeef".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 0,
answered: 0,
expected: 0,
clean: true,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
runner.gate().await.expect("gate");
assert!(
runner.state.gate_ran,
"zero configured commands is still a real attempt, not an unrun gate"
);
assert!(runner.state.gate.is_empty());
assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
assert_ne!(
runner.state.status,
RunStatus::Blocked,
"a gate with nothing to check must not read as failed"
);
runner.merge().await.expect("merge");
assert_eq!(
runner.state.status,
RunStatus::Ready,
"a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
);
}
/// `Config::cache_dir` is derived from `verify.e2e` as well as
/// `verify.gate` (so the e2e leg and the final gate never build against
/// different directories). With zero `verify.gate` commands but a
/// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
/// that lease before discovering it had nothing to run - so a repo with
/// no gate commands could come back `resource_blocked` (and therefore
/// still `gate_ran == false`) on nothing but an unrelated run holding the
/// cache, exactly the contention this run's own zero commands could
/// never have touched. `gate` must recognise there is nothing to check
/// before it ever asks for the lease.
#[tokio::test]
async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
crate::run::pin_test_home();
let home = crate::run::home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
// Unique to this test, so holding its lease cannot collide with
// another test sharing the same process-wide `home`.
let cache_dir = tmp.path().join("target");
let mut config = Config::default();
config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
// `verify.gate` stays empty (the default). Bounded so a regression
// that does start waiting fails the test in seconds, not hangs it.
config.graph.timeout_verify = Some(2);
let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
.expect("no io error acquiring directly")
{
crate::cache::AcquireOutcome::Acquired(g) => g,
crate::cache::AcquireOutcome::Busy(b) => {
panic!("expected the direct acquire to win the lease first: {b:?}")
}
};
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
state.reviews = vec![ReviewRound {
round: 1,
head: "deadbeef".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 0,
answered: 0,
expected: 0,
clean: true,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
let started = std::time::Instant::now();
runner.gate().await.expect("gate");
assert!(
started.elapsed() < Duration::from_secs(1),
"a gate with nothing to run must never wait on a lease it never needed"
);
assert!(
runner.state.gate_ran,
"zero commands is still a real, immediate attempt"
);
assert!(runner.state.gate.is_empty());
assert_ne!(
runner.state.status,
RunStatus::Blocked,
"must not read as resource-blocked on a lease it never asked for"
);
}
/// The addendum's second gap: a `verify.gate` command running for real
/// wall-clock time had nothing at all to show for it in `active` before
/// `run_commands` learned to record it — a run could sit in `Gating` for
/// minutes with `magi show` and `GET /api/runs/{id}` both silent about
/// what was actually happening. Proven with a genuinely still-running
/// command, not just a before/after check on the final state: a poller
/// task reads the same `run.json` `gate()` is writing, the same way the
/// phone or `magi show` would, while the shell command is still blocked
/// on its own release marker.
#[tokio::test]
async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let mut config = Config::default();
config.verify.gate = vec![
"printf started > started.marker; i=0; while [ ! -f release.marker ] && \
[ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
.to_owned(),
];
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
let run_id = state.id.clone();
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
state.reviews = vec![ReviewRound {
round: 1,
head: "deadbeef".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 0,
answered: 0,
expected: 0,
clean: true,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
let started_marker = repo.join("started.marker");
let release_marker = repo.join("release.marker");
let poller = tokio::spawn(async move {
// Bounded so a regression that never records the task entry
// fails this test in seconds instead of hanging the suite —
// the same shape `a_park_requested_while_a_seat_is_mid_call_
// does_not_cut_it_short` uses for the same reason.
for _ in 0..100 {
if started_marker.exists()
&& let Ok(s) = crate::run::RunState::load(&run_id)
&& let Some(a) = s.active.get("gate")
{
std::fs::write(&release_marker, b"go").expect("release marker");
return Some(a.clone());
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
None
});
runner.gate().await.expect("gate");
let captured = poller.await.expect("poller task");
let captured = captured.expect(
"the poller never saw a `gate` task entry in run.json while the command was \
still blocked on its own release marker",
);
assert_eq!(captured.task.as_deref(), Some("gate"));
assert_eq!(captured.node, "gate");
assert_eq!(captured.index, Some(1));
assert_eq!(captured.total, Some(1));
assert!(
captured
.command
.as_deref()
.is_some_and(|c| c.contains("started.marker")),
"{captured:?}"
);
assert!(
runner.state.active.is_empty(),
"the entry must be cleared once the command actually finished: {:?}",
runner.state.active
);
assert!(runner.state.gate_ran);
assert!(runner.state.gate.iter().all(CommandOutcome::ok));
}
/// The hand-off over a blocking finding a reviewer rejected on leaves a
/// record for `land`; one with only a Minor, or no reject, leaves none.
#[tokio::test]
async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
use crate::verdict::{Finding, ReviewVote, Severity};
crate::run::pin_test_home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
for (severity, vote, expect) in [
(Severity::Major, ReviewVote::Reject, true),
(Severity::Minor, ReviewVote::Reject, false),
(Severity::Major, ReviewVote::Approve, false),
] {
let mut round = review_round(false, 1, 1, 1, false, true);
round.reviews = vec![ReviewRecord {
reviewer: 1,
agent: "alpha".to_owned(),
summary: String::new(),
findings: vec![Finding {
id: "R1-1-1".to_owned(),
severity,
file: None,
line: None,
title: "t".to_owned(),
detail: String::new(),
}],
vote: Some(vote),
failed: None,
duration_ms: 0,
attempts: 0,
}];
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
Config::default(),
);
state.reviews = vec![round];
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
let shell = runner.state.config.shell();
runner
.stop_reviewing("round budget spent", &shell, &repo)
.await
.expect("stop_reviewing");
assert_eq!(runner.state.status, RunStatus::Gating);
assert_eq!(
runner.state.contested_handoff.is_some(),
expect,
"{severity:?} + {vote:?}"
);
}
}
/// The shape the incident this whole fix responds to actually had: the
/// round budget spent, the last round's own e2e blocked on the shared
/// build cache (held here by a live pid — this test process — exactly
/// `cache`'s own unit tests' pattern for "another owner, still alive"
/// without forking a process). `stop_reviewing` must retry it — not
/// silently leave the round looking untouched (the catch-up-only half of
/// the bug), and not read the contention as a red `e2e` and block the
/// run on it (the other half). Called directly, the same way
/// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
/// above exercises `gate`, so this never needs a real cargo build to
/// reach: the lease is never released, so `with_cache_lease` never gets
/// past acquiring it into anything that would need a real workspace.
#[tokio::test]
async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
crate::run::pin_test_home();
let home = crate::run::home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let head = crate::git::rev_parse(&repo, "HEAD")
.await
.expect("rev-parse");
// Unique to this test, so holding its lease cannot collide with
// another test sharing the same process-wide `home`.
let cache_dir = tmp.path().join("target");
let mut config = Config::default();
config.verify.e2e = vec![format!(
"CARGO_TARGET_DIR='{}' test -f README.md",
cache_dir.display()
)];
config.graph.review_rounds = 1;
// Bounded so a regression that does start waiting fails the test in
// seconds, not hangs it.
config.graph.timeout_verify = Some(2);
let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
.expect("no io error acquiring directly")
{
crate::cache::AcquireOutcome::Acquired(g) => g,
crate::cache::AcquireOutcome::Busy(b) => {
panic!("expected the direct acquire to win the lease first: {b:?}")
}
};
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
head.clone(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
// The round budget's last round, deferred: `needs_catchup_run`'s
// other trigger. `stop_reviewing`'s retry machinery must treat this
// exactly like a resource-blocked attempt once it actually runs.
state.reviews = vec![ReviewRound {
round: 1,
head: head.clone(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 1,
answered: 1,
expected: 1,
clean: false,
verify_retried: false,
e2e_deferred: true,
e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
let shell = runner.state.config.shell();
runner
.stop_reviewing("round budget spent", &shell, &repo)
.await
.expect("stop_reviewing");
let last = runner.state.reviews.last().expect("round record");
assert_eq!(
last.e2e_status(),
E2eStatus::ResourceBlocked,
"the shared cache is still held; the attempt must read as blocked, not deferred or \
failed: {last:?}"
);
assert_eq!(
last.verified_head.as_deref(),
Some(head.as_str()),
"which commit this attempt targeted is known even though nothing finished checking \
it"
);
let first_attempt_at = last
.verified_at
.expect("when this attempt ran is known too");
assert_ne!(
runner.state.status,
RunStatus::Blocked,
"contention is evidence about the machine, not the patch — it must not settle the \
run as blocked: {:?}",
runner.state.status
);
assert!(
!runner
.state
.events
.iter()
.any(|e| e.node == "review" && e.message.contains("e2e failed")),
"a resource-blocked attempt must never be logged as a failed e2e: {:?}",
runner.state.events
);
// The cache is still held: a later reentry must retry the same
// round's verification again — not leave it looking exactly as
// untouched as the first blocked attempt, which is indistinguishable
// from never having tried again at all.
runner
.stop_reviewing("round budget spent", &shell, &repo)
.await
.expect("stop_reviewing retry");
assert_eq!(
runner.state.reviews.len(),
1,
"no new round was started: {:?}",
runner.state.reviews
);
let last = runner.state.reviews.last().expect("round record");
assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
assert!(
last.verified_at.expect("still known") > first_attempt_at,
"a second reentry must be a fresh attempt, not a stale copy of the first"
);
assert_ne!(runner.state.status, RunStatus::Blocked);
held.release();
}
/// A resumed run — a fresh `Runner`, `self.state.reviews` already
/// holding the round `stop_reviewing` left `ResourceBlocked` from a
/// prior process — must not sit at `Reviewing` forever: `review_loop`'s
/// own top-of-function fast path (`review_conclusion`) correctly reads
/// this shape as `None` rather than guessing `Blocked`, and the loop's
/// own `for` range is empty once the round budget is spent, so
/// `review_loop` must retry the check itself rather than silently doing
/// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
/// above exercises directly, but through `review_loop`'s own entry point
/// this time, proving the wiring between the two rather than just the
/// retry logic in isolation.
#[tokio::test]
async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
crate::run::pin_test_home();
let home = crate::run::home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let head = crate::git::rev_parse(&repo, "HEAD")
.await
.expect("rev-parse");
let cache_dir = tmp.path().join("target");
let mut config = Config::default();
config.verify.e2e = vec![format!(
"CARGO_TARGET_DIR='{}' test -f README.md",
cache_dir.display()
)];
config.graph.review_rounds = 1;
config.graph.timeout_verify = Some(2);
let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
.expect("no io error acquiring directly")
{
crate::cache::AcquireOutcome::Acquired(g) => g,
crate::cache::AcquireOutcome::Busy(b) => {
panic!("expected the direct acquire to win the lease first: {b:?}")
}
};
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
head.clone(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
// The exact shape a prior process's `stop_reviewing` would have left
// on disk: the round budget's last round, a real attempt already
// made and already resource-blocked.
state.reviews = vec![ReviewRound {
round: 1,
head: head.clone(),
verified_head: Some(head.clone()),
verified_at: Some(jiff::Timestamp::now()),
reviews: Vec::new(),
e2e: vec![CommandOutcome {
command: format!(
"CARGO_TARGET_DIR='{}' test -f README.md",
cache_dir.display()
),
code: None,
output_tail: "waiting for the shared build cache".to_owned(),
duration_ms: 0,
resource_blocked: true,
}],
fix: None,
blocking: 1,
answered: 1,
expected: 1,
clean: false,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
let first_attempt_at = state.reviews[0].verified_at.expect("set above");
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
// The lease is still held throughout, so this reentry's own retry is
// also contended — proving `review_loop` actually tried again (not
// that it happened to succeed) is what the timestamp comparison
// below is for.
runner.review_loop().await.expect("review_loop");
assert_eq!(
runner.state.reviews.len(),
1,
"no new round was started on top of the unresolved one: {:?}",
runner.state.reviews
);
let last = &runner.state.reviews[0];
assert_eq!(
last.e2e_status(),
E2eStatus::ResourceBlocked,
"still contended: {last:?}"
);
assert!(
last.verified_at.expect("still known") > first_attempt_at,
"review_loop must have actually retried the check, not left it exactly as found"
);
assert_ne!(
runner.state.status,
RunStatus::Blocked,
"a resumed run must not read leftover contention as a verdict on the patch: {:?}",
runner.state.status
);
held.release();
}
#[tokio::test]
async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let mut config = Config::default();
config.merge.mode = MergeMode::Pr;
config.graph.land = true;
config.graph.land_approval = false;
let mut state = RunState::new(
repo.clone(),
"main".to_owned(),
"deadbeef".to_owned(),
"task".to_owned(),
config,
);
state.candidates = vec![Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "does-not-exist".to_owned(),
worktree: repo.clone(),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty: false,
failed: None,
verified_noop: None,
duration_ms: 0,
folded: false,
}];
state.tally = Some(Tally {
first_choice: BTreeMap::from([('A', 1)]),
borda: BTreeMap::new(),
winner: 'A',
rankings: 1,
unanimous_initial: true,
deliberated: false,
changed_votes: 0,
unanimous_final: true,
tie_break: None,
judges: 0,
present: 0,
quorum: 0,
met_quorum: true,
uncontested: Some("only candidate A produced a change".to_owned()),
});
state.reviews = vec![ReviewRound {
round: 1,
head: "deadbeef".to_owned(),
verified_head: None,
verified_at: None,
reviews: Vec::new(),
e2e: Vec::new(),
fix: None,
blocking: 0,
answered: 0,
expected: 0,
clean: true,
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}];
state.gate = vec![CommandOutcome {
command: "test".to_owned(),
code: Some(0),
output_tail: String::new(),
duration_ms: 0,
resource_blocked: false,
}];
state.gate_ran = true;
// A first pass through `merge` already pushed and opened this pull
// request; `status` is `Landing` because a previous call into `land`
// parked or was interrupted before it reached a terminal outcome.
state.status = RunStatus::Landing;
state.merge = Some(MergeOutcome {
mode: MergeMode::Pr,
ok: true,
detail: "https://example.invalid/x/y/pull/1".to_owned(),
empty: false,
});
// The Landing-resume shortcut calls `run_land` directly rather than
// through `merge`, which is exactly the call site that used to skip
// `settle_questions` - see the fixture below.
ask_test_home();
// A real resume loads a persisted run; driver ownership is checked
// against that record before the Landing shortcut is entered.
state.save().expect("save landing run");
let store = ask::Questions::open();
let q = ask_open_question(&store, &state.id);
let mut runner = Runner {
state,
roles: ResolvedRoles {
implementers: Vec::new(),
judges: Vec::new(),
reviewers: Vec::new(),
fixer: None,
conductor: conductor(),
implementer_roster: Vec::new(),
judge_roster: Vec::new(),
reviewer_roster: Vec::new(),
},
sem: Arc::new(Semaphore::new(1)),
pause: Pause::new(),
interrupt: Pause::new(),
};
// `execute`, not `merge` directly: the Landing-resume shortcut lives
// at the top of `execute`, not inside `merge` (see `execute`'s doc)
// exactly because `review_loop` would otherwise clobber the marker
// first.
runner.execute().await.expect("execute");
assert_eq!(
runner.state.merge.as_ref().map(|m| m.detail.as_str()),
Some("https://example.invalid/x/y/pull/1"),
"reentry must not push again or open a second pull request over the \
one `land` is already watching"
);
assert_ne!(
runner.state.status,
RunStatus::Landing,
"land could not actually reach the fake pull request, so it must \
have given up rather than left the run silently parked forever"
);
// `land` could not reach the fake pull request, so it gave up into
// `Blocked` - still resumable, so the question must not have been
// swept just because this branch now also calls `settle_questions`.
assert_eq!(runner.state.status, RunStatus::Blocked);
assert!(
store.get(&q.id).unwrap().status.open(),
"Blocked is still alive; settle_questions must have been a no-op here"
);
}
fn state_with_round(round: ReviewRound) -> RunState {
let mut s = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234".to_owned(),
"add retries".to_owned(),
Config::default(),
);
s.reviews = vec![round];
s
}
fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
crate::verdict::Finding {
id: id.to_owned(),
severity,
file: None,
line: None,
title: title.to_owned(),
detail: String::new(),
}
}
#[test]
fn pr_body_names_open_findings_and_declined_ones() {
let round = ReviewRound {
round: 2,
head: "deadbee".to_owned(),
verified_head: None,
verified_at: None,
reviews: vec![ReviewRecord {
attempts: 0,
reviewer: 1,
agent: "alpha".to_owned(),
summary: String::new(),
findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
vote: None,
failed: None,
duration_ms: 0,
}],
e2e: vec![CommandOutcome {
command: "cargo test".to_owned(),
code: Some(0),
output_tail: String::new(),
duration_ms: 0,
resource_blocked: false,
}],
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
fix: Some(FixRecord {
agent: "alpha".to_owned(),
addressed: Vec::new(),
rejected: vec![crate::verdict::Rejection {
id: "R1-1-1".to_owned(),
why: "not reachable from any caller".to_owned(),
}],
notes: String::new(),
committed: true,
failed: None,
duration_ms: 0,
continuation: None,
}),
blocking: 0,
answered: 1,
expected: 1,
clean: false,
progressed: true,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
};
let state = state_with_round(round);
let body = pr_message(&state, 'A').body;
assert!(body.contains("add retries"), "the task must still be there");
assert!(body.contains("R2-1-1"), "{body}");
assert!(!body.contains("unused import"), "{body}");
assert!(body.contains("R1-1-1"), "the declined finding: {body}");
assert!(
!body.contains("not reachable from any caller"),
"the reason it was declined: {body}"
);
}
#[test]
fn pr_body_says_nothing_extra_when_the_round_was_clean() {
let round = ReviewRound {
round: 1,
head: "deadbee".to_owned(),
verified_head: None,
verified_at: None,
reviews: vec![ReviewRecord {
attempts: 0,
reviewer: 1,
agent: "alpha".to_owned(),
summary: String::new(),
findings: Vec::new(),
vote: None,
failed: None,
duration_ms: 0,
}],
e2e: Vec::new(),
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
fix: None,
blocking: 0,
answered: 1,
expected: 1,
clean: true,
progressed: false,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
};
let state = state_with_round(round);
let body = pr_message(&state, 'A').body;
assert!(!body.contains("Open review findings"), "{body}");
assert!(!body.contains("Declined"), "{body}");
}
fn state_with_summary(instruction: &str, summary: &str) -> RunState {
let mut state = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234".to_owned(),
instruction.to_owned(),
Config::default(),
);
state.candidates.push(Candidate {
index: 0,
label: 'A',
agent: "alpha".to_owned(),
branch: "magi/x/A".to_owned(),
worktree: PathBuf::from("/wt"),
summary: summary.to_owned(),
stat: String::new(),
files: 1,
commits: 1,
empty: false,
failed: None,
verified_noop: None,
folded: false,
duration_ms: 0,
});
state
}
fn review_state(subjects: &[&str]) -> RunState {
let mut state = state_with_summary(
"Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
"",
);
state.candidates[0].agent = EXISTING_BRANCH.to_owned();
state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
state
}
/// A branch rebased onto a main that moved past the recorded
/// `base_commit` is titled from its own first commit, never main's.
#[tokio::test]
async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
ask_test_home();
let tmp = tempfile::tempdir().unwrap();
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let origin = tmp.path().join("origin.git");
let g = |dir: &Path, args: &[&str]| {
let out = std::process::Command::new("git")
.args(args)
.current_dir(dir)
.quiet()
.output()
.expect("spawn git");
assert!(
out.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&out.stderr)
);
};
g(
tmp.path(),
&[
"clone",
"--bare",
"-q",
repo.to_str().unwrap(),
origin.to_str().unwrap(),
],
);
g(
&repo,
&["remote", "add", "origin", origin.to_str().unwrap()],
);
let c1 = git::rev_parse(&repo, "main").await.unwrap();
// Main moves on; the branch is built on top of the new main.
std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
g(&repo, &["add", "-A"]);
g(
&repo,
&["commit", "-q", "-m", "chore(deps): update a crate"],
);
g(&repo, &["push", "-q", "origin", "main"]);
g(&repo, &["checkout", "-q", "-b", "feat/own"]);
std::fs::write(repo.join("own.txt"), "own\n").unwrap();
g(&repo, &["add", "-A"]);
g(
&repo,
&["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
);
g(&repo, &["checkout", "-q", "main"]);
let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
// Without a readable tracking ref the recorded base's merge base is used.
let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
assert_eq!(fallback, c1);
let mut state = review_state(&[
"chore(deps): update a crate",
"fix(daemon): apply a chosen action",
]);
state.repo = repo.clone();
state.base_branch = "main".to_owned();
state.base_commit = c1;
refresh_reviewed_commits(&mut state, "feat/own").await;
assert_eq!(
state.reviewed_commits,
Some(vec!["fix(daemon): apply a chosen action".to_owned()])
);
assert_eq!(
review_title(&state).as_deref(),
Some("fix(daemon): apply a chosen action")
);
assert_eq!(
leaked_subjects(&state, "feat/own").await,
Some(vec!["chore(deps): update a crate".to_owned()])
);
// A stale tracking ref is still used when the fetch fails, but the
// leak list is withheld.
state.config.merge.remote = "nowhere".to_owned();
assert_eq!(leaked_subjects(&state, "feat/own").await, None);
}
/// A review-only run whose winner is `feat/own`, checked out in `repo`,
/// with `origin/main` one commit behind it, and a summarizing agent that
/// prints whatever `reply` holds.
struct SummaryFixture {
_tmp: tempfile::TempDir,
repo: PathBuf,
reply: PathBuf,
runner: Runner,
}
fn summary_fixture() -> SummaryFixture {
ask_test_home();
let tmp = tempfile::tempdir().unwrap();
let repo = tmp.path().join("repo");
std::fs::create_dir_all(&repo).unwrap();
init_repo(&repo);
let origin = tmp.path().join("origin.git");
let g = |args: &[&str]| {
let out = std::process::Command::new("git")
.args(args)
.current_dir(&repo)
.quiet()
.output()
.expect("spawn git");
assert!(
out.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&out.stderr)
);
};
g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
g(&["remote", "add", "origin", origin.to_str().unwrap()]);
g(&["push", "-q", "origin", "main"]);
g(&["checkout", "-q", "-b", "feat/own"]);
std::fs::write(repo.join("own.txt"), "own\n").unwrap();
g(&["add", "-A"]);
g(&["commit", "-q", "-m", "feat(own): add the own file"]);
let reply = tmp.path().join("reply.txt");
std::fs::write(
&reply,
"TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
)
.unwrap();
let mut runner = runner_at(RunStatus::Gating);
let mut state = review_state(&["feat(own): add the own file"]);
state.repo = repo.clone();
state.base_branch = "main".to_owned();
state.candidates[0].branch = "feat/own".to_owned();
state.candidates[0].worktree = repo.clone();
let mut author = spec("alpha");
author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
author
.env
.insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
state.config.agents = vec![author];
runner.state = state;
SummaryFixture {
_tmp: tmp,
repo,
reply,
runner,
}
}
async fn posted(f: &mut SummaryFixture) -> PrMessage {
let winner = f.runner.state.candidates[0].clone();
f.runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("an English description never parks")
}
fn clean_round_with_minor_findings() -> ReviewRound {
let mut round = passing_round(2, None);
round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
round.reviews = vec![ReviewRecord {
attempts: 0,
reviewer: 1,
agent: "alpha".into(),
summary: "内部レビュー".into(),
findings: vec![
finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
],
vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
failed: None,
duration_ms: 0,
}];
round
}
#[tokio::test]
async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
for review_only in [false, true] {
let mut f = summary_fixture();
if !review_only {
f.runner.state.candidates[0].agent = "alpha".into();
}
f.runner.state.reviews = vec![clean_round_with_minor_findings()];
assert!(f.runner.state.reviews[0].clean);
assert!(f.runner.state.reviews[0].fix.is_none());
assert!(f.runner.state.open_findings().is_empty());
assert_eq!(
crate::pr_outcome::required_finding_ids(&f.runner.state),
["R2-1-1", "R2-1-2"]
);
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
let evidence = crate::pr_outcome::summary_evidence(
&f.runner.state,
crate::pr_outcome::FinalHead::At(&head),
);
assert!(
evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
);
// The ordinary reply omits both IDs: no cached description is adopted.
let omitted = posted(&mut f).await;
assert!(omitted.body.contains("No description"));
assert!(omitted.body.contains("`R2-1-1` [Minor]"));
assert!(omitted.body.contains("`R2-1-2` [Minor]"));
assert!(f.runner.state.change_summary.is_none());
std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
let described = posted(&mut f).await;
assert!(described.body.contains("improve documentation for callers"));
assert!(described.body.contains("remaining edge case"));
assert!(described.body.contains("## Open review findings"));
assert!(!described.body.contains("未修正"));
}
}
async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
let mut f = summary_fixture();
if !review_only {
f.runner.state.candidates[0].agent = "alpha".into();
}
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
fixed
.fix
.as_mut()
.unwrap()
.rejected
.push(crate::verdict::Rejection {
id: "R1-1-1".into(),
why: "呼び出されないコードのため辞退".into(),
});
let mut clean = clean_round_with_minor_findings();
clean.verified_head = Some(head.clone());
f.runner.state.reviews = vec![fixed, clean];
ok_gate(&mut f.runner.state, Some(&head));
std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
f.runner.state.config.agents.push(scripted("beta", reply));
f.runner.state.config.roles.language_judge =
Some(crate::config::AgentChoice::One("beta".into()));
f
}
#[tokio::test]
async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
for review_only in [false, true] {
let mut f = rewrite_fixture_with_findings(review_only,
"TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
let m = posted(&mut f).await;
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
let expected =
public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
assert!(m.body.contains(&expected));
assert!(m.body.contains("## Review rounds"));
assert!(
m.body
.contains("Gate (`verify.gate`): passed on the final commit")
);
assert!(
m.body
.contains("Tests (`verify.e2e`): passed on the final commit")
);
let c = f.runner.state.change_summary.as_ref().unwrap();
assert!(
m.body
.contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
);
assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
assert!(crate::pr_outcome::covers_required_findings(
&f.runner.state,
&m.body
));
assert!(m.body.contains("documentation remains incomplete"));
assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
assert!(f.runner.state.github_text.is_none());
}
}
#[tokio::test]
async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
for review_only in [false, true] {
for reply in [
"TITLE: fix: public description\nA short English description.",
"TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
] {
let mut f = rewrite_fixture_with_findings(review_only, reply).await;
let winner = f.runner.state.candidates[0].clone();
let posted = f
.runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap();
assert!(posted.is_none());
assert!(f.runner.state.parked);
let withheld = f.runner.state.github_text.as_ref().unwrap();
assert!(
withheld.rewrite.is_none(),
"an incomplete rewrite must not be selectable"
);
let q = ask::Questions::open()
.get(withheld.question.as_ref().unwrap())
.unwrap();
assert!(
!q.choices
.contains(&crate::github_text::USE_REWRITE.to_owned())
);
assert!(
q.detail.contains("R1-1-1")
&& q.detail.contains("R2-1-1")
&& q.detail.contains("R2-1-2")
);
}
}
}
#[tokio::test]
async fn japanese_reports_stay_local_in_both_public_run_modes() {
for review_only in [false, true] {
let mut f = summary_fixture();
if !review_only {
f.runner.state.candidates[0].agent = "alpha".into();
}
f.runner.state.instruction = "内部タスクの説明".into();
f.runner.state.candidates[0].summary = "内部の候補報告".into();
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
f.runner.state.reviews = vec![
round_with_notes(1, Some("コミットできません。テストは実行できません。")),
round_with_notes(2, Some("二回目の内部修正報告です。")),
passing_round(3, Some(&head)),
];
f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
ok_gate(&mut f.runner.state, Some(&head));
let captured = f.repo.join("public-prompt.txt");
let author = &mut f.runner.state.config.agents[0];
author.command = vec![
"sh".into(),
"-c".into(),
"cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
];
author.env.insert(
"PROMPT_FILE".into(),
captured.to_string_lossy().into_owned(),
);
let m = posted(&mut f).await;
assert!(m.body.contains("Adds `own.txt`"));
assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
assert!(m.body.contains("R1-1-1"));
assert!(
m.body
.contains("Gate (`verify.gate`): passed on the final commit")
);
assert!(
m.body
.contains("Tests (`verify.e2e`): passed on the final commit")
);
assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
assert!(!m.body.contains("内部") && !m.body.contains("できません"));
let prompt = std::fs::read_to_string(captured).unwrap();
assert!(prompt.contains(&format!(
"git diff {} {head}",
f.runner.state.change_summary.as_ref().unwrap().base
)));
assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
assert_eq!(
f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
"二回目の内部修正報告です。"
);
}
}
#[tokio::test]
async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
for review_only in [false, true] {
let mut f = summary_fixture();
if !review_only {
f.runner.state.candidates[0].agent = "alpha".into();
}
f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
std::fs::write(&f.reply, "no usable description").unwrap();
let failed = posted(&mut f).await;
assert!(
failed
.body
.contains("No description of what this change does could be generated")
);
assert!(failed.body.is_ascii());
assert!(f.runner.state.github_text.is_none());
assert!(f.runner.state.change_summary.is_none());
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
let evidence = crate::pr_outcome::summary_evidence(
&f.runner.state,
crate::pr_outcome::FinalHead::At(&head),
);
let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
assert!(
crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
.exists()
);
f.runner.state =
serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
std::fs::write(
&f.reply,
"TITLE: feat(own): add the own file\nAdds the committed own file.",
)
.unwrap();
let retried = posted(&mut f).await;
assert!(retried.body.contains("Adds the committed own file."));
assert_eq!(
f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
"日本語の修正報告"
);
}
}
#[tokio::test]
async fn same_head_verification_and_findings_invalidate_public_prose() {
let mut f = summary_fixture();
posted(&mut f).await;
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
let original = f
.runner
.state
.change_summary
.as_ref()
.unwrap()
.evidence
.clone();
ok_gate(&mut f.runner.state, Some(&head));
std::fs::write(
&f.reply,
"TITLE: feat(own): add the own file\nDescribes the now verified final change.",
)
.unwrap();
let verified = posted(&mut f).await;
assert!(verified.body.contains("now verified"));
assert_ne!(
f.runner.state.change_summary.as_ref().unwrap().evidence,
original
);
let mut round = round_with_notes(1, Some("内部報告"));
round.reviews.push(ReviewRecord {
attempts: 0,
reviewer: 1,
agent: "alpha".into(),
summary: "内部レビュー".into(),
findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
vote: None,
failed: None,
duration_ms: 0,
});
round
.fix
.as_mut()
.unwrap()
.rejected
.push(crate::verdict::Rejection {
id: "R1-1-1".into(),
why: "呼び出し元から到達できない".into(),
});
f.runner.state.reviews = vec![round];
// A description omitting the required ID is unusable, even with a title.
let omitted = posted(&mut f).await;
assert!(omitted.body.contains("No description"));
assert!(omitted.body.contains("R1-1-1"));
assert!(!omitted.body.contains("到達"));
std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
let described = posted(&mut f).await;
assert!(described.body.contains("unreachable from callers"));
assert!(described.body.contains("R1-1-1"));
assert!(!described.body.contains("到達"));
let evidence = f
.runner
.state
.change_summary
.as_ref()
.unwrap()
.evidence
.clone();
f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
assert!(posted(&mut f).await.body.contains("rationale updated"));
assert_ne!(
f.runner.state.change_summary.as_ref().unwrap().evidence,
evidence
);
// Old serialized summaries have no evidence fingerprint and must regenerate.
f.runner
.state
.change_summary
.as_mut()
.unwrap()
.evidence
.clear();
std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
assert!(posted(&mut f).await.body.contains("current specification"));
}
#[tokio::test]
async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
let mut f = summary_fixture();
let m = posted(&mut f).await;
assert_eq!(m.title, "feat(own): add the own file");
assert!(
m.body
.contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
"{}",
m.body
);
assert!(!m.body.contains("## Review fixes"), "{}", m.body);
assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
let kept = f.runner.state.change_summary.as_ref().unwrap();
assert_eq!(
(kept.head.as_str(), kept.base.as_str()),
(head.as_str(), base.as_str())
);
assert!(!f.runner.state.seats["change-summary"].key.is_empty());
}
#[tokio::test]
async fn review_only_summary_receives_the_configured_github_language() {
let mut f = summary_fixture();
f.runner.state.config.graph.github_language = "ja".into();
let captured = f.repo.join("summary-prompt.txt");
let author = &mut f.runner.state.config.agents[0];
author.command = vec![
"sh".into(),
"-c".into(),
"cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
];
author.env.insert(
"PROMPT_FILE".into(),
captured.to_string_lossy().into_owned(),
);
std::fs::write(
&f.reply,
"TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
)
.unwrap();
let m = posted(&mut f).await;
let prompt = std::fs::read_to_string(captured).unwrap();
assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
assert!(!prompt.contains("in English"));
assert_eq!(m.title, "feat(own): ファイルを追加");
assert!(m.body.contains("own.txt を追加します。"));
}
#[tokio::test]
async fn review_fixes_come_after_the_whole_change_summary() {
let mut f = summary_fixture();
f.runner.state.reviews = vec![
round_with_notes(1, Some("handled the PATH case")),
round_with_notes(2, None),
];
let m = posted(&mut f).await;
let summary = m.body.find("## Summary").expect("summary");
let fixes = m.body.find("## Review rounds").expect("review rounds");
assert!(summary < fixes, "{}", m.body);
assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
assert!(m.body.contains("## Final verification"), "{}", m.body);
}
#[tokio::test]
async fn a_changed_final_diff_invalidates_the_earlier_summary() {
let mut f = summary_fixture();
let first = posted(&mut f).await;
let first_head = f.runner.state.change_summary.clone().unwrap().head;
// The same head reuses the record: a different reply is not read.
std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
let again = posted(&mut f).await;
assert_eq!((again.title, again.body), (first.title, first.body));
let g = |args: &[&str]| {
let out = std::process::Command::new("git")
.args(args)
.current_dir(&f.repo)
.quiet()
.output()
.expect("spawn git");
assert!(out.status.success(), "git {args:?}");
};
std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
g(&["add", "-A"]);
g(&["commit", "-q", "-m", "fix(own): handle more"]);
let second = posted(&mut f).await;
let kept = f.runner.state.change_summary.clone().unwrap();
assert_ne!(kept.head, first_head);
assert!(second.body.contains("Not read yet."), "{}", second.body);
assert!(!second.body.contains("so the first summary is read"));
assert_eq!(second.title, "feat(own): other");
}
#[tokio::test]
async fn an_unavailable_summary_is_stated_and_never_invented() {
let mut f = summary_fixture();
std::fs::write(&f.reply, "no title line here").unwrap();
let m = posted(&mut f).await;
assert!(
m.body
.contains("No description of what this change does could be generated"),
"{}",
m.body
);
assert!(m.body.contains("internal reports remain in local records"));
assert!(f.runner.state.change_summary.is_none());
assert!(!m.body.contains("## Commits under review"), "{}", m.body);
// A preview asks nobody and claims nothing.
let winner = f.runner.state.candidates[0].clone();
let preview = f
.runner
.guarded_pr_message(&winner, None, false)
.await
.unwrap()
.unwrap();
assert!(!preview.body.contains("## Summary"), "{}", preview.body);
}
#[test]
fn pr_message_review_single_commit_uses_its_subject() {
let state = review_state(&["feat(nats): per-role user"]);
let m = pr_message(&state, 'A');
assert_eq!(m.title, "feat(nats): per-role user");
assert!(!m.body.contains("Review the work already"), "{}", m.body);
assert!(m.body.contains("## Commits under review"), "{}", m.body);
}
#[test]
fn pr_message_review_multi_commit_takes_the_oldest() {
let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
let m = pr_message(&state, 'A');
assert_eq!(m.title, "feat: the change");
for s in ["feat: the change", "fix: typo", "fix: again"] {
assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
}
}
#[test]
fn pr_message_review_without_a_usable_first_subject_is_neutral() {
for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
let mut state = review_state(&[first, "fix: later fixup"]);
state.candidates[0].branch = "機能/ログイン".to_owned();
let m = pr_message(&state, 'A');
assert!(
m.title.starts_with("chore: land candidate A of run"),
"{}",
m.title
);
}
}
fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
BranchFacts {
commits: commits
.iter()
.map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
.collect(),
stat: stat.to_owned(),
}
}
fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
let mut r = review_round(true, 0, 1, 1, true, true);
r.round = round;
r.fix = notes.map(|n| FixRecord {
agent: "fixer".to_owned(),
addressed: Vec::new(),
rejected: Vec::new(),
notes: n.to_owned(),
committed: true,
failed: None,
duration_ms: 0,
continuation: None,
});
r
}
#[test]
fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
let state = review_state(&["ignored"]);
let f = facts(
&[
(
"fix(login): resolve PATH on macOS",
"Login shells skip rc files.",
),
("fix: address review", ""),
],
" src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
);
let m = pr_message_with(&state, 'A', Some(&f));
assert_eq!(m.title, "fix(login): resolve PATH on macOS");
assert!(
m.body
.contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
"{}",
m.body
);
assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
assert!(m.body.contains("## Diff stat"), "{}", m.body);
assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
for banned in [
"Review the work already",
"no task statement",
"Original task",
] {
assert!(!m.body.contains(banned), "{banned}: {}", m.body);
}
assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
}
#[test]
fn pr_message_review_truncates_a_huge_first_commit_body() {
let state = review_state(&["ignored"]);
let f = facts(
&[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
"s",
);
let m = pr_message_with(&state, 'A', Some(&f));
assert!(m.body.len() < 30_000, "{}", m.body.len());
assert!(m.body.contains("(truncated)"), "{}", m.body.len());
assert!(
m.body.contains("- ... 1 more commit(s)"),
"{}",
m.body.len()
);
assert!(m.body.ends_with("magi:candidate-a\n"));
}
#[test]
fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
assert_eq!(m.title, "feat: x");
assert!(m.body.contains("- fix: y\n"), "{}", m.body);
assert!(!m.body.contains("Diff stat"), "{}", m.body);
assert!(!m.body.contains("no task statement"), "{}", m.body);
}
#[test]
fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
let mut state = review_state(&["日本語の件名"]);
state.candidates[0].branch = "fix/macos-login-path".to_owned();
assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
state.candidates[0].branch = "機能/ログイン".to_owned();
assert!(
pr_message(&state, 'A')
.title
.starts_with("chore: land candidate A")
);
}
#[test]
fn pr_message_review_fixes_survive_a_clean_final_round() {
let mut state = review_state(&["feat: x"]);
state.reviews = vec![
round_with_notes(1, Some("handled the PATH case")),
round_with_notes(2, None),
];
let body = pr_message(&state, 'A').body;
assert!(
body.contains("## Review rounds") && !body.contains("handled the PATH case"),
"{body}"
);
assert!(body.contains("### Round 1"), "{body}");
state.reviews = vec![
round_with_notes(1, Some("first fix")),
round_with_notes(2, Some("")),
round_with_notes(3, Some("second fix")),
round_with_notes(4, None),
];
let body = pr_message(&state, 'A').body;
assert!(
body.contains("### Round 1") && !body.contains("first fix"),
"{body}"
);
assert!(
body.contains("### Round 3") && !body.contains("second fix"),
"{body}"
);
assert!(body.contains("### Round 2"), "{body}");
}
const FINAL: &str = "abcdef1234567890";
fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
let mut r = round_with_notes(round, None);
r.verified_head = verified.map(str::to_owned);
r
}
fn ok_gate(state: &mut RunState, head: Option<&str>) {
state.gate = vec![CommandOutcome {
command: "gate".to_owned(),
code: Some(0),
output_tail: String::new(),
duration_ms: 0,
resource_blocked: false,
}];
state.gate_ran = true;
state.gate_head = head.map(str::to_owned);
}
fn body_at(state: &RunState) -> String {
pr_message_raw(
state,
'A',
None,
crate::pr_outcome::FinalHead::At(FINAL),
ChangeNote::NotAsked,
)
.body
}
const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
#[test]
fn rescued_commit_and_later_verification_override_the_fixers_limits() {
let mut state = review_state(&["feat: x"]);
state.reviews = vec![
round_with_notes(1, Some(SANDBOXED)),
passing_round(2, Some(FINAL)),
];
ok_gate(&mut state, Some(FINAL));
let body = body_at(&state);
assert!(!body.contains("Handled the PATH case."), "{body}");
assert!(!body.contains("could not commit"), "{body}");
assert!(!body.contains("could not run"), "{body}");
assert!(
body.contains("The fix is committed on the branch"),
"{body}"
);
assert!(!body.contains("sandbox"), "{body}");
assert!(
body.contains("Tests (`verify.e2e`): passed on the final commit"),
"{body}"
);
assert!(
body.contains("Gate (`verify.gate`): passed on the final commit"),
"{body}"
);
// The raw report is untouched in the record.
assert!(
state.reviews[0]
.fix
.as_ref()
.unwrap()
.notes
.contains("could not commit")
);
}
#[test]
fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
let mut state = review_state(&["feat: x"]);
let mut r1 = round_with_notes(1, Some(SANDBOXED));
r1.fix.as_mut().unwrap().committed = false;
let mut r2 = passing_round(2, Some(FINAL));
r2.e2e[0].code = Some(1);
state.reviews = vec![r1, r2];
let body = body_at(&state);
assert!(!body.contains("could not commit"), "{body}");
assert!(!body.contains("could not run"), "{body}");
assert!(body.contains("This round produced no commit"), "{body}");
assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
assert!(!body.contains("passed"), "{body}");
}
#[test]
fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
let mut state = review_state(&["feat: x"]);
state.reviews = vec![
round_with_notes(1, Some(SANDBOXED)),
passing_round(2, Some("0123456789abcdef")),
];
ok_gate(&mut state, Some("0123456789abcdef"));
let body = body_at(&state);
assert!(!body.contains("HTTP tests could not run"), "{body}");
// The commit claim is still settled by the structured record.
assert!(!body.contains("could not commit"), "{body}");
assert!(
body.contains("passed on an earlier commit (`0123456`)"),
"{body}"
);
assert!(!body.contains("passed on the final commit"), "{body}");
// A gate recorded before `gate_head` existed is "not recorded".
ok_gate(&mut state, None);
let body = body_at(&state);
assert!(
body.contains("passed on a commit that was not recorded"),
"{body}"
);
}
#[test]
fn pr_message_implementation_run_keeps_its_shape_and_marker() {
let state = state_with_summary(
"add retries to the client",
"TITLE: feat: retries\n\nDid it.",
);
let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
assert_eq!(m.title, "feat: retries");
assert!(
m.body.contains("<summary>Original task</summary>"),
"{}",
m.body
);
assert!(!m.body.contains("Commits under review"), "{}", m.body);
assert!(
m.body
.ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
"{}",
m.body
);
}
#[test]
fn pr_message_review_bounds_a_long_english_subject() {
let long = format!("feat: {}", "word ".repeat(100));
let m = pr_message(&review_state(&[&long]), 'A');
assert!(m.title.starts_with("feat: word"), "{}", m.title);
assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
}
#[test]
fn pr_message_implementation_run_is_unchanged_by_review_support() {
let state = state_with_summary("add retries\n\ndetails", "- did some things");
let m = pr_message(&state, 'A');
assert_eq!(m.title, "add retries");
assert!(m.body.contains("<summary>Original task</summary>"));
assert!(!m.body.contains("Commits under review"));
assert_eq!(landing_subject_source(&state), state.instruction);
}
#[test]
fn review_run_squash_subject_is_the_change_not_the_prompt() {
let state = review_state(&["feat: the change", "fix: typo"]);
let source = landing_subject_source(&state);
assert_eq!(land::merge_subject("", &source), "feat: the change");
assert_eq!(
land::merge_subject("magi: candidate A (uncommitted work)", &source),
"feat: the change"
);
// An operator's rename still wins.
assert_eq!(
land::merge_subject("feat: renamed by hand", &source),
"feat: renamed by hand"
);
let blank = review_state(&["日本語"]);
assert!(
land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
);
}
#[test]
fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
let state = review_state(&["feat: the change"]);
let source = landing_subject_source(&state);
let old = "Review the work already on branch `magi/x/A`. There is no task statement";
assert_eq!(
land::merge_subject(landing_title(&state, old), &source),
"feat: the change"
);
assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
let task = state_with_summary("add retries", "");
assert_eq!(landing_title(&task, old), old);
}
#[test]
fn pr_message_describes_the_change_not_the_task() {
let state = state_with_summary(
"今回やってほしいこと: results projector を直す",
"TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
);
let m = pr_message(&state, 'A');
assert_eq!(m.title, "fix(web): batch the runs list reads");
assert!(
m.body.starts_with("## Summary\n\n- reads run.json once"),
"{}",
m.body
);
assert!(!m.body.contains("TITLE:"), "{}", m.body);
let task_at = m.body.find("今回やってほしいこと").unwrap();
let details_at = m.body.find("<details>").unwrap();
assert!(
details_at < task_at,
"the task lives inside <details>: {}",
m.body
);
assert!(m.body.contains(&format!("magi:run/{}", state.id)));
assert!(m.body.contains("magi:candidate-a"));
}
#[test]
fn pr_message_falls_back_to_the_task_without_a_title_line() {
let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
let m = pr_message(&state, 'A');
assert_eq!(m.title, "add retries");
assert!(
m.body.contains("## Summary\n\n- did some things"),
"{}",
m.body
);
let none = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234".to_owned(),
"add retries".to_owned(),
Config::default(),
);
let m = pr_message(&none, 'A');
assert_eq!(m.title, "add retries");
assert!(!m.body.contains("## Summary"), "{}", m.body);
}
#[test]
fn pr_message_refuses_the_candidate_commit_subject() {
for bad in [
"TITLE: magi: candidate A (uncommitted work)",
"TITLE: chore: stuff (uncommitted work)",
"TITLE: ",
] {
let state = state_with_summary("add retries", bad);
assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
}
}
#[test]
fn pr_message_bounds_a_very_long_task_and_title() {
let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
let state = state_with_summary(&long, "- nothing");
let m = pr_message(&state, 'A');
assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
assert!(!m.title.contains('\n'));
let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
let m = pr_message(&state, 'A');
assert!(m.title.starts_with("feat: "));
assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
}
fn long_title_of(instruction: &str) -> String {
pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
}
#[test]
fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
let first = "Make the landing path keep a readable title for long tasks";
let line = format!(
"{first}. {}",
"And then keep going with more words ".repeat(20)
);
let t = long_title_of(&line);
assert_eq!(t, first);
assert!(!t.starts_with("chore: land"));
}
#[test]
fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
let line = "word ".repeat(200);
let t = long_title_of(&line);
assert!(t.ends_with("word..."), "{t}");
assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
}
#[test]
fn pr_message_long_non_english_or_letterless_line_is_neutral() {
for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
}
}
#[test]
fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
let mut state = state_with_summary("再試行を追加する", "- nothing");
assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
state.config.graph.github_language = "ja".into();
assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
}
#[test]
fn pr_message_title_limit_is_exact() {
let at = "a".repeat(PR_TITLE_MAX);
assert_eq!(long_title_of(&at), at);
let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
assert!(over.ends_with("..."), "{over}");
assert_eq!(over.chars().count(), PR_TITLE_MAX);
}
#[test]
fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
let t = long_title_of(&line);
assert!(t.ends_with("..."), "{t}");
assert!(t.is_ascii(), "{t}");
}
#[test]
fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
let line = format!(
"Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
"plus more filler words ".repeat(20)
);
let t = long_title_of(&line);
assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
}
#[test]
fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
let line = format!("\n\n# {}", "title words ".repeat(40));
let state = state_with_summary(&line, "- nothing");
let m = pr_message(&state, 'A');
assert!(m.title.starts_with("title words"), "{}", m.title);
assert_eq!(
land::merge_subject(&m.title, &landing_subject_source(&state)),
m.title
);
// An operator's rename wins untouched.
assert_eq!(
land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
"feat: renamed by hand"
);
}
#[tokio::test]
async fn github_text_rewrite_is_bounded_and_falls_back() {
crate::run::pin_test_home();
for (reply, accepted) in [
(
"TITLE: fix: retries\nAdd retries for failed requests.",
true,
),
(
"TITLE: fix: retries\n日本語の説明をもう一度書きます。",
false,
),
(
"TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
true,
),
(
"TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
true,
),
(
"TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
true,
),
(
"TITLE: fix: retries\n日本語の説明です。 token=secret",
false,
),
("TITLE: fix: retries\n", false),
] {
let dir = tempfile::tempdir().unwrap();
let mut runner = runner_at(RunStatus::Gating);
runner.state = state_with_summary(
"add retries",
"TITLE: fix: retries\n日本語の説明を書きます。",
);
runner.state.repo = dir.path().to_owned();
runner.state.candidates[0].worktree = dir.path().to_owned();
let mut author = spec("alpha");
author.command = vec![
"sh".into(),
"-c".into(),
"printf '%s' \"$REWRITE_REPLY\"".into(),
];
author.env.insert("REWRITE_REPLY".into(), reply.into());
runner.state.config.agents = vec![author];
seed_public_summary(&mut runner);
let winner = runner.state.candidates[0].clone();
let first = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap();
let message = if accepted {
first.expect("a passing rewrite never asks")
} else {
// Withheld: the run parks on a question and posts nothing yet.
assert!(first.is_none());
assert!(runner.state.parked);
let store = ask::Questions::open();
let asked: Vec<_> = store
.list()
.into_iter()
.filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
.collect();
assert_eq!(asked.len(), 1);
assert!(!asked[0].detail.contains("secret"));
let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
assert_eq!(
asked[0]
.choices
.contains(&crate::github_text::USE_REWRITE.to_owned()),
rewrite.is_some()
);
// Silence: the question lapses and the neutral text is used,
// without a second rewrite call or a second question.
store
.update(&asked[0].id, |q| {
q.abandon("test");
Ok(())
})
.unwrap();
let again = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("an abandoned question falls back");
let n = store
.list()
.into_iter()
.filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
.count();
assert_eq!(n, 1, "asked once per rejected text");
again
};
assert!(crate::github_text::check(&message.title, &message.body).is_empty());
assert!(!message.body.contains("secret"));
assert!(!message.body.contains("prefix"));
assert!(!message.title.contains("secret"));
assert_eq!(
message.body.contains("Add retries for failed requests."),
accepted
);
assert_eq!(
runner.state.seats["github-text-rewrite"].turns, 1,
"only one rewrite invocation"
);
assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
if accepted {
assert!(runner.state.github_text.is_none());
assert!(!runner.state.parked);
assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
assert!(message.body.contains("## Final verification"));
assert!(message.body.contains("Evidence: final diff"));
assert!(
message
.body
.contains(&format!("magi:run/{}", runner.state.id))
);
}
if !accepted {
assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
assert!(
message
.body
.contains(&format!("magi:run/{}", runner.state.id))
);
}
}
}
fn scripted(id: &str, reply: &str) -> AgentSpec {
let mut a = spec(id);
a.command = vec![
"sh".into(),
"-c".into(),
"printf '%s' \"$REWRITE_REPLY\"".into(),
];
a.env.insert("REWRITE_REPLY".into(), reply.into());
a
}
const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
/// Gate tests inject generated public output, not internal reports.
fn seed_public_summary(runner: &mut Runner) {
let repo = &runner.state.repo;
let g = |args: &[&str]| {
let out = std::process::Command::new("git")
.args(args)
.current_dir(repo)
.quiet()
.output()
.unwrap();
assert!(out.status.success(), "git {args:?}");
String::from_utf8_lossy(&out.stdout).trim().to_owned()
};
g(&["init", "-q", "-b", "main"]);
g(&[
"-c",
"user.name=Test",
"-c",
"user.email=test@example.invalid",
"commit",
"-q",
"--allow-empty",
"-m",
"feat: fixture",
]);
let head = g(&["rev-parse", "HEAD"]);
g(&["update-ref", "refs/remotes/origin/main", &head]);
let evidence = crate::pr_outcome::summary_evidence(
&runner.state,
crate::pr_outcome::FinalHead::At(&head),
);
let generated = &runner.state.candidates[0].summary;
runner.state.change_summary = Some(crate::run::ChangeSummary {
base: head.clone(),
head,
evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
title: summary_title(generated).unwrap(),
body: summary_without_title(generated),
});
}
fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
let mut runner = runner_at(RunStatus::Gating);
runner.state = state_with_summary("add retries", JA_SUMMARY);
if review_only {
runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
}
runner.state.repo = dir.to_owned();
runner.state.candidates[0].worktree = dir.to_owned();
runner.state.config.agents = agents;
seed_public_summary(&mut runner);
runner
}
fn gate_questions(runner: &Runner) -> usize {
ask::Questions::open()
.list()
.into_iter()
.filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
.count()
}
#[tokio::test]
async fn review_only_description_is_rewritten_by_the_language_judge_role() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut runner = rewrite_runner(
dir.path(),
true,
vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
);
runner.state.config.roles.language_judge =
Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
let winner = runner.state.candidates[0].clone();
let msg = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("an English rewrite posts without asking");
assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
assert!(msg.body.contains("<details>kept</details>"));
assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
assert_eq!(gate_questions(&runner), 0);
assert!(runner.state.github_text.is_none());
assert!(!runner.state.seats.contains_key("impl-A"));
assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
}
#[tokio::test]
async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut runner = rewrite_runner(
dir.path(),
false,
vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
);
runner.state.config.roles.language_judge =
Some(crate::config::AgentChoice::One("beta".into()));
runner.state.seats.insert(
"impl-A".into(),
SeatState::new("impl-A", "alpha", runner.state.seed),
);
let winner = runner.state.candidates[0].clone();
let msg = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("rewritten");
assert!(msg.body.contains("<details>kept</details>"));
assert_eq!(runner.state.seats["impl-A"].turns, 0);
assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
}
#[tokio::test]
async fn rewrite_reuses_a_resumable_implementer_session() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut runner = rewrite_runner(
dir.path(),
false,
vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
);
runner.state.config.roles.language_judge =
Some(crate::config::AgentChoice::One("beta".into()));
let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
seat.turns = 1;
runner.state.seats.insert("impl-A".into(), seat);
let winner = runner.state.candidates[0].clone();
runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("rewritten");
assert_eq!(runner.state.seats["impl-A"].turns, 2);
assert!(!runner.state.seats.contains_key("github-text-rewrite"));
}
#[tokio::test]
async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut broken = spec("bad");
broken.command = vec!["false".into()];
let mut runner = rewrite_runner(
dir.path(),
true,
vec![
broken,
scripted("noformat", "Add retries."),
scripted("good", EN_REWRITE),
],
);
runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
"bad".into(),
"noformat".into(),
"good".into(),
]));
let winner = runner.state.candidates[0].clone();
let msg = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("the third agent rewrites");
assert!(msg.body.contains("`src/net.rs`"));
assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
assert!(runner.state.handovers.is_empty(), "no handover is recorded");
}
#[tokio::test]
async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
crate::run::pin_test_home();
// (agents, role): still Japanese; no TITLE; nothing resolvable.
for (agents, role, saved) in [
(
vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
Some("beta"),
true,
),
(vec![scripted("beta", "Add retries.")], Some("beta"), false),
(Vec::new(), None, false),
] {
let dir = tempfile::tempdir().unwrap();
let mut runner = rewrite_runner(dir.path(), true, agents);
runner.state.config.roles.language_judge =
role.map(|r| crate::config::AgentChoice::One(r.into()));
let winner = runner.state.candidates[0].clone();
let first = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap();
assert!(first.is_none(), "withheld text parks the run");
assert!(runner.state.parked);
assert_eq!(gate_questions(&runner), 1);
let g = runner.state.github_text.as_ref().unwrap();
assert_eq!(g.rewrite.is_some(), saved);
assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
}
}
#[tokio::test]
async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut runner = runner_at(RunStatus::Gating);
runner.state =
state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
runner.state.repo = dir.path().to_owned();
runner.state.candidates[0].worktree = dir.path().to_owned();
let mut author = spec("alpha");
author.command = vec![
"sh".into(),
"-c".into(),
"printf '%s' \"$REWRITE_REPLY\"".into(),
];
author.env.insert(
"REWRITE_REPLY".into(),
format!(
"TITLE: fix: rewritten title\n{}\n日本語の説明です。",
"English sentence. ".repeat(600)
),
);
runner.state.config.agents = vec![author];
seed_public_summary(&mut runner);
let winner = runner.state.candidates[0].clone();
assert!(
runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.is_none()
);
let record = runner.state.github_text.clone().unwrap();
let rewrite = record.rewrite.clone().unwrap();
assert_eq!(rewrite.failed, ["body-language"]);
assert!(
rewrite
.body
.ends_with(&format!("magi:run/{}", runner.state.id))
);
let store = ask::Questions::open();
let q = store.get(record.question.as_ref().unwrap()).unwrap();
assert!(
q.choices
.contains(&crate::github_text::USE_REWRITE.to_owned())
);
for label in [
"ORIGINAL",
"FALLBACK",
"REWRITE",
"Rules still failed: body-language",
"truncated",
] {
assert!(q.detail.contains(label), "{label}");
}
let artifact = std::fs::read_to_string(crate::run::artifact_path(
&runner.state,
&crate::github_text::artifact_name(&record.fingerprint),
))
.unwrap();
assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
// Restart from serialized state, then retry after a bad owner title.
runner.state =
serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
store
.update(&q.id, |q| {
q.say("まだ日本語")?;
q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
})
.unwrap();
assert!(
runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.is_none()
);
let retry = runner.state.github_text.as_ref().unwrap();
assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
let q = store.get(retry.question.as_ref().unwrap()).unwrap();
assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
assert!(
q.posting_comparison
.as_ref()
.unwrap()
.matches_detail(&q.detail)
);
store
.update(&q.id, |q| {
q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
})
.unwrap();
let message = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.unwrap();
assert_eq!(message.title, rewrite.title);
assert_eq!(message.body, rewrite.body);
assert_eq!(
runner.state.github_text.as_ref().unwrap().choice,
Some(GithubTextChoice::Rewrite)
);
let again = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.unwrap();
assert_eq!(again.title, message.title);
assert_eq!(again.body, message.body);
assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
assert!(
!runner
.state
.events
.iter()
.any(|e| e.message.contains(&rewrite.title)
|| e.message.contains("English sentence."))
);
}
#[tokio::test]
async fn github_text_rewrite_choice_without_saved_text_falls_back() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut runner = runner_at(RunStatus::Gating);
runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
runner.state.repo = dir.path().to_owned();
runner.state.candidates[0].worktree = dir.path().to_owned();
seed_public_summary(&mut runner);
let winner = runner.state.candidates[0].clone();
assert!(
runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.is_none()
);
let record = runner.state.github_text.as_ref().unwrap();
assert!(record.rewrite.is_none());
// Records written before the additive field deserialize with no rewrite.
let mut old = serde_json::to_value(record).unwrap();
old.as_object_mut().unwrap().remove("rewrite");
let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
assert!(old.rewrite.is_none());
let id = record.question.clone().unwrap();
ask::Questions::open()
.update(&id, |q| {
q.choices.push(crate::github_text::USE_REWRITE.into());
q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
})
.unwrap();
let message = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.unwrap();
assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
assert_eq!(
runner.state.github_text.as_ref().unwrap().choice,
Some(GithubTextChoice::Fallback)
);
}
#[tokio::test]
async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
crate::run::pin_test_home();
let dir = tempfile::tempdir().unwrap();
let mut runner = runner_at(RunStatus::Gating);
runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
runner.state.repo = dir.path().to_owned();
runner.state.candidates[0].worktree = dir.path().to_owned();
seed_public_summary(&mut runner);
let winner = runner.state.candidates[0].clone();
assert!(
runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.is_none()
);
let store = ask::Questions::open();
let find = |run: &str| {
store
.list()
.into_iter()
.find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
.unwrap()
};
let run = runner.state.id.clone();
let q = find(&run);
let fp = runner
.state
.github_text
.as_ref()
.unwrap()
.fingerprint
.clone();
let name = crate::github_text::artifact_name(&fp);
let path = crate::run::artifact_path(&runner.state, &name);
assert!(q.detail.contains(&path.display().to_string()));
assert!(q.detail.contains("title-language: title"));
let saved = std::fs::read_to_string(&path).expect("artifact written");
assert!(saved.contains("日本語のタイトル"));
assert!(q.cwd.is_none());
assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
// A bad replacement is refused and asked about once more.
let say = |id: &str, text: &str, choice: &str| {
store
.update(id, |q| {
q.thread.push(ask::Turn {
who: ask::Who::Operator,
body: text.to_owned(),
at: jiff::Timestamp::now(),
note: None,
});
q.answer(ask::Answer::Choice(choice.to_owned()))
})
.unwrap();
};
say(&q.id, "まだ日本語", "use my text");
assert!(
runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.is_none()
);
let second = find(&run);
assert_ne!(second.id, q.id);
assert!(second.detail.contains("did not pass"));
// The retry still compares the original text; the rejected
// replacement is named by category only.
assert!(second.detail.contains("日本語のタイトル"));
assert!(!second.detail.contains("まだ日本語"));
assert!(second.detail.contains(&path.display().to_string()));
say(&second.id, "fix: retry failed requests", "use my text");
let message = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.expect("a vetted replacement is posted");
assert_eq!(message.title, "fix: retry failed requests");
assert_eq!(
runner
.state
.github_text
.as_ref()
.unwrap()
.chosen_title
.as_deref(),
Some("fix: retry failed requests")
);
assert!(runner.state.github_text.as_ref().unwrap().resolved);
// Resumed again: the saved decision is applied, nothing is re-asked.
let again = runner
.guarded_pr_message(&winner, None, true)
.await
.unwrap()
.unwrap();
assert_eq!(again.title, message.title);
assert_eq!(
store
.list()
.into_iter()
.filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
.count(),
2
);
}
#[test]
fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
// What magi itself writes stays English under any configured language,
// so a future localisation of these headings fails here. (The agents'
// own text is also checked by the posting gate.)
let mut state = state_with_summary(
"add retries",
"TITLE: fix(web): batch reads\n- reads run.json once",
);
state.config.graph.language = "ja".to_owned();
let m = pr_message(&state, 'A');
assert!(crate::github_text::check(&m.title, &m.body).is_empty());
assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
// The task is the operator's own text: it goes in untouched, and the
// fallback title (no summary) may be in its language too.
let task = "今回やってほしいこと: results projector を直す";
let mut state = state_with_summary(task, "- no title line");
state.config.graph.language = "ja".to_owned();
let m = pr_message(&state, 'A');
assert_eq!(
m.title,
format!("chore: land candidate A of run {}", state.id)
);
assert!(
m.body.contains(&format!(
"<summary>Original task</summary>\n\n{task}\n\n</details>"
)),
"{}",
m.body
);
}
#[test]
fn pr_message_scrubs_home_paths_and_addresses() {
let state = state_with_summary(
"fix it in /Users/someone/src/x",
"TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
);
let m = pr_message(&state, 'A');
for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
assert!(!m.body.contains(leak), "{}", m.body);
}
assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
}
#[test]
fn pr_message_survives_a_task_that_closes_details() {
let state = state_with_summary("a </details> b", "TITLE: fix: x");
let m = pr_message(&state, 'A');
assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
}
#[test]
fn manual_squash_subject_cannot_break_out_of_its_quotes() {
let cmd = manual_merge_command(
MergeStyle::Squash,
Path::new("/repo"),
"b",
"fix: \"quoted\" $(x) `y`\n\nbody",
);
assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
}
#[test]
fn manual_merge_command_matches_the_configured_style() {
let repo = Path::new("/repo");
let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
assert_eq!(
squash,
"git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
\"Merge magi run 0832 (candidate A)\""
);
let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
}
#[test]
fn a_nudge_gets_a_quarter_of_the_budget() {
// The judge and implement budgets magi ships with.
assert_eq!(retry_budget(secs(1200), true), secs(300));
assert_eq!(retry_budget(secs(3600), true), secs(900));
}
#[test]
fn a_resent_prompt_keeps_the_whole_budget() {
// The seat kept no context, so the retry is the original job again and
// shortening it would only guarantee a second failure.
assert_eq!(retry_budget(secs(1200), false), secs(1200));
assert_eq!(retry_budget(secs(60), false), secs(60));
}
#[test]
fn the_floor_never_exceeds_the_original_budget() {
// A short configured timeout must not be *raised* by the floor: the
// operator asked for a bound, and a retry may not outlast the attempt
// it is retrying.
assert_eq!(retry_budget(secs(60), true), secs(60));
assert_eq!(retry_budget(secs(480), true), secs(120));
assert_eq!(retry_budget(secs(0), true), secs(0));
}
fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
agent::CommandEvidence {
id: "item1".to_owned(),
description: "cargo test".to_owned(),
exit_code,
result_summary: String::new(),
source: "codex".to_owned(),
}
}
#[test]
fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
// No evidence is not the same fact as unconfirmed evidence: a
// backend with no adapter, or a reply that ran no commands at all,
// must not be misread as carrying a dangling job.
assert!(!has_unconfirmed_command(&[]));
}
#[test]
fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
// Deliberately not a check on the exit code's *value*: a fixer
// legitimately runs something that fails mid-iteration before it
// succeeds, and that must never by itself reopen a valid report.
assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
assert!(!has_unconfirmed_command(&[
evidence(Some(0)),
evidence(Some(101))
]));
}
#[test]
fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
assert!(has_unconfirmed_command(&[
evidence(Some(0)),
evidence(None)
]));
}
#[test]
fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
assert_eq!(
verified_noop_claim(true, &[], text).as_deref(),
Some("already fixed by b32cfc4, on main.")
);
}
#[test]
fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
// A timeout or a bad exit code reads as the ordinary loss it is,
// whatever the reply's own prose claims.
let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
assert!(verified_noop_claim(false, &[], text).is_none());
}
#[test]
fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
// A confirmed command alongside the marker is fine.
assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
}
#[test]
fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
}
/// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
/// pair, in order, labelled A, B, C, ...
fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
runner.state.candidates = shape
.iter()
.enumerate()
.map(|(i, &(empty, verified))| Candidate {
index: i,
label: (b'A' + i as u8) as char,
agent: "sonnet".to_owned(),
branch: format!("magi/x/{}", (b'A' + i as u8) as char),
worktree: PathBuf::from(format!("/wt/{i}")),
summary: String::new(),
stat: String::new(),
files: 0,
commits: 0,
empty,
failed: None,
verified_noop: verified.map(str::to_owned),
duration_ms: 0,
folded: false,
})
.collect();
}
#[test]
fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
ask_test_home();
let mut runner = runner_at(RunStatus::Implementing);
set_candidates(
&mut runner,
&[
(true, Some("already on main at b32cfc4")),
(true, Some("same fix, see the existing test")),
],
);
runner
.after_implement()
.expect("a verified no-op is not an error");
assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
}
#[test]
fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
ask_test_home();
let mut runner = runner_at(RunStatus::Implementing);
// Candidate A declares a verified no-op; candidate B simply wrote
// nothing and said nothing about why. One candidate's claim is not
// the whole run's agreement.
set_candidates(
&mut runner,
&[(true, Some("already on main at b32cfc4")), (true, None)],
);
let err = runner
.after_implement()
.expect_err("an unverified empty candidate must still fail the run");
assert!(
err.to_string().contains("no candidate produced a change"),
"{err}"
);
assert_eq!(runner.state.status, RunStatus::Failed);
}
#[test]
fn after_implement_still_fails_an_ordinary_all_empty_run() {
ask_test_home();
let mut runner = runner_at(RunStatus::Implementing);
set_candidates(&mut runner, &[(true, None), (true, None)]);
let err = runner
.after_implement()
.expect_err("no candidate declared anything; this is an ordinary failure");
assert!(
err.to_string().contains("no candidate produced a change"),
"{err}"
);
assert_eq!(runner.state.status, RunStatus::Failed);
}
}
/// A graft rollback whose restore could not be confirmed (see
/// [`Runner::graft`]).
#[derive(Debug)]
struct GraftRestoreFailed(String);
impl std::fmt::Display for GraftRestoreFailed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"could not restore the winner to its pre-graft commit: {}",
self.0
)
}
}
impl std::error::Error for GraftRestoreFailed {}
/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
/// not the agent: they never draw on an agent's own timeout, and running out of
/// them never reads as an agent failure.
#[cfg(test)]
mod resource_wait_tests {
use super::*;
use std::collections::BTreeMap;
fn sh(id: &str, script: &str) -> AgentSpec {
AgentSpec {
id: id.to_owned(),
kind: crate::config::AgentKind::Command,
model: None,
command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
extra_args: Vec::new(),
env: BTreeMap::new(),
prompt_delivery: None,
}
}
fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
SeatJob {
seat: SeatState::new("impl-A", &spec.id, 1),
spec,
cwd: dir.to_path_buf(),
prompt: "do it".to_owned(),
timeout,
allow_write: true,
sessions: false,
artifacts: dir.join("art"),
stem: "impl-A".to_owned(),
handover: None,
}
}
fn state(resource_wait_secs: u64) -> RunState {
let mut config = Config::default();
config.graph.timeout_resource_wait = resource_wait_secs;
RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234".to_owned(),
"task".to_owned(),
config,
)
}
fn hold(cache: &Path) -> crate::cache::Guard {
let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
crate::cache::AcquireOutcome::Acquired(g) => g,
crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
}
}
async fn run(
jobs: Vec<SeatJob>,
sem: Arc<Semaphore>,
cache: Option<&Path>,
st: &mut RunState,
) -> Vec<(usize, SeatState, AgentOutcome)> {
let prompts = Prompts::default();
let ctx = WaveCtx {
run: "r",
node: "implement",
prompts: &prompts,
cache,
round: None,
carry_seats: false,
pauses: None,
};
wave(jobs, sem, &ctx, st, 0).await
}
#[tokio::test]
async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().join("target-a");
let guard = hold(&cache);
// Held for longer than the agent's whole 1s budget.
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(1500)).await;
guard.release();
});
let mut st = state(60);
let results = run(
vec![job(
sh("alpha", "echo ok"),
tmp.path(),
Duration::from_secs(1),
)],
Arc::new(Semaphore::new(1)),
Some(&cache),
&mut st,
)
.await;
assert!(
matches!(results[0].2, AgentOutcome::Ok(_)),
"the wait must not be charged to the agent"
);
assert!(st.resource_block.is_none());
assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
assert!(st.resource_wait.is_none() && st.active.is_empty());
}
#[tokio::test]
async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().join("target-b");
let _held = hold(&cache);
let mut st = state(1);
let results = run(
vec![job(
sh("alpha", "echo ok"),
tmp.path(),
Duration::from_secs(1),
)],
Arc::new(Semaphore::new(1)),
Some(&cache),
&mut st,
)
.await;
let (_, seat, out) = &results[0];
assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
assert!(wave_blocked(&results));
assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
assert_eq!(seat.turns, 0, "nothing was asked");
assert!(
!tmp.path().join("art").join("impl-A.prompt.md").exists(),
"no agent was spawned"
);
assert!(FailClass::of(out).is_none(), "not a failure class");
let block = st.resource_block.as_ref().expect("recorded for the resume");
assert_eq!(block.kind, "shared build cache");
assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
assert!(st.active.is_empty());
}
#[tokio::test]
async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().unwrap();
let mut st = state(1);
let results = run(
vec![job(
sh("alpha", "echo ok"),
tmp.path(),
Duration::from_secs(1),
)],
Arc::new(Semaphore::new(0)),
None,
&mut st,
)
.await;
assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
}
#[tokio::test]
async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
crate::run::pin_test_home();
let tmp = tempfile::tempdir().unwrap();
let mut st = state(60);
let results = run(
vec![job(
sh("alpha", "sleep 30"),
tmp.path(),
Duration::from_secs(1),
)],
Arc::new(Semaphore::new(1)),
None,
&mut st,
)
.await;
let out = &results[0].2;
assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
let class = FailClass::of(out).expect("a failure");
assert_eq!(class, FailClass::Timeout);
assert!(should_hand_over(None, &class), "existing handover policy");
assert!(st.resource_block.is_none());
}
}