1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::refs;
46use crate::run::{
47 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
48 DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, GateFixRecord, Handover, JobRecord,
49 JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest,
50 Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally,
51 VoteRecord, tail, write_artifact,
52};
53use crate::verdict::{
54 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
55 ReviewVote, Severity,
56};
57
58const OUTPUT_TAIL: usize = 8_000;
60
61const EVENT_OUTPUT_TAIL: usize = 2_000;
64
65const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
68
69const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
86
87pub(crate) const STAGNANT_LIMIT: usize = 2;
101
102const BASE_SYNC_ROUNDS: usize = 4;
115
116const MAX_FIX_CONTINUATIONS: usize = 2;
132
133#[derive(Clone)]
139struct SeatJob {
140 spec: AgentSpec,
141 seat: SeatState,
142 cwd: PathBuf,
143 prompt: String,
144 timeout: Duration,
145 allow_write: bool,
146 sessions: bool,
147 artifacts: PathBuf,
148 stem: String,
149}
150
151enum AgentOutcome {
163 Ok(AgentOutput),
165 Quota(AgentOutput),
167 Dropped(AgentOutput),
170 Failed(String),
172}
173
174#[derive(Debug, Clone, Default)]
201pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
202
203impl Pause {
204 #[must_use]
206 pub fn new() -> Self {
207 Self::default()
208 }
209
210 pub fn park(&self) {
212 self.0.store(true, Ordering::SeqCst);
213 }
214
215 pub fn park_because(&self, reason: impl Into<String>) {
221 let mut reason_guard = self
222 .1
223 .lock()
224 .unwrap_or_else(std::sync::PoisonError::into_inner);
225 if reason_guard.is_none() {
226 *reason_guard = Some(reason.into());
227 }
228 drop(reason_guard);
229 self.park();
230 }
231
232 #[must_use]
234 pub fn parked(&self) -> bool {
235 self.0.load(Ordering::SeqCst)
236 }
237
238 #[must_use]
240 pub fn reason(&self) -> Option<String> {
241 self.1
242 .lock()
243 .unwrap_or_else(std::sync::PoisonError::into_inner)
244 .clone()
245 }
246}
247
248pub struct Runner {
250 pub state: RunState,
252 roles: ResolvedRoles,
253 sem: Arc<Semaphore>,
254 pause: Pause,
258 interrupt: Pause,
264}
265
266async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
295 let tracking = format!("{remote}/{branch}");
296 let fetched = git::fetch(repo, remote, branch).await;
297 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
298 let local_exists = git::branch_exists(repo, branch).await?;
299 if !fresh {
300 if !local_exists {
301 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
302 }
303 tracing::warn!(
304 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
305 );
306 return Ok(());
307 }
308 let remote_sha = git::rev_parse(repo, &tracking).await?;
309 if !local_exists {
310 git::git(repo, &["branch", branch, &tracking]).await?;
311 return Ok(());
312 }
313 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
314 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
315 return Ok(());
316 }
317 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
318 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
324 .await?
325 {
326 crate::reconcile::Reconciliation::Pushed => {
327 tracing::warn!(
328 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
329 short(&local_sha),
330 short(&remote_sha)
331 );
332 return Ok(());
333 }
334 crate::reconcile::Reconciliation::Placeholder => {}
335 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
336 }
337 }
338 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
339 if !out.ok() {
340 bail!(
341 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
342 short(&local_sha),
343 short(&remote_sha),
344 out.stderr
345 );
346 }
347 tracing::warn!(
348 "local `{branch}` was stale: fast-forwarded {} -> {}",
349 short(&local_sha),
350 short(&remote_sha)
351 );
352 Ok(())
353}
354
355async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
356 let tracking = format!("{remote}/{base_branch}");
357 let fetched = git::fetch(repo, remote, base_branch).await;
358 if let Ok(out) = &fetched
359 && out.ok()
360 && git::rev_exists(repo, &tracking).await
361 {
362 return git::rev_parse(repo, &tracking).await;
363 }
364 let why = match &fetched {
365 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
366 Ok(_) => format!("{remote} has no {base_branch}"),
367 Err(e) => e.to_string(),
368 };
369 tracing::warn!(
370 "could not read {tracking} ({why}); branching off the local \
371 {base_branch} instead, which may be behind"
372 );
373 git::rev_parse(repo, base_branch).await.with_context(|| {
374 format!(
375 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
376 branch that exists"
377 )
378 })
379}
380
381struct FixClaim {
398 path: PathBuf,
399}
400
401impl FixClaim {
402 fn acquire(dir: &Path) -> Result<Self> {
403 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
404 let path = dir.join("fix.lock");
405 match Self::create(&path) {
406 Ok(claim) => Ok(claim),
407 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
408 if Self::reclaim_if_dead(&path) {
409 Self::create(&path).with_context(|| format!("lock {}", path.display()))
410 } else {
411 bail!(
412 "another `magi fix` is already running for this run ({} exists)",
413 path.display()
414 )
415 }
416 }
417 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
418 }
419 }
420
421 fn create(path: &Path) -> std::io::Result<Self> {
422 let mut f = std::fs::OpenOptions::new()
423 .write(true)
424 .create_new(true)
425 .open(path)?;
426 use std::io::Write as _;
427 writeln!(f, "{}", std::process::id())?;
429 Ok(Self {
430 path: path.to_owned(),
431 })
432 }
433
434 fn reclaim_if_dead(path: &Path) -> bool {
438 let dead = std::fs::read_to_string(path)
439 .ok()
440 .and_then(|body| body.trim().parse::<u32>().ok())
441 .is_some_and(|pid| !crate::proc::pid_alive(pid));
442 dead && std::fs::remove_file(path).is_ok()
443 }
444}
445
446impl Drop for FixClaim {
447 fn drop(&mut self) {
448 let _ = std::fs::remove_file(&self.path);
449 }
450}
451
452impl Runner {
453 pub async fn start(
455 repo: &Path,
456 instruction: String,
457 config: Config,
458 origin: Origin,
459 ) -> Result<Self> {
460 Self::start_naming(repo, instruction, "", config, origin).await
461 }
462
463 pub async fn start_naming(
467 repo: &Path,
468 instruction: String,
469 also_scan: &str,
470 config: Config,
471 origin: Origin,
472 ) -> Result<Self> {
473 let repo = git::toplevel(repo).await?;
474 let missing = agent::missing_programs(&config.agents);
475 if !missing.is_empty() {
476 bail!(
477 "these agent programs are not on PATH: {}. Fix the roster in \
478 magi.toml or install them.",
479 missing.join(", ")
480 );
481 }
482 let base_branch = match config.merge.base.clone() {
483 Some(b) => b,
484 None => git::current_branch(&repo)
485 .await?
486 .context("HEAD is detached; set [merge] base in magi.toml")?,
487 };
488 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
489 if !git::is_clean(&repo).await? {
493 tracing::warn!(
494 "{} has uncommitted changes; they are not part of this run, \
495 which branches off {base_branch} ({})",
496 repo.display(),
497 &base_commit[..base_commit.len().min(8)]
498 );
499 }
500 let roles = config.resolve_roles()?;
501 let max_parallel = config.graph.max_parallel.max(1);
502 let seeds = refs::resolve(
505 &repo,
506 &base_commit,
507 &config.merge.remote,
508 &format!("{also_scan}\n{instruction}"),
509 )
510 .await;
511 refs::plan(&repo, &seeds).await?;
512 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
513 state.origin = Some(origin);
517 for seed in &seeds {
518 state.event(
519 "seed",
520 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
521 );
522 }
523 state.seeds = seeds;
524 state.event("start", format!("run {} created", state.id));
525 state.save()?;
526 Ok(Self {
527 state,
528 roles,
529 sem: Arc::new(Semaphore::new(max_parallel)),
530 pause: Pause::new(),
531 interrupt: Pause::new(),
532 })
533 }
534
535 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
549 Self::review_taking_over(repo, branch, config, None, origin).await
550 }
551
552 pub async fn review_taking_over(
559 repo: &Path,
560 branch: &str,
561 config: Config,
562 takeover: Option<crate::handover::Takeover>,
563 origin: Origin,
564 ) -> Result<Self> {
565 let repo = git::toplevel(repo).await?;
566 let missing = agent::missing_programs(&config.agents);
567 if !missing.is_empty() {
568 bail!(
569 "these agent programs are not on PATH: {}. Fix the roster in \
570 magi.toml or install them.",
571 missing.join(", ")
572 );
573 }
574 let base_branch = match config.merge.base.clone() {
575 Some(b) => b,
576 None => git::current_branch(&repo)
577 .await?
578 .context("HEAD is detached; set [merge] base in magi.toml")?,
579 };
580 if base_branch == branch {
581 bail!("`{branch}` is the base branch; there is nothing to review against");
582 }
583 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
584
585 let roles = config.resolve_roles()?;
586 let max_parallel = config.graph.max_parallel.max(1);
587 let mut state = RunState::new(
588 repo.clone(),
589 base_branch,
590 base_commit.clone(),
591 String::new(),
592 config,
593 );
594 state.origin = Some(origin);
595
596 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
603 earlier: Vec::new(),
604 home: crate::run::home(),
605 choice: None,
606 });
607 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
608 if let Some(released) = &released {
609 state.event(
610 "release",
611 format!(
612 "took `{branch}` over from run {}: its worktree was released: {}",
613 crate::run::short_of(&released.old_id),
614 released.audit
615 ),
616 );
617 }
618 if let Some(choice) = takeover.choice.as_ref()
622 && let Err(e) =
623 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
624 .await
625 {
626 if let Some(released) = &released {
627 released.restore(&repo, branch).await;
628 }
629 return Err(e.context("applying the owner's answer about the diverged branch"));
630 }
631 let opened =
632 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
633 if opened.is_err()
634 && let Some(released) = &released
635 {
636 released.restore(&repo, branch).await;
637 }
638 opened
639 }
640
641 async fn open_review(
644 repo: &Path,
645 branch: &str,
646 mut state: RunState,
647 roles: ResolvedRoles,
648 max_parallel: usize,
649 base_commit: String,
650 ) -> Result<Self> {
651 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
652 let log = git::log_oneline(repo, &base_commit, branch)
655 .await
656 .unwrap_or_default();
657 let instruction = format!(
658 "Review the work already on branch `{branch}`. There is no task \
659 statement: what the change claims to do is whatever its commits \
660 say.\n\n{}",
661 if log.trim().is_empty() {
662 "(no commit messages)"
663 } else {
664 log.trim()
665 }
666 );
667 state.instruction = instruction;
668 state.reviewed_commits = Some(
669 git::subjects(repo, &base_commit, branch)
670 .await
671 .unwrap_or_default(),
672 );
673
674 let worktree = state.worktree_root().join("under-review");
677 if let Some(parent) = worktree.parent() {
678 tokio::fs::create_dir_all(parent).await.ok();
679 }
680 let path = worktree.to_string_lossy().to_string();
681 git::git(repo, &["worktree", "add", &path, branch])
682 .await
683 .with_context(|| {
684 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
685 })?;
686
687 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
688 .await
689 .unwrap_or(0);
690 if commits == 0 {
691 git::worktree_remove(repo, &worktree).await.ok();
692 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
693 }
694 let files = git::changed_files(&worktree, &base_commit, "HEAD")
695 .await
696 .map(|f| f.len())
697 .unwrap_or(0);
698 if files == 0
699 && let (Ok(head_tree), Ok(base_tree)) = (
700 git::tree_of(&worktree, "HEAD").await,
701 git::tree_of(&worktree, &base_commit).await,
702 )
703 && head_tree == base_tree
704 {
705 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
706 git::worktree_remove(repo, &worktree).await.ok();
707 bail!(
708 "`{branch}` at {} has a tree identical to base {}; this usually means \
709 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
710 against `{}/{branch}`) rather than an empty change",
711 short(&head),
712 short(&base_commit),
713 state.config.merge.remote
714 );
715 }
716 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
717 .await
718 .unwrap_or_default();
719
720 state.candidates.push(Candidate {
721 index: 0,
722 label: 'A',
723 agent: EXISTING_BRANCH.to_owned(),
726 branch: branch.to_owned(),
727 worktree,
728 summary: String::new(),
729 stat,
730 files,
731 commits,
732 empty: false,
733 failed: None,
734 verified_noop: None,
735 duration_ms: 0,
736 folded: false,
737 });
738 state.tally = Some(Tally {
739 first_choice: BTreeMap::from([('A', 0)]),
740 borda: BTreeMap::new(),
741 winner: 'A',
742 rankings: 0,
743 unanimous_initial: false,
744 deliberated: false,
745 changed_votes: 0,
746 unanimous_final: false,
747 tie_break: None,
748 judges: 0,
752 present: 0,
753 quorum: 0,
754 met_quorum: true,
755 uncontested: Some("review-only run: nothing competed".to_owned()),
756 });
757 state.status = RunStatus::Reviewing;
758 state.event(
759 "start",
760 format!(
761 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
762 state.id
763 ),
764 );
765 state.save()?;
766 Ok(Self {
767 state,
768 roles,
769 sem: Arc::new(Semaphore::new(max_parallel)),
770 pause: Pause::new(),
771 interrupt: Pause::new(),
772 })
773 }
774
775 pub fn resume(id: &str) -> Result<Self> {
777 let state = RunState::load(id)?;
778 if let Some(to) = &state.released_to {
779 bail!(
780 "run {} cannot be resumed: its worktree was released to run {}",
781 state.short(),
782 crate::run::short_of(to)
783 );
784 }
785 let roles = state.config.resolve_roles()?;
786 let max_parallel = state.config.graph.max_parallel.max(1);
787 Ok(Self {
788 state,
789 roles,
790 sem: Arc::new(Semaphore::new(max_parallel)),
791 pause: Pause::new(),
792 interrupt: Pause::new(),
793 })
794 }
795
796 pub async fn execute(&mut self) -> Result<()> {
803 let result = self.execute_graph().await;
804 self.mark_driver_exited();
805 let ended = if result.is_err() {
806 Some(crate::notices::run_stopped(&self.state.id, &self.state))
807 } else {
808 crate::notices::run_ended(&self.state)
809 };
810 if let Some(notice) = ended {
811 crate::notices::raise(notice);
812 }
813 result
814 }
815
816 fn mark_driver_exited(&mut self) {
825 self.state.driver_exited = true;
826 let pid = std::process::id();
827 let Ok(mut disk) = RunState::load(&self.state.id) else {
828 return;
829 };
830 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
831 return;
832 }
833 disk.driver_exited = true;
834 if let Err(e) = disk.save() {
835 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
836 }
837 }
838
839 async fn execute_graph(&mut self) -> Result<()> {
840 self.state.parked = false;
845 self.state.clear_active();
852 if let Ok(disk) = RunState::load(&self.state.id)
870 && let Some(to) = &disk.released_to
871 {
872 bail!(
873 "run {} cannot continue: its worktree was released to run {}",
874 self.state.short(),
875 crate::run::short_of(to)
876 );
877 }
878 let pid = std::process::id();
879 self.state.driver_pid = Some(pid);
880 self.state.driver_started_at = crate::proc::process_started_at(pid);
881 self.state.driver_exited = false;
882 self.state.save()?;
883 if self.state.status == RunStatus::Stalled {
896 if self.recover_stall().await? {
897 self.finish_after_tally().await?;
898 } else {
899 self.state.save()?;
901 }
902 return Ok(());
903 }
904 if self.state.status == RunStatus::Landing {
914 self.run_land().await?;
915 self.settle_questions();
920 return Ok(());
921 }
922 self.prep().await?;
923 if self.park_here()? {
924 return Ok(());
925 }
926 self.advise().await?;
927 if self.park_here()? {
928 return Ok(());
929 }
930 self.implement().await?;
931 if self.park_here()? {
932 return Ok(());
933 }
934 if self.state.status == RunStatus::VerifiedNoop {
938 return Ok(());
939 }
940 self.judge().await?;
941 if self.park_here()? {
942 return Ok(());
943 }
944 self.deliberate().await?;
945 if self.park_here()? {
946 return Ok(());
947 }
948 self.vote().await?;
949 if self.park_here()? {
950 return Ok(());
951 }
952 self.tally()?;
953 if self.state.status == RunStatus::Stalled {
958 self.state.save()?;
962 return Ok(());
963 }
964 self.finish_after_tally().await?;
965 Ok(())
966 }
967
968 fn park_here(&mut self) -> Result<bool> {
975 if !self.pause.parked() && !self.interrupt.parked() {
980 return Ok(false);
981 }
982 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
983 Some(reason) => format!(
984 "parked after `{}` ({reason}) — resume to carry on from here",
985 self.state.status.as_str()
986 ),
987 None => format!(
988 "parked after `{}` — resume to carry on from here",
989 self.state.status.as_str()
990 ),
991 };
992 self.state.event("park", why);
993 self.state.parked = true;
994 self.state.save()?;
995 Ok(true)
996 }
997
998 pub fn on_pause(&mut self, pause: Pause) {
1000 self.pause = pause;
1001 }
1002
1003 pub fn watch_interrupt(&mut self, pause: Pause) {
1009 self.interrupt = pause;
1010 }
1011
1012 fn settle_questions(&mut self) {
1032 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1033 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1034 }
1035 }
1036
1037 async fn finish_after_tally(&mut self) -> Result<()> {
1040 self.fold_losers().await?;
1041 self.sync_to_base().await?;
1046 if self.state.status == RunStatus::AlreadyInBase {
1047 return Ok(());
1048 }
1049 self.review_loop().await?;
1050 self.sync_to_base().await?;
1051 if self.state.status == RunStatus::AlreadyInBase {
1052 return Ok(());
1053 }
1054 self.gate().await?;
1055 self.merge().await?;
1056 self.state.save()?;
1057 Ok(())
1058 }
1059
1060 async fn prep(&mut self) -> Result<()> {
1063 if !self.state.candidates.is_empty() {
1064 return Ok(());
1065 }
1066 self.state.status = RunStatus::Prep;
1067 let repo = self.state.repo.clone();
1068 let base = self.state.base_commit.clone();
1069 let plan = refs::plan(&repo, &self.state.seeds).await?;
1070 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1071 let root = self.state.worktree_root();
1072 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1073
1074 let hooks_dir = self.state.dir().join("hooks");
1077 if self.state.config.blind.commit_msg_hook {
1078 std::fs::create_dir_all(&hooks_dir)
1079 .with_context(|| format!("create {}", hooks_dir.display()))?;
1080 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1081 let path = hooks_dir.join("commit-msg");
1082 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1083 make_executable(&path)?;
1084 git::acquire_worktree_config(&repo).await?;
1092 self.state.enabled_worktree_config = true;
1093 }
1094
1095 for (index, (spec, label)) in self
1096 .roles
1097 .implementers
1098 .clone()
1099 .into_iter()
1100 .zip(labels)
1101 .enumerate()
1102 {
1103 let branch = self.state.branch_for(label);
1104 let worktree = root.join(format!("cand-{label}"));
1105 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1106 if self.state.config.blind.commit_msg_hook {
1107 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1108 }
1109 git::local_exclude(&worktree, "/.magi/").await?;
1110 for pick in &plan.picks {
1111 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1112 self.state.status = RunStatus::Blocked;
1113 self.state
1114 .event("prep", format!("cannot apply referenced commit: {e}"));
1115 self.state.save()?;
1116 return Err(e);
1117 }
1118 }
1119 self.state.candidates.push(Candidate {
1120 index,
1121 label,
1122 agent: spec.id.clone(),
1123 branch,
1124 worktree,
1125 summary: String::new(),
1126 stat: String::new(),
1127 files: 0,
1128 commits: 0,
1129 empty: false,
1130 failed: None,
1131 verified_noop: None,
1132 duration_ms: 0,
1133 folded: false,
1134 });
1135 }
1136
1137 for j in 1..=self.roles.judges.len() {
1138 let wt = root.join(format!("judge-{j}"));
1139 if !wt.exists() {
1140 git::worktree_add_detached(&repo, &wt, &base).await?;
1141 }
1142 }
1143
1144 if self.state.config.graph.advise {
1152 for k in 1..=self.state.config.graph.advisors {
1153 let wt = root.join(format!("advisor-{k}"));
1154 if !wt.exists() {
1155 git::worktree_add_detached(&repo, &wt, &base).await?;
1156 }
1157 }
1158 }
1159
1160 let authors: Vec<&str> = self
1165 .roles
1166 .implementers
1167 .iter()
1168 .map(|a| a.id.as_str())
1169 .collect();
1170 let overlap: Vec<String> = self
1171 .roles
1172 .judges
1173 .iter()
1174 .enumerate()
1175 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1176 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1177 .collect();
1178 if !overlap.is_empty() {
1179 let note = format!(
1180 "{} also authored a candidate; blind, but the panel is less \
1181 independent than {} distinct agents would be",
1182 overlap.join(", "),
1183 self.roles.judges.len()
1184 );
1185 self.state.event("prep", note);
1186 }
1187
1188 self.state.event(
1189 "prep",
1190 format!(
1191 "{} candidates, {} judges, base {} ({})",
1192 self.state.candidates.len(),
1193 self.roles.judges.len(),
1194 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1195 self.state.base_branch
1196 ),
1197 );
1198 self.state.status = RunStatus::Implementing;
1199 self.state.save()?;
1200 Ok(())
1201 }
1202
1203 async fn advise(&mut self) -> Result<()> {
1236 let implement_untouched = self
1237 .state
1238 .candidates
1239 .iter()
1240 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1241 if !self.state.config.graph.advise || self.state.advise_attempted {
1242 return Ok(());
1243 }
1244 if !implement_untouched {
1245 self.state.event(
1246 "advise",
1247 "skipping the design-deliberation stage: at least one \
1248 candidate already shows implementation progress, so this \
1249 run is past the point the stage exists to run before"
1250 .to_owned(),
1251 );
1252 self.state.advise_attempted = true;
1253 self.state.save()?;
1254 return Ok(());
1255 }
1256 let run_id = self.state.id.clone();
1257 let prompts = self.state.config.prompts.clone();
1258 let instruction = self.state.instruction.clone();
1259 let language = self.state.config.graph.language.clone();
1260 let root = self.state.worktree_root();
1261 let n = self.state.config.graph.advisors;
1262 let where_recorded = self.state.dir().join("run.json");
1263
1264 let seats = match self.state.config.advisors() {
1265 Ok(seats) if !seats.is_empty() => seats,
1266 Ok(_) => {
1267 self.state.event(
1268 "advise",
1269 format!(
1270 "[graph] advisors is 0; skipping the design-deliberation \
1271 stage and continuing without a synthesis brief (see {})",
1272 where_recorded.display()
1273 ),
1274 );
1275 self.state.advise_attempted = true;
1276 self.state.save()?;
1277 return Ok(());
1278 }
1279 Err(e) => {
1280 self.state.event(
1281 "advise",
1282 format!(
1283 "could not resolve advisor seats ({e:#}); continuing \
1284 without a design-deliberation brief (see {})",
1285 where_recorded.display()
1286 ),
1287 );
1288 self.state.advise_attempted = true;
1289 self.state.save()?;
1290 return Ok(());
1291 }
1292 };
1293
1294 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1295 let artifacts = agent::artifacts_dir(&self.state.dir());
1296 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1297
1298 let mut jobs = Vec::new();
1299 for (i, spec) in seats.iter().cloned().enumerate() {
1300 let seat_key = format!("advisor-{}", i + 1);
1301 let seat = self.seat(&seat_key, &spec.id);
1302 jobs.push(SeatJob {
1303 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1304 spec,
1305 seat,
1306 cwd: worktrees[i % worktrees.len()].clone(),
1307 timeout,
1308 allow_write: false,
1309 sessions: false,
1310 artifacts: artifacts.clone(),
1311 stem: seat_key,
1312 });
1313 }
1314
1315 self.state.event(
1316 "advise",
1317 format!(
1318 "{} advisor seat(s) sketching a design in parallel",
1319 jobs.len()
1320 ),
1321 );
1322 let mut quota_losses = Vec::new();
1323 let cache = self.state.config.cache_dir();
1324 let ctx = WaveCtx {
1325 run: &run_id,
1326 node: "advise",
1327 prompts: &prompts,
1328 cache: cache.as_deref(),
1329 round: None,
1330 };
1331 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1332 let results = ask_json_wave::<Proposal>(
1333 jobs,
1334 Arc::clone(&self.sem),
1335 self.state.config.graph.retries,
1336 &advisor_roster,
1337 &ctx,
1338 &mut quota_losses,
1339 &mut self.state,
1340 &|p: &Proposal| p.validate(),
1341 )
1342 .await;
1343 self.state.quota.extend(quota_losses);
1344
1345 let mut records = Vec::with_capacity(results.len());
1346 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1347 let agent_id = seat.agent.clone();
1348 self.state.seats.insert(seat.key.clone(), seat);
1349 match res {
1350 Ok((proposal, out)) => {
1351 self.state
1352 .event("advise", format!("advisor-{} proposed a design", i + 1));
1353 records.push(advise::AdvisorRecord::proposed(
1354 i + 1,
1355 agent_id,
1356 proposal,
1357 out.duration_ms,
1358 ));
1359 }
1360 Err(e) => {
1361 self.state.event(
1362 "advise",
1363 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1364 );
1365 records.push(advise::AdvisorRecord::failed(
1366 i + 1,
1367 agent_id,
1368 e.to_string(),
1369 ));
1370 }
1371 }
1372 }
1373
1374 let mut advice = advise::Advice {
1375 records,
1376 synthesis: None,
1377 };
1378 if advice.proposals().is_empty() {
1379 self.state.event(
1380 "advise",
1381 "no advisor produced a usable proposal; continuing without a \
1382 synthesis brief"
1383 .to_owned(),
1384 );
1385 } else {
1386 match self
1387 .synthesize_brief(
1388 &advice,
1389 &instruction,
1390 &language,
1391 &worktrees[0],
1392 &artifacts,
1393 &run_id,
1394 &prompts,
1395 cache.as_deref(),
1396 )
1397 .await
1398 {
1399 Ok(Some(text)) => {
1400 self.state.event(
1401 "advise",
1402 "synthesized a design brief for the implementer".to_owned(),
1403 );
1404 advice.synthesis = Some(text);
1405 }
1406 Ok(None) => {
1407 self.state.event(
1408 "advise",
1409 "the synthesis seat produced nothing usable; continuing \
1410 without a design brief"
1411 .to_owned(),
1412 );
1413 }
1414 Err(e) => {
1415 self.state.event(
1416 "advise",
1417 format!("could not synthesize a design brief: {e:#}"),
1418 );
1419 }
1420 }
1421 }
1422 advise::apply_reflection(&mut advice);
1423
1424 self.state.advice = Some(advice);
1425 self.state.advise_attempted = true;
1426 self.state.save()?;
1427 Ok(())
1428 }
1429
1430 #[allow(clippy::too_many_arguments)]
1442 async fn synthesize_brief(
1443 &mut self,
1444 advice: &advise::Advice,
1445 instruction: &str,
1446 language: &str,
1447 cwd: &Path,
1448 artifacts: &Path,
1449 run_id: &str,
1450 prompts: &Prompts,
1451 cache: Option<&Path>,
1452 ) -> Result<Option<String>> {
1453 let chain = agent::pick_chain(
1454 &self.state.config.agents,
1455 self.state.config.roles.synthesizer.as_ref(),
1456 &agent::installed,
1457 "synthesizer",
1458 )?;
1459 let proposals = advice.proposals();
1460 let mut prompt = prompt::with_overlay(
1461 prompt::synthesize_brief(instruction, &proposals, language),
1462 prompts.overlay("advise"),
1463 );
1464 if cache.is_some() {
1465 prompt.push('\n');
1470 prompt.push_str(&prompt::build_cache_note("advise", false));
1471 }
1472 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1473 let mut last = None;
1478 for (n, spec) in chain.iter().enumerate() {
1479 if n > 0 {
1480 self.state
1481 .event("advise", format!("synthesis falling back to {}", spec.id));
1482 }
1483 let mut seat = self.seat("advise-synthesis", &spec.id);
1484 let outcome = agent::invoke(
1485 spec,
1486 &mut seat,
1487 &Invocation {
1488 cwd,
1489 prompt: &prompt,
1490 timeout,
1491 allow_write: false,
1492 sessions: false,
1493 artifacts,
1494 stem: &if n == 0 {
1495 "advise-synthesis".to_owned()
1496 } else {
1497 format!("advise-synthesis-{}", spec.id)
1498 },
1499 run: run_id,
1500 node: "advise",
1501 cache_dir: None,
1502 attachments: &[],
1503 writable: &[],
1504 },
1505 )
1506 .await;
1507 if outcome.is_ok() {
1508 self.state.seats.insert(seat.key.clone(), seat);
1509 }
1510 let advance = agent::chain_advances(&outcome);
1511 last = Some(outcome);
1512 if !advance {
1513 break;
1514 }
1515 }
1516 let out = last.expect("a chain holds at least one agent")?;
1519 if !out.usable() {
1520 return Ok(None);
1521 }
1522 let text =
1523 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1524 Ok((!text.trim().is_empty()).then_some(text))
1525 }
1526
1527 async fn implement(&mut self) -> Result<()> {
1530 let run_id = self.state.id.clone();
1535 let prompts = self.state.config.prompts.clone();
1536 let todo: Vec<usize> = self
1537 .state
1538 .candidates
1539 .iter()
1540 .enumerate()
1541 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1542 .map(|(i, _)| i)
1543 .collect();
1544 if todo.is_empty() {
1545 return self.after_implement();
1546 }
1547 self.state.status = RunStatus::Implementing;
1548
1549 let language = self.state.config.graph.language.clone();
1550 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1551 let sessions = self.state.config.graph.sessions;
1552 let artifacts = agent::artifacts_dir(&self.state.dir());
1553 let brief = self
1557 .state
1558 .advice
1559 .as_ref()
1560 .and_then(|a| a.synthesis.as_deref())
1561 .map(str::to_owned);
1562 let attachments = self.state.attachments.clone();
1563
1564 let mut jobs = Vec::new();
1565 for &i in &todo {
1566 let (index, label, worktree) = {
1567 let c = &self.state.candidates[i];
1568 (c.index, c.label, c.worktree.clone())
1569 };
1570 let spec = self.roles.implementers[index].clone();
1571 let seat_key = format!("impl-{label}");
1572 let seat = self.seat(&seat_key, &spec.id);
1573 let instruction = seeded_instruction(&self.state);
1574 jobs.push(SeatJob {
1575 spec,
1576 seat,
1577 prompt: prompt::implement(
1578 &instruction,
1579 &worktree.to_string_lossy(),
1580 &language,
1581 brief.as_deref(),
1582 &attachments,
1583 ),
1584 cwd: worktree,
1585 timeout,
1586 allow_write: true,
1587 sessions,
1588 artifacts: artifacts.clone(),
1589 stem: format!("impl-{label}"),
1590 });
1591 }
1592
1593 self.state.event(
1594 "implement",
1595 format!("{} candidates in parallel", jobs.len()),
1596 );
1597 let mut sent = jobs.clone();
1603 let cache = self.state.config.cache_dir();
1604 let ctx = WaveCtx {
1605 run: &run_id,
1606 node: "implement",
1607 prompts: &prompts,
1608 cache: cache.as_deref(),
1609 round: None,
1610 };
1611 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1612 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1613 .await;
1614 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1615 .await;
1616 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1617 .await;
1618
1619 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1620 let seat_key = seat.key.clone();
1621 let agent = seat.agent.clone();
1630 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1631 self.state.seats.insert(seat.key.clone(), seat);
1632 let label = self.state.candidates[i].label;
1633 let worktree = self.state.candidates[i].worktree.clone();
1634 let base = self.state.base_commit.clone();
1635
1636 let (summary, duration, failed, verified_claim) = match out {
1637 AgentOutcome::Ok(o) => {
1638 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1639 let failed = (!o.usable()).then(|| {
1640 if o.timed_out {
1641 "agent timed out".to_owned()
1642 } else {
1643 format!("agent exited with {:?}", o.exit_code)
1644 }
1645 });
1646 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1647 (text, o.duration_ms, failed, verified_claim)
1648 }
1649 AgentOutcome::Dropped(o) => {
1655 let why = o
1656 .dropped
1657 .as_ref()
1658 .map(|d| d.why.as_str())
1659 .unwrap_or("the CLI ended the stream without delivering its answer");
1660 (
1661 String::new(),
1662 o.duration_ms,
1663 Some(format!("the CLI dropped the stream ({why})")),
1664 None,
1665 )
1666 }
1667 AgentOutcome::Quota(o) => {
1668 self.state.quota.push(QuotaLoss {
1669 seat: seat_key,
1670 node: "implement".to_owned(),
1671 at: Timestamp::now(),
1672 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1673 });
1674 (
1675 String::new(),
1676 o.duration_ms,
1677 Some("rate limited (quota); produced no change".to_owned()),
1678 None,
1679 )
1680 }
1681 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1682 };
1683
1684 let rescued = match git::rescue_commit(
1687 &worktree,
1688 &format!("magi: candidate {label} (uncommitted work)"),
1689 )
1690 .await
1691 {
1692 Ok(r) => {
1693 self.state.note_withheld("implement", &r.withheld);
1694 r.committed
1695 }
1696 Err(_) => false,
1697 };
1698 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1699 .await
1700 .unwrap_or(0);
1701 let patch = git::diff(&worktree, &base, "HEAD")
1702 .await
1703 .unwrap_or_default();
1704 let stat = git::diff_stat(&worktree, &base, "HEAD")
1705 .await
1706 .unwrap_or_default();
1707 let files = git::changed_files(&worktree, &base, "HEAD")
1708 .await
1709 .map(|f| f.len())
1710 .unwrap_or(0);
1711 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1712
1713 let c = &mut self.state.candidates[i];
1714 if !exhausted_the_fallback_chain {
1715 c.agent = agent;
1716 }
1717 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1718 c.stat = stat;
1719 c.files = files;
1720 c.commits = commits;
1721 c.duration_ms = duration;
1722 c.empty = commits == 0 || patch.trim().is_empty();
1723 c.failed = match failed {
1726 Some(_) if c.empty => failed,
1727 _ => None,
1728 };
1729 c.verified_noop = if c.empty { verified_claim } else { None };
1734 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1735 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1736 (None, true, Some(_), _) => {
1737 format!("candidate {label}: no change produced (agent-verified no-op)")
1738 }
1739 (None, true, None, _) => format!("candidate {label}: no change produced"),
1740 (None, false, _, true) => {
1741 format!(
1742 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1743 )
1744 }
1745 (None, false, _, false) => {
1746 format!("candidate {label}: {files} files, {commits} commits")
1747 }
1748 };
1749 self.state.event("implement", note);
1750 self.state.save()?;
1751 }
1752
1753 self.after_implement()
1754 }
1755
1756 async fn resume_undelivered(
1784 &mut self,
1785 results: &mut [(usize, SeatState, AgentOutcome)],
1786 sent: &[SeatJob],
1787 prompts: &Prompts,
1788 run_id: &str,
1789 ) {
1790 for (wi, seat, out) in results.iter_mut() {
1791 let Some(dropped) = (match &*out {
1792 AgentOutcome::Dropped(o) => o.dropped.clone(),
1793 _ => None,
1794 }) else {
1795 continue;
1796 };
1797 let Some(job) = sent.get(*wi) else { continue };
1798 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1800 self.state.event(
1801 "implement",
1802 format!(
1803 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1804 work is in the tree",
1805 seat.key, dropped.output_tokens, dropped.why
1806 ),
1807 );
1808 continue;
1809 }
1810 if !has_context(&job.spec, seat, job.sessions) {
1818 self.state.event(
1819 "implement",
1820 format!(
1821 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1822 is no session left to resume",
1823 seat.key, dropped.output_tokens, dropped.why
1824 ),
1825 );
1826 continue;
1827 }
1828 self.state.event(
1829 "implement",
1830 format!(
1831 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1832 conversation",
1833 seat.key, dropped.output_tokens, dropped.why
1834 ),
1835 );
1836 let mut retry = job.clone();
1837 retry.seat = seat.clone();
1838 retry.prompt = prompt::resume_after_drop(&dropped.why);
1839 retry.timeout = retry_budget(job.timeout, true);
1840 retry.stem = format!("{}-resume", job.stem);
1841 let cache = self.state.config.cache_dir();
1842 let ctx = WaveCtx {
1843 run: run_id,
1844 node: "implement",
1845 prompts,
1846 cache: cache.as_deref(),
1847 round: None,
1848 };
1849 let (resumed_seat, resumed) =
1850 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1851 *seat = resumed_seat;
1852 *out = resumed;
1853 }
1854 }
1855
1856 async fn resume_seat_handovers(
1921 &mut self,
1922 results: &mut [(usize, SeatState, AgentOutcome)],
1923 sent: &mut [SeatJob],
1924 prompts: &Prompts,
1925 run_id: &str,
1926 ) {
1927 let instruction = seeded_instruction(&self.state);
1928 let language = self.state.config.graph.language.clone();
1929 let brief = self
1930 .state
1931 .advice
1932 .as_ref()
1933 .and_then(|a| a.synthesis.as_deref())
1934 .map(str::to_owned);
1935 let attachments = self.state.attachments.clone();
1936 for (wi, seat, out) in results.iter_mut() {
1937 let Some(job) = sent.get_mut(*wi) else {
1938 continue;
1939 };
1940 let start = self
1945 .roles
1946 .implementer_roster
1947 .iter()
1948 .position(|s| s.id == job.spec.id)
1949 .unwrap_or(0);
1950 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1951 let mut fallback_attempt = 0usize;
1952 let mut prev: Option<FailClass> = None;
1953 while let Some(cur) = FailClass::of(&*out) {
1954 if !should_hand_over(prev.as_ref(), &cur) {
1955 break;
1956 }
1957 let Some(next) =
1958 next_untried_in_roster(&self.roles.implementer_roster, start, &tried).cloned()
1959 else {
1960 break;
1961 };
1962 tried.insert(next.id.clone());
1963 fallback_attempt += 1;
1964
1965 if let Ok(r) = git::rescue_commit(
1966 &job.cwd,
1967 &format!(
1968 "magi: candidate {} (uncommitted work before {} fallback)",
1969 seat.key,
1970 if cur == FailClass::Quota {
1971 "quota"
1972 } else {
1973 "handover"
1974 }
1975 ),
1976 )
1977 .await
1978 {
1979 self.state.note_withheld("implement", &r.withheld);
1980 }
1981
1982 record_handover(
1983 &mut self.state,
1984 "implement",
1985 &seat.key,
1986 &seat.agent,
1987 &next.id,
1988 &cur,
1989 &fail_reason(&*out),
1990 );
1991 prev = Some(cur.clone());
1992
1993 let new_seat = handover_seat(&seat.key, &next.id, self.state.seed);
1994 self.state.seats.insert(seat.key.clone(), new_seat.clone());
1995 job.spec = next.clone();
2003 let mut retry = job.clone();
2004 retry.seat = new_seat;
2005 retry.prompt = prompt::implement(
2006 &instruction,
2007 &job.cwd.to_string_lossy(),
2008 &language,
2009 brief.as_deref(),
2010 &attachments,
2011 );
2012 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2013 let cache = self.state.config.cache_dir();
2014 let ctx = WaveCtx {
2015 run: run_id,
2016 node: "implement",
2017 prompts,
2018 cache: cache.as_deref(),
2019 round: None,
2020 };
2021 let (fallback_seat, fallback_out) = run_one(
2022 retry,
2023 Arc::clone(&self.sem),
2024 &ctx,
2025 &mut self.state,
2026 fallback_attempt,
2027 )
2028 .await;
2029 *seat = fallback_seat;
2030 *out = fallback_out;
2031 }
2032 }
2033 }
2034
2035 async fn resume_unconfirmed_commands(
2059 &mut self,
2060 results: &mut [(usize, SeatState, AgentOutcome)],
2061 sent: &[SeatJob],
2062 prompts: &Prompts,
2063 run_id: &str,
2064 ) {
2065 for (wi, seat, out) in results.iter_mut() {
2066 let AgentOutcome::Ok(o) = &*out else {
2067 continue;
2068 };
2069 if !has_unconfirmed_command(&o.commands) {
2070 continue;
2071 }
2072 let Some(job) = sent.get(*wi) else { continue };
2073 if !has_context(&job.spec, seat, job.sessions) {
2074 self.state.event(
2075 "implement",
2076 format!(
2077 "{}: the reply named a command whose own CLI never confirmed the exit \
2078 status of, but there is no session left to resume",
2079 seat.key
2080 ),
2081 );
2082 continue;
2083 }
2084 self.state.event(
2085 "implement",
2086 format!(
2087 "{}: the reply named a command whose own CLI never confirmed the exit \
2088 status of; resuming the conversation",
2089 seat.key
2090 ),
2091 );
2092 let mut retry = job.clone();
2093 retry.seat = seat.clone();
2094 retry.prompt = prompt::resume_incomplete(
2095 "a command in your last reply had no confirmed exit status",
2096 );
2097 retry.timeout = retry_budget(job.timeout, true);
2098 retry.stem = format!("{}-confirm", job.stem);
2099 let cache = self.state.config.cache_dir();
2100 let ctx = WaveCtx {
2101 run: run_id,
2102 node: "implement",
2103 prompts,
2104 cache: cache.as_deref(),
2105 round: None,
2106 };
2107 let (resumed_seat, resumed) =
2108 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2109 *seat = resumed_seat;
2110 *out = resumed;
2111 }
2112 }
2113
2114 async fn continue_fix_report(
2135 &mut self,
2136 mut seat: SeatState,
2137 parse_err: String,
2138 job: &SeatJob,
2139 prompts: &Prompts,
2140 run_id: &str,
2141 round: usize,
2142 ) -> (
2143 SeatState,
2144 Option<FixReport>,
2145 Option<String>,
2146 ContinuationRecord,
2147 ) {
2148 let mut last_err = parse_err;
2149 let mut cumulative_wait_ms = 0u64;
2150 let mut attempts = 0usize;
2151 loop {
2152 if !has_context(&job.spec, &seat, job.sessions) {
2153 self.state.event(
2154 "fix",
2155 format!(
2156 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2157 session left to resume into"
2158 ),
2159 );
2160 let outcome = if attempts == 0 {
2161 ContinuationOutcome::NoSession
2162 } else {
2163 ContinuationOutcome::Exhausted
2164 };
2165 return (
2166 seat,
2167 None,
2168 Some(format!("unparsable fix report: {last_err}")),
2169 ContinuationRecord {
2170 attempts,
2171 cumulative_wait_ms,
2172 outcome,
2173 },
2174 );
2175 }
2176 if attempts >= MAX_FIX_CONTINUATIONS {
2177 self.state.event(
2178 "fix",
2179 format!(
2180 "round {round}: fixer's reply still had no adoption report after \
2181 {attempts} continuation(s) ({last_err}); giving up"
2182 ),
2183 );
2184 return (
2185 seat,
2186 None,
2187 Some(format!(
2188 "unparsable fix report after {attempts} continuation(s): {last_err}"
2189 )),
2190 ContinuationRecord {
2191 attempts,
2192 cumulative_wait_ms,
2193 outcome: ContinuationOutcome::Exhausted,
2194 },
2195 );
2196 }
2197 attempts += 1;
2198 self.state.event(
2199 "fix",
2200 format!(
2201 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2202 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2203 ),
2204 );
2205 let mut retry = job.clone();
2206 retry.seat = seat.clone();
2207 retry.prompt = prompt::resume_incomplete(&last_err);
2208 retry.timeout = retry_budget(job.timeout, true);
2209 retry.stem = format!("{}-continue{attempts}", job.stem);
2210 let cache = self.state.config.cache_dir();
2211 let ctx = WaveCtx {
2212 run: run_id,
2213 node: "fix",
2214 prompts,
2215 cache: cache.as_deref(),
2216 round: Some(round),
2217 };
2218 let (resumed_seat, resumed_out) = run_one(
2219 retry,
2220 Arc::clone(&self.sem),
2221 &ctx,
2222 &mut self.state,
2223 attempts,
2224 )
2225 .await;
2226 seat = resumed_seat;
2227 match resumed_out {
2228 AgentOutcome::Ok(o) => {
2229 cumulative_wait_ms += o.duration_ms;
2230 match verdict::extract_json::<FixReport>(&o.text) {
2231 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2232 self.state.event(
2233 "fix",
2234 format!(
2235 "round {round}: fixer's adoption report recovered after \
2236 {attempts} continuation(s)"
2237 ),
2238 );
2239 return (
2240 seat,
2241 Some(report),
2242 None,
2243 ContinuationRecord {
2244 attempts,
2245 cumulative_wait_ms,
2246 outcome: ContinuationOutcome::Resumed,
2247 },
2248 );
2249 }
2250 Ok(_) => {
2258 last_err = "the reply parsed, but it reported a command whose own CLI \
2259 never confirmed an exit status"
2260 .to_owned();
2261 }
2262 Err(e) => last_err = e.to_string(),
2263 }
2264 }
2265 AgentOutcome::Quota(o) => {
2266 cumulative_wait_ms += o.duration_ms;
2267 self.state.quota.push(QuotaLoss {
2268 seat: seat.key.clone(),
2269 node: "fix".to_owned(),
2270 at: Timestamp::now(),
2271 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2272 });
2273 self.state.event(
2274 "fix",
2275 format!(
2276 "round {round}: continuation rate limited (quota); not retrying now"
2277 ),
2278 );
2279 return (
2280 seat,
2281 None,
2282 Some("rate limited (quota) while recovering the fix report".to_owned()),
2283 ContinuationRecord {
2284 attempts,
2285 cumulative_wait_ms,
2286 outcome: ContinuationOutcome::QuotaLost,
2287 },
2288 );
2289 }
2290 AgentOutcome::Dropped(o) => {
2291 cumulative_wait_ms += o.duration_ms;
2292 let why = o
2293 .dropped
2294 .as_ref()
2295 .map(|d| d.why.as_str())
2296 .unwrap_or("the CLI ended the stream without delivering its answer");
2297 last_err = format!("the CLI dropped the stream ({why})");
2298 }
2299 AgentOutcome::Failed(e) => last_err = e,
2300 }
2301 }
2302 }
2303
2304 fn after_implement(&mut self) -> Result<()> {
2305 if self.state.leaks.is_empty() {
2307 let cfg = self.state.config.blind.clone();
2308 let mut leaks = Vec::new();
2309 for c in &self.state.candidates {
2310 let Some(patch) =
2311 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2312 else {
2313 continue;
2314 };
2315 leaks.extend(blind::scan(
2316 &format!("candidate {} patch", c.label),
2317 &patch,
2318 &cfg.vendor_tokens,
2319 ));
2320 }
2321 if !leaks.is_empty() {
2322 let summary = leaks
2323 .iter()
2324 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2325 .collect::<Vec<_>>()
2326 .join(", ");
2327 match cfg.on_leak {
2328 LeakPolicy::Fail => {
2329 self.state.status = RunStatus::Failed;
2330 self.state
2331 .event("blind", format!("vendor text in a patch: {summary}"));
2332 self.state.leaks = leaks;
2333 self.state.save()?;
2334 self.settle_questions();
2335 bail!(
2336 "blind.on_leak = \"fail\" and vendor text reached a \
2337 judged patch: {summary}"
2338 );
2339 }
2340 LeakPolicy::Redact => self.state.event(
2341 "blind",
2342 format!("redacting vendor text for judging: {summary}"),
2343 ),
2344 LeakPolicy::Warn => self.state.event(
2345 "blind",
2346 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2347 ),
2348 }
2349 self.state.leaks = leaks;
2350 }
2351 }
2352
2353 if self.state.viable().is_empty() {
2354 if self.state.all_candidates_verified_noop() {
2355 self.state.status = RunStatus::VerifiedNoop;
2366 self.state.save()?;
2367 self.settle_questions();
2368 return Ok(());
2369 }
2370 self.state.status = RunStatus::Failed;
2371 self.state.save()?;
2372 self.settle_questions();
2373 bail!("no candidate produced a change; nothing to judge");
2374 }
2375 self.state.status = RunStatus::Judging;
2376 self.state.save()?;
2377 Ok(())
2378 }
2379
2380 async fn judge(&mut self) -> Result<()> {
2383 let run_id = self.state.id.clone();
2388 let prompts = self.state.config.prompts.clone();
2389 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2390 return Ok(());
2391 }
2392 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2393 if viable.len() == 1 {
2394 self.state.judge_skipped = true;
2401 self.state.event(
2402 "judge",
2403 format!(
2404 "only candidate {} produced a change; judging skipped",
2405 viable[0].label
2406 ),
2407 );
2408 self.state.save()?;
2409 return Ok(());
2410 }
2411 self.state.status = RunStatus::Judging;
2412
2413 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2414 let language = self.state.config.graph.language.clone();
2415 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2416 let sessions = self.state.config.graph.sessions;
2417 let artifacts = agent::artifacts_dir(&self.state.dir());
2418 let root = self.state.worktree_root();
2419 let base_short = short(&self.state.base_commit);
2420
2421 let mut jobs = Vec::new();
2422 let mut orders = Vec::new();
2423 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2424 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2425 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2426 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2427 let seat_key = format!("judge-{}", j + 1);
2428 let seat = self.seat(&seat_key, &spec.id);
2429 jobs.push(SeatJob {
2430 prompt: prompt::judge(
2431 &self.state.instruction,
2432 &views,
2433 self.roles.judges.len(),
2434 &base_short,
2435 &language,
2436 ),
2437 spec,
2438 seat,
2439 cwd: root.join(format!("judge-{}", j + 1)),
2440 timeout,
2441 allow_write: false,
2442 sessions,
2443 artifacts: artifacts.clone(),
2444 stem: format!("judge-{}", j + 1),
2445 });
2446 }
2447
2448 self.state.event(
2449 "judge",
2450 format!(
2451 "{} judges ranking {} candidates blind",
2452 jobs.len(),
2453 viable.len()
2454 ),
2455 );
2456 let labels_for_check = labels.clone();
2457 let mut quota_losses = Vec::new();
2458 let cache = self.state.config.cache_dir();
2459 let ctx = WaveCtx {
2460 run: &run_id,
2461 node: "judge",
2462 prompts: &prompts,
2463 cache: cache.as_deref(),
2464 round: None,
2465 };
2466 let results = ask_json_wave::<Ranking>(
2467 jobs,
2468 Arc::clone(&self.sem),
2469 self.state.config.graph.retries,
2470 &self.roles.judge_roster,
2471 &ctx,
2472 &mut quota_losses,
2473 &mut self.state,
2474 &move |r: &Ranking| r.validate(&labels_for_check),
2475 )
2476 .await;
2477 self.state.quota.extend(quota_losses);
2478
2479 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2480 let agent_id = seat.agent.clone();
2481 self.state.seats.insert(seat.key.clone(), seat);
2482 let mut record = Judgement {
2483 judge: j + 1,
2484 seat: format!("judge-{}", j + 1),
2485 agent: agent_id,
2486 ranking: Vec::new(),
2487 reasons: BTreeMap::new(),
2488 confidence: None,
2489 order: orders[j].clone(),
2490 failed: None,
2491 duration_ms: 0,
2492 };
2493 match res {
2494 Ok((ranking, out)) => {
2495 record.ranking = ranking.normalized();
2496 record.reasons = ranking.reasons;
2497 record.confidence = ranking.confidence;
2498 record.duration_ms = out.duration_ms;
2499 self.state.event(
2500 "judge",
2501 format!(
2502 "judge {} ranked {}",
2503 j + 1,
2504 record.ranking.iter().collect::<String>()
2505 ),
2506 );
2507 }
2508 Err(e) => {
2509 record.failed = Some(e.to_string());
2510 self.state
2511 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2512 }
2513 }
2514 self.state.judgements.push(record);
2515 self.state.save()?;
2516 }
2517 Ok(())
2518 }
2519
2520 async fn deliberate(&mut self) -> Result<()> {
2523 let run_id = self.state.id.clone();
2528 let prompts = self.state.config.prompts.clone();
2529 if !self.state.deliberation.is_empty() {
2530 return Ok(());
2531 }
2532 let tops: Vec<char> = self
2533 .state
2534 .judgements
2535 .iter()
2536 .filter_map(|j| j.ranking.first().copied())
2537 .collect();
2538 let rounds = self.state.config.graph.deliberate_rounds;
2539 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2540 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2541 self.state.event(
2542 "deliberate",
2543 format!("judges agreed on {} outright; no deliberation", tops[0]),
2544 );
2545 }
2546 self.state.status = RunStatus::Voting;
2547 self.state.save()?;
2548 return Ok(());
2549 }
2550
2551 self.state.status = RunStatus::Deliberating;
2552 self.state.event(
2553 "deliberate",
2554 format!(
2555 "split: first choices were {} — opening {rounds} round(s)",
2556 tops.iter().collect::<String>()
2557 ),
2558 );
2559
2560 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2561 let language = self.state.config.graph.language.clone();
2562 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2563 let sessions = self.state.config.graph.sessions;
2564 let artifacts = agent::artifacts_dir(&self.state.dir());
2565 let root = self.state.worktree_root();
2566 let base_short = short(&self.state.base_commit);
2567
2568 for round in 1..=rounds {
2572 let mut turns: Vec<DeliberationTurn> = Vec::new();
2573 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2574 if self.state.judgements[j].failed.is_some() {
2575 continue;
2576 }
2577 let seat_key = format!("judge-{}", j + 1);
2578 let spec = self.occupant(&seat_key, spec);
2579 let mut seat = self.seat(&seat_key, &spec.id);
2580 let transcript = self.transcript(&turns, j);
2581 let context = if has_context(&spec, &seat, sessions) {
2582 None
2583 } else {
2584 Some(self.candidate_block(&viable, &base_short))
2585 };
2586 let text = prompt::deliberate(
2587 &self.state.instruction,
2588 context.as_deref(),
2589 &transcript,
2590 round,
2591 rounds,
2592 &language,
2593 );
2594 let job = SeatJob {
2595 spec,
2596 seat: seat.clone(),
2597 prompt: text,
2598 cwd: root.join(format!("judge-{}", j + 1)),
2599 timeout,
2600 allow_write: false,
2601 sessions,
2602 artifacts: artifacts.clone(),
2603 stem: format!("delib-{round}-judge-{}", j + 1),
2604 };
2605 let cache = self.state.config.cache_dir();
2606 let ctx = WaveCtx {
2607 run: &run_id,
2608 node: "deliberate",
2609 prompts: &prompts,
2610 cache: cache.as_deref(),
2611 round: None,
2612 };
2613 let (updated, out) =
2614 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2615 seat = updated;
2616 let agent_id = seat.agent.clone();
2617 let seat_key = seat.key.clone();
2618 self.state.seats.insert(seat.key.clone(), seat);
2619 let body = match out {
2620 AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2621 AgentOutcome::Dropped(o) => {
2625 let why =
2626 o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2627 "the CLI ended the stream without delivering its answer",
2628 );
2629 self.state.event(
2630 "deliberate",
2631 format!(
2632 "judge {} skipped: the CLI dropped the stream ({why})",
2633 j + 1
2634 ),
2635 );
2636 continue;
2637 }
2638 AgentOutcome::Quota(o) => {
2639 self.state.quota.push(QuotaLoss {
2640 seat: seat_key,
2641 node: "deliberate".to_owned(),
2642 at: Timestamp::now(),
2643 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2644 });
2645 self.state.event(
2646 "deliberate",
2647 format!("judge {} skipped: rate limited (quota)", j + 1),
2648 );
2649 continue;
2650 }
2651 AgentOutcome::Failed(e) => {
2652 self.state
2653 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2654 continue;
2655 }
2656 };
2657 let tentative = verdict::extract_json::<Position>(&body)
2658 .ok()
2659 .and_then(|p| p.tentative)
2660 .and_then(|s| s.trim().chars().next())
2661 .map(|c| c.to_ascii_uppercase());
2662 self.state.event(
2663 "deliberate",
2664 format!(
2665 "round {round}: judge {} now favours {}",
2666 j + 1,
2667 tentative.map_or("—".to_owned(), |c| c.to_string())
2668 ),
2669 );
2670 turns.push(DeliberationTurn {
2671 judge: j + 1,
2672 agent: agent_id,
2673 body: blind::sanitize_prose(&body, &self.state.config.blind),
2674 tentative,
2675 });
2676 }
2677 self.state
2678 .deliberation
2679 .push(DeliberationRound { round, turns });
2680 self.state.save()?;
2681 }
2682
2683 self.state.status = RunStatus::Voting;
2684 self.state.save()?;
2685 Ok(())
2686 }
2687
2688 async fn vote(&mut self) -> Result<()> {
2691 let run_id = self.state.id.clone();
2696 let prompts = self.state.config.prompts.clone();
2697 if !self.state.votes.is_empty() {
2698 return Ok(());
2699 }
2700 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2701 if viable.len() == 1 {
2702 return Ok(());
2703 }
2704 self.state.status = RunStatus::Voting;
2705
2706 let language = self.state.config.graph.language.clone();
2707 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2708 let sessions = self.state.config.graph.sessions;
2709 let artifacts = agent::artifacts_dir(&self.state.dir());
2710 let root = self.state.worktree_root();
2711 let base_short = short(&self.state.base_commit);
2712 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2713
2714 let mut jobs = Vec::new();
2715 let mut seats_at = Vec::new();
2716 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2717 if self
2718 .state
2719 .judgements
2720 .get(j)
2721 .is_some_and(|r| r.failed.is_some())
2722 {
2723 continue;
2724 }
2725 let seat_key = format!("judge-{}", j + 1);
2726 let spec = self.occupant(&seat_key, spec);
2727 let seat = self.seat(&seat_key, &spec.id);
2728 let mut text = prompt::final_vote(&viable, &language);
2729 if !has_context(&spec, &seat, sessions) {
2730 text = format!(
2731 "{}\n\n# Candidates\n\n{}",
2732 text,
2733 self.candidate_block(&candidates, &base_short)
2734 );
2735 }
2736 jobs.push(SeatJob {
2737 spec,
2738 seat,
2739 prompt: text,
2740 cwd: root.join(format!("judge-{}", j + 1)),
2741 timeout,
2742 allow_write: false,
2743 sessions,
2744 artifacts: artifacts.clone(),
2745 stem: format!("vote-judge-{}", j + 1),
2746 });
2747 seats_at.push(j);
2748 }
2749
2750 self.state.event(
2751 "vote",
2752 format!(
2753 "collecting {} final votes one by one, privately",
2754 jobs.len()
2755 ),
2756 );
2757 let allowed = viable.clone();
2758 let mut quota_losses = Vec::new();
2759 let cache = self.state.config.cache_dir();
2760 let ctx = WaveCtx {
2761 run: &run_id,
2762 node: "vote",
2763 prompts: &prompts,
2764 cache: cache.as_deref(),
2765 round: None,
2766 };
2767 let results = ask_json_wave::<FinalVote>(
2768 jobs,
2769 Arc::clone(&self.sem),
2770 self.state.config.graph.retries,
2771 &[],
2772 &ctx,
2773 &mut quota_losses,
2774 &mut self.state,
2775 &move |v: &FinalVote| match v.label() {
2776 Some(c) if allowed.contains(&c) => Ok(()),
2777 other => bail!("vote {other:?} is not one of {allowed:?}"),
2778 },
2779 )
2780 .await;
2781 self.state.quota.extend(quota_losses);
2782
2783 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2784 let agent_id = seat.agent.clone();
2785 self.state.seats.insert(seat.key.clone(), seat);
2786 let initial = self
2787 .state
2788 .judgements
2789 .get(j)
2790 .and_then(|r| r.ranking.first().copied());
2791 let mut record = VoteRecord {
2792 judge: j + 1,
2793 agent: agent_id,
2794 vote: None,
2795 reason: String::new(),
2796 changed: false,
2797 };
2798 match res {
2799 Ok((v, _)) => {
2800 record.vote = v.label();
2801 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2802 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2803 self.state.event(
2804 "vote",
2805 format!(
2806 "judge {} voted {}{}",
2807 j + 1,
2808 record.vote.unwrap_or('?'),
2809 if record.changed { " (changed)" } else { "" }
2810 ),
2811 );
2812 }
2813 Err(e) => {
2814 self.state
2815 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2816 }
2817 }
2818 self.state.votes.push(record);
2819 self.state.save()?;
2820 }
2821 Ok(())
2822 }
2823
2824 fn tally(&mut self) -> Result<()> {
2827 if self.state.tally.is_some() {
2828 return Ok(());
2829 }
2830 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2831 let tops: Vec<char> = self
2832 .state
2833 .judgements
2834 .iter()
2835 .filter_map(|j| j.ranking.first().copied())
2836 .collect();
2837 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2838
2839 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2842 let mut cast: Vec<char> = Vec::new();
2843 for (i, j) in self.state.judgements.iter().enumerate() {
2844 let vote = self
2845 .state
2846 .votes
2847 .iter()
2848 .find(|v| v.judge == i + 1)
2849 .and_then(|v| v.vote)
2850 .or_else(|| j.ranking.first().copied());
2851 if let Some(v) = vote {
2852 *first_choice.entry(v).or_insert(0) += 1;
2853 cast.push(v);
2854 }
2855 }
2856
2857 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2858 for j in &self.state.judgements {
2859 let n = j.ranking.len();
2860 for (pos, label) in j.ranking.iter().enumerate() {
2861 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2862 }
2863 }
2864
2865 let best = first_choice.values().copied().max().unwrap_or(0);
2866 let mut leaders: Vec<char> = first_choice
2867 .iter()
2868 .filter(|(_, v)| **v == best)
2869 .map(|(k, _)| *k)
2870 .collect();
2871 let mut tie_break = None;
2872 if leaders.len() > 1 {
2873 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2874 let borda_leaders: Vec<char> = leaders
2875 .iter()
2876 .copied()
2877 .filter(|l| borda[l] == top_borda)
2878 .collect();
2879 tie_break = Some(if borda_leaders.len() == 1 {
2880 format!(
2881 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2882 leaders.len()
2883 )
2884 } else {
2885 format!(
2886 "{} way tie on both first-choice votes and Borda points, broken by label order",
2887 leaders.len()
2888 )
2889 });
2890 leaders = borda_leaders;
2891 leaders.sort_unstable();
2892 }
2893 let winner = *leaders
2894 .first()
2895 .or(viable.first())
2896 .context("no candidate to declare a winner from")?;
2897
2898 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2899 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2900 let deliberated = !self.state.deliberation.is_empty();
2901
2902 let quota_seats: std::collections::BTreeSet<&str> =
2906 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2907 let mut present = 0usize;
2908 for (i, j) in self.state.judgements.iter().enumerate() {
2909 if quota_seats.contains(j.seat.as_str()) {
2910 continue;
2911 }
2912 let ranked = !j.ranking.is_empty() && j.failed.is_none();
2913 let voted = self
2914 .state
2915 .votes
2916 .iter()
2917 .any(|v| v.judge == i + 1 && v.vote.is_some());
2918 if ranked || voted {
2919 present += 1;
2920 }
2921 }
2922 let needs_quorum = viable.len() > 1;
2928 let judges_total = if needs_quorum {
2929 self.roles.judges.len()
2930 } else {
2931 0
2932 };
2933 let quorum = if needs_quorum {
2934 judges_total / 2 + 1
2935 } else {
2936 0
2937 };
2938 let met_quorum = !needs_quorum || present >= quorum;
2939 let uncontested = (!needs_quorum).then(|| {
2940 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2941 });
2942
2943 self.state.event(
2944 "tally",
2945 match &uncontested {
2946 Some(reason) => format!("winner {winner} — {reason}"),
2947 None => format!(
2948 "winner {winner} — votes {} | initial {} | {} changed | \
2949 {present}/{judges_total} judges{}",
2950 first_choice
2951 .iter()
2952 .map(|(k, v)| format!("{k}:{v}"))
2953 .collect::<Vec<_>>()
2954 .join(" "),
2955 if unanimous_initial {
2956 "unanimous"
2957 } else {
2958 "split"
2959 },
2960 changed_votes,
2961 if met_quorum {
2962 String::new()
2963 } else {
2964 format!(" — below quorum ({quorum} required)")
2965 },
2966 ),
2967 },
2968 );
2969 if !met_quorum {
2970 self.state.event(
2971 "stall",
2972 format!(
2973 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2974 the run stops here, resumable"
2975 ),
2976 );
2977 }
2978 self.state.tally = Some(Tally {
2979 first_choice,
2980 borda,
2981 winner,
2982 rankings: tops.len(),
2983 unanimous_initial,
2984 deliberated,
2985 changed_votes,
2986 unanimous_final,
2987 tie_break,
2988 judges: judges_total,
2989 present,
2990 quorum,
2991 met_quorum,
2992 uncontested,
2993 });
2994 self.state.status = if met_quorum {
2995 RunStatus::Reviewing
2996 } else {
2997 RunStatus::Stalled
2998 };
2999 self.state.save()?;
3000 Ok(())
3001 }
3002
3003 #[allow(clippy::too_many_lines)]
3024 async fn recover_stall(&mut self) -> Result<bool> {
3025 let run_id = self.state.id.clone();
3030 let prompts = self.state.config.prompts.clone();
3031 let quota_seats: BTreeSet<&str> =
3036 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3037 let absent: Vec<String> = self
3038 .state
3039 .judgements
3040 .iter()
3041 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3042 .map(|j| j.seat.clone())
3043 .collect();
3044 if absent.is_empty() {
3045 return Ok(false);
3046 }
3047 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3048 if viable.len() <= 1 {
3049 return Ok(false);
3050 }
3051 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3052 let language = self.state.config.graph.language.clone();
3053 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3054 let sessions = self.state.config.graph.sessions;
3055 let artifacts = agent::artifacts_dir(&self.state.dir());
3056 let root = self.state.worktree_root();
3057 let base_short = short(&self.state.base_commit);
3058 let candidates: Vec<Candidate> = viable.clone();
3059
3060 let mut positions: Vec<usize> = absent
3062 .iter()
3063 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3064 .collect();
3065 if positions.is_empty() {
3066 return Ok(false);
3067 }
3068 positions.sort_unstable();
3069 positions.dedup();
3070
3071 let mut judge_jobs = Vec::new();
3073 for &j in &positions {
3074 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3075 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3076 let seat_key = format!("judge-{}", j + 1);
3077 let spec = self.roles.judges[j].clone();
3078 let seat = self.seat(&seat_key, &spec.id);
3079 judge_jobs.push(SeatJob {
3080 spec,
3081 seat,
3082 prompt: prompt::judge(
3083 &self.state.instruction,
3084 &views,
3085 self.roles.judges.len(),
3086 &base_short,
3087 &language,
3088 ),
3089 cwd: root.join(seat_key),
3090 timeout,
3091 allow_write: false,
3092 sessions,
3093 artifacts: artifacts.clone(),
3094 stem: format!("judge-{}-recover", j + 1),
3095 });
3096 }
3097
3098 let labels_for_check = labels.clone();
3099 let mut judge_losses = Vec::new();
3100 let retries = self.state.config.graph.retries;
3101 let cache = self.state.config.cache_dir();
3102 let ctx = WaveCtx {
3103 run: &run_id,
3104 node: "judge",
3105 prompts: &prompts,
3106 cache: cache.as_deref(),
3107 round: None,
3108 };
3109 let results = ask_json_wave::<Ranking>(
3110 judge_jobs,
3111 Arc::clone(&self.sem),
3112 retries,
3113 &[],
3114 &ctx,
3115 &mut judge_losses,
3116 &mut self.state,
3117 &move |r: &Ranking| r.validate(&labels_for_check),
3118 )
3119 .await;
3120
3121 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3123 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3124 self.state.seats.insert(seat.key.clone(), seat);
3125 let record = &mut self.state.judgements[j];
3126 match res {
3127 Ok((ranking, out)) => {
3128 record.ranking = ranking.normalized();
3129 record.reasons = ranking.reasons;
3130 record.confidence = ranking.confidence;
3131 record.failed = None;
3132 record.duration_ms = out.duration_ms;
3133 recovered.insert(j);
3134 self.state.event(
3135 "recover",
3136 format!("judge {} ranked again after the limit", j + 1),
3137 );
3138 }
3139 Err(e) => {
3140 self.state
3141 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3142 }
3143 }
3144 }
3145
3146 let mut vote_jobs = Vec::new();
3148 let mut vote_pos: Vec<usize> = Vec::new();
3149 for &j in &recovered {
3150 let seat_key = format!("judge-{}", j + 1);
3151 let spec = self.roles.judges[j].clone();
3152 let seat = self.seat(&seat_key, &spec.id);
3153 let mut text = prompt::final_vote(&labels, &language);
3154 if !has_context(&spec, &seat, sessions) {
3155 text = format!(
3156 "{}\n\n# Candidates\n\n{}",
3157 text,
3158 self.candidate_block(&candidates, &base_short)
3159 );
3160 }
3161 vote_jobs.push(SeatJob {
3162 spec,
3163 seat,
3164 prompt: text,
3165 cwd: root.join(seat_key),
3166 timeout,
3167 allow_write: false,
3168 sessions,
3169 artifacts: artifacts.clone(),
3170 stem: format!("vote-judge-{}-recover", j + 1),
3171 });
3172 vote_pos.push(j);
3173 }
3174 let allowed = labels.clone();
3175 let mut vote_losses = Vec::new();
3176 let vote_retries = self.state.config.graph.retries;
3177 let vote_cache = self.state.config.cache_dir();
3178 let ctx = WaveCtx {
3179 run: &run_id,
3180 node: "vote",
3181 prompts: &prompts,
3182 cache: vote_cache.as_deref(),
3183 round: None,
3184 };
3185 let votes = ask_json_wave::<FinalVote>(
3186 vote_jobs,
3187 Arc::clone(&self.sem),
3188 vote_retries,
3189 &[],
3190 &ctx,
3191 &mut vote_losses,
3192 &mut self.state,
3193 &move |v: &FinalVote| match v.label() {
3194 Some(c) if allowed.contains(&c) => Ok(()),
3195 other => bail!("vote {other:?} is not one of {allowed:?}"),
3196 },
3197 )
3198 .await;
3199 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3200 let agent_id = seat.agent.clone();
3201 self.state.seats.insert(seat.key.clone(), seat);
3202 match res {
3203 Ok((v, _)) => {
3204 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3205 rec.vote = v.label();
3206 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3207 } else {
3208 self.state.votes.push(VoteRecord {
3209 judge: j + 1,
3210 agent: agent_id,
3211 vote: v.label(),
3212 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3213 changed: false,
3214 });
3215 }
3216 self.state.event(
3217 "recover",
3218 format!("judge {} voted again after the limit", j + 1),
3219 );
3220 }
3221 Err(e) => {
3222 self.state
3223 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3224 }
3225 }
3226 }
3227
3228 let recovered_keys: BTreeSet<String> = recovered
3232 .iter()
3233 .map(|&j| format!("judge-{}", j + 1))
3234 .collect();
3235 self.state
3236 .quota
3237 .retain(|q| !recovered_keys.contains(&q.seat));
3238 for loss in judge_losses.into_iter().chain(vote_losses) {
3242 if recovered_keys.contains(&loss.seat) {
3243 continue;
3244 }
3245 self.state.quota.retain(|q| q.seat != loss.seat);
3246 self.state.quota.push(loss);
3247 }
3248
3249 self.state.tally = None;
3251 self.tally()?;
3252 Ok(self
3253 .state
3254 .tally
3255 .as_ref()
3256 .map(|t| t.met_quorum)
3257 .unwrap_or(false))
3258 }
3259
3260 async fn fold_losers(&mut self) -> Result<()> {
3263 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3264 return Ok(());
3265 };
3266 let repo = self.state.repo.clone();
3267 let mut folded = Vec::new();
3268 for i in 0..self.state.candidates.len() {
3269 let c = &self.state.candidates[i];
3270 if c.label == winner || c.folded {
3271 continue;
3272 }
3273 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3274 git::worktree_remove(&repo, &wt).await.ok();
3275 git::branch_delete(&repo, &branch).await.ok();
3276 self.state.candidates[i].folded = true;
3277 folded.push(label.to_string());
3278 }
3279 let root = self.state.worktree_root();
3281 for j in 1..=self.roles.judges.len() {
3282 let wt = root.join(format!("judge-{j}"));
3283 if wt.exists() {
3284 git::worktree_remove(&repo, &wt).await.ok();
3285 }
3286 }
3287 if self.state.config.graph.advise {
3290 for k in 1..=self.state.config.graph.advisors {
3291 let wt = root.join(format!("advisor-{k}"));
3292 if wt.exists() {
3293 git::worktree_remove(&repo, &wt).await.ok();
3294 }
3295 }
3296 }
3297 if !folded.is_empty() {
3298 self.state
3299 .event("fold", format!("folded candidates {}", folded.join(", ")));
3300 self.state.save()?;
3301 }
3302 Ok(())
3303 }
3304
3305 async fn sync_to_base(&mut self) -> Result<()> {
3345 if self.state.status == RunStatus::AlreadyInBase {
3346 return Ok(());
3347 }
3348 let conflicted = self
3349 .state
3350 .base_sync
3351 .as_ref()
3352 .is_some_and(|s| s.conflict.is_some());
3353 let Some(winner) = self.state.winner().cloned() else {
3354 return Ok(());
3355 };
3356
3357 let repo = self.state.repo.clone();
3358 let remote = self.state.config.merge.remote.clone();
3359 let base_branch = self.state.base_branch.clone();
3360 let tracking = format!("{remote}/{base_branch}");
3361
3362 git::fetch(&repo, &remote, &base_branch).await.ok();
3363 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3367 return Ok(());
3368 };
3369
3370 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3371 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3372 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3373
3374 if (behind > 0 || conflicted || head == tip)
3383 && self
3384 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3385 .await?
3386 {
3387 return Ok(());
3388 }
3389 if conflicted {
3390 return Ok(());
3391 }
3392
3393 if behind == 0 {
3394 if let Some(from) = self
3403 .state
3404 .rebase_fixes
3405 .iter()
3406 .rev()
3407 .find_map(|r| r.from.clone())
3408 && from != head
3409 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3410 .await
3411 .is_ok_and(|o| o.ok())
3412 {
3413 git::sync_to_head(&winner.worktree).await?;
3414 }
3415 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3421 if let Some(why) = &conflict {
3422 self.state.status = RunStatus::Blocked;
3423 self.state.event("land", why.clone());
3424 }
3425 self.state.base_sync = Some(BaseSync {
3426 tip,
3427 behind: 0,
3428 attempts,
3429 conflict,
3430 already_in: None,
3431 });
3432 self.state.save()?;
3433 return Ok(());
3434 }
3435
3436 if attempts >= BASE_SYNC_ROUNDS {
3437 let why = format!(
3438 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3439 rebase(s); rebasing again would only race it",
3440 winner.branch
3441 );
3442 self.state.status = RunStatus::Blocked;
3443 self.state.base_sync = Some(BaseSync {
3444 tip,
3445 behind,
3446 attempts,
3447 conflict: Some(why.clone()),
3448 already_in: None,
3449 });
3450 self.state.event("land", why);
3451 self.state.save()?;
3452 return Ok(());
3453 }
3454
3455 self.state.event(
3456 "land",
3457 format!(
3458 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3459 winner.branch
3460 ),
3461 );
3462 self.state.save()?;
3463
3464 let branch_tracking = format!("{remote}/{}", winner.branch);
3470 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3471 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3472 git::rev_parse(&repo, &branch_tracking).await.ok()
3473 } else {
3474 None
3475 };
3476 if let Some(theirs) = &remote_tip
3480 && !git::is_ancestor(&repo, theirs, &head).await
3481 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3482 .await
3483 .is_ok_and(|missing| missing.is_empty())
3484 {
3485 let why = format!(
3486 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3487 them",
3488 short(theirs),
3489 winner.branch
3490 );
3491 self.state.status = RunStatus::Blocked;
3492 self.state.base_sync = Some(BaseSync {
3493 tip,
3494 behind,
3495 attempts,
3496 conflict: Some(why.clone()),
3497 already_in: None,
3498 });
3499 self.state.event("land", why);
3500 self.state.save()?;
3501 return Ok(());
3502 }
3503
3504 let scratch = self.state.dir().join("base-sync");
3505 let rebased = match crate::rebase::rebase_with_fixer(
3506 &mut self.state,
3507 &scratch,
3508 &winner.branch,
3509 &tracking,
3510 )
3511 .await
3512 {
3513 Ok(crate::rebase::Rebased::Applied) => Ok(None),
3514 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3515 Err(e) => Err(e),
3516 };
3517 let attempts = attempts + 1;
3518 match rebased {
3519 Ok(None) => {
3520 git::sync_to_head(&winner.worktree).await?;
3524 let mut conflict = None;
3525 if let Some(pinned) = &remote_tip {
3526 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3527 match pushed {
3528 Ok(o) if o.ok() => self.state.event(
3529 "land",
3530 format!("pushed rebased {} to {remote}", winner.branch),
3531 ),
3532 Ok(o) => {
3533 conflict = Some(format!(
3534 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
3535 winner.branch,
3536 short(pinned),
3537 o.stderr.chars().take(600).collect::<String>()
3538 ));
3539 }
3540 Err(e) => {
3541 conflict = Some(format!(
3542 "rebased {} locally but could not push it: {e:#}",
3543 winner.branch
3544 ));
3545 }
3546 }
3547 }
3548 if let Some(why) = &conflict {
3549 self.state.status = RunStatus::Blocked;
3550 self.state.event("land", why.clone());
3551 }
3552 self.state.base_sync = Some(BaseSync {
3553 tip: tip.clone(),
3554 behind: 0,
3555 attempts,
3556 conflict,
3557 already_in: None,
3558 });
3559 self.state
3560 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3561 }
3562 Ok(Some(conflict)) => {
3563 let why = format!(
3564 "{} conflicts with {tracking} and did not rebase: {}",
3565 winner.branch,
3566 conflict.chars().take(600).collect::<String>()
3567 );
3568 self.state.status = RunStatus::Blocked;
3569 self.state.base_sync = Some(BaseSync {
3570 tip,
3571 behind,
3572 attempts,
3573 conflict: Some(why.clone()),
3574 already_in: None,
3575 });
3576 self.state.event("land", why);
3577 }
3578 Err(e) => {
3579 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3580 self.state.status = RunStatus::Blocked;
3581 self.state.base_sync = Some(BaseSync {
3582 tip,
3583 behind,
3584 attempts,
3585 conflict: Some(why.clone()),
3586 already_in: None,
3587 });
3588 self.state.event("land", why);
3589 }
3590 }
3591 self.state.save()?;
3592 Ok(())
3593 }
3594
3595 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
3601 let repo = self.state.repo.clone();
3602 let remote = self.state.config.merge.remote.clone();
3603 let fetched = git::fetch(&repo, &remote, branch).await;
3604 if !matches!(&fetched, Ok(o) if o.ok()) {
3605 return None;
3606 }
3607 let branch_tracking = format!("{remote}/{branch}");
3608 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
3609 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
3610 return None;
3611 }
3612 if !crate::reconcile::origin_missing(&repo, head, &theirs)
3613 .await
3614 .is_ok_and(|missing| missing.is_empty())
3615 {
3616 return Some(format!(
3617 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
3618 them",
3619 short(&theirs)
3620 ));
3621 }
3622 match git::push_pinned(&repo, &remote, branch, &theirs).await {
3623 Ok(o) if o.ok() => {
3624 self.state
3625 .event("land", format!("pushed rebased {branch} to {remote}"));
3626 None
3627 }
3628 Ok(o) => Some(format!(
3629 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
3630 someone may have pushed): {}",
3631 short(&theirs),
3632 o.stderr.chars().take(600).collect::<String>()
3633 )),
3634 Err(e) => Some(format!(
3635 "{branch} is rebased locally but could not be pushed: {e:#}"
3636 )),
3637 }
3638 }
3639
3640 async fn settle_already_in(
3657 &mut self,
3658 branch: &str,
3659 tip: &str,
3660 head: &str,
3661 attempts: usize,
3662 behind: usize,
3663 ) -> Result<bool> {
3664 let repo = self.state.repo.clone();
3665 let remote = self.state.config.merge.remote.clone();
3666 let start = self.state.base_commit.clone();
3667 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3668 Ok(Some(e)) => e,
3669 Ok(None) => return Ok(false),
3670 Err(e) => {
3671 tracing::warn!("already-in-base check for {branch}: {e:#}");
3672 return Ok(false);
3673 }
3674 };
3675 let mut verified = vec![head.to_owned()];
3676 let fetched = git::fetch(&repo, &remote, branch).await;
3677 if matches!(&fetched, Ok(o) if o.ok())
3678 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3679 && theirs != head
3680 {
3681 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3682 Ok(Some(_)) => verified.push(theirs),
3683 _ => return Ok(false),
3684 }
3685 }
3686 let base_branch = self.state.base_branch.clone();
3687 let message = format!(
3688 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3689 land",
3690 evidence.names(),
3691 evidence.proof.as_str()
3692 );
3693 let closed =
3694 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3695 match closed {
3696 Ok(Ok(url)) => self
3697 .state
3698 .event("land", format!("closed {url}: superseded on {base_branch}")),
3699 Ok(Err(why)) => self
3700 .state
3701 .event("land", format!("did not close a pull request: {why}")),
3702 Err(e) => {
3703 let why = format!(
3704 "{branch} is already in {remote}/{base_branch}, but its pull request could \
3705 not be closed ({e:#}); resume to retry"
3706 );
3707 self.state.status = RunStatus::Blocked;
3708 self.state.base_sync = Some(BaseSync {
3709 tip: tip.to_owned(),
3710 behind,
3711 attempts,
3712 conflict: Some(why.clone()),
3713 already_in: None,
3714 });
3715 self.state.event("land", why);
3716 self.state.save()?;
3717 return Ok(true);
3718 }
3719 }
3720 self.state.status = RunStatus::AlreadyInBase;
3721 self.state.base_sync = Some(BaseSync {
3722 tip: tip.to_owned(),
3723 behind,
3724 attempts,
3725 conflict: None,
3726 already_in: Some(evidence),
3727 });
3728 self.state.event("land", message);
3729 self.state.save()?;
3730 self.settle_questions();
3731 Ok(true)
3732 }
3733
3734 fn landing_base(&self) -> String {
3744 self.state
3745 .base_sync
3746 .as_ref()
3747 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3748 }
3749
3750 pub async fn fix_selected(
3783 &mut self,
3784 ids: &[String],
3785 reason: &str,
3786 allow_stale: bool,
3787 ) -> Result<()> {
3788 let reason = reason.trim();
3789 if reason.is_empty() {
3790 bail!("a fix request needs a reason — that is the operator's own record of why");
3791 }
3792 if ids.is_empty() {
3793 bail!("no finding id given");
3794 }
3795 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3796 bail!(
3797 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3798 has already concluded — can be given a targeted fix. A run still \
3799 in progress should simply be resumed; a `merged` run's branch has \
3800 already landed, so its answer is a fresh `magi review <branch>`, \
3801 not reopening this run's own record",
3802 self.state.id,
3803 self.state.status.as_str()
3804 );
3805 }
3806 let Some(winner) = self.state.winner().cloned() else {
3807 bail!("run {} has no winning candidate to fix", self.state.id);
3808 };
3809 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3810 bail!(
3811 "branch `{}` no longer exists; this run cannot be extended",
3812 winner.branch
3813 );
3814 }
3815 let home = crate::run::home();
3816 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3817 bail!(
3818 "run {} is currently being worked on by another magi process",
3819 self.state.id
3820 );
3821 }
3822 let _claim = FixClaim::acquire(&self.state.dir())?;
3828
3829 let mut seen = BTreeSet::new();
3833 let mut findings = Vec::new();
3834 let mut missing = Vec::new();
3835 for id in ids {
3836 if !seen.insert(id.clone()) {
3837 continue;
3838 }
3839 match self.state.finding(id) {
3840 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3841 id: f.id.clone(),
3842 severity: f.severity,
3843 reviewer_vote: rec.vote,
3844 round: round.round,
3845 round_head: round.head.clone(),
3846 reviewer: rec.reviewer,
3847 agent: rec.agent.clone(),
3848 file: f.file.clone(),
3849 line: f.line,
3850 title: f.title.clone(),
3851 detail: f.detail.clone(),
3852 outcome: OperatorFixOutcome::Pending,
3853 }),
3854 None => missing.push(id.clone()),
3855 }
3856 }
3857 if !missing.is_empty() {
3858 bail!(
3859 "unknown finding id(s): {}; nothing was changed",
3860 missing.join(", ")
3861 );
3862 }
3863
3864 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3865 let stale_details: Vec<(String, String)> = findings
3866 .iter()
3867 .filter(|f| f.round_head != head_at_request)
3868 .map(|f| (f.id.clone(), f.round_head.clone()))
3869 .collect();
3870 let stale = !stale_details.is_empty();
3871 if stale && !allow_stale {
3872 bail!(
3873 "the branch has moved since some finding(s) were raised — {} — now \
3874 at {}; pass --allow-stale to fix anyway, or re-run review first",
3875 stale_details
3876 .iter()
3877 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3878 .collect::<Vec<_>>()
3879 .join(", "),
3880 short(&head_at_request)
3881 );
3882 }
3883
3884 let request = OperatorFixRequest {
3885 requested_at: Timestamp::now(),
3886 reason: reason.to_owned(),
3887 findings,
3888 head_at_request: head_at_request.clone(),
3889 allow_stale,
3890 stale,
3891 fix: None,
3892 result_head: None,
3893 follow_up_review_run: None,
3894 };
3895 self.state.event(
3896 "fix",
3897 format!(
3898 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3899 request.findings.len(),
3900 request
3901 .findings
3902 .iter()
3903 .map(|f| f.id.as_str())
3904 .collect::<Vec<_>>()
3905 .join(", "),
3906 ),
3907 );
3908 self.state.operator_fixes.push(request);
3915 self.state.save()?;
3916 let request_index = self.state.operator_fixes.len() - 1;
3917
3918 if winner.worktree.exists() {
3927 let dirty = git::git(
3930 &winner.worktree,
3931 &["status", "--porcelain", "--untracked-files=all"],
3932 )
3933 .await?;
3934 let only_withheld = dirty.lines().all(|l| {
3935 l.strip_prefix("?? ")
3936 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3937 });
3938 if !only_withheld {
3939 bail!(
3940 "`{}` has uncommitted changes; refusing to touch it — commit or \
3941 discard them first",
3942 winner.worktree.display()
3943 );
3944 }
3945 git::worktree_remove(&self.state.repo, &winner.worktree)
3946 .await
3947 .ok();
3948 }
3949 let fix_worktree = self.state.worktree_root().join("operator-fix");
3950 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3951 git::git(
3952 &self.state.repo,
3953 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3954 )
3955 .await
3956 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3957 if !git::is_clean(&fix_worktree).await? {
3958 git::worktree_remove(&self.state.repo, &fix_worktree)
3959 .await
3960 .ok();
3961 bail!(
3962 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3963 winner.branch
3964 );
3965 }
3966
3967 let run_id = self.state.id.clone();
3968 let prompts = self.state.config.prompts.clone();
3969 let language = self.state.config.graph.language.clone();
3970 let sessions = self.state.config.graph.sessions;
3971 let artifacts = agent::artifacts_dir(&self.state.dir());
3972 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3973 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3974 _ => (
3975 self.state
3976 .config
3977 .agent(&winner.agent)
3978 .cloned()
3979 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3980 format!("impl-{}", winner.label),
3981 ),
3982 };
3983 let seat = self.seat(&fix_seat_key, &fix_spec.id);
3984 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3985 .findings
3986 .iter()
3987 .map(|f| Finding {
3988 id: f.id.clone(),
3989 severity: f.severity,
3990 file: f.file.clone(),
3991 line: f.line,
3992 title: f.title.clone(),
3993 detail: f.detail.clone(),
3994 })
3995 .collect();
3996 let job = SeatJob {
3997 prompt: prompt::operator_fix(
3998 &self.state.instruction,
3999 &finding_list,
4000 reason,
4001 &stale_details,
4002 &head_at_request,
4003 &language,
4004 ),
4005 spec: fix_spec.clone(),
4006 seat,
4007 cwd: fix_worktree.clone(),
4008 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4009 allow_write: true,
4010 sessions,
4011 artifacts: artifacts.clone(),
4012 stem: "operator-fix".to_owned(),
4013 };
4014 let cache = self.state.config.cache_dir();
4015 let ctx = WaveCtx {
4016 run: &run_id,
4017 node: "fix",
4018 prompts: &prompts,
4019 cache: cache.as_deref(),
4020 round: None,
4021 };
4022 let (seat, out) =
4023 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4024 let agent_id = seat.agent.clone();
4025
4026 let mut fix = FixRecord {
4027 agent: agent_id,
4028 addressed: Vec::new(),
4029 rejected: Vec::new(),
4030 notes: String::new(),
4031 committed: false,
4032 failed: None,
4033 duration_ms: 0,
4034 continuation: None,
4035 };
4036 let mut final_seat = seat.clone();
4037 match out {
4038 AgentOutcome::Ok(o) => {
4039 fix.duration_ms = o.duration_ms;
4040 let parsed = verdict::extract_json::<FixReport>(&o.text);
4041 let incomplete_reason = match &parsed {
4042 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4043 "the reply parsed, but it reported a command whose own CLI \
4044 never confirmed an exit status"
4045 .to_owned(),
4046 ),
4047 Ok(_) => None,
4048 Err(e) => Some(e.to_string()),
4049 };
4050 match incomplete_reason {
4051 None => {
4052 let report = parsed.expect("checked Ok above");
4053 fix.addressed = report.addressed;
4054 fix.rejected = report.rejected;
4055 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4056 }
4057 Some(reason) => {
4058 let (resumed_seat, resolved, failure, cont) = self
4059 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4060 .await;
4061 fix.duration_ms += cont.cumulative_wait_ms;
4062 fix.continuation = Some(cont);
4063 final_seat = resumed_seat;
4064 match resolved {
4065 Some(report) => {
4066 fix.addressed = report.addressed;
4067 fix.rejected = report.rejected;
4068 fix.notes =
4069 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4070 }
4071 None => fix.failed = failure,
4072 }
4073 }
4074 }
4075 }
4076 AgentOutcome::Dropped(o) => {
4077 fix.duration_ms = o.duration_ms;
4078 let why = o
4079 .dropped
4080 .as_ref()
4081 .map(|d| d.why.as_str())
4082 .unwrap_or("the CLI ended the stream without delivering its answer");
4083 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4084 }
4085 AgentOutcome::Quota(o) => {
4086 self.state.quota.push(QuotaLoss {
4087 seat: final_seat.key.clone(),
4088 node: "fix".to_owned(),
4089 at: Timestamp::now(),
4090 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4091 });
4092 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4093 }
4094 AgentOutcome::Failed(e) => fix.failed = Some(e),
4095 }
4096 if fix.continuation.is_none() {
4097 fix.continuation = Some(ContinuationRecord::not_needed());
4098 }
4099 self.state.seats.insert(final_seat.key.clone(), final_seat);
4100
4101 let rescue_message = format!(
4102 "magi: operator-selected fix ({}) (uncommitted work)",
4103 self.state.operator_fixes[request_index]
4104 .findings
4105 .iter()
4106 .map(|f| f.id.as_str())
4107 .collect::<Vec<_>>()
4108 .join(", ")
4109 );
4110 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4111 self.state.note_withheld("fix", &r.withheld);
4112 }
4113 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4114 fix.committed = after != head_at_request;
4115 git::worktree_remove(&self.state.repo, &fix_worktree)
4116 .await
4117 .ok();
4118
4119 self.state.event(
4120 "fix",
4121 match &fix.failed {
4122 Some(reason) => format!(
4123 "operator fix: adoption report was lost ({reason}); {}",
4124 if fix.committed {
4125 "committed"
4126 } else {
4127 "NO new commit"
4128 }
4129 ),
4130 None => format!(
4131 "operator fix: {} addressed, {} rejected, {}",
4132 fix.addressed.len(),
4133 fix.rejected.len(),
4134 if fix.committed {
4135 "committed"
4136 } else {
4137 "NO new commit"
4138 }
4139 ),
4140 },
4141 );
4142
4143 for f in &mut self.state.operator_fixes[request_index].findings {
4150 f.outcome = if fix.failed.is_some() {
4151 OperatorFixOutcome::Unreported
4152 } else if fix.addressed.contains(&f.id) {
4153 OperatorFixOutcome::Addressed
4154 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4155 OperatorFixOutcome::Rejected { why: r.why.clone() }
4156 } else {
4157 OperatorFixOutcome::Unreported
4158 };
4159 }
4160
4161 let committed = fix.committed;
4162 if committed {
4163 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4164 }
4165 self.state.operator_fixes[request_index].fix = Some(fix);
4166 self.state.save()?;
4169
4170 if committed {
4171 self.state.event(
4172 "fix",
4173 format!(
4174 "operator fix committed {}; opening a follow-up review-only run",
4175 short(&after)
4176 ),
4177 );
4178 let origin =
4181 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4182 match Self::review(
4183 &self.state.repo,
4184 &winner.branch,
4185 self.state.config.clone(),
4186 origin,
4187 )
4188 .await
4189 {
4190 Ok(mut follow_up) => {
4191 follow_up.state.event(
4192 "start",
4193 format!(
4194 "requested by an operator fix on run {} for finding(s) {}",
4195 self.state.id,
4196 self.state.operator_fixes[request_index]
4197 .findings
4198 .iter()
4199 .map(|f| f.id.as_str())
4200 .collect::<Vec<_>>()
4201 .join(", "),
4202 ),
4203 );
4204 follow_up.state.save()?;
4205 let follow_up_id = follow_up.state.id.clone();
4206 if let Err(e) = follow_up.execute().await {
4207 self.state.event(
4208 "fix",
4209 format!(
4210 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4211 ),
4212 );
4213 }
4214 self.state.operator_fixes[request_index].follow_up_review_run =
4215 Some(follow_up_id);
4216 }
4217 Err(e) => {
4218 self.state.event(
4219 "fix",
4220 format!("committed the fix but could not open a follow-up review: {e:#}"),
4221 );
4222 }
4223 }
4224 self.state.save()?;
4225 }
4226
4227 Ok(())
4228 }
4229
4230 fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
4237 match &self.roles.fixer {
4238 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4239 _ => (
4240 self.state
4241 .config
4242 .agent(&winner.agent)
4243 .cloned()
4244 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4245 format!("impl-{}", winner.label),
4246 ),
4247 }
4248 }
4249
4250 async fn review_loop(&mut self) -> Result<()> {
4251 if self
4256 .state
4257 .base_sync
4258 .as_ref()
4259 .is_some_and(|s| s.conflict.is_some())
4260 {
4261 return Ok(());
4262 }
4263 let run_id = self.state.id.clone();
4268 let prompts = self.state.config.prompts.clone();
4269 let Some(winner) = self.state.winner().cloned() else {
4270 return Ok(());
4271 };
4272 let max_rounds = self.state.config.graph.review_rounds;
4273 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4283 if status == RunStatus::Gating {
4286 self.record_contested_handoff();
4287 }
4288 self.state.status = status;
4289 self.state.save()?;
4290 return Ok(());
4291 }
4292 self.state.status = RunStatus::Reviewing;
4293 if self
4303 .state
4304 .reviews
4305 .last()
4306 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4307 {
4308 let shell = self.state.config.shell();
4309 return self
4310 .stop_reviewing(
4311 "the last round's own verification never resolved",
4312 &shell,
4313 &winner.worktree,
4314 )
4315 .await;
4316 }
4317
4318 let repo = self.state.repo.clone();
4319 let root = self.state.worktree_root();
4320 let language = self.state.config.graph.language.clone();
4321 let sessions = self.state.config.graph.sessions;
4322 let artifacts = agent::artifacts_dir(&self.state.dir());
4323 let base = self.landing_base();
4324 let base_short = short(&base);
4325 let reviewers = self.roles.reviewers.clone();
4326 let shell = self.state.config.shell();
4327
4328 for round in (self.state.reviews.len() + 1)..=max_rounds {
4329 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4330 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4331 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4332 let prev_verification = self
4341 .state
4342 .reviews
4343 .last()
4344 .and_then(|r| r.verification_summary(&head));
4345
4346 let mut jobs = Vec::new();
4350 for (r, spec) in reviewers.iter().cloned().enumerate() {
4351 let wt = root.join(format!("review-{}", r + 1));
4352 if wt.exists() {
4353 git::reset_detached(&wt, &head).await?;
4354 } else {
4355 git::worktree_add_detached(&repo, &wt, &head).await?;
4356 }
4357 let seat_key = format!("review-{}", r + 1);
4358 let seat = self.seat(&seat_key, &spec.id);
4359 jobs.push(SeatJob {
4360 prompt: prompt::review(&prompt::ReviewCtx {
4361 instruction: &self.state.instruction,
4362 branch: &winner.branch,
4363 base_short: &base_short,
4364 stat: &stat,
4365 patch: &patch,
4366 verification: prev_verification.as_ref(),
4367 reviewers: reviewers.len(),
4368 round,
4369 rounds: max_rounds,
4370 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4373 lens: Lens::for_seat(r),
4374 language: &language,
4375 }),
4376 spec,
4377 seat,
4378 cwd: wt,
4379 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4380 allow_write: false,
4381 sessions,
4382 artifacts: artifacts.clone(),
4383 stem: format!("review-{round}-{}", r + 1),
4384 });
4385 }
4386
4387 self.state.event(
4388 "review",
4389 format!(
4390 "round {round}: {} reviewers on {}",
4391 jobs.len(),
4392 short(&head)
4393 ),
4394 );
4395 let mut quota_losses = Vec::new();
4396 let review_retries = self.state.config.graph.retries;
4397 let review_cache = self.state.config.cache_dir();
4398 let ctx = WaveCtx {
4399 run: &run_id,
4400 node: "review",
4401 prompts: &prompts,
4402 cache: review_cache.as_deref(),
4403 round: Some(round),
4404 };
4405 let results = ask_json_wave::<Review>(
4406 jobs,
4407 Arc::clone(&self.sem),
4408 review_retries,
4409 &self.roles.reviewer_roster,
4410 &ctx,
4411 &mut quota_losses,
4412 &mut self.state,
4413 &|_: &Review| Ok(()),
4414 )
4415 .await;
4416 let round_quota_missing = quota_losses.len();
4420 self.state.quota.extend(quota_losses);
4421
4422 let mut records = Vec::new();
4423 let mut all_findings = Vec::new();
4424 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4425 let agent_id = seat.agent.clone();
4426 self.state.seats.insert(seat.key.clone(), seat);
4427 let mut record = ReviewRecord {
4428 reviewer: r + 1,
4429 agent: agent_id,
4430 summary: String::new(),
4431 findings: Vec::new(),
4432 vote: None,
4433 failed: None,
4434 duration_ms: 0,
4435 attempts,
4441 };
4442 match res {
4443 Ok((review, out)) => {
4444 record.summary =
4452 blind::sanitize_prose(&review.summary, &self.state.config.blind);
4453 record.vote = Some(review.vote);
4454 record.duration_ms = out.duration_ms;
4455 for (n, mut f) in review.findings.into_iter().enumerate() {
4456 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4459 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4460 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4461 f.file = f
4467 .file
4468 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4469 all_findings.push(f.clone());
4470 record.findings.push(f);
4471 }
4472 self.state.event(
4473 "review",
4474 format!(
4475 "round {round}: reviewer {} voted {} with {} finding(s)",
4476 r + 1,
4477 review.vote.label(),
4478 record.findings.len()
4479 ),
4480 );
4481 }
4482 Err(e) => {
4483 record.failed = Some(e.to_string());
4484 self.state.event(
4485 "review",
4486 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4487 );
4488 }
4489 }
4490 records.push(record);
4491 }
4492
4493 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4500 let vote_split =
4501 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4502 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4503 if vote_split {
4504 self.state.event(
4505 "review",
4506 format!(
4507 "round {round}: votes split ({}) — one round of reconsideration",
4508 initial_votes
4509 .iter()
4510 .map(|v| v.label())
4511 .collect::<Vec<_>>()
4512 .join(", ")
4513 ),
4514 );
4515 let panel: Vec<ReviewSeatReport<'_>> = records
4518 .iter()
4519 .filter_map(|r| {
4520 r.vote.map(|vote| ReviewSeatReport {
4521 reviewer: r.reviewer,
4522 vote,
4523 summary: &r.summary,
4524 findings: &r.findings,
4525 })
4526 })
4527 .collect();
4528
4529 let mut jobs = Vec::new();
4530 let mut seats_at = Vec::new();
4531 for (r, spec) in reviewers.iter().cloned().enumerate() {
4532 if records[r].vote.is_none() {
4536 continue;
4537 }
4538 let wt = root.join(format!("review-{}", r + 1));
4539 let seat_key = format!("review-{}", r + 1);
4540 let spec = self.occupant(&seat_key, spec);
4541 let seat = self.seat(&seat_key, &spec.id);
4542 let patch_ctx = if has_context(&spec, &seat, sessions) {
4547 None
4548 } else {
4549 Some(ReviewPatch {
4550 branch: &winner.branch,
4551 base_short: &base_short,
4552 stat: &stat,
4553 patch: &patch,
4554 })
4555 };
4556 let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
4557 instruction: &self.state.instruction,
4558 reviewer: r + 1,
4559 lens: Lens::for_seat(r),
4560 panel: &panel,
4561 patch: patch_ctx,
4562 round,
4563 rounds: max_rounds,
4564 language: &language,
4565 });
4566 jobs.push(SeatJob {
4567 prompt,
4568 spec,
4569 seat,
4570 cwd: wt,
4571 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4572 allow_write: false,
4573 sessions,
4574 artifacts: artifacts.clone(),
4575 stem: format!("review-{round}-reconsider-{}", r + 1),
4576 });
4577 seats_at.push(r);
4578 }
4579
4580 let mut recon_quota_losses = Vec::new();
4581 let recon_cache = self.state.config.cache_dir();
4582 let recon_ctx = WaveCtx {
4583 run: &run_id,
4584 node: "review",
4585 prompts: &prompts,
4586 cache: recon_cache.as_deref(),
4587 round: Some(round),
4588 };
4589 let recon_results = ask_json_wave::<ReviewRevote>(
4590 jobs,
4591 Arc::clone(&self.sem),
4592 review_retries,
4593 &[],
4594 &recon_ctx,
4595 &mut recon_quota_losses,
4596 &mut self.state,
4597 &|_: &ReviewRevote| Ok(()),
4598 )
4599 .await;
4600 self.state.quota.extend(recon_quota_losses);
4601
4602 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4603 let agent_id = seat.agent.clone();
4604 self.state.seats.insert(seat.key.clone(), seat);
4605 let mut rec = ReviewRevoteRecord {
4606 reviewer: r + 1,
4607 agent: agent_id,
4608 vote: None,
4609 reason: String::new(),
4610 failed: None,
4611 };
4612 match res {
4613 Ok((rv, _)) => {
4614 rec.vote = Some(rv.vote);
4615 rec.reason =
4616 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4617 self.state.event(
4618 "review",
4619 format!(
4620 "round {round}: reviewer {} revoted {}",
4621 r + 1,
4622 rv.vote.label()
4623 ),
4624 );
4625 }
4626 Err(e) => {
4627 rec.failed = Some(e.to_string());
4628 self.state.event(
4629 "review",
4630 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4631 );
4632 }
4633 }
4634 reconsideration.push(rec);
4635 }
4636 } else if initial_votes.len() > 1 {
4637 self.state.event(
4638 "review",
4639 format!(
4640 "round {round}: votes agreed ({}) — no reconsideration",
4641 initial_votes[0].label()
4642 ),
4643 );
4644 }
4645
4646 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4647 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4648 let defer_e2e =
4659 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4660 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4661 let reason =
4662 format!("{blocking} blocking finding(s) already required a fix this round");
4663 self.state.event(
4664 "verify",
4665 format!(
4666 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4667 {}); it will run once a round has none left",
4668 short(&head)
4669 ),
4670 );
4671 (Vec::new(), false, true, Some(reason))
4672 } else {
4673 let e2e_commands = self.state.config.verify.e2e.clone();
4674 let cache_dir = self.state.config.cache_dir();
4675 let context = format!("round {round}");
4676 let (e2e, verify_retried) = with_cache_lease(
4677 &mut self.state,
4678 cache_dir.as_deref(),
4679 "e2e",
4680 "e2e",
4681 &winner.worktree,
4682 &head,
4683 verify_timeout,
4684 &context,
4685 |state, budget| {
4686 let shell = shell.clone();
4687 let e2e_commands = e2e_commands.clone();
4688 let worktree = winner.worktree.clone();
4689 let context = context.clone();
4690 async move {
4691 run_e2e_with_retry(
4692 state,
4693 &shell,
4694 &e2e_commands,
4695 &worktree,
4696 budget,
4697 &context,
4698 )
4699 .await
4700 }
4701 },
4702 )
4703 .await;
4704 (e2e, verify_retried, false, None)
4705 };
4706
4707 let expected = records.len();
4708 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4709 let incomplete = answered < expected;
4710 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4711 let policy = self.state.config.graph.incomplete_review;
4712 let clean = round_is_clean(
4713 blocking,
4714 e2e_ok,
4715 answered,
4716 expected,
4717 round_quota_missing,
4718 policy,
4719 );
4720
4721 let mut round_record = ReviewRound {
4722 round,
4723 head: head.clone(),
4724 verified_head: None,
4725 verified_at: None,
4726 reviews: records,
4727 e2e,
4728 verify_retried,
4729 e2e_deferred,
4730 e2e_defer_reason,
4731 fix: None,
4732 blocking,
4733 answered,
4734 expected,
4735 clean,
4736 progressed: false,
4737 vote_split,
4738 reconsideration,
4739 verdict: None,
4740 };
4741 round_record.verdict = ReviewVote::worst(
4745 round_record
4746 .final_votes()
4747 .into_iter()
4748 .map(|(_, _, vote)| vote),
4749 );
4750 if !matches!(
4761 round_record.e2e_status(),
4762 E2eStatus::Deferred | E2eStatus::NotConfigured
4763 ) {
4764 round_record.verified_head = Some(head.clone());
4765 round_record.verified_at = Some(Timestamp::now());
4766 }
4767 let this_round_verification = round_record.verification_summary(&head);
4768
4769 if incomplete {
4770 let missing: Vec<String> = round_record
4771 .reviews
4772 .iter()
4773 .filter(|r| r.failed.is_some())
4774 .map(|r| format!("review-{}", r.reviewer))
4775 .collect();
4776 self.state.event(
4777 "review",
4778 format!(
4779 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4780 missing.join(", ")
4781 ),
4782 );
4783 }
4784
4785 if clean {
4786 self.state.event(
4787 "review",
4788 if incomplete && policy == IncompleteReviewPolicy::Warn {
4789 format!(
4790 "round {round}: clean (warn policy, incomplete panel) — no \
4791 blocking findings from the seats that answered, verification green"
4792 )
4793 } else if incomplete {
4794 format!(
4795 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4796 quorum) — no blocking findings from the seats that answered, \
4797 verification green",
4798 expected - answered
4799 )
4800 } else {
4801 format!("round {round}: clean — no blocking findings, verification green")
4802 },
4803 );
4804 self.state.reviews.push(round_record);
4805 self.state.status = RunStatus::Gating;
4806 self.state.save()?;
4807 return Ok(());
4808 }
4809
4810 if incomplete && blocking == 0 && e2e_ok {
4818 self.state.reviews.push(round_record);
4819 self.state.save()?;
4820 if round == max_rounds {
4821 self.state.status = RunStatus::Blocked;
4822 self.state.event(
4823 "review",
4824 format!(
4825 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4826 refusing to call it clean",
4827 expected - answered
4828 ),
4829 );
4830 return Ok(());
4831 }
4832 continue;
4833 }
4834
4835 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4847 self.state.reviews.push(round_record);
4848 return self
4849 .stop_reviewing(
4850 "the round's own verification could not run",
4851 &shell,
4852 &winner.worktree,
4853 )
4854 .await;
4855 }
4856
4857 if round == max_rounds {
4858 self.state.reviews.push(round_record);
4859 return self
4860 .stop_reviewing(
4861 &format!(
4862 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4863 ),
4864 &shell,
4865 &winner.worktree,
4866 )
4867 .await;
4868 }
4869
4870 let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4873 let seat = self.seat(&fix_seat_key, &fix_spec.id);
4874 let blocking_findings: Vec<_> = all_findings
4875 .iter()
4876 .filter(|f| f.severity.blocks())
4877 .cloned()
4878 .collect();
4879 let job = SeatJob {
4880 prompt: prompt::fix(
4881 &self.state.instruction,
4882 &blocking_findings,
4883 this_round_verification.as_ref(),
4884 round,
4885 max_rounds,
4886 &language,
4887 ),
4888 spec: fix_spec.clone(),
4889 seat,
4890 cwd: winner.worktree.clone(),
4891 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4892 allow_write: true,
4893 sessions,
4894 artifacts: artifacts.clone(),
4895 stem: format!("fix-{round}"),
4896 };
4897 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4898 let cache = self.state.config.cache_dir();
4899 let ctx = WaveCtx {
4900 run: &run_id,
4901 node: "fix",
4902 prompts: &prompts,
4903 cache: cache.as_deref(),
4904 round: Some(round),
4905 };
4906 let (seat, out) =
4907 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4908 let agent_id = seat.agent.clone();
4909
4910 let mut fix = FixRecord {
4911 agent: agent_id,
4912 addressed: Vec::new(),
4913 rejected: Vec::new(),
4914 notes: String::new(),
4915 committed: false,
4916 failed: None,
4917 duration_ms: 0,
4918 continuation: None,
4919 };
4920 let mut continuation = ContinuationRecord::not_needed();
4921 let mut final_seat = seat.clone();
4922 match out {
4923 AgentOutcome::Ok(o) => {
4924 fix.duration_ms = o.duration_ms;
4925 let parsed = verdict::extract_json::<FixReport>(&o.text);
4926 let incomplete_reason = match &parsed {
4933 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4934 "the reply parsed, but it reported a command whose own CLI never \
4935 confirmed an exit status"
4936 .to_owned(),
4937 ),
4938 Ok(_) => None,
4939 Err(e) => Some(e.to_string()),
4940 };
4941 match incomplete_reason {
4942 None => {
4943 let report = parsed.expect("checked Ok above");
4944 fix.addressed = report.addressed;
4945 fix.rejected = report.rejected;
4946 fix.notes =
4947 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4948 }
4949 Some(reason) => {
4950 let (resumed_seat, resolved, failure, cont) = self
4951 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4952 .await;
4953 fix.duration_ms += cont.cumulative_wait_ms;
4954 continuation = cont;
4955 final_seat = resumed_seat;
4956 match resolved {
4957 Some(report) => {
4958 fix.addressed = report.addressed;
4959 fix.rejected = report.rejected;
4960 fix.notes = blind::sanitize_prose(
4961 &report.notes,
4962 &self.state.config.blind,
4963 );
4964 }
4965 None => fix.failed = failure,
4966 }
4967 }
4968 }
4969 }
4970 AgentOutcome::Dropped(o) => {
4972 fix.duration_ms = o.duration_ms;
4973 let why = o
4974 .dropped
4975 .as_ref()
4976 .map(|d| d.why.as_str())
4977 .unwrap_or("the CLI ended the stream without delivering its answer");
4978 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4979 }
4980 AgentOutcome::Quota(o) => {
4981 self.state.quota.push(QuotaLoss {
4982 seat: final_seat.key.clone(),
4983 node: "fix".to_owned(),
4984 at: Timestamp::now(),
4985 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4986 });
4987 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4988 }
4989 AgentOutcome::Failed(e) => fix.failed = Some(e),
4990 }
4991 fix.continuation = Some(continuation);
4992 self.state.seats.insert(final_seat.key.clone(), final_seat);
4993 if let Ok(r) = git::rescue_commit(
4994 &winner.worktree,
4995 &format!("magi: review round {round} fixes (uncommitted work)"),
4996 )
4997 .await
4998 {
4999 self.state.note_withheld("fix", &r.withheld);
5000 }
5001 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5002 fix.committed = after != before;
5003 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5011 let progressed = diff_after != patch;
5012 let commit_note = if fix.committed {
5013 "committed"
5014 } else {
5015 "NO new commit"
5016 };
5017 let tree_note = if progressed {
5018 "changed vs base"
5019 } else {
5020 "unchanged vs base"
5021 };
5022 self.state.event(
5023 "fix",
5024 match &fix.failed {
5025 Some(reason) => {
5031 format!(
5032 "round {round}: fixer's adoption report was lost ({reason}); \
5033 {commit_note}, tree {tree_note}"
5034 )
5035 }
5036 None => format!(
5037 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5038 {tree_note}{}",
5039 fix.addressed.len(),
5040 fix.rejected.len(),
5041 if continuation.outcome == ContinuationOutcome::Resumed {
5042 format!(
5043 " (adoption report recovered after {} continuation(s))",
5044 continuation.attempts
5045 )
5046 } else {
5047 String::new()
5048 },
5049 ),
5050 },
5051 );
5052 round_record.fix = Some(fix);
5053 round_record.progressed = progressed;
5054 self.state.reviews.push(round_record);
5055 self.state.save()?;
5056
5057 if matches!(
5070 continuation.outcome,
5071 ContinuationOutcome::Exhausted
5072 | ContinuationOutcome::QuotaLost
5073 | ContinuationOutcome::NoSession
5074 ) {
5075 return self
5076 .stop_reviewing(
5077 "the fixer's adoption report never came back, even after resuming its \
5078 own seat; refusing to start another round against the same worktree \
5079 while that is unresolved",
5080 &shell,
5081 &winner.worktree,
5082 )
5083 .await;
5084 }
5085
5086 let streak = self
5087 .state
5088 .reviews
5089 .iter()
5090 .rev()
5091 .take_while(|r| !r.progressed)
5092 .count();
5093 if streak >= STAGNANT_LIMIT {
5094 return self
5095 .stop_reviewing(
5096 &format!(
5097 "the tree has not moved against base for {streak} round(s) in a row"
5098 ),
5099 &shell,
5100 &winner.worktree,
5101 )
5102 .await;
5103 }
5104 }
5105 Ok(())
5106 }
5107
5108 fn record_contested_handoff(&mut self) {
5142 if self.state.contested_handoff.is_some() {
5143 return;
5144 }
5145 let Some(contested) = self
5146 .state
5147 .reviews
5148 .last()
5149 .and_then(ReviewRound::contested_handoff)
5150 else {
5151 return;
5152 };
5153 self.state.event(
5154 "review",
5155 format!(
5156 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5157 wait for the owner's approval",
5158 contested.findings.len(),
5159 contested.rejecters.len()
5160 ),
5161 );
5162 self.state.contested_handoff = Some(contested);
5163 }
5164
5165 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5166 let round_idx = self.state.reviews.len() - 1;
5167 let needs_catchup_run = matches!(
5175 self.state.reviews[round_idx].e2e_status(),
5176 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5177 );
5178 if needs_catchup_run {
5179 let round = self.state.reviews[round_idx].round;
5180 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5181 let commands = self.state.config.verify.e2e.clone();
5182 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5183 let cache_dir = self.state.config.cache_dir();
5184 let context = format!(
5185 "round {round}: verification unresolved, catching up before the final decision"
5186 );
5187 let (outcomes, verify_retried) = with_cache_lease(
5188 &mut self.state,
5189 cache_dir.as_deref(),
5190 "e2e",
5191 "e2e",
5192 worktree,
5193 &attempted_head,
5194 timeout,
5195 &context,
5196 |state, budget| {
5197 let shell = shell.to_vec();
5198 let commands = commands.clone();
5199 let context = context.clone();
5200 async move {
5201 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5202 .await
5203 }
5204 },
5205 )
5206 .await;
5207 let last = &mut self.state.reviews[round_idx];
5208 last.e2e = outcomes;
5209 last.verify_retried = verify_retried;
5210 last.verified_head = Some(attempted_head);
5217 last.verified_at = Some(Timestamp::now());
5218 if verify_inconclusive(&last.e2e) {
5219 self.state.save()?;
5226 return Ok(());
5227 }
5228 last.e2e_deferred = false;
5229 }
5230 let last = &self.state.reviews[round_idx];
5231 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5232
5233 match last.e2e_status() {
5234 E2eStatus::Failed => {
5235 let red: Vec<String> = last
5236 .e2e
5237 .iter()
5238 .filter(|o| !o.ok())
5239 .map(|o| {
5240 format!(
5241 "`{}` -> {:?}\n{}",
5242 o.command,
5243 o.code,
5244 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5245 )
5246 })
5247 .collect();
5248 self.state
5249 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5250 self.state.status = RunStatus::Blocked;
5251 }
5252 E2eStatus::ResourceBlocked => {
5257 self.state.event(
5258 "review",
5259 format!(
5260 "{why}; e2e could not run (shared build cache unavailable); not \
5261 deciding yet"
5262 ),
5263 );
5264 }
5265 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5266 self.state.event(
5267 "review",
5268 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5269 );
5270 self.record_contested_handoff();
5271 self.state.status = RunStatus::Gating;
5272 }
5273 }
5274 self.state.save()?;
5275 Ok(())
5276 }
5277
5278 async fn gate(&mut self) -> Result<()> {
5281 if self.state.status == RunStatus::Failed
5293 || self
5294 .state
5295 .base_sync
5296 .as_ref()
5297 .is_some_and(|s| s.conflict.is_some())
5298 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5299 != Some(RunStatus::Gating)
5300 {
5301 return Ok(());
5302 }
5303 if self.state.gate_ran {
5304 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5315 self.state.status = RunStatus::Blocked;
5316 self.state.save()?;
5317 }
5318 return Ok(());
5319 }
5320 let Some(winner) = self.state.winner().cloned() else {
5321 return Ok(());
5322 };
5323 self.state.status = RunStatus::Gating;
5324 let mut outcomes = self.run_gate(&winner).await?;
5325 loop {
5326 if verify_inconclusive(&outcomes) {
5337 self.state.save()?;
5338 return Ok(());
5339 }
5340 if outcomes.iter().all(CommandOutcome::ok) {
5341 break;
5342 }
5343 match self.gate_fix_round(&winner, &outcomes).await? {
5344 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5345 GateFix::Stop => break,
5346 GateFix::Defer => {
5347 self.state.save()?;
5348 return Ok(());
5349 }
5350 }
5351 }
5352 let passed = outcomes.iter().all(CommandOutcome::ok);
5353 self.state.gate = outcomes;
5354 self.state.gate_ran = true;
5355 if !passed {
5356 self.state.status = RunStatus::Blocked;
5357 let spent = self.state.gate_fixes.len();
5358 self.state.event(
5359 "gate",
5360 if spent == 0 {
5361 "gate failed; not merging".to_owned()
5362 } else {
5363 format!("gate failed after {spent} gate-fix round(s); not merging")
5364 },
5365 );
5366 }
5367 self.state.save()?;
5368 Ok(())
5369 }
5370
5371 async fn run_pre_gate(&mut self, winner: &Candidate) {
5381 let commands = self.state.config.verify.pre_gate.clone();
5382 if commands.is_empty() {
5383 return;
5384 }
5385 let shell = self.state.config.shell();
5386 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5387 let (outcomes, _) = run_commands(
5388 &mut self.state,
5389 "pre_gate",
5390 "pre_gate",
5391 0,
5392 &shell,
5393 &commands,
5394 &winner.worktree,
5395 timeout,
5396 )
5397 .await;
5398 for o in &outcomes {
5399 if !o.ok() {
5400 tracing::warn!(
5401 "pre_gate `{}` failed ({:?}); the gate decides",
5402 o.command,
5403 o.code
5404 );
5405 }
5406 self.state.event(
5407 "pre_gate",
5408 format!(
5409 "`{}` -> {}",
5410 o.command,
5411 if o.ok() {
5412 "pass".to_owned()
5413 } else {
5414 format!(
5415 "FAIL ({:?})\n{}",
5416 o.code,
5417 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5418 )
5419 }
5420 ),
5421 );
5422 }
5423 self.state.pre_gate = outcomes;
5424 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5425 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5426 Ok(head) => {
5427 self.state
5428 .event("pre_gate", format!("committed mechanical fixes ({head})"));
5429 self.state.pre_gate_commit = Some(head);
5430 }
5431 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5432 },
5433 Ok(false) => {}
5434 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5435 }
5436 if let Err(e) = self.state.save() {
5437 tracing::warn!("could not persist the pre_gate record: {e:#}");
5438 }
5439 }
5440
5441 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5444 self.run_pre_gate(winner).await;
5445 let shell = self.state.config.shell();
5446 let gate_commands = self.state.config.verify.gate.clone();
5447 let outcomes = if gate_commands.is_empty() {
5456 Vec::new()
5457 } else {
5458 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5459 let cache_dir = self.state.config.cache_dir();
5460 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5461 let (outcomes, _) = with_cache_lease(
5462 &mut self.state,
5463 cache_dir.as_deref(),
5464 "gate",
5465 "gate",
5466 &winner.worktree,
5467 &head,
5468 timeout,
5469 "final gate",
5470 |state, budget| {
5471 let shell = shell.clone();
5472 let gate_commands = gate_commands.clone();
5473 let worktree = winner.worktree.clone();
5474 async move {
5475 let (outcomes, timed_out_pids) = run_commands(
5476 state,
5477 "gate",
5478 "gate",
5479 0,
5480 &shell,
5481 &gate_commands,
5482 &worktree,
5483 budget,
5484 )
5485 .await;
5486 (outcomes, false, timed_out_pids)
5487 }
5488 },
5489 )
5490 .await;
5491 outcomes
5492 };
5493 if outcomes.is_empty() {
5494 self.state.event(
5499 "gate",
5500 "no gate commands configured; nothing to check, passing",
5501 );
5502 }
5503 for o in &outcomes {
5504 self.state.event(
5505 "gate",
5506 format!(
5507 "`{}` -> {}",
5508 o.command,
5509 if o.ok() {
5510 "pass".to_owned()
5511 } else {
5512 format!(
5513 "FAIL ({:?})\n{}",
5514 o.code,
5515 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5516 )
5517 }
5518 ),
5519 );
5520 }
5521 Ok(outcomes)
5522 }
5523
5524 async fn gate_fix_round(
5537 &mut self,
5538 winner: &Candidate,
5539 outcomes: &[CommandOutcome],
5540 ) -> Result<GateFix> {
5541 let cap = self.state.config.graph.gate_fix_rounds;
5542 let spent = self.state.gate_fixes.len();
5543 if spent >= cap {
5544 if cap > 0 {
5545 self.state.event(
5546 "gate",
5547 format!("{spent} gate-fix round(s) spent and the gate still fails"),
5548 );
5549 }
5550 return Ok(GateFix::Stop);
5551 }
5552 if !gate_fixable(outcomes) {
5553 self.state.event(
5554 "gate",
5555 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5556 command or similar); not spending a fix round on it",
5557 );
5558 return Ok(GateFix::Stop);
5559 }
5560 let min_free = self.state.config.disk.min_free_bytes;
5561 if min_free > 0 {
5562 match crate::disk::free_bytes(&winner.worktree) {
5563 Ok(free) if crate::disk::enough_space(free, min_free) => {}
5564 Ok(free) => {
5565 self.state.event(
5566 "gate",
5567 format!(
5568 "only {free} bytes free ({min_free} required by `[disk] \
5569 min_free_bytes`); not spending a fix round on a failure the disk \
5570 may explain"
5571 ),
5572 );
5573 return Ok(GateFix::Stop);
5574 }
5575 Err(e) => {
5576 self.state.event(
5577 "gate",
5578 format!("free disk space could not be measured ({e:#}); no fix round"),
5579 );
5580 return Ok(GateFix::Stop);
5581 }
5582 }
5583 }
5584
5585 let attempt = spent + 1;
5586 let run_id = self.state.id.clone();
5587 let prompts = self.state.config.prompts.clone();
5588 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5589 let base = self.landing_base();
5590 let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5591 let seat = self.seat(&fix_seat_key, &fix_spec.id);
5592 let job = SeatJob {
5593 prompt: prompt::gate_fix(
5594 &self.state.instruction,
5595 &failed,
5596 attempt,
5597 cap,
5598 &self.state.config.graph.language,
5599 ),
5600 spec: fix_spec,
5601 seat,
5602 cwd: winner.worktree.clone(),
5603 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5604 allow_write: true,
5605 sessions: self.state.config.graph.sessions,
5606 artifacts: agent::artifacts_dir(&self.state.dir()),
5607 stem: format!("gate-fix-{attempt}"),
5608 };
5609 self.state.event(
5610 "gate",
5611 format!("gate failed; gate-fix round {attempt} of {cap}"),
5612 );
5613 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5614 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5615 let cache = self.state.config.cache_dir();
5616 let ctx = WaveCtx {
5617 run: &run_id,
5618 node: "gate-fix",
5619 prompts: &prompts,
5620 cache: cache.as_deref(),
5621 round: None,
5622 };
5623 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5624 let mut record = GateFixRecord {
5625 agent: seat.agent.clone(),
5626 failed,
5627 notes: String::new(),
5628 committed: false,
5629 error: None,
5630 };
5631 match out {
5632 AgentOutcome::Ok(o) => {
5633 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5636 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5637 }
5638 }
5639 AgentOutcome::Dropped(_) => {
5640 record.error = Some("the CLI dropped the stream".to_owned());
5641 }
5642 AgentOutcome::Quota(o) => {
5643 self.state.quota.push(QuotaLoss {
5644 seat: seat.key.clone(),
5645 node: "gate-fix".to_owned(),
5646 at: Timestamp::now(),
5647 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5648 });
5649 record.error = Some("rate limited (quota); fixer could not run".to_owned());
5650 }
5651 AgentOutcome::Failed(e) => record.error = Some(e),
5652 }
5653 self.state.seats.insert(seat.key.clone(), seat);
5654 if let Ok(r) = git::rescue_commit(
5655 &winner.worktree,
5656 &format!("magi: gate fix {attempt} (uncommitted work)"),
5657 )
5658 .await
5659 {
5660 self.state.note_withheld("gate-fix", &r.withheld);
5661 }
5662 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5663 record.committed = after != before;
5664 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5665 let note = record.error.clone();
5666 self.state.gate_fixes.push(record);
5667 self.state.save()?;
5668 if !changed {
5669 self.state.event(
5670 "gate",
5671 match note {
5672 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5673 None => format!("gate-fix round {attempt}: the tree did not change"),
5674 },
5675 );
5676 return Ok(GateFix::Stop);
5677 }
5678 self.state.event(
5679 "gate",
5680 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5681 );
5682
5683 let commands = self.state.config.verify.e2e.clone();
5684 if !commands.is_empty() {
5685 let shell = self.state.config.shell();
5686 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5687 let cache_dir = self.state.config.cache_dir();
5688 let context = format!("gate-fix round {attempt}");
5689 let (e2e, _) = with_cache_lease(
5690 &mut self.state,
5691 cache_dir.as_deref(),
5692 "e2e",
5693 "e2e",
5694 &winner.worktree,
5695 &after,
5696 timeout,
5697 &context,
5698 |state, budget| {
5699 let shell = shell.clone();
5700 let commands = commands.clone();
5701 let context = context.clone();
5702 let worktree = winner.worktree.clone();
5703 async move {
5704 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5705 .await
5706 }
5707 },
5708 )
5709 .await;
5710 if verify_inconclusive(&e2e) {
5711 return Ok(GateFix::Defer);
5712 }
5713 if e2e.iter().any(|o| !o.ok()) {
5714 self.state.event(
5715 "gate",
5716 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5717 );
5718 return Ok(GateFix::Stop);
5719 }
5720 }
5721 Ok(GateFix::Retry)
5722 }
5723
5724 async fn merge(&mut self) -> Result<()> {
5727 if self
5742 .state
5743 .base_sync
5744 .as_ref()
5745 .is_some_and(|s| s.conflict.is_some())
5746 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5747 != Some(RunStatus::Gating)
5748 || !self.state.gate_status().ok()
5757 {
5758 return Ok(());
5759 }
5760 if self.state.merge.is_some() {
5769 return Ok(());
5770 }
5771 let Some(winner) = self.state.winner().cloned() else {
5772 return Ok(());
5773 };
5774 let repo = self.state.repo.clone();
5775 let base = self.state.base_branch.clone();
5776 let mode = self.state.config.merge.mode;
5777 let style = self.state.config.merge.style;
5778 let pr = pr_message(&self.state, winner.label);
5779 let message = pr.commit_message();
5780
5781 let outcome = match mode {
5782 MergeMode::None => MergeOutcome {
5783 mode,
5784 ok: true,
5785 detail: manual_merge_command(style, &repo, &winner.branch, &message),
5786 empty: false,
5787 },
5788 MergeMode::Pr | MergeMode::Local
5789 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
5790 {
5791 MergeOutcome {
5792 mode,
5793 ok: false,
5794 detail: empty_candidate_detail(&self.state, &base),
5795 empty: true,
5796 }
5797 }
5798 MergeMode::Local => {
5799 let on = git::current_branch(&repo).await?;
5800 if on.as_deref() != Some(base.as_str()) {
5801 MergeOutcome {
5802 mode,
5803 ok: false,
5804 detail: format!(
5805 "{} has {} checked out, not the base branch {base}",
5806 repo.display(),
5807 on.unwrap_or_else(|| "a detached HEAD".to_owned())
5808 ),
5809 empty: false,
5810 }
5811 } else if !git::is_clean(&repo).await? {
5812 MergeOutcome {
5813 mode,
5814 ok: false,
5815 detail: format!("{} is dirty; refusing to merge", repo.display()),
5816 empty: false,
5817 }
5818 } else {
5819 let out = match style {
5820 MergeStyle::Merge => {
5821 git::merge_no_ff(&repo, &winner.branch, &message).await?
5822 }
5823 MergeStyle::Squash => {
5824 git::merge_squash(&repo, &winner.branch, &message).await?
5825 }
5826 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5827 };
5828 MergeOutcome {
5829 mode,
5830 ok: out.ok(),
5831 detail: if out.ok() { out.stdout } else { out.stderr },
5832 empty: false,
5833 }
5834 }
5835 }
5836 MergeMode::Pr => {
5837 let remote = self.state.config.merge.remote.clone();
5838 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5839 if !pushed.ok() {
5840 MergeOutcome {
5841 mode,
5842 ok: false,
5843 detail: pushed.stderr,
5844 empty: false,
5845 }
5846 } else {
5847 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
5853 let out = match pr_merge_plan(found) {
5854 PrPlan::Create => {
5855 gh_pr_create(
5856 &winner.worktree,
5857 &base,
5858 &winner.branch,
5859 &pr.title,
5860 &pr.body,
5861 )
5862 .await
5863 }
5864 PrPlan::Adopt { url, title } => {
5865 self.state
5866 .event("merge", format!("Pr: adopted open pull request {url}"));
5867 if title != pr.title
5868 && let Err(e) =
5869 land::set_pr_title(&winner.worktree, &url, &pr.title).await
5870 {
5871 tracing::warn!("could not refresh title of {url}: {e:#}");
5872 self.state
5873 .event("merge", format!("Pr: title refresh failed: {e:#}"));
5874 }
5875 Ok(url)
5876 }
5877 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
5878 };
5879 match out {
5880 Ok(url) => MergeOutcome {
5881 mode,
5882 ok: true,
5883 detail: url,
5884 empty: false,
5885 },
5886 Err(e) => MergeOutcome {
5887 mode,
5888 ok: false,
5889 detail: e.to_string(),
5890 empty: false,
5891 },
5892 }
5893 }
5894 }
5895 };
5896
5897 self.state.status = match (mode, outcome.ok) {
5898 (MergeMode::None, _) => RunStatus::Ready,
5899 (_, true) => RunStatus::Merged,
5900 (_, false) => RunStatus::Blocked,
5901 };
5902 self.state.event(
5903 "merge",
5904 format!(
5905 "{:?}: {}",
5906 mode,
5907 outcome.detail.lines().next().unwrap_or("")
5908 ),
5909 );
5910 self.state.merge = Some(outcome);
5911 self.state.save()?;
5912
5913 if self.state.config.graph.land
5919 && mode == MergeMode::Pr
5920 && self.state.status == RunStatus::Merged
5921 {
5922 self.run_land().await?;
5923 }
5924 self.settle_questions();
5929 Ok(())
5930 }
5931
5932 async fn run_land(&mut self) -> Result<()> {
5943 let url = self
5944 .state
5945 .merge
5946 .as_ref()
5947 .map(|m| m.detail.clone())
5948 .unwrap_or_default();
5949 let url = url.lines().next().unwrap_or("").trim().to_owned();
5950 if !url.starts_with("http") {
5951 return Ok(());
5952 }
5953 match land::land(&mut self.state, &url).await {
5956 Ok(pr) if self.state.parked => {
5957 let _ = pr;
5961 }
5962 Ok(pr) => {
5963 self.state.status = match pr.state {
5964 land::PrLifecycle::Merged => RunStatus::Merged,
5965 _ => RunStatus::Blocked,
5966 };
5967 if bump::should_release_bump(self.state.status)
5974 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
5975 {
5976 self.state
5982 .event("bump", format!("release bump skipped: {e:#}"));
5983 }
5984 if self.state.status == RunStatus::Merged {
5987 crate::followup::after_merge(&mut self.state, &pr.url).await;
5988 }
5989 self.state.save()?;
5990 }
5991 Err(e) => {
5992 self.state.status = RunStatus::Blocked;
5993 self.state.event("land", format!("gave up: {e}"));
5994 self.state.save()?;
5995 }
5996 }
5997 Ok(())
5998 }
5999
6000 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
6004 if let Some(existing) = self.state.seats.get(key)
6005 && existing.agent == agent
6006 {
6007 return existing.clone();
6008 }
6009 let fresh = SeatState::new(key, agent, self.state.seed);
6010 self.state.seats.insert(key.to_owned(), fresh.clone());
6011 fresh
6012 }
6013
6014 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
6020 match self.state.seats.get(key) {
6021 Some(s) if s.agent != spec.id => {
6022 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
6023 }
6024 _ => spec,
6025 }
6026 }
6027
6028 fn view(&self, c: &Candidate) -> CandidateView {
6030 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
6031 .unwrap_or_default();
6032 let (patch, _) = blind::sanitize_patch(
6033 &format!("candidate {} patch", c.label),
6034 &raw,
6035 &self.state.config.blind,
6036 );
6037 CandidateView {
6038 label: c.label,
6039 branch: c.branch.clone(),
6040 summary: c.summary.clone(),
6041 stat: c.stat.clone(),
6042 patch,
6043 }
6044 }
6045
6046 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
6048 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
6049 prompt::judge(
6050 "(see above)",
6051 &views,
6052 self.roles.judges.len(),
6053 base_short,
6054 "en",
6055 )
6056 }
6057
6058 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6065 let mut turns = Vec::new();
6066 for j in &self.state.judgements {
6067 if j.ranking.is_empty() {
6068 continue;
6069 }
6070 let reasons = j
6071 .reasons
6072 .iter()
6073 .map(|(k, v)| format!("- {k}: {v}"))
6074 .collect::<Vec<_>>()
6075 .join("\n");
6076 turns.push(Turn {
6077 who: format!("Judge {} (opening ranking)", j.judge),
6078 is_self: j.judge == self_idx + 1,
6079 body: format!(
6080 "Ranked {}{}{reasons}",
6081 j.ranking.iter().collect::<String>(),
6082 if reasons.is_empty() {
6083 ""
6084 } else {
6085 ", because:\n"
6086 }
6087 ),
6088 });
6089 }
6090 for t in self
6091 .state
6092 .deliberation
6093 .iter()
6094 .flat_map(|r| r.turns.iter())
6095 .chain(current)
6096 {
6097 turns.push(Turn {
6098 who: format!("Judge {}", t.judge),
6099 is_self: t.judge == self_idx + 1,
6100 body: t.body.clone(),
6101 });
6102 }
6103 turns
6104 }
6105}
6106
6107fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6109 agent::has_session(spec.kind, seat, sessions)
6110}
6111
6112fn next_untried_in_roster<'a>(
6133 roster: &'a [AgentSpec],
6134 start: usize,
6135 tried: &BTreeSet<String>,
6136) -> Option<&'a AgentSpec> {
6137 roster
6138 .get(start + 1..)?
6139 .iter()
6140 .find(|s| !tried.contains(&s.id))
6141}
6142
6143fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6147 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6148}
6149
6150const TIMED_OUT: &str = "timed out";
6153
6154#[derive(Debug, Clone, PartialEq, Eq)]
6157enum FailClass {
6158 Quota,
6159 Timeout,
6160 Other(String),
6162}
6163
6164impl FailClass {
6165 fn of(out: &AgentOutcome) -> Option<Self> {
6167 match out {
6168 AgentOutcome::Ok(_) => None,
6169 AgentOutcome::Quota(_) => Some(Self::Quota),
6170 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6171 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6172 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6173 }
6174 }
6175
6176 fn stem_word(&self) -> &'static str {
6178 match self {
6179 Self::Quota => "quota",
6180 _ => "handover",
6181 }
6182 }
6183}
6184
6185fn failure_signature(msg: &str) -> String {
6189 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6190 let mut out = Vec::new();
6191 for word in line.split_whitespace() {
6192 if word.contains('/') || word.contains('\\') {
6193 out.push("<path>".to_owned());
6194 continue;
6195 }
6196 let mut w = String::new();
6197 let mut in_digits = false;
6198 for c in word.chars() {
6199 if c.is_ascii_digit() {
6200 if !in_digits {
6201 w.push('#');
6202 }
6203 in_digits = true;
6204 } else {
6205 in_digits = false;
6206 w.push(c);
6207 }
6208 }
6209 out.push(w);
6210 }
6211 out.join(" ").chars().take(120).collect()
6212}
6213
6214fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6221 match cur {
6222 FailClass::Quota | FailClass::Timeout => true,
6223 FailClass::Other(_) => prev != Some(cur),
6224 }
6225}
6226
6227fn fail_reason(out: &AgentOutcome) -> String {
6229 match out {
6230 AgentOutcome::Ok(_) => String::new(),
6231 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6232 AgentOutcome::Dropped(o) => format!(
6233 "the CLI dropped the stream ({})",
6234 o.dropped
6235 .as_ref()
6236 .map(|d| d.why.as_str())
6237 .unwrap_or("it ended without delivering its answer")
6238 ),
6239 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6240 }
6241}
6242
6243fn record_handover(
6246 state: &mut RunState,
6247 node: &str,
6248 seat: &str,
6249 from: &str,
6250 to: &str,
6251 class: &FailClass,
6252 reason: &str,
6253) {
6254 let message = if *class == FailClass::Quota {
6255 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6256 } else {
6257 format!("{seat}: handed over {from} -> {to} ({reason})")
6258 };
6259 state.event(node, message);
6260 state.handovers.push(Handover {
6261 at: Timestamp::now(),
6262 node: node.to_owned(),
6263 seat: seat.to_owned(),
6264 from: from.to_owned(),
6265 to: to.to_owned(),
6266 reason: reason.to_owned(),
6267 });
6268}
6269
6270fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6284 commands.iter().any(|c| c.exit_code.is_none())
6285}
6286
6287fn verified_noop_claim(
6300 usable: bool,
6301 commands: &[agent::CommandEvidence],
6302 text: &str,
6303) -> Option<String> {
6304 (usable && !has_unconfirmed_command(commands))
6305 .then(|| verdict::verified_noop(text))
6306 .flatten()
6307}
6308
6309fn short(commit: &str) -> String {
6310 commit.chars().take(7).collect()
6311}
6312
6313fn make_executable(path: &Path) -> Result<()> {
6314 #[cfg(unix)]
6315 {
6316 use std::os::unix::fs::PermissionsExt as _;
6317 let mut perms = std::fs::metadata(path)?.permissions();
6318 perms.set_mode(0o755);
6319 std::fs::set_permissions(path, perms)?;
6320 }
6321 #[cfg(not(unix))]
6322 {
6323 let _ = path;
6324 }
6325 Ok(())
6326}
6327
6328struct WaveCtx<'a> {
6335 run: &'a str,
6338 node: &'a str,
6340 prompts: &'a Prompts,
6341 cache: Option<&'a Path>,
6343 round: Option<usize>,
6346}
6347
6348async fn run_one(
6350 job: SeatJob,
6351 sem: Arc<Semaphore>,
6352 ctx: &WaveCtx<'_>,
6353 state: &mut RunState,
6354 attempt: usize,
6355) -> (SeatState, AgentOutcome) {
6356 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6357 .await
6358 .pop()
6359 .expect("one job in, one result out");
6360 (seat, out)
6361}
6362
6363async fn wave(
6369 jobs: Vec<SeatJob>,
6370 sem: Arc<Semaphore>,
6371 ctx: &WaveCtx<'_>,
6372 state: &mut RunState,
6373 attempt: usize,
6374) -> Vec<(usize, SeatState, AgentOutcome)> {
6375 let WaveCtx {
6376 run,
6377 node,
6378 prompts,
6379 cache,
6380 round,
6381 } = *ctx;
6382 for job in &jobs {
6383 state.seat_started(node, &job.seat.key, job.timeout, attempt);
6384 }
6385 if let Err(e) = state.save() {
6386 tracing::warn!("could not persist in-progress seats: {e:#}");
6391 }
6392 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6408 let wait_started = Instant::now();
6409 let cache_guard = if let Some(cache_dir) = cache {
6410 if jobs_had_a_writer {
6411 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6412 let budget = jobs
6413 .iter()
6414 .map(|j| j.timeout)
6415 .max()
6416 .unwrap_or(Duration::from_secs(60));
6417 acquire_cache_lease(state, cache_dir, &owner, budget, node)
6418 .await
6419 .ok()
6420 } else {
6421 None
6422 }
6423 } else {
6424 None
6425 };
6426 let waited_for_lease = wait_started.elapsed();
6433 let mut set = tokio::task::JoinSet::new();
6434 let overlay = prompts.overlay(node);
6435 for (i, mut job) in jobs.into_iter().enumerate() {
6436 job.timeout = job.timeout.saturating_sub(waited_for_lease);
6437 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6438 if cache.is_some() {
6439 job.prompt.push('\n');
6440 job.prompt
6441 .push_str(&prompt::build_cache_note(node, job.allow_write));
6442 }
6443 let sem = Arc::clone(&sem);
6444 let run = run.to_owned();
6445 let node = node.to_owned();
6446 let attachments = if node == "implement" {
6449 state.attachments.clone()
6450 } else {
6451 Vec::new()
6452 };
6453 let cache = cache
6464 .filter(|_| job.allow_write && cache_guard.is_some())
6465 .map(Path::to_path_buf);
6466 set.spawn(async move {
6467 let _permit = sem.acquire().await;
6468 let mut seat = job.seat;
6469 let out = agent::invoke(
6470 &job.spec,
6471 &mut seat,
6472 &Invocation {
6473 cwd: &job.cwd,
6474 prompt: &job.prompt,
6475 timeout: job.timeout,
6476 allow_write: job.allow_write,
6477 sessions: job.sessions,
6478 artifacts: &job.artifacts,
6479 stem: &job.stem,
6480 run: &run,
6481 node: &node,
6482 cache_dir: cache.as_deref(),
6483 attachments: &attachments,
6484 writable: &[],
6485 },
6486 )
6487 .await;
6488 let out = match out {
6489 Ok(o) if o.usable() => AgentOutcome::Ok(o),
6490 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
6491 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
6499 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
6500 Ok(o) => AgentOutcome::Failed(format!(
6501 "exited with {:?} and no usable output",
6502 o.exit_code
6503 )),
6504 Err(e) => AgentOutcome::Failed(e.to_string()),
6505 };
6506 (i, seat, out)
6507 });
6508 }
6509 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
6510 while let Some(joined) = set.join_next().await {
6511 let (i, seat, out) = match joined {
6512 Ok(v) => v,
6513 Err(e) => {
6517 tracing::error!("agent task panicked: {e}");
6518 continue;
6519 }
6520 };
6521 state.seat_finished(&seat.key);
6522 record_jobs(state, node, round, &seat.key, &out);
6523 if let Err(e) = state.save() {
6524 tracing::warn!("could not persist a seat's completion: {e:#}");
6525 }
6526 if collected.len() <= i {
6527 collected.resize_with(i + 1, || None);
6528 }
6529 collected[i] = Some((i, seat, out));
6530 }
6531 if state
6537 .active
6538 .values()
6539 .any(|a| a.node == node && a.attempt == attempt)
6540 {
6541 state
6542 .active
6543 .retain(|_, a| !(a.node == node && a.attempt == attempt));
6544 if let Err(e) = state.save() {
6545 tracing::warn!("could not persist the end of a wave: {e:#}");
6546 }
6547 }
6548 if let Some(cache_dir) = cache
6555 && jobs_had_a_writer
6556 {
6557 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
6558 }
6559 if let Some(guard) = cache_guard {
6560 guard.release();
6561 }
6562 collected.into_iter().flatten().collect()
6563}
6564
6565fn record_jobs(
6576 state: &mut RunState,
6577 node: &str,
6578 round: Option<usize>,
6579 seat: &str,
6580 out: &AgentOutcome,
6581) {
6582 let commands: &[agent::CommandEvidence] = match out {
6583 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
6584 AgentOutcome::Failed(_) => &[],
6585 };
6586 let checked_at = Timestamp::now();
6587 for c in commands {
6588 state.jobs.push(JobRecord {
6589 node: node.to_owned(),
6590 round,
6591 seat: seat.to_owned(),
6592 id: c.id.clone(),
6593 description: c.description.clone(),
6594 checked_at,
6595 status: match c.exit_code {
6596 Some(0) => JobStatus::Completed,
6597 Some(_) => JobStatus::Failed,
6598 None => JobStatus::Unknown,
6599 },
6600 exit_code: c.exit_code,
6601 result_summary: c.result_summary.clone(),
6602 source: c.source.clone(),
6603 });
6604 }
6605}
6606
6607fn round_is_clean(
6628 blocking: usize,
6629 e2e_ok: bool,
6630 answered: usize,
6631 expected: usize,
6632 quota_missing: usize,
6633 policy: IncompleteReviewPolicy,
6634) -> bool {
6635 if blocking != 0 || !e2e_ok {
6636 return false;
6637 }
6638 if answered == expected || policy == IncompleteReviewPolicy::Warn {
6639 return true;
6640 }
6641 answered > 0 && expected - answered <= quota_missing
6642}
6643
6644fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6668 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6669 return Some(RunStatus::Gating);
6670 }
6671 let last = reviews.last()?;
6672 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6673 if reviews.len() < max_rounds && !stagnant {
6674 return None;
6675 }
6676 if last.incomplete() && last.blocking == 0 {
6677 return Some(RunStatus::Blocked);
6678 }
6679 if last.e2e_status() == E2eStatus::ResourceBlocked {
6680 return None;
6681 }
6682 Some(if last.e2e.iter().all(CommandOutcome::ok) {
6683 RunStatus::Gating
6684 } else {
6685 RunStatus::Blocked
6686 })
6687}
6688
6689fn retry_budget(full: Duration, nudged: bool) -> Duration {
6704 if nudged {
6705 (full / 4).max(Duration::from_secs(120)).min(full)
6706 } else {
6707 full
6708 }
6709}
6710
6711#[allow(clippy::too_many_arguments)]
6735async fn ask_json_wave<T>(
6736 jobs: Vec<SeatJob>,
6737 sem: Arc<Semaphore>,
6738 retries: usize,
6739 roster: &[AgentSpec],
6740 ctx: &WaveCtx<'_>,
6741 losses: &mut Vec<QuotaLoss>,
6742 state: &mut RunState,
6743 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
6744) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6745where
6746 T: serde::de::DeserializeOwned + Send + 'static,
6747{
6748 let n = jobs.len();
6749 let originals: Vec<SeatJob> = jobs;
6750 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
6751 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
6752 let mut nudges: Vec<usize> = vec![0; n];
6759 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
6763 let mut tried: Vec<BTreeSet<String>> = specs
6764 .iter()
6765 .map(|s| BTreeSet::from([s.id.clone()]))
6766 .collect();
6767 let starts: Vec<usize> = specs
6768 .iter()
6769 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
6770 .collect();
6771 let mut prev: Vec<Option<FailClass>> = vec![None; n];
6772 let mut fresh: Vec<Option<String>> = vec![None; n];
6773 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
6774 let mut pending: Vec<usize> = (0..n).collect();
6775
6776 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
6779 for round in 0..max_rounds {
6780 if pending.is_empty() {
6781 break;
6782 }
6783 let mut batch = Vec::with_capacity(pending.len());
6784 let mut renudged: Vec<&str> = Vec::new();
6785 for &i in &pending {
6786 let src = &originals[i];
6787 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
6790 (
6791 src.prompt.clone(),
6792 src.timeout,
6793 format!("{}-{word}-{}", src.stem, specs[i].id),
6794 )
6795 } else if nudges[i] == 0 {
6796 (src.prompt.clone(), src.timeout, src.stem.clone())
6797 } else {
6798 renudged.push(src.seat.key.as_str());
6799 let why = done[i]
6800 .as_ref()
6801 .and_then(|r| r.as_ref().err().map(ToString::to_string))
6802 .unwrap_or_else(|| "no parsable answer".to_owned());
6803 let nudge = prompt::nudge(&why);
6804 let nudged = has_context(&specs[i], &seats[i], src.sessions);
6805 let prompt = if nudged {
6806 nudge
6807 } else {
6808 format!("{}\n\n---\n\n{}", src.prompt, nudge)
6809 };
6810 (
6811 prompt,
6812 retry_budget(src.timeout, nudged),
6813 format!("{}-retry{}", src.stem, nudges[i]),
6814 )
6815 };
6816 batch.push(SeatJob {
6817 spec: specs[i].clone(),
6818 seat: seats[i].clone(),
6819 cwd: src.cwd.clone(),
6820 prompt,
6821 timeout,
6822 allow_write: src.allow_write,
6823 sessions: src.sessions,
6824 artifacts: src.artifacts.clone(),
6825 stem,
6826 });
6827 }
6828
6829 if !renudged.is_empty() {
6830 state.event(
6831 ctx.node,
6832 format!("retry {round}: re-asking {}", renudged.join(", ")),
6833 );
6834 }
6835 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
6836 let mut still = Vec::new();
6837 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
6838 seats[i] = seat;
6839 let class = FailClass::of(&out);
6840 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
6843 && nudges[i] < retries
6844 && !roster.is_empty();
6845 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
6846 let next = should_hand_over(prev[i].as_ref(), &cur)
6847 .then(|| next_untried_in_roster(roster, starts[i], &tried[i]).cloned())
6848 .flatten();
6849 if let Some(next) = next {
6850 record_handover(
6851 state,
6852 ctx.node,
6853 &originals[i].seat.key,
6854 &specs[i].id,
6855 &next.id,
6856 &cur,
6857 &fail_reason(&out),
6858 );
6859 tried[i].insert(next.id.clone());
6860 prev[i] = Some(cur.clone());
6861 seats[i] = handover_seat(&originals[i].seat.key, &next.id, state.seed);
6862 specs[i] = next;
6863 fresh[i] = Some(cur.stem_word().to_owned());
6864 nudges[i] = 0;
6865 done[i] = Some(Err(anyhow::anyhow!(
6866 "handed over after: {}",
6867 fail_reason(&out)
6868 )));
6869 still.push(i);
6870 continue;
6871 }
6872 }
6873 let parsed = match out {
6874 AgentOutcome::Ok(o) => match verdict::extract_json::<T>(&o.text) {
6875 Ok(v) => match validate(&v) {
6876 Ok(()) => Ok((v, o)),
6877 Err(e) => Err(e),
6878 },
6879 Err(e) => Err(e),
6880 },
6881 AgentOutcome::Quota(o) => {
6882 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
6883 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
6884 }
6885 AgentOutcome::Dropped(o) => {
6889 let why = o
6890 .dropped
6891 .as_ref()
6892 .map(|d| d.why.as_str())
6893 .unwrap_or("the CLI ended the stream without delivering its answer");
6894 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
6895 }
6896 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
6897 };
6898 let quota = class == Some(FailClass::Quota);
6899 let failed = parsed.is_err();
6900 done[i] = Some(parsed);
6901 let agent_failure = class.is_some() && !roster.is_empty() && !nudge_first;
6910 if failed && !quota && !agent_failure && nudges[i] < retries {
6911 nudges[i] += 1;
6912 still.push(i);
6913 }
6914 }
6915 pending = still;
6916 }
6917
6918 for (i, q) in last_quota.into_iter().enumerate() {
6921 if let Some(reset) = q {
6922 losses.push(QuotaLoss {
6923 seat: originals[i].seat.key.clone(),
6924 node: ctx.node.to_owned(),
6925 at: Timestamp::now(),
6926 reset,
6927 });
6928 }
6929 }
6930
6931 seats
6932 .into_iter()
6933 .zip(done)
6934 .zip(nudges)
6935 .map(|((seat, res), attempts)| {
6936 (
6937 seat,
6938 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
6939 attempts,
6940 )
6941 })
6942 .collect()
6943}
6944
6945async fn acquire_cache_lease(
6958 state: &mut RunState,
6959 cache_dir: &Path,
6960 owner: &crate::cache::Owner,
6961 budget: Duration,
6962 context: &str,
6963) -> Result<crate::cache::Guard> {
6964 let home = crate::run::home();
6965 let started = Instant::now();
6966 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
6967 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
6968 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
6969 Err(e) => {
6970 state.event(
6971 "verify",
6972 format!("{context}: could not check the shared build cache: {e:#}"),
6973 );
6974 if let Err(e2) = state.save() {
6975 tracing::warn!("could not persist a cache-check failure: {e2:#}");
6976 }
6977 return Err(e);
6978 }
6979 };
6980 state.event(
6981 "verify",
6982 format!(
6983 "{context}: waiting for the shared build cache at {} ({})",
6984 cache_dir.display(),
6985 busy.describe()
6986 ),
6987 );
6988 if let Err(e) = state.save() {
6989 tracing::warn!("could not persist a cache wait: {e:#}");
6990 }
6991 let remaining = budget.saturating_sub(started.elapsed());
6992 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
6993 Ok(g) => Ok(g),
6994 Err(e) => {
6995 state.event("verify", format!("{context}: {e:#}"));
6996 if let Err(e2) = state.save() {
6997 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
6998 }
6999 Err(e)
7000 }
7001 }
7002}
7003
7004#[allow(clippy::too_many_arguments)]
7025async fn with_cache_lease<'s, F, Fut>(
7026 state: &'s mut RunState,
7027 cache_dir: Option<&Path>,
7028 node: &str,
7029 seat: &str,
7030 worktree: &Path,
7031 head: &str,
7032 budget: Duration,
7033 context: &str,
7034 body: F,
7035) -> (Vec<CommandOutcome>, bool)
7036where
7037 F: FnOnce(&'s mut RunState, Duration) -> Fut,
7038 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
7039{
7040 let Some(cache_dir) = cache_dir else {
7041 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
7042 return (outcomes, retried);
7043 };
7044 let home = crate::run::home();
7045 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
7046 let started = Instant::now();
7047 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
7048 Ok(g) => g,
7049 Err(e) => {
7050 return (
7051 vec![CommandOutcome {
7052 command: "(waiting for the shared build cache)".to_owned(),
7053 code: None,
7054 output_tail: e.to_string(),
7055 duration_ms: started.elapsed().as_millis() as u64,
7056 resource_blocked: true,
7057 }],
7058 false,
7059 );
7060 }
7061 };
7062 let identity = crate::cache::Identity::new(worktree, head);
7063 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7064 state.event(
7072 "verify",
7073 format!(
7074 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7075 worktree.display(),
7076 short(head)
7077 ),
7078 );
7079 guard.release();
7080 return (
7081 vec![CommandOutcome {
7082 command: "(confirming the shared build cache is fresh)".to_owned(),
7083 code: None,
7084 output_tail: e.to_string(),
7085 duration_ms: started.elapsed().as_millis() as u64,
7086 resource_blocked: true,
7087 }],
7088 false,
7089 );
7090 }
7091 let remaining = budget.saturating_sub(started.elapsed());
7092 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7093 if !timed_out_pids.is_empty() {
7098 wait_for_timed_out_children_to_die(&timed_out_pids).await;
7099 }
7100 guard.release();
7101 (outcomes, retried)
7102}
7103
7104async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7116 wait_for_pids_with(
7117 pids,
7118 crate::proc::pid_alive,
7119 LEASE_RELEASE_POLL,
7120 LEASE_RELEASE_MAX_WAIT,
7121 )
7122 .await;
7123}
7124
7125async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7131 pids: &[u32],
7132 alive: F,
7133 poll: Duration,
7134 max_wait: Duration,
7135) {
7136 let deadline = Instant::now() + max_wait;
7137 loop {
7138 if pids.iter().all(|&pid| !alive(pid)) {
7139 return;
7140 }
7141 if Instant::now() >= deadline {
7142 return;
7143 }
7144 tokio::time::sleep(poll).await;
7145 }
7146}
7147
7148fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7155 outcomes.iter().any(|o| o.resource_blocked)
7156}
7157
7158enum GateFix {
7160 Retry,
7162 Stop,
7165 Defer,
7168}
7169
7170fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7178 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7179 red.peek().is_some()
7180 && red.all(|o| {
7181 !o.resource_blocked
7182 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7183 && !o.output_tail.trim().is_empty()
7184 })
7185}
7186
7187fn e2e_outcome_label(o: &CommandOutcome) -> String {
7191 if o.ok() {
7192 return "pass".to_owned();
7193 }
7194 let reason = if o.build_failed() {
7195 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7196 } else {
7197 format!("FAIL ({:?})", o.code)
7198 };
7199 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7200}
7201
7202async fn run_e2e_with_retry(
7210 state: &mut RunState,
7211 shell: &[String],
7212 commands: &[String],
7213 worktree: &Path,
7214 timeout: Duration,
7215 context: &str,
7216) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7217 let (mut e2e, mut timed_out_pids) = run_commands(
7218 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7219 )
7220 .await;
7221 for o in &e2e {
7222 state.event(
7223 "verify",
7224 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7225 );
7226 }
7227 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7231 if verify_retried {
7232 state.event(
7233 "verify",
7234 format!(
7235 "{context}: verify could not build/link, not a test result — retrying once \
7236 before concluding"
7237 ),
7238 );
7239 let retried = run_commands(
7240 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7241 )
7242 .await;
7243 e2e = retried.0;
7244 timed_out_pids.extend(retried.1);
7247 for o in &e2e {
7248 state.event(
7249 "verify",
7250 format!(
7251 "{context}: retry `{}` -> {}",
7252 o.command,
7253 e2e_outcome_label(o)
7254 ),
7255 );
7256 }
7257 }
7258 (e2e, verify_retried, timed_out_pids)
7259}
7260
7261#[allow(clippy::too_many_arguments)]
7277async fn run_commands(
7278 state: &mut RunState,
7279 node: &str,
7280 task: &str,
7281 attempt: usize,
7282 shell: &[String],
7283 commands: &[String],
7284 cwd: &Path,
7285 timeout: Duration,
7286) -> (Vec<CommandOutcome>, Vec<u32>) {
7287 if commands.is_empty() {
7288 return (Vec::new(), Vec::new());
7293 }
7294 let mut out = Vec::new();
7295 let mut timed_out_pids = Vec::new();
7296 let total = commands.len();
7297 for (idx, command) in commands.iter().enumerate() {
7298 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7299 if let Err(e) = state.save() {
7300 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7301 }
7302 let started = Instant::now();
7303 let mut cmd = tokio::process::Command::new(&shell[0]);
7304 cmd.quiet();
7305 cmd.args(&shell[1..])
7306 .arg(command)
7307 .current_dir(cwd)
7308 .stdin(std::process::Stdio::null())
7309 .stdout(std::process::Stdio::piped())
7310 .stderr(std::process::Stdio::piped())
7311 .kill_on_drop(true);
7312 let spawned = cmd.spawn();
7313 let (code, body) = match spawned {
7314 Ok(child) => {
7315 let pid = child.id();
7320 match tokio::time::timeout(timeout, child.wait_with_output()).await {
7321 Ok(Ok(o)) => {
7322 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7323 body.push_str(&String::from_utf8_lossy(&o.stderr));
7324 (o.status.code(), body)
7325 }
7326 Ok(Err(e)) => (None, format!("failed to run: {e}")),
7327 Err(_) => {
7328 if let Some(pid) = pid {
7329 timed_out_pids.push(pid);
7330 }
7331 (None, format!("timed out after {}s", timeout.as_secs()))
7332 }
7333 }
7334 }
7335 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7336 };
7337 out.push(CommandOutcome {
7338 command: command.clone(),
7339 code,
7340 output_tail: tail(&body, OUTPUT_TAIL),
7341 duration_ms: started.elapsed().as_millis() as u64,
7342 resource_blocked: false,
7343 });
7344 }
7345 state.task_finished(task);
7346 if let Err(e) = state.save() {
7347 tracing::warn!("could not persist the end of {task}: {e:#}");
7348 }
7349 (out, timed_out_pids)
7350}
7351
7352fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
7364 let repo = repo.display();
7365 match style {
7366 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
7367 MergeStyle::Squash => {
7368 let subject = message
7371 .lines()
7372 .next()
7373 .unwrap_or(branch)
7374 .replace(['\\', '"', '$', '`'], "");
7375 format!(
7376 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
7377 )
7378 }
7379 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
7380 }
7381}
7382
7383const PR_TITLE_MAX: usize = 240;
7394
7395struct PrMessage {
7400 title: String,
7401 body: String,
7402}
7403
7404impl PrMessage {
7405 fn commit_message(&self) -> String {
7409 format!("{}\n\n{}", self.title, self.body)
7410 }
7411}
7412
7413fn title_marker(line: &str) -> Option<&str> {
7415 let line = line.trim();
7416 let head = line.get(..6)?;
7417 head.eq_ignore_ascii_case("title:")
7418 .then(|| line[6..].trim())
7419}
7420
7421fn summary_title(summary: &str) -> Option<String> {
7426 let first = summary.lines().find(|l| !l.trim().is_empty())?;
7427 let raw = title_marker(first)?;
7428 if raw.is_empty() {
7429 return None;
7430 }
7431 let title = queue::title_from(raw, PR_TITLE_MAX);
7432 let lower = title.to_ascii_lowercase();
7433 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7434 return None;
7435 }
7436 Some(title)
7437}
7438
7439const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
7441
7442const EXISTING_BRANCH: &str = "(existing branch)";
7444
7445fn is_review_run(state: &RunState) -> bool {
7449 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
7450}
7451
7452fn review_title(state: &RunState) -> Option<String> {
7457 let raw = state.reviewed_commits.as_ref()?.first()?.trim();
7458 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
7459 return None;
7460 }
7461 let title = queue::title_from(raw, PR_TITLE_MAX);
7462 let lower = title.to_ascii_lowercase();
7463 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7464 return None;
7465 }
7466 Some(title)
7467}
7468
7469fn neutral_title(state: &RunState, winner: char) -> String {
7470 format!(
7471 "chore: land candidate {} of run {}",
7472 winner.to_ascii_uppercase(),
7473 state.id
7474 )
7475}
7476
7477pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
7483 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
7484 ""
7485 } else {
7486 pr_title
7487 }
7488}
7489
7490pub fn landing_subject_source(state: &RunState) -> String {
7494 if is_review_run(state) {
7495 let winner = state.candidates.first().map_or('A', |c| c.label);
7496 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
7497 }
7498 state.instruction.clone()
7499}
7500
7501fn summary_without_title(summary: &str) -> String {
7504 let mut lines = summary.trim().lines().peekable();
7505 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
7506 lines.next();
7507 }
7508 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
7509}
7510
7511fn pr_message(state: &RunState, winner: char) -> PrMessage {
7525 let summary = state
7526 .candidates
7527 .iter()
7528 .find(|c| c.label == winner)
7529 .map(|c| c.summary.as_str())
7530 .unwrap_or_default();
7531 let review = is_review_run(state);
7534 let title = if review {
7535 review_title(state).unwrap_or_else(|| neutral_title(state, winner))
7536 } else {
7537 summary_title(summary).unwrap_or_else(|| {
7538 let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
7539 if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
7540 t
7541 } else {
7542 neutral_title(state, winner)
7543 }
7544 })
7545 };
7546
7547 let mut body = String::new();
7548 let what = summary_without_title(summary);
7549 if !what.is_empty() {
7550 body.push_str("## Summary\n\n");
7551 body.push_str(&what);
7552 body.push_str("\n\n");
7553 }
7554
7555 let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
7556 if let Some(fix) = fix
7557 && !fix.notes.trim().is_empty()
7558 {
7559 body.push_str("## Review fixes\n\n");
7560 body.push_str(fix.notes.trim());
7561 body.push_str("\n\n");
7562 }
7563
7564 let open = state.open_findings();
7565 if !open.is_empty() {
7566 body.push_str("## Open review findings\n\n");
7567 for f in &open {
7568 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
7569 }
7570 body.push('\n');
7571 }
7572
7573 if let Some(fix) = fix
7574 && !fix.rejected.is_empty()
7575 {
7576 body.push_str("## Declined by the fixer\n\n");
7577 for r in &fix.rejected {
7578 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
7579 }
7580 body.push('\n');
7581 }
7582
7583 if review {
7584 body.push_str("## Commits under review\n\n");
7586 match &state.reviewed_commits {
7587 Some(subjects) => {
7588 for s in subjects {
7589 body.push_str(&format!("- {}\n", s.trim()));
7590 }
7591 }
7592 None => {
7593 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
7596 body.push_str(rest.trim());
7597 body.push('\n');
7598 }
7599 }
7600 } else {
7601 let task = state.instruction.trim();
7602 let task = if task.is_empty() {
7603 "(empty task)"
7604 } else {
7605 task
7606 };
7607 body.push_str(&format!(
7608 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
7609 task.replace("</details>", "</details>")
7610 ));
7611 }
7612
7613 body.push_str(&format!(
7614 "\n---\nmagi:run/{} magi:candidate-{}\n",
7615 state.id,
7616 winner.to_ascii_lowercase()
7617 ));
7618
7619 let id = crate::scrub::Identity::current();
7622 PrMessage {
7623 title: crate::scrub::scrub(&title, &id),
7624 body: crate::scrub::scrub(&body, &id),
7625 }
7626}
7627
7628fn seeded_instruction(state: &RunState) -> String {
7632 match refs::describe(&state.seeds) {
7633 Some(facts) => format!(
7634 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
7635 Candidates start from the unmerged branch named above, when there \
7636 is one, and carry any unmerged commit named by sha as a \
7637 cherry-pick. Check that this is what the task meant before \
7638 building on it.",
7639 state.instruction
7640 ),
7641 None => state.instruction.clone(),
7642 }
7643}
7644
7645async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
7649 let base = &state.base_branch;
7650 let mut against = base.clone();
7651 if mode == MergeMode::Pr {
7652 let remote = &state.config.merge.remote;
7653 let tracking = format!("{remote}/{base}");
7654 let fetched = git::fetch(repo, remote, base).await;
7655 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
7656 against = tracking;
7657 }
7658 }
7659 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
7660}
7661
7662fn empty_candidate_detail(state: &RunState, base: &str) -> String {
7665 let mut detail = format!(
7666 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
7667 nothing to open a pull request for"
7668 );
7669 match refs::describe(&state.seeds) {
7670 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
7671 None => detail.push_str(
7672 "\nThe task names no existing branch or commit; if it means to land work \
7673 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
7674 ),
7675 }
7676 detail
7677}
7678
7679#[derive(Debug, PartialEq, Eq)]
7682enum PrPlan {
7683 Create,
7684 Adopt { url: String, title: String },
7685 Stop(String),
7686}
7687
7688fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
7691 match found {
7692 Ok(land::OpenPr::None) => PrPlan::Create,
7693 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
7694 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
7695 "several open pull requests exist for this branch, not picking one: {}",
7696 urls.join(" ")
7697 )),
7698 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
7699 }
7700}
7701
7702async fn gh_pr_create(
7704 cwd: &Path,
7705 base: &str,
7706 head: &str,
7707 title: &str,
7708 body: &str,
7709) -> Result<String> {
7710 let out = tokio::process::Command::new("gh")
7711 .args([
7712 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
7713 ])
7714 .current_dir(cwd)
7715 .quiet()
7716 .stdin(std::process::Stdio::null())
7717 .output()
7718 .await
7719 .context("spawn gh")?;
7720 if out.status.success() {
7721 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
7722 } else {
7723 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
7724 }
7725}
7726
7727pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
7736 let repo = state.repo.clone();
7737 let root = state.worktree_root();
7738 let winner = state.tally.as_ref().map(|t| t.winner);
7739 let mut removed = Vec::new();
7740
7741 for i in 0..state.candidates.len() {
7742 let c = state.candidates[i].clone();
7743 let is_winner = Some(c.label) == winner;
7744 if is_winner && !drop_winner {
7745 continue;
7746 }
7747 if c.worktree.exists() {
7748 git::worktree_remove(&repo, &c.worktree).await.ok();
7749 removed.push(c.worktree.to_string_lossy().into_owned());
7750 }
7751 let handed_over = state.released_branches.contains(&c.branch);
7754 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
7755 git::branch_delete(&repo, &c.branch).await.ok();
7756 removed.push(c.branch.clone());
7757 }
7758 state.candidates[i].folded = true;
7759 }
7760
7761 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
7762 let path = name.path();
7763 let keep = !drop_winner
7764 && winner.is_some_and(|w| {
7765 path.file_name()
7766 .is_some_and(|n| n == format!("cand-{w}").as_str())
7767 });
7768 if keep {
7769 continue;
7770 }
7771 git::worktree_remove(&repo, &path).await.ok();
7772 removed.push(path.to_string_lossy().into_owned());
7773 }
7774
7775 remove_if_empty(&root);
7784
7785 if state.enabled_worktree_config && drop_winner {
7786 git::release_worktree_config(&repo).await.ok();
7790 state.enabled_worktree_config = false;
7791 }
7792 state.save_under(home)?;
7793 Ok(removed)
7794}
7795
7796fn remove_if_empty(dir: &Path) {
7807 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
7808 std::fs::remove_dir(dir).ok();
7809 }
7810}
7811
7812pub fn worst_open(state: &RunState) -> Option<Severity> {
7814 state
7815 .reviews
7816 .last()?
7817 .reviews
7818 .iter()
7819 .flat_map(|r| r.findings.iter())
7820 .map(|f| f.severity)
7821 .max()
7822}
7823
7824#[cfg(test)]
7825mod tests {
7826 #[test]
7827 fn pr_merge_plan_creates_adopts_or_stops() {
7828 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
7829 assert_eq!(
7830 pr_merge_plan(Ok(land::OpenPr::One {
7831 url: "u".into(),
7832 title: "t".into()
7833 })),
7834 PrPlan::Adopt {
7835 url: "u".into(),
7836 title: "t".into()
7837 }
7838 );
7839 let PrPlan::Stop(many) =
7840 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
7841 else {
7842 panic!("many must stop");
7843 };
7844 assert!(many.contains('a') && many.contains('b'));
7845 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
7846 panic!("a failed lookup must stop");
7847 };
7848 assert!(err.contains("bad token"));
7849 }
7850
7851 use super::*;
7852 use crate::run::GateStatus;
7853 use std::collections::BTreeMap;
7854 use std::time::Duration;
7855
7856 fn conductor() -> AgentSpec {
7857 AgentSpec {
7858 id: "conductor".to_owned(),
7859 kind: crate::config::AgentKind::Command,
7860 model: None,
7861 command: vec!["true".to_owned()],
7862 extra_args: Vec::new(),
7863 env: BTreeMap::new(),
7864 prompt_delivery: None,
7865 }
7866 }
7867
7868 fn spec(id: &str) -> AgentSpec {
7869 AgentSpec {
7870 id: id.to_owned(),
7871 kind: crate::config::AgentKind::Command,
7872 model: None,
7873 command: vec!["true".to_owned()],
7874 extra_args: Vec::new(),
7875 env: BTreeMap::new(),
7876 prompt_delivery: None,
7877 }
7878 }
7879
7880 #[test]
7887 fn failure_signature_ignores_numbers_and_paths() {
7888 assert_eq!(
7889 failure_signature("exited with Some(2) and no usable output"),
7890 failure_signature("exited with Some(137) and no usable output")
7891 );
7892 assert_eq!(
7893 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
7894 failure_signature("cannot open /var/x.txt: denied")
7895 );
7896 assert_ne!(failure_signature("boom"), failure_signature("bang"));
7897 }
7898
7899 #[test]
7900 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
7901 let other = FailClass::Other("x".into());
7902 assert!(should_hand_over(None, &FailClass::Quota));
7903 assert!(should_hand_over(Some(&other), &FailClass::Quota));
7904 assert!(should_hand_over(
7905 Some(&FailClass::Timeout),
7906 &FailClass::Timeout
7907 ));
7908 assert!(should_hand_over(None, &other));
7909 assert!(!should_hand_over(Some(&other), &other));
7910 assert!(should_hand_over(
7911 Some(&other),
7912 &FailClass::Other("y".into())
7913 ));
7914 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
7916 assert!(should_hand_over(Some(&FailClass::Quota), &other));
7917 }
7918
7919 #[test]
7920 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
7921 let a = SeatState::new("judge-1", "alpha", 7);
7922 let b = handover_seat("judge-1", "beta", 7);
7923 assert_ne!(a.claude_session, b.claude_session);
7924 assert_eq!(b.turns, 0);
7925 }
7926
7927 #[test]
7928 fn a_timeout_is_classified_apart_from_other_failures() {
7929 assert_eq!(
7930 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
7931 Some(FailClass::Timeout)
7932 );
7933 assert!(matches!(
7934 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
7935 Some(FailClass::Other(_))
7936 ));
7937 }
7938
7939 #[test]
7940 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
7941 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
7942 let tried = BTreeSet::from(["beta".to_owned()]);
7943 let next = next_untried_in_roster(&roster, 1, &tried);
7946 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
7947 }
7948
7949 #[test]
7950 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
7951 let roster = vec![spec("alpha"), spec("beta")];
7952 let tried = BTreeSet::from(["beta".to_owned()]);
7953 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
7957 }
7958
7959 #[test]
7960 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
7961 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
7962 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
7963 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
7967 }
7968
7969 #[test]
7970 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
7971 let roster = vec![spec("a"), spec("a"), spec("b")];
7972 let tried = BTreeSet::from(["a".to_owned()]);
7973 let next = next_untried_in_roster(&roster, 0, &tried);
7974 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
7975 }
7976
7977 #[test]
7978 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
7979 let roster = vec![spec("a"), spec("b")];
7980 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
7981 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
7982 }
7983
7984 #[test]
7985 fn remove_if_empty_only_ever_takes_a_bare_directory() {
7986 let dir = tempfile::tempdir().unwrap();
7987 let bay = dir.path().join("ffff");
7988
7989 remove_if_empty(&bay);
7991 assert!(!bay.exists());
7992
7993 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
7996 remove_if_empty(&bay);
7997 assert!(bay.exists(), "non-empty directory must survive");
7998
7999 std::fs::remove_dir(bay.join("cand-A")).unwrap();
8001 remove_if_empty(&bay);
8002 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
8003 }
8004
8005 #[test]
8014 fn a_full_panel_that_found_nothing_is_clean() {
8015 assert!(round_is_clean(
8016 0,
8017 true,
8018 2,
8019 2,
8020 0,
8021 IncompleteReviewPolicy::Block
8022 ));
8023 }
8024
8025 #[test]
8026 fn a_missing_seat_is_never_clean_under_the_default_policy() {
8027 assert!(!round_is_clean(
8028 0,
8029 true,
8030 1,
8031 2,
8032 0,
8033 IncompleteReviewPolicy::Block
8034 ));
8035 }
8036
8037 #[test]
8038 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
8039 assert!(!round_is_clean(
8040 1,
8041 true,
8042 1,
8043 2,
8044 0,
8045 IncompleteReviewPolicy::Warn
8046 ));
8047 }
8048
8049 #[test]
8050 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
8051 assert!(round_is_clean(
8052 0,
8053 true,
8054 1,
8055 2,
8056 0,
8057 IncompleteReviewPolicy::Warn
8058 ));
8059 }
8060
8061 #[test]
8062 fn a_full_panel_with_an_open_finding_is_not_clean() {
8063 assert!(!round_is_clean(
8064 1,
8065 true,
8066 2,
8067 2,
8068 0,
8069 IncompleteReviewPolicy::Block
8070 ));
8071 }
8072
8073 #[test]
8074 fn a_full_panel_with_a_red_e2e_is_not_clean() {
8075 assert!(!round_is_clean(
8076 0,
8077 false,
8078 2,
8079 2,
8080 0,
8081 IncompleteReviewPolicy::Block
8082 ));
8083 }
8084
8085 #[test]
8092 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
8093 assert!(round_is_clean(
8096 0,
8097 true,
8098 1,
8099 2,
8100 1,
8101 IncompleteReviewPolicy::Block
8102 ));
8103 }
8104
8105 #[test]
8106 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
8107 assert!(!round_is_clean(
8110 0,
8111 true,
8112 1,
8113 2,
8114 0,
8115 IncompleteReviewPolicy::Block
8116 ));
8117 }
8118
8119 #[test]
8120 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
8121 assert!(!round_is_clean(
8122 1,
8123 true,
8124 1,
8125 2,
8126 1,
8127 IncompleteReviewPolicy::Block
8128 ));
8129 assert!(!round_is_clean(
8130 0,
8131 false,
8132 1,
8133 2,
8134 1,
8135 IncompleteReviewPolicy::Block
8136 ));
8137 }
8138
8139 #[test]
8140 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
8141 assert!(!round_is_clean(
8145 0,
8146 true,
8147 0,
8148 2,
8149 2,
8150 IncompleteReviewPolicy::Block
8151 ));
8152 }
8153
8154 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
8155 CommandOutcome {
8156 command: "test".to_owned(),
8157 code,
8158 output_tail: String::new(),
8159 duration_ms: 0,
8160 resource_blocked,
8161 }
8162 }
8163
8164 #[test]
8165 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
8166 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
8167 assert!(
8168 !verify_inconclusive(&[outcome(Some(1), false)]),
8169 "an ordinary failure is still evidence about the patch"
8170 );
8171 assert!(verify_inconclusive(&[outcome(None, true)]));
8172 assert!(
8173 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
8174 "one inconclusive outcome taints the whole batch"
8175 );
8176 assert!(!verify_inconclusive(&[]));
8177 }
8178
8179 #[tokio::test]
8180 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
8181 let calls = std::sync::atomic::AtomicUsize::new(0);
8185 let started = Instant::now();
8186 wait_for_pids_with(
8187 &[123],
8188 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
8189 Duration::from_millis(5),
8190 Duration::from_secs(5),
8191 )
8192 .await;
8193 assert!(
8194 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
8195 "must keep checking rather than deciding on the first answer"
8196 );
8197 assert!(
8198 started.elapsed() < Duration::from_secs(1),
8199 "must return the moment it is confirmed dead, not wait out the ceiling"
8200 );
8201 }
8202
8203 #[tokio::test]
8204 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
8205 let started = Instant::now();
8206 wait_for_pids_with(
8207 &[123],
8208 |_| true, Duration::from_millis(5),
8210 Duration::from_millis(30),
8211 )
8212 .await;
8213 let elapsed = started.elapsed();
8214 assert!(
8215 elapsed >= Duration::from_millis(30),
8216 "must not give up before its own ceiling: {elapsed:?}"
8217 );
8218 assert!(
8219 elapsed < Duration::from_secs(1),
8220 "must not wait past its own ceiling either: {elapsed:?}"
8221 );
8222 }
8223
8224 #[tokio::test]
8225 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
8226 let started = Instant::now();
8227 wait_for_pids_with(
8228 &[],
8229 |_| true,
8230 Duration::from_secs(5),
8231 Duration::from_secs(5),
8232 )
8233 .await;
8234 assert!(
8235 started.elapsed() < Duration::from_millis(200),
8236 "an empty pid list has nothing to confirm"
8237 );
8238 }
8239
8240 fn review_round(
8246 clean: bool,
8247 blocking: usize,
8248 answered: usize,
8249 expected: usize,
8250 progressed: bool,
8251 e2e_ok: bool,
8252 ) -> ReviewRound {
8253 ReviewRound {
8254 round: 1,
8255 head: "h".to_owned(),
8256 verified_head: None,
8257 verified_at: None,
8258 reviews: Vec::new(),
8259 e2e: vec![CommandOutcome {
8260 command: "test".to_owned(),
8261 code: Some(if e2e_ok { 0 } else { 1 }),
8262 output_tail: String::new(),
8263 duration_ms: 0,
8264 resource_blocked: false,
8265 }],
8266 verify_retried: false,
8267 e2e_deferred: false,
8268 e2e_defer_reason: None,
8269 fix: None,
8270 blocking,
8271 answered,
8272 expected,
8273 clean,
8274 progressed,
8275 vote_split: false,
8276 reconsideration: Vec::new(),
8277 verdict: None,
8278 }
8279 }
8280
8281 #[test]
8282 fn review_conclusion_is_none_when_nothing_has_run() {
8283 assert_eq!(review_conclusion(&[], 3), None);
8284 }
8285
8286 #[test]
8287 fn review_conclusion_is_none_while_rounds_remain() {
8288 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
8289 assert_eq!(review_conclusion(&rounds, 3), None);
8290 }
8291
8292 #[test]
8293 fn review_conclusion_is_gating_once_a_round_is_clean() {
8294 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
8295 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
8296 }
8297
8298 #[test]
8299 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
8300 let rounds = vec![
8301 review_round(false, 1, 2, 2, true, true),
8302 review_round(false, 1, 2, 2, true, true),
8303 ];
8304 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
8305 }
8306
8307 #[test]
8308 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
8309 let rounds = vec![
8310 review_round(false, 1, 2, 2, true, true),
8311 review_round(false, 1, 2, 2, true, false),
8312 ];
8313 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
8314 }
8315
8316 #[test]
8317 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
8318 let mut blocked = review_round(false, 1, 2, 2, true, false);
8325 blocked.e2e[0].resource_blocked = true;
8326 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
8327 assert_eq!(review_conclusion(&rounds, 2), None);
8328 }
8329
8330 #[test]
8331 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
8332 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
8334 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
8335 }
8336
8337 #[test]
8338 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
8339 let rounds = vec![
8340 review_round(false, 1, 2, 2, false, true),
8341 review_round(false, 1, 2, 2, false, true),
8342 ];
8343 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
8344 }
8345
8346 fn secs(n: u64) -> Duration {
8347 Duration::from_secs(n)
8348 }
8349
8350 fn init_repo(dir: &Path) {
8353 let run = |args: &[&str]| {
8354 let out = std::process::Command::new("git")
8355 .args(args)
8356 .current_dir(dir)
8357 .quiet()
8358 .output()
8359 .expect("spawn git");
8360 assert!(
8361 out.status.success(),
8362 "git {args:?} failed: {}",
8363 String::from_utf8_lossy(&out.stderr)
8364 );
8365 };
8366 run(&["init", "-b", "main"]);
8367 run(&["config", "user.name", "magi test"]);
8368 run(&["config", "user.email", "magi@example.com"]);
8369 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
8370 run(&["add", "-A"]);
8371 run(&["commit", "-m", "init"]);
8372 }
8373
8374 fn ask_test_home() {
8382 crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
8383 }
8384
8385 fn runner_at(status: RunStatus) -> Runner {
8388 let mut state = RunState::new(
8389 PathBuf::from("/nonexistent/repo"),
8390 "main".to_owned(),
8391 "deadbeef".to_owned(),
8392 "task".to_owned(),
8393 Config::default(),
8394 );
8395 state.status = status;
8396 Runner {
8397 state,
8398 roles: ResolvedRoles {
8399 implementers: Vec::new(),
8400 judges: Vec::new(),
8401 reviewers: Vec::new(),
8402 fixer: None,
8403 conductor: conductor(),
8404 implementer_roster: Vec::new(),
8405 judge_roster: Vec::new(),
8406 reviewer_roster: Vec::new(),
8407 },
8408 sem: Arc::new(Semaphore::new(1)),
8409 pause: Pause::new(),
8410 interrupt: Pause::new(),
8411 }
8412 }
8413
8414 #[test]
8418 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
8419 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8420 let mut runner = runner_at(RunStatus::Implementing);
8421 let interrupt = Pause::new();
8422 runner.watch_interrupt(interrupt.clone());
8423
8424 interrupt.park_because("task a1b2 asked to run first");
8425
8426 assert!(runner.park_here().expect("park_here"));
8427 assert!(runner.state.parked);
8428 let last = runner.state.events.last().expect("a park event");
8429 assert_eq!(last.node, "park");
8430 assert!(
8431 last.message.contains("task a1b2 asked to run first"),
8432 "expected the interrupt reason in {:?}",
8433 last.message
8434 );
8435 }
8436
8437 #[test]
8445 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
8446 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8447 let mut runner = runner_at(RunStatus::Implementing);
8448 let shutdown = Pause::new();
8449 runner.on_pause(shutdown.clone());
8450 let interrupt = Pause::new();
8451 runner.watch_interrupt(interrupt.clone());
8452
8453 assert!(!runner.park_here().expect("park_here"));
8455 assert!(!runner.state.parked);
8456
8457 interrupt.park_because("test");
8459 assert!(!shutdown.parked());
8460 assert!(runner.park_here().expect("park_here"));
8461 }
8462
8463 #[tokio::test]
8477 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
8478 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8479 let mut runner = runner_at(RunStatus::Implementing);
8480 let interrupt = Pause::new();
8481 runner.watch_interrupt(interrupt.clone());
8482
8483 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
8484 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
8485
8486 let node = async move {
8490 started_tx.send(()).expect("send started");
8491 finish_rx.await.expect("recv finish");
8492 "node finished"
8493 };
8494
8495 let interrupter = async move {
8496 started_rx.await.expect("recv started");
8497 interrupt.park_because("higher-priority task waiting");
8499 tokio::task::yield_now().await;
8503 finish_tx.send(()).expect("send finish");
8504 };
8505
8506 let (node_result, ()) = tokio::join!(node, interrupter);
8507 assert_eq!(
8508 node_result, "node finished",
8509 "the in-flight call ran to completion"
8510 );
8511
8512 assert!(runner.park_here().expect("park_here"));
8515 assert!(runner.state.parked);
8516 }
8517
8518 #[test]
8524 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
8525 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8526 let mut runner = runner_at(RunStatus::Judging);
8527 runner.state.config.agents = vec![conductor()];
8531 runner.state.candidates = vec![Candidate {
8532 index: 0,
8533 label: 'A',
8534 agent: "alpha".to_owned(),
8535 branch: "magi/x/A".to_owned(),
8536 worktree: PathBuf::from("/nonexistent/worktree"),
8537 summary: "did the thing".to_owned(),
8538 stat: "1 file changed".to_owned(),
8539 files: 1,
8540 commits: 1,
8541 empty: false,
8542 failed: None,
8543 verified_noop: None,
8544 duration_ms: 1234,
8545 folded: false,
8546 }];
8547 let run_id = runner.state.id.clone();
8548
8549 let interrupt = Pause::new();
8550 runner.watch_interrupt(interrupt.clone());
8551 interrupt.park_because("task c3d4 asked to run first");
8552 assert!(runner.park_here().expect("park_here"));
8553
8554 let resumed = Runner::resume(&run_id).expect("resume");
8555 assert_eq!(resumed.state.candidates.len(), 1);
8556 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
8557 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
8558 assert_eq!(resumed.state.status, runner.state.status);
8559 assert!(
8560 resumed.state.parked,
8561 "still parked until `execute` actually walks the graph again"
8562 );
8563 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
8564 }
8565
8566 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
8568 let mut q = ask::Question::new(
8569 run.to_owned(),
8570 "implement".to_owned(),
8571 "impl-A".to_owned(),
8572 "Which storage backend should the cache use?".to_owned(),
8573 String::new(),
8574 vec!["SQLite".to_owned(), "Redis".to_owned()],
8575 );
8576 store.put(&mut q).unwrap();
8577 q
8578 }
8579
8580 #[test]
8581 fn a_failed_runs_open_question_is_abandoned() {
8582 ask_test_home();
8583 let store = ask::Questions::open();
8584 let mut runner = runner_at(RunStatus::Failed);
8585 let run = runner.state.id.clone();
8586 let q = ask_open_question(&store, &run);
8587
8588 runner.settle_questions();
8589
8590 let back = store.get(&q.id).unwrap();
8591 assert!(
8592 !back.status.open(),
8593 "the seat that asked died with the run; nobody is left to read an answer"
8594 );
8595 assert!(
8596 back.detail.contains(&run) && back.detail.contains("failed"),
8597 "the reason names what the run became, not just that it is gone: {}",
8598 back.detail
8599 );
8600 }
8601
8602 #[test]
8603 fn a_merged_runs_open_question_is_abandoned_too() {
8604 ask_test_home();
8605 let store = ask::Questions::open();
8606 for status in [RunStatus::Merged, RunStatus::Ready] {
8609 let mut runner = runner_at(status);
8610 let run = runner.state.id.clone();
8611 let q = ask_open_question(&store, &run);
8612
8613 runner.settle_questions();
8614
8615 let back = store.get(&q.id).unwrap();
8616 assert!(
8617 !back.status.open(),
8618 "{status:?} run's question must not outlive the run"
8619 );
8620 }
8621 }
8622
8623 #[test]
8624 fn a_still_resumable_runs_open_question_is_left_alone() {
8625 ask_test_home();
8626 let store = ask::Questions::open();
8627 for status in [RunStatus::Blocked, RunStatus::Stalled] {
8633 let mut runner = runner_at(status);
8634 let run = runner.state.id.clone();
8635 let q = ask_open_question(&store, &run);
8636
8637 runner.settle_questions();
8638
8639 let back = store.get(&q.id).unwrap();
8640 assert!(
8641 back.status.open(),
8642 "{status:?} is still alive; the question must still be waiting"
8643 );
8644 }
8645 }
8646
8647 #[test]
8648 fn settle_questions_never_touches_an_already_answered_question() {
8649 ask_test_home();
8650 let store = ask::Questions::open();
8651 let mut runner = runner_at(RunStatus::Failed);
8652 let run = runner.state.id.clone();
8653 let mut q = ask_open_question(&store, &run);
8654 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
8655 .unwrap();
8656 store.put(&mut q).unwrap();
8657
8658 runner.settle_questions();
8663 runner.settle_questions();
8664
8665 let back = store.get(&q.id).unwrap();
8666 assert_eq!(
8667 back.status,
8668 ask::QuestionStatus::Answered,
8669 "a real answer is a decision on record, never overwritten by a sweep"
8670 );
8671 }
8672
8673 #[tokio::test]
8684 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
8685 crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
8686 let tmp = tempfile::tempdir().expect("tempdir");
8687 let repo = tmp.path().join("repo");
8688 std::fs::create_dir_all(&repo).unwrap();
8689 init_repo(&repo);
8690
8691 let mut config = Config::default();
8692 config.graph.worktree_root = Some(tmp.path().join("wt"));
8693
8694 let mut state = RunState::new(
8695 repo.clone(),
8696 "main".to_owned(),
8697 "deadbeef".to_owned(),
8698 "task".to_owned(),
8699 config,
8700 );
8701 let root = state.worktree_root();
8702 let wt_a = root.join("cand-A");
8703 let wt_b = root.join("cand-B");
8704 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
8705 .await
8706 .expect("worktree A");
8707 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
8708 .await
8709 .expect("worktree B");
8710
8711 state.candidates = vec![
8712 Candidate {
8713 index: 0,
8714 label: 'A',
8715 agent: "alpha".to_owned(),
8716 branch: "magi/x/A".to_owned(),
8717 worktree: wt_a.clone(),
8718 summary: String::new(),
8719 stat: String::new(),
8720 files: 0,
8721 commits: 0,
8722 empty: false,
8723 failed: None,
8724 verified_noop: None,
8725 duration_ms: 0,
8726 folded: false,
8727 },
8728 Candidate {
8729 index: 1,
8730 label: 'B',
8731 agent: "beta".to_owned(),
8732 branch: "magi/x/B".to_owned(),
8733 worktree: wt_b.clone(),
8734 summary: String::new(),
8735 stat: String::new(),
8736 files: 0,
8737 commits: 0,
8738 empty: false,
8739 failed: None,
8740 verified_noop: None,
8741 duration_ms: 0,
8742 folded: false,
8743 },
8744 ];
8745 state.tally = Some(Tally {
8746 first_choice: BTreeMap::from([('A', 1)]),
8747 borda: BTreeMap::new(),
8748 winner: 'A',
8749 rankings: 1,
8750 unanimous_initial: true,
8751 deliberated: false,
8752 changed_votes: 0,
8753 unanimous_final: true,
8754 tie_break: None,
8755 judges: 1,
8756 present: 1,
8757 quorum: 1,
8758 met_quorum: true,
8759 uncontested: None,
8760 });
8761 state.status = RunStatus::Ready;
8762
8763 fold_run(&mut state, false, &crate::run::home())
8764 .await
8765 .expect("fold_run");
8766
8767 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
8768 assert!(
8769 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
8770 "the unmerged winner's branch survives"
8771 );
8772 assert!(
8773 !state.candidates[0].folded,
8774 "the winner is not marked folded"
8775 );
8776
8777 assert!(!wt_b.exists(), "the loser's worktree is removed");
8778 assert!(
8779 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
8780 "the loser's branch is removed"
8781 );
8782 assert!(state.candidates[1].folded, "the loser is marked folded");
8783 }
8784
8785 #[tokio::test]
8788 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
8789 let tmp = tempfile::tempdir().expect("tempdir");
8790 let repo = tmp.path().join("repo");
8791 std::fs::create_dir_all(&repo).unwrap();
8792 init_repo(&repo);
8793 let home = tmp.path().join("home");
8794
8795 let mut config = Config::default();
8796 config.graph.worktree_root = Some(tmp.path().join("wt"));
8797 let mut state = RunState::new(
8798 repo.clone(),
8799 "main".to_owned(),
8800 "deadbeef".to_owned(),
8801 "task".to_owned(),
8802 config,
8803 );
8804 git::git(&repo, &["branch", "magi/x/A", "main"])
8806 .await
8807 .expect("branch");
8808 state.candidates = vec![Candidate {
8809 index: 0,
8810 label: 'A',
8811 agent: "alpha".to_owned(),
8812 branch: "magi/x/A".to_owned(),
8813 worktree: state.worktree_root().join("cand-A"),
8814 summary: String::new(),
8815 stat: String::new(),
8816 files: 0,
8817 commits: 0,
8818 empty: false,
8819 failed: None,
8820 verified_noop: None,
8821 duration_ms: 0,
8822 folded: true,
8823 }];
8824 state.released_to = Some("20260901-000000-new1".to_owned());
8825 state.released_branches = vec!["magi/x/A".to_owned()];
8826
8827 fold_run(&mut state, true, &home).await.expect("fold_run");
8828
8829 assert!(
8830 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
8831 "the handed-over branch survives a fold"
8832 );
8833 }
8834
8835 #[tokio::test]
8839 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
8840 let tmp = tempfile::tempdir().expect("tempdir");
8841 let repo = tmp.path().join("repo");
8842 std::fs::create_dir_all(&repo).unwrap();
8843 init_repo(&repo);
8844 let run = |args: &[&str]| {
8845 let out = std::process::Command::new("git")
8846 .quiet()
8847 .args(args)
8848 .current_dir(&repo)
8849 .output()
8850 .expect("spawn git");
8851 assert!(out.status.success(), "git {args:?}");
8852 };
8853 run(&["branch", "magi/x/A"]);
8854 run(&["checkout", "-q", "-b", "magi/x/B"]);
8855 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
8856 run(&["add", "-A"]);
8857 run(&["commit", "-q", "-m", "work"]);
8858 run(&["checkout", "-q", "main"]);
8859
8860 let mut state = RunState::new(
8861 repo.clone(),
8862 "main".to_owned(),
8863 "deadbeef".to_owned(),
8864 "task".to_owned(),
8865 Config::default(),
8866 );
8867 state.seeds = vec![refs::Seed {
8868 token: "magi/27b2/A".to_owned(),
8869 kind: refs::SeedKind::Unresolved,
8870 sha: String::new(),
8871 branch: true,
8872 detail: "no branch or commit named magi/27b2/A".to_owned(),
8873 }];
8874
8875 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
8876 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
8877 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
8878 let detail = empty_candidate_detail(&state, "main");
8879 assert!(detail.starts_with("empty candidate"), "{detail}");
8880 assert!(detail.contains("magi/27b2/A"), "{detail}");
8881 }
8882
8883 #[tokio::test]
8892 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
8893 let tmp = tempfile::tempdir().expect("tempdir");
8894 let repo = tmp.path().join("repo");
8895 std::fs::create_dir_all(&repo).unwrap();
8896 init_repo(&repo);
8897
8898 let mut config = Config::default();
8899 config.merge.mode = MergeMode::Local;
8900
8901 let mut state = RunState::new(
8902 repo.clone(),
8903 "main".to_owned(),
8904 "deadbeef".to_owned(),
8905 "task".to_owned(),
8906 config,
8907 );
8908 state.candidates = vec![Candidate {
8909 index: 0,
8910 label: 'A',
8911 agent: "alpha".to_owned(),
8912 branch: "does-not-exist".to_owned(),
8913 worktree: repo.clone(),
8914 summary: String::new(),
8915 stat: String::new(),
8916 files: 0,
8917 commits: 0,
8918 empty: false,
8919 failed: None,
8920 verified_noop: None,
8921 duration_ms: 0,
8922 folded: false,
8923 }];
8924 state.tally = Some(Tally {
8925 first_choice: BTreeMap::from([('A', 1)]),
8926 borda: BTreeMap::new(),
8927 winner: 'A',
8928 rankings: 1,
8929 unanimous_initial: true,
8930 deliberated: false,
8931 changed_votes: 0,
8932 unanimous_final: true,
8933 tie_break: None,
8934 judges: 0,
8935 present: 0,
8936 quorum: 0,
8937 met_quorum: true,
8938 uncontested: Some("only candidate A produced a change".to_owned()),
8939 });
8940 state.reviews = vec![ReviewRound {
8941 round: 1,
8942 head: "deadbeef".to_owned(),
8943 verified_head: None,
8944 verified_at: None,
8945 reviews: Vec::new(),
8946 e2e: Vec::new(),
8947 fix: None,
8948 blocking: 0,
8949 answered: 0,
8950 expected: 0,
8951 clean: true,
8952 verify_retried: false,
8953 e2e_deferred: false,
8954 e2e_defer_reason: None,
8955 progressed: false,
8956 vote_split: false,
8957 reconsideration: Vec::new(),
8958 verdict: None,
8959 }];
8960 state.gate = vec![CommandOutcome {
8961 command: "test".to_owned(),
8962 code: Some(0),
8963 output_tail: String::new(),
8964 duration_ms: 0,
8965 resource_blocked: false,
8966 }];
8967 state.gate_ran = true;
8968 state.status = RunStatus::Ready;
8973 state.merge = Some(MergeOutcome {
8974 mode: MergeMode::Local,
8975 ok: false,
8976 detail: "already concluded".to_owned(),
8977 empty: false,
8978 });
8979
8980 let mut runner = Runner {
8981 state,
8982 roles: ResolvedRoles {
8983 implementers: Vec::new(),
8984 judges: Vec::new(),
8985 reviewers: Vec::new(),
8986 fixer: None,
8987 conductor: conductor(),
8988 implementer_roster: Vec::new(),
8989 judge_roster: Vec::new(),
8990 reviewer_roster: Vec::new(),
8991 },
8992 sem: Arc::new(Semaphore::new(1)),
8993 pause: Pause::new(),
8994 interrupt: Pause::new(),
8995 };
8996
8997 runner.merge().await.expect("merge");
8998
8999 assert_eq!(
9000 runner.state.status,
9001 RunStatus::Ready,
9002 "a concluded run's status must not change on reentry"
9003 );
9004 assert_eq!(
9005 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
9006 Some("already concluded"),
9007 "merge must not run again once the node already recorded an outcome"
9008 );
9009 }
9010
9011 #[tokio::test]
9020 async fn merge_refuses_a_gate_that_has_not_actually_run() {
9021 let tmp = tempfile::tempdir().expect("tempdir");
9022 let repo = tmp.path().join("repo");
9023 std::fs::create_dir_all(&repo).unwrap();
9024 init_repo(&repo);
9025
9026 let mut config = Config::default();
9027 config.merge.mode = MergeMode::Local;
9028
9029 let mut state = RunState::new(
9030 repo.clone(),
9031 "main".to_owned(),
9032 "deadbeef".to_owned(),
9033 "task".to_owned(),
9034 config,
9035 );
9036 state.candidates = vec![Candidate {
9037 index: 0,
9038 label: 'A',
9039 agent: "alpha".to_owned(),
9040 branch: "does-not-exist".to_owned(),
9041 worktree: repo.clone(),
9042 summary: String::new(),
9043 stat: String::new(),
9044 files: 0,
9045 commits: 0,
9046 empty: false,
9047 failed: None,
9048 verified_noop: None,
9049 duration_ms: 0,
9050 folded: false,
9051 }];
9052 state.tally = Some(Tally {
9053 first_choice: BTreeMap::from([('A', 1)]),
9054 borda: BTreeMap::new(),
9055 winner: 'A',
9056 rankings: 1,
9057 unanimous_initial: true,
9058 deliberated: false,
9059 changed_votes: 0,
9060 unanimous_final: true,
9061 tie_break: None,
9062 judges: 0,
9063 present: 0,
9064 quorum: 0,
9065 met_quorum: true,
9066 uncontested: Some("only candidate A produced a change".to_owned()),
9067 });
9068 state.reviews = vec![ReviewRound {
9069 round: 1,
9070 head: "deadbeef".to_owned(),
9071 verified_head: None,
9072 verified_at: None,
9073 reviews: Vec::new(),
9074 e2e: Vec::new(),
9075 fix: None,
9076 blocking: 0,
9077 answered: 0,
9078 expected: 0,
9079 clean: true,
9080 verify_retried: false,
9081 e2e_deferred: false,
9082 e2e_defer_reason: None,
9083 progressed: false,
9084 vote_split: false,
9085 reconsideration: Vec::new(),
9086 verdict: None,
9087 }];
9088 state.gate = Vec::new();
9090 state.gate_ran = false;
9091 state.status = RunStatus::Gating;
9092
9093 let mut runner = Runner {
9094 state,
9095 roles: ResolvedRoles {
9096 implementers: Vec::new(),
9097 judges: Vec::new(),
9098 reviewers: Vec::new(),
9099 fixer: None,
9100 conductor: conductor(),
9101 implementer_roster: Vec::new(),
9102 judge_roster: Vec::new(),
9103 reviewer_roster: Vec::new(),
9104 },
9105 sem: Arc::new(Semaphore::new(1)),
9106 pause: Pause::new(),
9107 interrupt: Pause::new(),
9108 };
9109
9110 runner.merge().await.expect("merge");
9111
9112 assert!(
9113 runner.state.merge.is_none(),
9114 "an empty gate must never be read as a passing one: {:?}",
9115 runner.state.merge
9116 );
9117 }
9118
9119 #[tokio::test]
9126 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
9127 let tmp = tempfile::tempdir().expect("tempdir");
9128 let repo = tmp.path().join("repo");
9129 std::fs::create_dir_all(&repo).unwrap();
9130 init_repo(&repo);
9131
9132 let config = Config::default();
9134
9135 let mut state = RunState::new(
9136 repo.clone(),
9137 "main".to_owned(),
9138 "deadbeef".to_owned(),
9139 "task".to_owned(),
9140 config,
9141 );
9142 state.candidates = vec![Candidate {
9143 index: 0,
9144 label: 'A',
9145 agent: "alpha".to_owned(),
9146 branch: "does-not-exist".to_owned(),
9147 worktree: repo.clone(),
9148 summary: String::new(),
9149 stat: String::new(),
9150 files: 0,
9151 commits: 0,
9152 empty: false,
9153 failed: None,
9154 verified_noop: None,
9155 duration_ms: 0,
9156 folded: false,
9157 }];
9158 state.tally = Some(Tally {
9159 first_choice: BTreeMap::from([('A', 1)]),
9160 borda: BTreeMap::new(),
9161 winner: 'A',
9162 rankings: 1,
9163 unanimous_initial: true,
9164 deliberated: false,
9165 changed_votes: 0,
9166 unanimous_final: true,
9167 tie_break: None,
9168 judges: 0,
9169 present: 0,
9170 quorum: 0,
9171 met_quorum: true,
9172 uncontested: Some("only candidate A produced a change".to_owned()),
9173 });
9174 state.reviews = vec![ReviewRound {
9175 round: 1,
9176 head: "deadbeef".to_owned(),
9177 verified_head: None,
9178 verified_at: None,
9179 reviews: Vec::new(),
9180 e2e: Vec::new(),
9181 fix: None,
9182 blocking: 0,
9183 answered: 0,
9184 expected: 0,
9185 clean: true,
9186 verify_retried: false,
9187 e2e_deferred: false,
9188 e2e_defer_reason: None,
9189 progressed: false,
9190 vote_split: false,
9191 reconsideration: Vec::new(),
9192 verdict: None,
9193 }];
9194
9195 let mut runner = Runner {
9196 state,
9197 roles: ResolvedRoles {
9198 implementers: Vec::new(),
9199 judges: Vec::new(),
9200 reviewers: Vec::new(),
9201 fixer: None,
9202 conductor: conductor(),
9203 implementer_roster: Vec::new(),
9204 judge_roster: Vec::new(),
9205 reviewer_roster: Vec::new(),
9206 },
9207 sem: Arc::new(Semaphore::new(1)),
9208 pause: Pause::new(),
9209 interrupt: Pause::new(),
9210 };
9211
9212 runner.gate().await.expect("gate");
9213 assert!(
9214 runner.state.gate_ran,
9215 "zero configured commands is still a real attempt, not an unrun gate"
9216 );
9217 assert!(runner.state.gate.is_empty());
9218 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
9219 assert_ne!(
9220 runner.state.status,
9221 RunStatus::Blocked,
9222 "a gate with nothing to check must not read as failed"
9223 );
9224
9225 runner.merge().await.expect("merge");
9226 assert_eq!(
9227 runner.state.status,
9228 RunStatus::Ready,
9229 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
9230 );
9231 }
9232
9233 #[tokio::test]
9244 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
9245 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9246 let home = crate::run::home();
9247
9248 let tmp = tempfile::tempdir().expect("tempdir");
9249 let repo = tmp.path().join("repo");
9250 std::fs::create_dir_all(&repo).unwrap();
9251 init_repo(&repo);
9252 let cache_dir = tmp.path().join("target");
9255
9256 let mut config = Config::default();
9257 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
9258 config.graph.timeout_verify = Some(2);
9261
9262 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9263 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9264 .expect("no io error acquiring directly")
9265 {
9266 crate::cache::AcquireOutcome::Acquired(g) => g,
9267 crate::cache::AcquireOutcome::Busy(b) => {
9268 panic!("expected the direct acquire to win the lease first: {b:?}")
9269 }
9270 };
9271
9272 let mut state = RunState::new(
9273 repo.clone(),
9274 "main".to_owned(),
9275 "deadbeef".to_owned(),
9276 "task".to_owned(),
9277 config,
9278 );
9279 state.candidates = vec![Candidate {
9280 index: 0,
9281 label: 'A',
9282 agent: "alpha".to_owned(),
9283 branch: "does-not-exist".to_owned(),
9284 worktree: repo.clone(),
9285 summary: String::new(),
9286 stat: String::new(),
9287 files: 0,
9288 commits: 0,
9289 empty: false,
9290 failed: None,
9291 verified_noop: None,
9292 duration_ms: 0,
9293 folded: false,
9294 }];
9295 state.tally = Some(Tally {
9296 first_choice: BTreeMap::from([('A', 1)]),
9297 borda: BTreeMap::new(),
9298 winner: 'A',
9299 rankings: 1,
9300 unanimous_initial: true,
9301 deliberated: false,
9302 changed_votes: 0,
9303 unanimous_final: true,
9304 tie_break: None,
9305 judges: 0,
9306 present: 0,
9307 quorum: 0,
9308 met_quorum: true,
9309 uncontested: Some("only candidate A produced a change".to_owned()),
9310 });
9311 state.reviews = vec![ReviewRound {
9312 round: 1,
9313 head: "deadbeef".to_owned(),
9314 verified_head: None,
9315 verified_at: None,
9316 reviews: Vec::new(),
9317 e2e: Vec::new(),
9318 fix: None,
9319 blocking: 0,
9320 answered: 0,
9321 expected: 0,
9322 clean: true,
9323 verify_retried: false,
9324 e2e_deferred: false,
9325 e2e_defer_reason: None,
9326 progressed: false,
9327 vote_split: false,
9328 reconsideration: Vec::new(),
9329 verdict: None,
9330 }];
9331
9332 let mut runner = Runner {
9333 state,
9334 roles: ResolvedRoles {
9335 implementers: Vec::new(),
9336 judges: Vec::new(),
9337 reviewers: Vec::new(),
9338 fixer: None,
9339 conductor: conductor(),
9340 implementer_roster: Vec::new(),
9341 judge_roster: Vec::new(),
9342 reviewer_roster: Vec::new(),
9343 },
9344 sem: Arc::new(Semaphore::new(1)),
9345 pause: Pause::new(),
9346 interrupt: Pause::new(),
9347 };
9348
9349 let started = std::time::Instant::now();
9350 runner.gate().await.expect("gate");
9351 assert!(
9352 started.elapsed() < Duration::from_secs(1),
9353 "a gate with nothing to run must never wait on a lease it never needed"
9354 );
9355 assert!(
9356 runner.state.gate_ran,
9357 "zero commands is still a real, immediate attempt"
9358 );
9359 assert!(runner.state.gate.is_empty());
9360 assert_ne!(
9361 runner.state.status,
9362 RunStatus::Blocked,
9363 "must not read as resource-blocked on a lease it never asked for"
9364 );
9365 }
9366
9367 #[tokio::test]
9377 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
9378 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9379
9380 let tmp = tempfile::tempdir().expect("tempdir");
9381 let repo = tmp.path().join("repo");
9382 std::fs::create_dir_all(&repo).unwrap();
9383 init_repo(&repo);
9384
9385 let mut config = Config::default();
9386 config.verify.gate = vec![
9387 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
9388 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
9389 .to_owned(),
9390 ];
9391
9392 let mut state = RunState::new(
9393 repo.clone(),
9394 "main".to_owned(),
9395 "deadbeef".to_owned(),
9396 "task".to_owned(),
9397 config,
9398 );
9399 let run_id = state.id.clone();
9400 state.candidates = vec![Candidate {
9401 index: 0,
9402 label: 'A',
9403 agent: "alpha".to_owned(),
9404 branch: "does-not-exist".to_owned(),
9405 worktree: repo.clone(),
9406 summary: String::new(),
9407 stat: String::new(),
9408 files: 0,
9409 commits: 0,
9410 empty: false,
9411 failed: None,
9412 verified_noop: None,
9413 duration_ms: 0,
9414 folded: false,
9415 }];
9416 state.tally = Some(Tally {
9417 first_choice: BTreeMap::from([('A', 1)]),
9418 borda: BTreeMap::new(),
9419 winner: 'A',
9420 rankings: 1,
9421 unanimous_initial: true,
9422 deliberated: false,
9423 changed_votes: 0,
9424 unanimous_final: true,
9425 tie_break: None,
9426 judges: 0,
9427 present: 0,
9428 quorum: 0,
9429 met_quorum: true,
9430 uncontested: Some("only candidate A produced a change".to_owned()),
9431 });
9432 state.reviews = vec![ReviewRound {
9433 round: 1,
9434 head: "deadbeef".to_owned(),
9435 verified_head: None,
9436 verified_at: None,
9437 reviews: Vec::new(),
9438 e2e: Vec::new(),
9439 fix: None,
9440 blocking: 0,
9441 answered: 0,
9442 expected: 0,
9443 clean: true,
9444 verify_retried: false,
9445 e2e_deferred: false,
9446 e2e_defer_reason: None,
9447 progressed: false,
9448 vote_split: false,
9449 reconsideration: Vec::new(),
9450 verdict: None,
9451 }];
9452
9453 let mut runner = Runner {
9454 state,
9455 roles: ResolvedRoles {
9456 implementers: Vec::new(),
9457 judges: Vec::new(),
9458 reviewers: Vec::new(),
9459 fixer: None,
9460 conductor: conductor(),
9461 implementer_roster: Vec::new(),
9462 judge_roster: Vec::new(),
9463 reviewer_roster: Vec::new(),
9464 },
9465 sem: Arc::new(Semaphore::new(1)),
9466 pause: Pause::new(),
9467 interrupt: Pause::new(),
9468 };
9469
9470 let started_marker = repo.join("started.marker");
9471 let release_marker = repo.join("release.marker");
9472 let poller = tokio::spawn(async move {
9473 for _ in 0..100 {
9478 if started_marker.exists()
9479 && let Ok(s) = crate::run::RunState::load(&run_id)
9480 && let Some(a) = s.active.get("gate")
9481 {
9482 std::fs::write(&release_marker, b"go").expect("release marker");
9483 return Some(a.clone());
9484 }
9485 tokio::time::sleep(Duration::from_millis(50)).await;
9486 }
9487 None
9488 });
9489
9490 runner.gate().await.expect("gate");
9491 let captured = poller.await.expect("poller task");
9492 let captured = captured.expect(
9493 "the poller never saw a `gate` task entry in run.json while the command was \
9494 still blocked on its own release marker",
9495 );
9496
9497 assert_eq!(captured.task.as_deref(), Some("gate"));
9498 assert_eq!(captured.node, "gate");
9499 assert_eq!(captured.index, Some(1));
9500 assert_eq!(captured.total, Some(1));
9501 assert!(
9502 captured
9503 .command
9504 .as_deref()
9505 .is_some_and(|c| c.contains("started.marker")),
9506 "{captured:?}"
9507 );
9508
9509 assert!(
9510 runner.state.active.is_empty(),
9511 "the entry must be cleared once the command actually finished: {:?}",
9512 runner.state.active
9513 );
9514 assert!(runner.state.gate_ran);
9515 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
9516 }
9517
9518 #[tokio::test]
9521 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
9522 use crate::verdict::{Finding, ReviewVote, Severity};
9523 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9524 let tmp = tempfile::tempdir().expect("tempdir");
9525 let repo = tmp.path().join("repo");
9526 std::fs::create_dir_all(&repo).unwrap();
9527 init_repo(&repo);
9528
9529 for (severity, vote, expect) in [
9530 (Severity::Major, ReviewVote::Reject, true),
9531 (Severity::Minor, ReviewVote::Reject, false),
9532 (Severity::Major, ReviewVote::Approve, false),
9533 ] {
9534 let mut round = review_round(false, 1, 1, 1, false, true);
9535 round.reviews = vec![ReviewRecord {
9536 reviewer: 1,
9537 agent: "alpha".to_owned(),
9538 summary: String::new(),
9539 findings: vec![Finding {
9540 id: "R1-1-1".to_owned(),
9541 severity,
9542 file: None,
9543 line: None,
9544 title: "t".to_owned(),
9545 detail: String::new(),
9546 }],
9547 vote: Some(vote),
9548 failed: None,
9549 duration_ms: 0,
9550 attempts: 0,
9551 }];
9552 let mut state = RunState::new(
9553 repo.clone(),
9554 "main".to_owned(),
9555 "deadbeef".to_owned(),
9556 "task".to_owned(),
9557 Config::default(),
9558 );
9559 state.reviews = vec![round];
9560 let mut runner = Runner {
9561 state,
9562 roles: ResolvedRoles {
9563 implementers: Vec::new(),
9564 judges: Vec::new(),
9565 reviewers: Vec::new(),
9566 fixer: None,
9567 conductor: conductor(),
9568 implementer_roster: Vec::new(),
9569 judge_roster: Vec::new(),
9570 reviewer_roster: Vec::new(),
9571 },
9572 sem: Arc::new(Semaphore::new(1)),
9573 pause: Pause::new(),
9574 interrupt: Pause::new(),
9575 };
9576 let shell = runner.state.config.shell();
9577 runner
9578 .stop_reviewing("round budget spent", &shell, &repo)
9579 .await
9580 .expect("stop_reviewing");
9581 assert_eq!(runner.state.status, RunStatus::Gating);
9582 assert_eq!(
9583 runner.state.contested_handoff.is_some(),
9584 expect,
9585 "{severity:?} + {vote:?}"
9586 );
9587 }
9588 }
9589
9590 #[tokio::test]
9603 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
9604 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9605 let home = crate::run::home();
9606
9607 let tmp = tempfile::tempdir().expect("tempdir");
9608 let repo = tmp.path().join("repo");
9609 std::fs::create_dir_all(&repo).unwrap();
9610 init_repo(&repo);
9611 let head = crate::git::rev_parse(&repo, "HEAD")
9612 .await
9613 .expect("rev-parse");
9614 let cache_dir = tmp.path().join("target");
9617
9618 let mut config = Config::default();
9619 config.verify.e2e = vec![format!(
9620 "CARGO_TARGET_DIR='{}' test -f README.md",
9621 cache_dir.display()
9622 )];
9623 config.graph.review_rounds = 1;
9624 config.graph.timeout_verify = Some(2);
9627
9628 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9629 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9630 .expect("no io error acquiring directly")
9631 {
9632 crate::cache::AcquireOutcome::Acquired(g) => g,
9633 crate::cache::AcquireOutcome::Busy(b) => {
9634 panic!("expected the direct acquire to win the lease first: {b:?}")
9635 }
9636 };
9637
9638 let mut state = RunState::new(
9639 repo.clone(),
9640 "main".to_owned(),
9641 head.clone(),
9642 "task".to_owned(),
9643 config,
9644 );
9645 state.candidates = vec![Candidate {
9646 index: 0,
9647 label: 'A',
9648 agent: "alpha".to_owned(),
9649 branch: "does-not-exist".to_owned(),
9650 worktree: repo.clone(),
9651 summary: String::new(),
9652 stat: String::new(),
9653 files: 0,
9654 commits: 0,
9655 empty: false,
9656 failed: None,
9657 verified_noop: None,
9658 duration_ms: 0,
9659 folded: false,
9660 }];
9661 state.tally = Some(Tally {
9662 first_choice: BTreeMap::from([('A', 1)]),
9663 borda: BTreeMap::new(),
9664 winner: 'A',
9665 rankings: 1,
9666 unanimous_initial: true,
9667 deliberated: false,
9668 changed_votes: 0,
9669 unanimous_final: true,
9670 tie_break: None,
9671 judges: 0,
9672 present: 0,
9673 quorum: 0,
9674 met_quorum: true,
9675 uncontested: Some("only candidate A produced a change".to_owned()),
9676 });
9677 state.reviews = vec![ReviewRound {
9681 round: 1,
9682 head: head.clone(),
9683 verified_head: None,
9684 verified_at: None,
9685 reviews: Vec::new(),
9686 e2e: Vec::new(),
9687 fix: None,
9688 blocking: 1,
9689 answered: 1,
9690 expected: 1,
9691 clean: false,
9692 verify_retried: false,
9693 e2e_deferred: true,
9694 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
9695 progressed: false,
9696 vote_split: false,
9697 reconsideration: Vec::new(),
9698 verdict: None,
9699 }];
9700
9701 let mut runner = Runner {
9702 state,
9703 roles: ResolvedRoles {
9704 implementers: Vec::new(),
9705 judges: Vec::new(),
9706 reviewers: Vec::new(),
9707 fixer: None,
9708 conductor: conductor(),
9709 implementer_roster: Vec::new(),
9710 judge_roster: Vec::new(),
9711 reviewer_roster: Vec::new(),
9712 },
9713 sem: Arc::new(Semaphore::new(1)),
9714 pause: Pause::new(),
9715 interrupt: Pause::new(),
9716 };
9717
9718 let shell = runner.state.config.shell();
9719 runner
9720 .stop_reviewing("round budget spent", &shell, &repo)
9721 .await
9722 .expect("stop_reviewing");
9723
9724 let last = runner.state.reviews.last().expect("round record");
9725 assert_eq!(
9726 last.e2e_status(),
9727 E2eStatus::ResourceBlocked,
9728 "the shared cache is still held; the attempt must read as blocked, not deferred or \
9729 failed: {last:?}"
9730 );
9731 assert_eq!(
9732 last.verified_head.as_deref(),
9733 Some(head.as_str()),
9734 "which commit this attempt targeted is known even though nothing finished checking \
9735 it"
9736 );
9737 let first_attempt_at = last
9738 .verified_at
9739 .expect("when this attempt ran is known too");
9740 assert_ne!(
9741 runner.state.status,
9742 RunStatus::Blocked,
9743 "contention is evidence about the machine, not the patch — it must not settle the \
9744 run as blocked: {:?}",
9745 runner.state.status
9746 );
9747 assert!(
9748 !runner
9749 .state
9750 .events
9751 .iter()
9752 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
9753 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
9754 runner.state.events
9755 );
9756
9757 runner
9762 .stop_reviewing("round budget spent", &shell, &repo)
9763 .await
9764 .expect("stop_reviewing retry");
9765 assert_eq!(
9766 runner.state.reviews.len(),
9767 1,
9768 "no new round was started: {:?}",
9769 runner.state.reviews
9770 );
9771 let last = runner.state.reviews.last().expect("round record");
9772 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
9773 assert!(
9774 last.verified_at.expect("still known") > first_attempt_at,
9775 "a second reentry must be a fresh attempt, not a stale copy of the first"
9776 );
9777 assert_ne!(runner.state.status, RunStatus::Blocked);
9778
9779 held.release();
9780 }
9781
9782 #[tokio::test]
9794 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
9795 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9796 let home = crate::run::home();
9797
9798 let tmp = tempfile::tempdir().expect("tempdir");
9799 let repo = tmp.path().join("repo");
9800 std::fs::create_dir_all(&repo).unwrap();
9801 init_repo(&repo);
9802 let head = crate::git::rev_parse(&repo, "HEAD")
9803 .await
9804 .expect("rev-parse");
9805 let cache_dir = tmp.path().join("target");
9806
9807 let mut config = Config::default();
9808 config.verify.e2e = vec![format!(
9809 "CARGO_TARGET_DIR='{}' test -f README.md",
9810 cache_dir.display()
9811 )];
9812 config.graph.review_rounds = 1;
9813 config.graph.timeout_verify = Some(2);
9814
9815 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9816 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9817 .expect("no io error acquiring directly")
9818 {
9819 crate::cache::AcquireOutcome::Acquired(g) => g,
9820 crate::cache::AcquireOutcome::Busy(b) => {
9821 panic!("expected the direct acquire to win the lease first: {b:?}")
9822 }
9823 };
9824
9825 let mut state = RunState::new(
9826 repo.clone(),
9827 "main".to_owned(),
9828 head.clone(),
9829 "task".to_owned(),
9830 config,
9831 );
9832 state.candidates = vec![Candidate {
9833 index: 0,
9834 label: 'A',
9835 agent: "alpha".to_owned(),
9836 branch: "does-not-exist".to_owned(),
9837 worktree: repo.clone(),
9838 summary: String::new(),
9839 stat: String::new(),
9840 files: 0,
9841 commits: 0,
9842 empty: false,
9843 failed: None,
9844 verified_noop: None,
9845 duration_ms: 0,
9846 folded: false,
9847 }];
9848 state.tally = Some(Tally {
9849 first_choice: BTreeMap::from([('A', 1)]),
9850 borda: BTreeMap::new(),
9851 winner: 'A',
9852 rankings: 1,
9853 unanimous_initial: true,
9854 deliberated: false,
9855 changed_votes: 0,
9856 unanimous_final: true,
9857 tie_break: None,
9858 judges: 0,
9859 present: 0,
9860 quorum: 0,
9861 met_quorum: true,
9862 uncontested: Some("only candidate A produced a change".to_owned()),
9863 });
9864 state.reviews = vec![ReviewRound {
9868 round: 1,
9869 head: head.clone(),
9870 verified_head: Some(head.clone()),
9871 verified_at: Some(jiff::Timestamp::now()),
9872 reviews: Vec::new(),
9873 e2e: vec![CommandOutcome {
9874 command: format!(
9875 "CARGO_TARGET_DIR='{}' test -f README.md",
9876 cache_dir.display()
9877 ),
9878 code: None,
9879 output_tail: "waiting for the shared build cache".to_owned(),
9880 duration_ms: 0,
9881 resource_blocked: true,
9882 }],
9883 fix: None,
9884 blocking: 1,
9885 answered: 1,
9886 expected: 1,
9887 clean: false,
9888 verify_retried: false,
9889 e2e_deferred: false,
9890 e2e_defer_reason: None,
9891 progressed: false,
9892 vote_split: false,
9893 reconsideration: Vec::new(),
9894 verdict: None,
9895 }];
9896
9897 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
9898 let mut runner = Runner {
9899 state,
9900 roles: ResolvedRoles {
9901 implementers: Vec::new(),
9902 judges: Vec::new(),
9903 reviewers: Vec::new(),
9904 fixer: None,
9905 conductor: conductor(),
9906 implementer_roster: Vec::new(),
9907 judge_roster: Vec::new(),
9908 reviewer_roster: Vec::new(),
9909 },
9910 sem: Arc::new(Semaphore::new(1)),
9911 pause: Pause::new(),
9912 interrupt: Pause::new(),
9913 };
9914
9915 runner.review_loop().await.expect("review_loop");
9920
9921 assert_eq!(
9922 runner.state.reviews.len(),
9923 1,
9924 "no new round was started on top of the unresolved one: {:?}",
9925 runner.state.reviews
9926 );
9927 let last = &runner.state.reviews[0];
9928 assert_eq!(
9929 last.e2e_status(),
9930 E2eStatus::ResourceBlocked,
9931 "still contended: {last:?}"
9932 );
9933 assert!(
9934 last.verified_at.expect("still known") > first_attempt_at,
9935 "review_loop must have actually retried the check, not left it exactly as found"
9936 );
9937 assert_ne!(
9938 runner.state.status,
9939 RunStatus::Blocked,
9940 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
9941 runner.state.status
9942 );
9943
9944 held.release();
9945 }
9946
9947 #[tokio::test]
9948 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
9949 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9950 let tmp = tempfile::tempdir().expect("tempdir");
9951 let repo = tmp.path().join("repo");
9952 std::fs::create_dir_all(&repo).unwrap();
9953 init_repo(&repo);
9954
9955 let mut config = Config::default();
9956 config.merge.mode = MergeMode::Pr;
9957 config.graph.land = true;
9958 config.graph.land_approval = false;
9959
9960 let mut state = RunState::new(
9961 repo.clone(),
9962 "main".to_owned(),
9963 "deadbeef".to_owned(),
9964 "task".to_owned(),
9965 config,
9966 );
9967 state.candidates = vec![Candidate {
9968 index: 0,
9969 label: 'A',
9970 agent: "alpha".to_owned(),
9971 branch: "does-not-exist".to_owned(),
9972 worktree: repo.clone(),
9973 summary: String::new(),
9974 stat: String::new(),
9975 files: 0,
9976 commits: 0,
9977 empty: false,
9978 failed: None,
9979 verified_noop: None,
9980 duration_ms: 0,
9981 folded: false,
9982 }];
9983 state.tally = Some(Tally {
9984 first_choice: BTreeMap::from([('A', 1)]),
9985 borda: BTreeMap::new(),
9986 winner: 'A',
9987 rankings: 1,
9988 unanimous_initial: true,
9989 deliberated: false,
9990 changed_votes: 0,
9991 unanimous_final: true,
9992 tie_break: None,
9993 judges: 0,
9994 present: 0,
9995 quorum: 0,
9996 met_quorum: true,
9997 uncontested: Some("only candidate A produced a change".to_owned()),
9998 });
9999 state.reviews = vec![ReviewRound {
10000 round: 1,
10001 head: "deadbeef".to_owned(),
10002 verified_head: None,
10003 verified_at: None,
10004 reviews: Vec::new(),
10005 e2e: Vec::new(),
10006 fix: None,
10007 blocking: 0,
10008 answered: 0,
10009 expected: 0,
10010 clean: true,
10011 verify_retried: false,
10012 e2e_deferred: false,
10013 e2e_defer_reason: None,
10014 progressed: false,
10015 vote_split: false,
10016 reconsideration: Vec::new(),
10017 verdict: None,
10018 }];
10019 state.gate = vec![CommandOutcome {
10020 command: "test".to_owned(),
10021 code: Some(0),
10022 output_tail: String::new(),
10023 duration_ms: 0,
10024 resource_blocked: false,
10025 }];
10026 state.gate_ran = true;
10027 state.status = RunStatus::Landing;
10031 state.merge = Some(MergeOutcome {
10032 mode: MergeMode::Pr,
10033 ok: true,
10034 detail: "https://example.invalid/x/y/pull/1".to_owned(),
10035 empty: false,
10036 });
10037
10038 ask_test_home();
10042 let store = ask::Questions::open();
10043 let q = ask_open_question(&store, &state.id);
10044
10045 let mut runner = Runner {
10046 state,
10047 roles: ResolvedRoles {
10048 implementers: Vec::new(),
10049 judges: Vec::new(),
10050 reviewers: Vec::new(),
10051 fixer: None,
10052 conductor: conductor(),
10053 implementer_roster: Vec::new(),
10054 judge_roster: Vec::new(),
10055 reviewer_roster: Vec::new(),
10056 },
10057 sem: Arc::new(Semaphore::new(1)),
10058 pause: Pause::new(),
10059 interrupt: Pause::new(),
10060 };
10061
10062 runner.execute().await.expect("execute");
10067
10068 assert_eq!(
10069 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10070 Some("https://example.invalid/x/y/pull/1"),
10071 "reentry must not push again or open a second pull request over the \
10072 one `land` is already watching"
10073 );
10074 assert_ne!(
10075 runner.state.status,
10076 RunStatus::Landing,
10077 "land could not actually reach the fake pull request, so it must \
10078 have given up rather than left the run silently parked forever"
10079 );
10080 assert_eq!(runner.state.status, RunStatus::Blocked);
10084 assert!(
10085 store.get(&q.id).unwrap().status.open(),
10086 "Blocked is still alive; settle_questions must have been a no-op here"
10087 );
10088 }
10089
10090 fn state_with_round(round: ReviewRound) -> RunState {
10091 let mut s = RunState::new(
10092 PathBuf::from("/repo"),
10093 "main".to_owned(),
10094 "abc1234".to_owned(),
10095 "add retries".to_owned(),
10096 Config::default(),
10097 );
10098 s.reviews = vec![round];
10099 s
10100 }
10101
10102 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
10103 crate::verdict::Finding {
10104 id: id.to_owned(),
10105 severity,
10106 file: None,
10107 line: None,
10108 title: title.to_owned(),
10109 detail: String::new(),
10110 }
10111 }
10112
10113 #[test]
10114 fn pr_body_names_open_findings_and_declined_ones() {
10115 let round = ReviewRound {
10116 round: 2,
10117 head: "deadbee".to_owned(),
10118 verified_head: None,
10119 verified_at: None,
10120 reviews: vec![ReviewRecord {
10121 attempts: 0,
10122 reviewer: 1,
10123 agent: "alpha".to_owned(),
10124 summary: String::new(),
10125 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
10126 vote: None,
10127 failed: None,
10128 duration_ms: 0,
10129 }],
10130 e2e: vec![CommandOutcome {
10131 command: "cargo test".to_owned(),
10132 code: Some(0),
10133 output_tail: String::new(),
10134 duration_ms: 0,
10135 resource_blocked: false,
10136 }],
10137 verify_retried: false,
10138 e2e_deferred: false,
10139 e2e_defer_reason: None,
10140 fix: Some(FixRecord {
10141 agent: "alpha".to_owned(),
10142 addressed: Vec::new(),
10143 rejected: vec![crate::verdict::Rejection {
10144 id: "R1-1-1".to_owned(),
10145 why: "not reachable from any caller".to_owned(),
10146 }],
10147 notes: String::new(),
10148 committed: true,
10149 failed: None,
10150 duration_ms: 0,
10151 continuation: None,
10152 }),
10153 blocking: 0,
10154 answered: 1,
10155 expected: 1,
10156 clean: false,
10157 progressed: true,
10158 vote_split: false,
10159 reconsideration: Vec::new(),
10160 verdict: None,
10161 };
10162 let state = state_with_round(round);
10163 let body = pr_message(&state, 'A').body;
10164
10165 assert!(body.contains("add retries"), "the task must still be there");
10166 assert!(body.contains("R2-1-1"), "{body}");
10167 assert!(body.contains("unused import"), "{body}");
10168 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
10169 assert!(
10170 body.contains("not reachable from any caller"),
10171 "the reason it was declined: {body}"
10172 );
10173 }
10174
10175 #[test]
10176 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
10177 let round = ReviewRound {
10178 round: 1,
10179 head: "deadbee".to_owned(),
10180 verified_head: None,
10181 verified_at: None,
10182 reviews: vec![ReviewRecord {
10183 attempts: 0,
10184 reviewer: 1,
10185 agent: "alpha".to_owned(),
10186 summary: String::new(),
10187 findings: Vec::new(),
10188 vote: None,
10189 failed: None,
10190 duration_ms: 0,
10191 }],
10192 e2e: Vec::new(),
10193 verify_retried: false,
10194 e2e_deferred: false,
10195 e2e_defer_reason: None,
10196 fix: None,
10197 blocking: 0,
10198 answered: 1,
10199 expected: 1,
10200 clean: true,
10201 progressed: false,
10202 vote_split: false,
10203 reconsideration: Vec::new(),
10204 verdict: None,
10205 };
10206 let state = state_with_round(round);
10207 let body = pr_message(&state, 'A').body;
10208 assert!(!body.contains("Open review findings"), "{body}");
10209 assert!(!body.contains("Declined"), "{body}");
10210 }
10211
10212 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
10213 let mut state = RunState::new(
10214 PathBuf::from("/repo"),
10215 "main".to_owned(),
10216 "abc1234".to_owned(),
10217 instruction.to_owned(),
10218 Config::default(),
10219 );
10220 state.candidates.push(Candidate {
10221 index: 0,
10222 label: 'A',
10223 agent: "alpha".to_owned(),
10224 branch: "magi/x/A".to_owned(),
10225 worktree: PathBuf::from("/wt"),
10226 summary: summary.to_owned(),
10227 stat: String::new(),
10228 files: 1,
10229 commits: 1,
10230 empty: false,
10231 failed: None,
10232 verified_noop: None,
10233 folded: false,
10234 duration_ms: 0,
10235 });
10236 state
10237 }
10238
10239 fn review_state(subjects: &[&str]) -> RunState {
10240 let mut state = state_with_summary(
10241 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
10242 "",
10243 );
10244 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
10245 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
10246 state
10247 }
10248
10249 #[test]
10250 fn pr_message_review_single_commit_uses_its_subject() {
10251 let state = review_state(&["feat(nats): per-role user"]);
10252 let m = pr_message(&state, 'A');
10253 assert_eq!(m.title, "feat(nats): per-role user");
10254 assert!(!m.body.contains("Review the work already"), "{}", m.body);
10255 assert!(m.body.contains("## Commits under review"), "{}", m.body);
10256 }
10257
10258 #[test]
10259 fn pr_message_review_multi_commit_takes_the_oldest() {
10260 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
10261 let m = pr_message(&state, 'A');
10262 assert_eq!(m.title, "feat: the change");
10263 for s in ["feat: the change", "fix: typo", "fix: again"] {
10264 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
10265 }
10266 }
10267
10268 #[test]
10269 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
10270 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
10271 let state = review_state(&[first, "fix: later fixup"]);
10272 let m = pr_message(&state, 'A');
10273 assert!(
10274 m.title.starts_with("chore: land candidate A of run"),
10275 "{}",
10276 m.title
10277 );
10278 }
10279 }
10280
10281 #[test]
10282 fn pr_message_review_bounds_a_long_english_subject() {
10283 let long = format!("feat: {}", "word ".repeat(100));
10284 let m = pr_message(&review_state(&[&long]), 'A');
10285 assert!(m.title.starts_with("feat: word"), "{}", m.title);
10286 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10287 }
10288
10289 #[test]
10290 fn pr_message_implementation_run_is_unchanged_by_review_support() {
10291 let state = state_with_summary("add retries\n\ndetails", "- did some things");
10292 let m = pr_message(&state, 'A');
10293 assert_eq!(m.title, "add retries");
10294 assert!(m.body.contains("<summary>Original task</summary>"));
10295 assert!(!m.body.contains("Commits under review"));
10296 assert_eq!(landing_subject_source(&state), state.instruction);
10297 }
10298
10299 #[test]
10300 fn review_run_squash_subject_is_the_change_not_the_prompt() {
10301 let state = review_state(&["feat: the change", "fix: typo"]);
10302 let source = landing_subject_source(&state);
10303 assert_eq!(land::merge_subject("", &source), "feat: the change");
10304 assert_eq!(
10305 land::merge_subject("magi: candidate A (uncommitted work)", &source),
10306 "feat: the change"
10307 );
10308 assert_eq!(
10310 land::merge_subject("feat: renamed by hand", &source),
10311 "feat: renamed by hand"
10312 );
10313 let blank = review_state(&["日本語"]);
10314 assert!(
10315 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
10316 );
10317 }
10318
10319 #[test]
10320 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
10321 let state = review_state(&["feat: the change"]);
10322 let source = landing_subject_source(&state);
10323 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
10324 assert_eq!(
10325 land::merge_subject(landing_title(&state, old), &source),
10326 "feat: the change"
10327 );
10328 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
10329 let task = state_with_summary("add retries", "");
10330 assert_eq!(landing_title(&task, old), old);
10331 }
10332
10333 #[test]
10334 fn pr_message_describes_the_change_not_the_task() {
10335 let state = state_with_summary(
10336 "今回やってほしいこと: results projector を直す",
10337 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
10338 );
10339 let m = pr_message(&state, 'A');
10340 assert_eq!(m.title, "fix(web): batch the runs list reads");
10341 assert!(
10342 m.body.starts_with("## Summary\n\n- reads run.json once"),
10343 "{}",
10344 m.body
10345 );
10346 assert!(!m.body.contains("TITLE:"), "{}", m.body);
10347 let task_at = m.body.find("今回やってほしいこと").unwrap();
10348 let details_at = m.body.find("<details>").unwrap();
10349 assert!(
10350 details_at < task_at,
10351 "the task lives inside <details>: {}",
10352 m.body
10353 );
10354 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
10355 assert!(m.body.contains("magi:candidate-a"));
10356 }
10357
10358 #[test]
10359 fn pr_message_falls_back_to_the_task_without_a_title_line() {
10360 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
10361 let m = pr_message(&state, 'A');
10362 assert_eq!(m.title, "add retries");
10363 assert!(
10364 m.body.contains("## Summary\n\n- did some things"),
10365 "{}",
10366 m.body
10367 );
10368
10369 let none = RunState::new(
10370 PathBuf::from("/repo"),
10371 "main".to_owned(),
10372 "abc1234".to_owned(),
10373 "add retries".to_owned(),
10374 Config::default(),
10375 );
10376 let m = pr_message(&none, 'A');
10377 assert_eq!(m.title, "add retries");
10378 assert!(!m.body.contains("## Summary"), "{}", m.body);
10379 }
10380
10381 #[test]
10382 fn pr_message_refuses_the_candidate_commit_subject() {
10383 for bad in [
10384 "TITLE: magi: candidate A (uncommitted work)",
10385 "TITLE: chore: stuff (uncommitted work)",
10386 "TITLE: ",
10387 ] {
10388 let state = state_with_summary("add retries", bad);
10389 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
10390 }
10391 }
10392
10393 #[test]
10394 fn pr_message_bounds_a_very_long_task_and_title() {
10395 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
10396 let state = state_with_summary(&long, "- nothing");
10397 let m = pr_message(&state, 'A');
10398 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10399 assert!(!m.title.contains('\n'));
10400
10401 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
10402 let m = pr_message(&state, 'A');
10403 assert!(m.title.starts_with("feat: "));
10404 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10405 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
10406 }
10407
10408 #[test]
10409 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
10410 let mut state = state_with_summary(
10414 "add retries",
10415 "TITLE: fix(web): batch reads\n- reads run.json once",
10416 );
10417 state.config.graph.language = "ja".to_owned();
10418 let m = pr_message(&state, 'A');
10419 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
10420
10421 let task = "今回やってほしいこと: results projector を直す";
10424 let mut state = state_with_summary(task, "- no title line");
10425 state.config.graph.language = "ja".to_owned();
10426 let m = pr_message(&state, 'A');
10427 assert_eq!(
10428 m.title,
10429 format!("chore: land candidate A of run {}", state.id)
10430 );
10431 assert!(
10432 m.body.contains(&format!(
10433 "<summary>Original task</summary>\n\n{task}\n\n</details>"
10434 )),
10435 "{}",
10436 m.body
10437 );
10438 }
10439
10440 #[test]
10441 fn pr_message_scrubs_home_paths_and_addresses() {
10442 let state = state_with_summary(
10443 "fix it in /Users/someone/src/x",
10444 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
10445 );
10446 let m = pr_message(&state, 'A');
10447 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
10448 assert!(!m.body.contains(leak), "{}", m.body);
10449 }
10450 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
10451 }
10452
10453 #[test]
10454 fn pr_message_survives_a_task_that_closes_details() {
10455 let state = state_with_summary("a </details> b", "TITLE: fix: x");
10456 let m = pr_message(&state, 'A');
10457 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
10458 }
10459
10460 #[test]
10461 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
10462 let cmd = manual_merge_command(
10463 MergeStyle::Squash,
10464 Path::new("/repo"),
10465 "b",
10466 "fix: \"quoted\" $(x) `y`\n\nbody",
10467 );
10468 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
10469 }
10470
10471 #[test]
10472 fn manual_merge_command_matches_the_configured_style() {
10473 let repo = Path::new("/repo");
10474 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
10475
10476 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
10477 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
10478
10479 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
10480 assert_eq!(
10481 squash,
10482 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
10483 \"Merge magi run 0832 (candidate A)\""
10484 );
10485
10486 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
10487 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
10488 }
10489
10490 #[test]
10491 fn a_nudge_gets_a_quarter_of_the_budget() {
10492 assert_eq!(retry_budget(secs(1200), true), secs(300));
10494 assert_eq!(retry_budget(secs(3600), true), secs(900));
10495 }
10496
10497 #[test]
10498 fn a_resent_prompt_keeps_the_whole_budget() {
10499 assert_eq!(retry_budget(secs(1200), false), secs(1200));
10502 assert_eq!(retry_budget(secs(60), false), secs(60));
10503 }
10504
10505 #[test]
10506 fn the_floor_never_exceeds_the_original_budget() {
10507 assert_eq!(retry_budget(secs(60), true), secs(60));
10511 assert_eq!(retry_budget(secs(480), true), secs(120));
10512 assert_eq!(retry_budget(secs(0), true), secs(0));
10513 }
10514
10515 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
10516 agent::CommandEvidence {
10517 id: "item1".to_owned(),
10518 description: "cargo test".to_owned(),
10519 exit_code,
10520 result_summary: String::new(),
10521 source: "codex".to_owned(),
10522 }
10523 }
10524
10525 #[test]
10526 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
10527 assert!(!has_unconfirmed_command(&[]));
10531 }
10532
10533 #[test]
10534 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
10535 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
10539 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
10540 assert!(!has_unconfirmed_command(&[
10541 evidence(Some(0)),
10542 evidence(Some(101))
10543 ]));
10544 }
10545
10546 #[test]
10547 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
10548 assert!(has_unconfirmed_command(&[
10549 evidence(Some(0)),
10550 evidence(None)
10551 ]));
10552 }
10553
10554 #[test]
10555 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
10556 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
10557 assert_eq!(
10558 verified_noop_claim(true, &[], text).as_deref(),
10559 Some("already fixed by b32cfc4, on main.")
10560 );
10561 }
10562
10563 #[test]
10564 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
10565 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
10568 assert!(verified_noop_claim(false, &[], text).is_none());
10569 }
10570
10571 #[test]
10572 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
10573 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
10574 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
10575 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
10577 }
10578
10579 #[test]
10580 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
10581 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
10582 }
10583
10584 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
10587 runner.state.candidates = shape
10588 .iter()
10589 .enumerate()
10590 .map(|(i, &(empty, verified))| Candidate {
10591 index: i,
10592 label: (b'A' + i as u8) as char,
10593 agent: "sonnet".to_owned(),
10594 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
10595 worktree: PathBuf::from(format!("/wt/{i}")),
10596 summary: String::new(),
10597 stat: String::new(),
10598 files: 0,
10599 commits: 0,
10600 empty,
10601 failed: None,
10602 verified_noop: verified.map(str::to_owned),
10603 duration_ms: 0,
10604 folded: false,
10605 })
10606 .collect();
10607 }
10608
10609 #[test]
10610 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
10611 ask_test_home();
10612 let mut runner = runner_at(RunStatus::Implementing);
10613 set_candidates(
10614 &mut runner,
10615 &[
10616 (true, Some("already on main at b32cfc4")),
10617 (true, Some("same fix, see the existing test")),
10618 ],
10619 );
10620
10621 runner
10622 .after_implement()
10623 .expect("a verified no-op is not an error");
10624
10625 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
10626 }
10627
10628 #[test]
10629 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
10630 ask_test_home();
10631 let mut runner = runner_at(RunStatus::Implementing);
10632 set_candidates(
10636 &mut runner,
10637 &[(true, Some("already on main at b32cfc4")), (true, None)],
10638 );
10639
10640 let err = runner
10641 .after_implement()
10642 .expect_err("an unverified empty candidate must still fail the run");
10643
10644 assert!(
10645 err.to_string().contains("no candidate produced a change"),
10646 "{err}"
10647 );
10648 assert_eq!(runner.state.status, RunStatus::Failed);
10649 }
10650
10651 #[test]
10652 fn after_implement_still_fails_an_ordinary_all_empty_run() {
10653 ask_test_home();
10654 let mut runner = runner_at(RunStatus::Implementing);
10655 set_candidates(&mut runner, &[(true, None), (true, None)]);
10656
10657 let err = runner
10658 .after_implement()
10659 .expect_err("no candidate declared anything; this is an ordinary failure");
10660
10661 assert!(
10662 err.to_string().contains("no candidate produced a change"),
10663 "{err}"
10664 );
10665 assert_eq!(runner.state.status, RunStatus::Failed);
10666 }
10667}