1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::refs;
46use crate::run::{
47 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
48 DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, GateFixRecord, Handover, JobRecord,
49 JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest,
50 Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally,
51 VoteRecord, tail, write_artifact,
52};
53use crate::verdict::{
54 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
55 ReviewVote, Severity,
56};
57
58const OUTPUT_TAIL: usize = 8_000;
60
61const EVENT_OUTPUT_TAIL: usize = 2_000;
64
65const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
68
69const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
86
87pub(crate) const STAGNANT_LIMIT: usize = 2;
101
102const BASE_SYNC_ROUNDS: usize = 4;
115
116const MAX_FIX_CONTINUATIONS: usize = 2;
132
133#[derive(Clone)]
139struct SeatJob {
140 spec: AgentSpec,
141 seat: SeatState,
142 cwd: PathBuf,
143 prompt: String,
144 timeout: Duration,
145 allow_write: bool,
146 sessions: bool,
147 artifacts: PathBuf,
148 stem: String,
149}
150
151enum AgentOutcome {
163 Ok(AgentOutput),
165 Quota(AgentOutput),
167 Dropped(AgentOutput),
170 Failed(String),
172}
173
174#[derive(Debug, Clone, Default)]
201pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
202
203impl Pause {
204 #[must_use]
206 pub fn new() -> Self {
207 Self::default()
208 }
209
210 pub fn park(&self) {
212 self.0.store(true, Ordering::SeqCst);
213 }
214
215 pub fn park_because(&self, reason: impl Into<String>) {
221 let mut reason_guard = self
222 .1
223 .lock()
224 .unwrap_or_else(std::sync::PoisonError::into_inner);
225 if reason_guard.is_none() {
226 *reason_guard = Some(reason.into());
227 }
228 drop(reason_guard);
229 self.park();
230 }
231
232 #[must_use]
234 pub fn parked(&self) -> bool {
235 self.0.load(Ordering::SeqCst)
236 }
237
238 #[must_use]
240 pub fn reason(&self) -> Option<String> {
241 self.1
242 .lock()
243 .unwrap_or_else(std::sync::PoisonError::into_inner)
244 .clone()
245 }
246}
247
248pub struct Runner {
250 pub state: RunState,
252 roles: ResolvedRoles,
253 sem: Arc<Semaphore>,
254 pause: Pause,
258 interrupt: Pause,
264}
265
266async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
295 let tracking = format!("{remote}/{branch}");
296 let fetched = git::fetch(repo, remote, branch).await;
297 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
298 let local_exists = git::branch_exists(repo, branch).await?;
299 if !fresh {
300 if !local_exists {
301 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
302 }
303 tracing::warn!(
304 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
305 );
306 return Ok(());
307 }
308 let remote_sha = git::rev_parse(repo, &tracking).await?;
309 if !local_exists {
310 git::git(repo, &["branch", branch, &tracking]).await?;
311 return Ok(());
312 }
313 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
314 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
315 return Ok(());
316 }
317 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
318 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
324 .await?
325 {
326 crate::reconcile::Reconciliation::Pushed => {
327 tracing::warn!(
328 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
329 short(&local_sha),
330 short(&remote_sha)
331 );
332 return Ok(());
333 }
334 crate::reconcile::Reconciliation::Placeholder => {}
335 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
336 }
337 }
338 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
339 if !out.ok() {
340 bail!(
341 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
342 short(&local_sha),
343 short(&remote_sha),
344 out.stderr
345 );
346 }
347 tracing::warn!(
348 "local `{branch}` was stale: fast-forwarded {} -> {}",
349 short(&local_sha),
350 short(&remote_sha)
351 );
352 Ok(())
353}
354
355async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
356 let tracking = format!("{remote}/{base_branch}");
357 let fetched = git::fetch(repo, remote, base_branch).await;
358 if let Ok(out) = &fetched
359 && out.ok()
360 && git::rev_exists(repo, &tracking).await
361 {
362 return git::rev_parse(repo, &tracking).await;
363 }
364 let why = match &fetched {
365 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
366 Ok(_) => format!("{remote} has no {base_branch}"),
367 Err(e) => e.to_string(),
368 };
369 tracing::warn!(
370 "could not read {tracking} ({why}); branching off the local \
371 {base_branch} instead, which may be behind"
372 );
373 git::rev_parse(repo, base_branch).await.with_context(|| {
374 format!(
375 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
376 branch that exists"
377 )
378 })
379}
380
381struct FixClaim {
398 path: PathBuf,
399}
400
401impl FixClaim {
402 fn acquire(dir: &Path) -> Result<Self> {
403 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
404 let path = dir.join("fix.lock");
405 match Self::create(&path) {
406 Ok(claim) => Ok(claim),
407 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
408 if Self::reclaim_if_dead(&path) {
409 Self::create(&path).with_context(|| format!("lock {}", path.display()))
410 } else {
411 bail!(
412 "another `magi fix` is already running for this run ({} exists)",
413 path.display()
414 )
415 }
416 }
417 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
418 }
419 }
420
421 fn create(path: &Path) -> std::io::Result<Self> {
422 let mut f = std::fs::OpenOptions::new()
423 .write(true)
424 .create_new(true)
425 .open(path)?;
426 use std::io::Write as _;
427 writeln!(f, "{}", std::process::id())?;
429 Ok(Self {
430 path: path.to_owned(),
431 })
432 }
433
434 fn reclaim_if_dead(path: &Path) -> bool {
438 let dead = std::fs::read_to_string(path)
439 .ok()
440 .and_then(|body| body.trim().parse::<u32>().ok())
441 .is_some_and(|pid| !crate::proc::pid_alive(pid));
442 dead && std::fs::remove_file(path).is_ok()
443 }
444}
445
446impl Drop for FixClaim {
447 fn drop(&mut self) {
448 let _ = std::fs::remove_file(&self.path);
449 }
450}
451
452impl Runner {
453 pub async fn start(
455 repo: &Path,
456 instruction: String,
457 config: Config,
458 origin: Origin,
459 ) -> Result<Self> {
460 Self::start_naming(repo, instruction, "", config, origin).await
461 }
462
463 pub async fn start_naming(
467 repo: &Path,
468 instruction: String,
469 also_scan: &str,
470 config: Config,
471 origin: Origin,
472 ) -> Result<Self> {
473 let repo = git::toplevel(repo).await?;
474 let missing = agent::missing_programs(&config.agents);
475 if !missing.is_empty() {
476 bail!(
477 "these agent programs are not on PATH: {}. Fix the roster in \
478 magi.toml or install them.",
479 missing.join(", ")
480 );
481 }
482 let base_branch = match config.merge.base.clone() {
483 Some(b) => b,
484 None => git::current_branch(&repo)
485 .await?
486 .context("HEAD is detached; set [merge] base in magi.toml")?,
487 };
488 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
489 if !git::is_clean(&repo).await? {
493 tracing::warn!(
494 "{} has uncommitted changes; they are not part of this run, \
495 which branches off {base_branch} ({})",
496 repo.display(),
497 &base_commit[..base_commit.len().min(8)]
498 );
499 }
500 let roles = config.resolve_roles()?;
501 let max_parallel = config.graph.max_parallel.max(1);
502 let seeds = refs::resolve(
505 &repo,
506 &base_commit,
507 &config.merge.remote,
508 &format!("{also_scan}\n{instruction}"),
509 )
510 .await;
511 refs::plan(&repo, &seeds).await?;
512 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
513 state.origin = Some(origin);
517 for seed in &seeds {
518 state.event(
519 "seed",
520 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
521 );
522 }
523 state.seeds = seeds;
524 state.event("start", format!("run {} created", state.id));
525 state.save()?;
526 Ok(Self {
527 state,
528 roles,
529 sem: Arc::new(Semaphore::new(max_parallel)),
530 pause: Pause::new(),
531 interrupt: Pause::new(),
532 })
533 }
534
535 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
549 Self::review_taking_over(repo, branch, config, None, origin).await
550 }
551
552 pub async fn review_taking_over(
559 repo: &Path,
560 branch: &str,
561 config: Config,
562 takeover: Option<crate::handover::Takeover>,
563 origin: Origin,
564 ) -> Result<Self> {
565 let repo = git::toplevel(repo).await?;
566 let missing = agent::missing_programs(&config.agents);
567 if !missing.is_empty() {
568 bail!(
569 "these agent programs are not on PATH: {}. Fix the roster in \
570 magi.toml or install them.",
571 missing.join(", ")
572 );
573 }
574 let base_branch = match config.merge.base.clone() {
575 Some(b) => b,
576 None => git::current_branch(&repo)
577 .await?
578 .context("HEAD is detached; set [merge] base in magi.toml")?,
579 };
580 if base_branch == branch {
581 bail!("`{branch}` is the base branch; there is nothing to review against");
582 }
583 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
584
585 let roles = config.resolve_roles()?;
586 let max_parallel = config.graph.max_parallel.max(1);
587 let mut state = RunState::new(
588 repo.clone(),
589 base_branch,
590 base_commit.clone(),
591 String::new(),
592 config,
593 );
594 state.origin = Some(origin);
595
596 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
603 earlier: Vec::new(),
604 home: crate::run::home(),
605 choice: None,
606 });
607 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
608 if let Some(released) = &released {
609 state.event(
610 "release",
611 format!(
612 "took `{branch}` over from run {}: its worktree was released: {}",
613 crate::run::short_of(&released.old_id),
614 released.audit
615 ),
616 );
617 }
618 if let Some(choice) = takeover.choice.as_ref()
622 && let Err(e) =
623 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
624 .await
625 {
626 if let Some(released) = &released {
627 released.restore(&repo, branch).await;
628 }
629 return Err(e.context("applying the owner's answer about the diverged branch"));
630 }
631 let opened =
632 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
633 if opened.is_err()
634 && let Some(released) = &released
635 {
636 released.restore(&repo, branch).await;
637 }
638 opened
639 }
640
641 async fn open_review(
644 repo: &Path,
645 branch: &str,
646 mut state: RunState,
647 roles: ResolvedRoles,
648 max_parallel: usize,
649 base_commit: String,
650 ) -> Result<Self> {
651 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
652 let log = git::log_oneline(repo, &base_commit, branch)
655 .await
656 .unwrap_or_default();
657 let instruction = format!(
658 "Review the work already on branch `{branch}`. There is no task \
659 statement: what the change claims to do is whatever its commits \
660 say.\n\n{}",
661 if log.trim().is_empty() {
662 "(no commit messages)"
663 } else {
664 log.trim()
665 }
666 );
667 state.instruction = instruction;
668 state.reviewed_commits = Some(
669 git::subjects(repo, &base_commit, branch)
670 .await
671 .unwrap_or_default(),
672 );
673
674 let worktree = state.worktree_root().join("under-review");
677 if let Some(parent) = worktree.parent() {
678 tokio::fs::create_dir_all(parent).await.ok();
679 }
680 let path = worktree.to_string_lossy().to_string();
681 git::git(repo, &["worktree", "add", &path, branch])
682 .await
683 .with_context(|| {
684 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
685 })?;
686
687 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
688 .await
689 .unwrap_or(0);
690 if commits == 0 {
691 git::worktree_remove(repo, &worktree).await.ok();
692 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
693 }
694 let files = git::changed_files(&worktree, &base_commit, "HEAD")
695 .await
696 .map(|f| f.len())
697 .unwrap_or(0);
698 if files == 0
699 && let (Ok(head_tree), Ok(base_tree)) = (
700 git::tree_of(&worktree, "HEAD").await,
701 git::tree_of(&worktree, &base_commit).await,
702 )
703 && head_tree == base_tree
704 {
705 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
706 git::worktree_remove(repo, &worktree).await.ok();
707 bail!(
708 "`{branch}` at {} has a tree identical to base {}; this usually means \
709 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
710 against `{}/{branch}`) rather than an empty change",
711 short(&head),
712 short(&base_commit),
713 state.config.merge.remote
714 );
715 }
716 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
717 .await
718 .unwrap_or_default();
719
720 state.candidates.push(Candidate {
721 index: 0,
722 label: 'A',
723 agent: EXISTING_BRANCH.to_owned(),
726 branch: branch.to_owned(),
727 worktree,
728 summary: String::new(),
729 stat,
730 files,
731 commits,
732 empty: false,
733 failed: None,
734 verified_noop: None,
735 duration_ms: 0,
736 folded: false,
737 });
738 state.tally = Some(Tally {
739 first_choice: BTreeMap::from([('A', 0)]),
740 borda: BTreeMap::new(),
741 winner: 'A',
742 rankings: 0,
743 unanimous_initial: false,
744 deliberated: false,
745 changed_votes: 0,
746 unanimous_final: false,
747 tie_break: None,
748 judges: 0,
752 present: 0,
753 quorum: 0,
754 met_quorum: true,
755 uncontested: Some("review-only run: nothing competed".to_owned()),
756 });
757 state.status = RunStatus::Reviewing;
758 state.event(
759 "start",
760 format!(
761 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
762 state.id
763 ),
764 );
765 state.save()?;
766 Ok(Self {
767 state,
768 roles,
769 sem: Arc::new(Semaphore::new(max_parallel)),
770 pause: Pause::new(),
771 interrupt: Pause::new(),
772 })
773 }
774
775 pub fn resume(id: &str) -> Result<Self> {
777 let state = RunState::load(id)?;
778 if let Some(to) = &state.released_to {
779 bail!(
780 "run {} cannot be resumed: its worktree was released to run {}",
781 state.short(),
782 crate::run::short_of(to)
783 );
784 }
785 let roles = state.config.resolve_roles()?;
786 let max_parallel = state.config.graph.max_parallel.max(1);
787 Ok(Self {
788 state,
789 roles,
790 sem: Arc::new(Semaphore::new(max_parallel)),
791 pause: Pause::new(),
792 interrupt: Pause::new(),
793 })
794 }
795
796 pub async fn execute(&mut self) -> Result<()> {
803 let result = self.execute_graph().await;
804 self.mark_driver_exited();
805 let ended = if result.is_err() {
806 Some(crate::notices::run_stopped(&self.state.id, &self.state))
807 } else {
808 crate::notices::run_ended(&self.state)
809 };
810 if let Some(notice) = ended {
811 crate::notices::raise(notice);
812 }
813 result
814 }
815
816 fn mark_driver_exited(&mut self) {
825 self.state.driver_exited = true;
826 let pid = std::process::id();
827 let Ok(mut disk) = RunState::load(&self.state.id) else {
828 return;
829 };
830 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
831 return;
832 }
833 disk.driver_exited = true;
834 if let Err(e) = disk.save() {
835 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
836 }
837 }
838
839 async fn execute_graph(&mut self) -> Result<()> {
840 self.state.parked = false;
845 self.state.clear_active();
852 if let Ok(disk) = RunState::load(&self.state.id)
870 && let Some(to) = &disk.released_to
871 {
872 bail!(
873 "run {} cannot continue: its worktree was released to run {}",
874 self.state.short(),
875 crate::run::short_of(to)
876 );
877 }
878 let pid = std::process::id();
879 self.state.driver_pid = Some(pid);
880 self.state.driver_started_at = crate::proc::process_started_at(pid);
881 self.state.driver_exited = false;
882 self.state.save()?;
883 if self.state.status == RunStatus::Stalled {
896 if self.recover_stall().await? {
897 self.finish_after_tally().await?;
898 } else {
899 self.state.save()?;
901 }
902 return Ok(());
903 }
904 if self.state.status == RunStatus::Landing {
914 self.run_land().await?;
915 self.settle_questions();
920 return Ok(());
921 }
922 self.prep().await?;
923 if self.park_here()? {
924 return Ok(());
925 }
926 self.advise().await?;
927 if self.park_here()? {
928 return Ok(());
929 }
930 self.implement().await?;
931 if self.park_here()? {
932 return Ok(());
933 }
934 if self.state.status == RunStatus::VerifiedNoop {
938 return Ok(());
939 }
940 self.judge().await?;
941 if self.park_here()? {
942 return Ok(());
943 }
944 self.deliberate().await?;
945 if self.park_here()? {
946 return Ok(());
947 }
948 self.vote().await?;
949 if self.park_here()? {
950 return Ok(());
951 }
952 self.tally()?;
953 if self.state.status == RunStatus::Stalled {
958 self.state.save()?;
962 return Ok(());
963 }
964 self.finish_after_tally().await?;
965 Ok(())
966 }
967
968 fn park_here(&mut self) -> Result<bool> {
975 if !self.pause.parked() && !self.interrupt.parked() {
980 return Ok(false);
981 }
982 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
983 Some(reason) => format!(
984 "parked after `{}` ({reason}) — resume to carry on from here",
985 self.state.status.as_str()
986 ),
987 None => format!(
988 "parked after `{}` — resume to carry on from here",
989 self.state.status.as_str()
990 ),
991 };
992 self.state.event("park", why);
993 self.state.parked = true;
994 self.state.save()?;
995 Ok(true)
996 }
997
998 pub fn on_pause(&mut self, pause: Pause) {
1000 self.pause = pause;
1001 }
1002
1003 pub fn watch_interrupt(&mut self, pause: Pause) {
1009 self.interrupt = pause;
1010 }
1011
1012 fn settle_questions(&mut self) {
1032 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1033 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1034 }
1035 }
1036
1037 async fn finish_after_tally(&mut self) -> Result<()> {
1040 self.fold_losers().await?;
1041 self.sync_to_base().await?;
1046 if self.state.status == RunStatus::AlreadyInBase {
1047 return Ok(());
1048 }
1049 self.review_loop().await?;
1050 self.sync_to_base().await?;
1051 if self.state.status == RunStatus::AlreadyInBase {
1052 return Ok(());
1053 }
1054 self.gate().await?;
1055 self.merge().await?;
1056 self.state.save()?;
1057 Ok(())
1058 }
1059
1060 async fn prep(&mut self) -> Result<()> {
1063 if !self.state.candidates.is_empty() {
1064 return Ok(());
1065 }
1066 self.state.status = RunStatus::Prep;
1067 let repo = self.state.repo.clone();
1068 let base = self.state.base_commit.clone();
1069 let plan = refs::plan(&repo, &self.state.seeds).await?;
1070 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1071 let root = self.state.worktree_root();
1072 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1073
1074 let hooks_dir = self.state.dir().join("hooks");
1077 if self.state.config.blind.commit_msg_hook {
1078 std::fs::create_dir_all(&hooks_dir)
1079 .with_context(|| format!("create {}", hooks_dir.display()))?;
1080 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1081 let path = hooks_dir.join("commit-msg");
1082 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1083 make_executable(&path)?;
1084 git::acquire_worktree_config(&repo).await?;
1092 self.state.enabled_worktree_config = true;
1093 }
1094
1095 for (index, (spec, label)) in self
1096 .roles
1097 .implementers
1098 .clone()
1099 .into_iter()
1100 .zip(labels)
1101 .enumerate()
1102 {
1103 let branch = self.state.branch_for(label);
1104 let worktree = root.join(format!("cand-{label}"));
1105 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1106 if self.state.config.blind.commit_msg_hook {
1107 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1108 }
1109 git::local_exclude(&worktree, "/.magi/").await?;
1110 for pick in &plan.picks {
1111 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1112 self.state.status = RunStatus::Blocked;
1113 self.state
1114 .event("prep", format!("cannot apply referenced commit: {e}"));
1115 self.state.save()?;
1116 return Err(e);
1117 }
1118 }
1119 self.state.candidates.push(Candidate {
1120 index,
1121 label,
1122 agent: spec.id.clone(),
1123 branch,
1124 worktree,
1125 summary: String::new(),
1126 stat: String::new(),
1127 files: 0,
1128 commits: 0,
1129 empty: false,
1130 failed: None,
1131 verified_noop: None,
1132 duration_ms: 0,
1133 folded: false,
1134 });
1135 }
1136
1137 for j in 1..=self.roles.judges.len() {
1138 let wt = root.join(format!("judge-{j}"));
1139 if !wt.exists() {
1140 git::worktree_add_detached(&repo, &wt, &base).await?;
1141 }
1142 }
1143
1144 if self.state.config.graph.advise {
1152 for k in 1..=self.state.config.graph.advisors {
1153 let wt = root.join(format!("advisor-{k}"));
1154 if !wt.exists() {
1155 git::worktree_add_detached(&repo, &wt, &base).await?;
1156 }
1157 }
1158 }
1159
1160 let authors: Vec<&str> = self
1165 .roles
1166 .implementers
1167 .iter()
1168 .map(|a| a.id.as_str())
1169 .collect();
1170 let overlap: Vec<String> = self
1171 .roles
1172 .judges
1173 .iter()
1174 .enumerate()
1175 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1176 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1177 .collect();
1178 if !overlap.is_empty() {
1179 let note = format!(
1180 "{} also authored a candidate; blind, but the panel is less \
1181 independent than {} distinct agents would be",
1182 overlap.join(", "),
1183 self.roles.judges.len()
1184 );
1185 self.state.event("prep", note);
1186 }
1187
1188 self.state.event(
1189 "prep",
1190 format!(
1191 "{} candidates, {} judges, base {} ({})",
1192 self.state.candidates.len(),
1193 self.roles.judges.len(),
1194 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1195 self.state.base_branch
1196 ),
1197 );
1198 self.state.status = RunStatus::Implementing;
1199 self.state.save()?;
1200 Ok(())
1201 }
1202
1203 async fn advise(&mut self) -> Result<()> {
1236 let implement_untouched = self
1237 .state
1238 .candidates
1239 .iter()
1240 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1241 if !self.state.config.graph.advise || self.state.advise_attempted {
1242 return Ok(());
1243 }
1244 if !implement_untouched {
1245 self.state.event(
1246 "advise",
1247 "skipping the design-deliberation stage: at least one \
1248 candidate already shows implementation progress, so this \
1249 run is past the point the stage exists to run before"
1250 .to_owned(),
1251 );
1252 self.state.advise_attempted = true;
1253 self.state.save()?;
1254 return Ok(());
1255 }
1256 let run_id = self.state.id.clone();
1257 let prompts = self.state.config.prompts.clone();
1258 let instruction = self.state.instruction.clone();
1259 let language = self.state.config.graph.language.clone();
1260 let root = self.state.worktree_root();
1261 let n = self.state.config.graph.advisors;
1262 let where_recorded = self.state.dir().join("run.json");
1263
1264 let seats = match self.state.config.advisors() {
1265 Ok(seats) if !seats.is_empty() => seats,
1266 Ok(_) => {
1267 self.state.event(
1268 "advise",
1269 format!(
1270 "[graph] advisors is 0; skipping the design-deliberation \
1271 stage and continuing without a synthesis brief (see {})",
1272 where_recorded.display()
1273 ),
1274 );
1275 self.state.advise_attempted = true;
1276 self.state.save()?;
1277 return Ok(());
1278 }
1279 Err(e) => {
1280 self.state.event(
1281 "advise",
1282 format!(
1283 "could not resolve advisor seats ({e:#}); continuing \
1284 without a design-deliberation brief (see {})",
1285 where_recorded.display()
1286 ),
1287 );
1288 self.state.advise_attempted = true;
1289 self.state.save()?;
1290 return Ok(());
1291 }
1292 };
1293
1294 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1295 let artifacts = agent::artifacts_dir(&self.state.dir());
1296 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1297
1298 let mut jobs = Vec::new();
1299 for (i, spec) in seats.iter().cloned().enumerate() {
1300 let seat_key = format!("advisor-{}", i + 1);
1301 let seat = self.seat(&seat_key, &spec.id);
1302 jobs.push(SeatJob {
1303 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1304 spec,
1305 seat,
1306 cwd: worktrees[i % worktrees.len()].clone(),
1307 timeout,
1308 allow_write: false,
1309 sessions: false,
1310 artifacts: artifacts.clone(),
1311 stem: seat_key,
1312 });
1313 }
1314
1315 self.state.event(
1316 "advise",
1317 format!(
1318 "{} advisor seat(s) sketching a design in parallel",
1319 jobs.len()
1320 ),
1321 );
1322 let mut quota_losses = Vec::new();
1323 let cache = self.state.config.cache_dir();
1324 let ctx = WaveCtx {
1325 run: &run_id,
1326 node: "advise",
1327 prompts: &prompts,
1328 cache: cache.as_deref(),
1329 round: None,
1330 };
1331 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1332 let results = ask_json_wave::<Proposal>(
1333 jobs,
1334 Arc::clone(&self.sem),
1335 self.state.config.graph.retries,
1336 &advisor_roster,
1337 &ctx,
1338 &mut quota_losses,
1339 &mut self.state,
1340 &|p: &Proposal| p.validate(),
1341 )
1342 .await;
1343 self.state.quota.extend(quota_losses);
1344
1345 let mut records = Vec::with_capacity(results.len());
1346 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1347 let agent_id = seat.agent.clone();
1348 self.state.seats.insert(seat.key.clone(), seat);
1349 match res {
1350 Ok((proposal, out)) => {
1351 self.state
1352 .event("advise", format!("advisor-{} proposed a design", i + 1));
1353 records.push(advise::AdvisorRecord::proposed(
1354 i + 1,
1355 agent_id,
1356 proposal,
1357 out.duration_ms,
1358 ));
1359 }
1360 Err(e) => {
1361 self.state.event(
1362 "advise",
1363 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1364 );
1365 records.push(advise::AdvisorRecord::failed(
1366 i + 1,
1367 agent_id,
1368 e.to_string(),
1369 ));
1370 }
1371 }
1372 }
1373
1374 let mut advice = advise::Advice {
1375 records,
1376 synthesis: None,
1377 };
1378 if advice.proposals().is_empty() {
1379 self.state.event(
1380 "advise",
1381 "no advisor produced a usable proposal; continuing without a \
1382 synthesis brief"
1383 .to_owned(),
1384 );
1385 } else {
1386 match self
1387 .synthesize_brief(
1388 &advice,
1389 &instruction,
1390 &language,
1391 &worktrees[0],
1392 &artifacts,
1393 &run_id,
1394 &prompts,
1395 cache.as_deref(),
1396 )
1397 .await
1398 {
1399 Ok(Some(text)) => {
1400 self.state.event(
1401 "advise",
1402 "synthesized a design brief for the implementer".to_owned(),
1403 );
1404 advice.synthesis = Some(text);
1405 }
1406 Ok(None) => {
1407 self.state.event(
1408 "advise",
1409 "the synthesis seat produced nothing usable; continuing \
1410 without a design brief"
1411 .to_owned(),
1412 );
1413 }
1414 Err(e) => {
1415 self.state.event(
1416 "advise",
1417 format!("could not synthesize a design brief: {e:#}"),
1418 );
1419 }
1420 }
1421 }
1422 advise::apply_reflection(&mut advice);
1423
1424 self.state.advice = Some(advice);
1425 self.state.advise_attempted = true;
1426 self.state.save()?;
1427 Ok(())
1428 }
1429
1430 #[allow(clippy::too_many_arguments)]
1442 async fn synthesize_brief(
1443 &mut self,
1444 advice: &advise::Advice,
1445 instruction: &str,
1446 language: &str,
1447 cwd: &Path,
1448 artifacts: &Path,
1449 run_id: &str,
1450 prompts: &Prompts,
1451 cache: Option<&Path>,
1452 ) -> Result<Option<String>> {
1453 let chain = agent::pick_chain(
1454 &self.state.config.agents,
1455 self.state.config.roles.synthesizer.as_ref(),
1456 &agent::installed,
1457 "synthesizer",
1458 )?;
1459 let proposals = advice.proposals();
1460 let mut prompt = prompt::with_overlay(
1461 prompt::synthesize_brief(instruction, &proposals, language),
1462 prompts.overlay("advise"),
1463 );
1464 if cache.is_some() {
1465 prompt.push('\n');
1470 prompt.push_str(&prompt::build_cache_note("advise", false));
1471 }
1472 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1473 let mut last = None;
1478 for (n, spec) in chain.iter().enumerate() {
1479 if n > 0 {
1480 self.state
1481 .event("advise", format!("synthesis falling back to {}", spec.id));
1482 }
1483 let mut seat = self.seat("advise-synthesis", &spec.id);
1484 let outcome = agent::invoke(
1485 spec,
1486 &mut seat,
1487 &Invocation {
1488 cwd,
1489 prompt: &prompt,
1490 timeout,
1491 allow_write: false,
1492 sessions: false,
1493 artifacts,
1494 stem: &if n == 0 {
1495 "advise-synthesis".to_owned()
1496 } else {
1497 format!("advise-synthesis-{}", spec.id)
1498 },
1499 run: run_id,
1500 node: "advise",
1501 cache_dir: None,
1502 attachments: &[],
1503 writable: &[],
1504 },
1505 )
1506 .await;
1507 if outcome.is_ok() {
1508 self.state.seats.insert(seat.key.clone(), seat);
1509 }
1510 let advance = agent::chain_advances(&outcome);
1511 last = Some(outcome);
1512 if !advance {
1513 break;
1514 }
1515 }
1516 let out = last.expect("a chain holds at least one agent")?;
1519 if !out.usable() {
1520 return Ok(None);
1521 }
1522 let text =
1523 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1524 Ok((!text.trim().is_empty()).then_some(text))
1525 }
1526
1527 async fn implement(&mut self) -> Result<()> {
1530 let run_id = self.state.id.clone();
1535 let prompts = self.state.config.prompts.clone();
1536 let todo: Vec<usize> = self
1537 .state
1538 .candidates
1539 .iter()
1540 .enumerate()
1541 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1542 .map(|(i, _)| i)
1543 .collect();
1544 if todo.is_empty() {
1545 return self.after_implement();
1546 }
1547 self.state.status = RunStatus::Implementing;
1548
1549 let language = self.state.config.graph.language.clone();
1550 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1551 let sessions = self.state.config.graph.sessions;
1552 let artifacts = agent::artifacts_dir(&self.state.dir());
1553 let brief = self
1557 .state
1558 .advice
1559 .as_ref()
1560 .and_then(|a| a.synthesis.as_deref())
1561 .map(str::to_owned);
1562 let attachments = self.state.attachments.clone();
1563
1564 let mut jobs = Vec::new();
1565 for &i in &todo {
1566 let (index, label, worktree) = {
1567 let c = &self.state.candidates[i];
1568 (c.index, c.label, c.worktree.clone())
1569 };
1570 let spec = self.roles.implementers[index].clone();
1571 let seat_key = format!("impl-{label}");
1572 let seat = self.seat(&seat_key, &spec.id);
1573 let instruction = seeded_instruction(&self.state);
1574 jobs.push(SeatJob {
1575 spec,
1576 seat,
1577 prompt: prompt::implement(
1578 &instruction,
1579 &worktree.to_string_lossy(),
1580 &language,
1581 brief.as_deref(),
1582 &attachments,
1583 ),
1584 cwd: worktree,
1585 timeout,
1586 allow_write: true,
1587 sessions,
1588 artifacts: artifacts.clone(),
1589 stem: format!("impl-{label}"),
1590 });
1591 }
1592
1593 self.state.event(
1594 "implement",
1595 format!("{} candidates in parallel", jobs.len()),
1596 );
1597 let mut sent = jobs.clone();
1603 let cache = self.state.config.cache_dir();
1604 let ctx = WaveCtx {
1605 run: &run_id,
1606 node: "implement",
1607 prompts: &prompts,
1608 cache: cache.as_deref(),
1609 round: None,
1610 };
1611 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1612 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1613 .await;
1614 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1615 .await;
1616 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1617 .await;
1618
1619 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1620 let seat_key = seat.key.clone();
1621 let agent = seat.agent.clone();
1630 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1631 self.state.seats.insert(seat.key.clone(), seat);
1632 let label = self.state.candidates[i].label;
1633 let worktree = self.state.candidates[i].worktree.clone();
1634 let base = self.state.base_commit.clone();
1635
1636 let (summary, duration, failed, verified_claim) = match out {
1637 AgentOutcome::Ok(o) => {
1638 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1639 let failed = (!o.usable()).then(|| {
1640 if o.timed_out {
1641 "agent timed out".to_owned()
1642 } else {
1643 format!("agent exited with {:?}", o.exit_code)
1644 }
1645 });
1646 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1647 (text, o.duration_ms, failed, verified_claim)
1648 }
1649 AgentOutcome::Dropped(o) => {
1655 let why = o
1656 .dropped
1657 .as_ref()
1658 .map(|d| d.why.as_str())
1659 .unwrap_or("the CLI ended the stream without delivering its answer");
1660 (
1661 String::new(),
1662 o.duration_ms,
1663 Some(format!("the CLI dropped the stream ({why})")),
1664 None,
1665 )
1666 }
1667 AgentOutcome::Quota(o) => {
1668 self.state.quota.push(QuotaLoss {
1669 seat: seat_key,
1670 node: "implement".to_owned(),
1671 at: Timestamp::now(),
1672 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1673 });
1674 (
1675 String::new(),
1676 o.duration_ms,
1677 Some("rate limited (quota); produced no change".to_owned()),
1678 None,
1679 )
1680 }
1681 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1682 };
1683
1684 let rescued = match git::rescue_commit(
1687 &worktree,
1688 &format!("magi: candidate {label} (uncommitted work)"),
1689 )
1690 .await
1691 {
1692 Ok(r) => {
1693 self.state.note_withheld("implement", &r.withheld);
1694 r.committed
1695 }
1696 Err(_) => false,
1697 };
1698 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1699 .await
1700 .unwrap_or(0);
1701 let patch = git::diff(&worktree, &base, "HEAD")
1702 .await
1703 .unwrap_or_default();
1704 let stat = git::diff_stat(&worktree, &base, "HEAD")
1705 .await
1706 .unwrap_or_default();
1707 let files = git::changed_files(&worktree, &base, "HEAD")
1708 .await
1709 .map(|f| f.len())
1710 .unwrap_or(0);
1711 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1712
1713 let c = &mut self.state.candidates[i];
1714 if !exhausted_the_fallback_chain {
1715 c.agent = agent;
1716 }
1717 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1718 c.stat = stat;
1719 c.files = files;
1720 c.commits = commits;
1721 c.duration_ms = duration;
1722 c.empty = commits == 0 || patch.trim().is_empty();
1723 c.failed = match failed {
1726 Some(_) if c.empty => failed,
1727 _ => None,
1728 };
1729 c.verified_noop = if c.empty { verified_claim } else { None };
1734 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1735 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1736 (None, true, Some(_), _) => {
1737 format!("candidate {label}: no change produced (agent-verified no-op)")
1738 }
1739 (None, true, None, _) => format!("candidate {label}: no change produced"),
1740 (None, false, _, true) => {
1741 format!(
1742 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1743 )
1744 }
1745 (None, false, _, false) => {
1746 format!("candidate {label}: {files} files, {commits} commits")
1747 }
1748 };
1749 self.state.event("implement", note);
1750 self.state.save()?;
1751 }
1752
1753 self.after_implement()
1754 }
1755
1756 async fn resume_undelivered(
1784 &mut self,
1785 results: &mut [(usize, SeatState, AgentOutcome)],
1786 sent: &[SeatJob],
1787 prompts: &Prompts,
1788 run_id: &str,
1789 ) {
1790 for (wi, seat, out) in results.iter_mut() {
1791 let Some(dropped) = (match &*out {
1792 AgentOutcome::Dropped(o) => o.dropped.clone(),
1793 _ => None,
1794 }) else {
1795 continue;
1796 };
1797 let Some(job) = sent.get(*wi) else { continue };
1798 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1800 self.state.event(
1801 "implement",
1802 format!(
1803 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1804 work is in the tree",
1805 seat.key, dropped.output_tokens, dropped.why
1806 ),
1807 );
1808 continue;
1809 }
1810 if !has_context(&job.spec, seat, job.sessions) {
1818 self.state.event(
1819 "implement",
1820 format!(
1821 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1822 is no session left to resume",
1823 seat.key, dropped.output_tokens, dropped.why
1824 ),
1825 );
1826 continue;
1827 }
1828 self.state.event(
1829 "implement",
1830 format!(
1831 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1832 conversation",
1833 seat.key, dropped.output_tokens, dropped.why
1834 ),
1835 );
1836 let mut retry = job.clone();
1837 retry.seat = seat.clone();
1838 retry.prompt = prompt::resume_after_drop(&dropped.why);
1839 retry.timeout = retry_budget(job.timeout, true);
1840 retry.stem = format!("{}-resume", job.stem);
1841 let cache = self.state.config.cache_dir();
1842 let ctx = WaveCtx {
1843 run: run_id,
1844 node: "implement",
1845 prompts,
1846 cache: cache.as_deref(),
1847 round: None,
1848 };
1849 let (resumed_seat, resumed) =
1850 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1851 *seat = resumed_seat;
1852 *out = resumed;
1853 }
1854 }
1855
1856 async fn resume_seat_handovers(
1921 &mut self,
1922 results: &mut [(usize, SeatState, AgentOutcome)],
1923 sent: &mut [SeatJob],
1924 prompts: &Prompts,
1925 run_id: &str,
1926 ) {
1927 let instruction = seeded_instruction(&self.state);
1928 let language = self.state.config.graph.language.clone();
1929 let brief = self
1930 .state
1931 .advice
1932 .as_ref()
1933 .and_then(|a| a.synthesis.as_deref())
1934 .map(str::to_owned);
1935 let attachments = self.state.attachments.clone();
1936 for (wi, seat, out) in results.iter_mut() {
1937 let Some(job) = sent.get_mut(*wi) else {
1938 continue;
1939 };
1940 let start = self
1945 .roles
1946 .implementer_roster
1947 .iter()
1948 .position(|s| s.id == job.spec.id)
1949 .unwrap_or(0);
1950 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1951 let mut fallback_attempt = 0usize;
1952 let mut prev: Option<FailClass> = None;
1953 while let Some(cur) = FailClass::of(&*out) {
1954 if !should_hand_over(prev.as_ref(), &cur) {
1955 break;
1956 }
1957 let Some(next) =
1958 next_untried_in_roster(&self.roles.implementer_roster, start, &tried).cloned()
1959 else {
1960 break;
1961 };
1962 tried.insert(next.id.clone());
1963 fallback_attempt += 1;
1964
1965 if let Ok(r) = git::rescue_commit(
1966 &job.cwd,
1967 &format!(
1968 "magi: candidate {} (uncommitted work before {} fallback)",
1969 seat.key,
1970 if cur == FailClass::Quota {
1971 "quota"
1972 } else {
1973 "handover"
1974 }
1975 ),
1976 )
1977 .await
1978 {
1979 self.state.note_withheld("implement", &r.withheld);
1980 }
1981
1982 record_handover(
1983 &mut self.state,
1984 "implement",
1985 &seat.key,
1986 &seat.agent,
1987 &next.id,
1988 &cur,
1989 &fail_reason(&*out),
1990 );
1991 prev = Some(cur.clone());
1992
1993 let new_seat = handover_seat(&seat.key, &next.id, self.state.seed);
1994 self.state.seats.insert(seat.key.clone(), new_seat.clone());
1995 job.spec = next.clone();
2003 let mut retry = job.clone();
2004 retry.seat = new_seat;
2005 retry.prompt = prompt::implement(
2006 &instruction,
2007 &job.cwd.to_string_lossy(),
2008 &language,
2009 brief.as_deref(),
2010 &attachments,
2011 );
2012 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2013 let cache = self.state.config.cache_dir();
2014 let ctx = WaveCtx {
2015 run: run_id,
2016 node: "implement",
2017 prompts,
2018 cache: cache.as_deref(),
2019 round: None,
2020 };
2021 let (fallback_seat, fallback_out) = run_one(
2022 retry,
2023 Arc::clone(&self.sem),
2024 &ctx,
2025 &mut self.state,
2026 fallback_attempt,
2027 )
2028 .await;
2029 *seat = fallback_seat;
2030 *out = fallback_out;
2031 }
2032 }
2033 }
2034
2035 async fn resume_unconfirmed_commands(
2059 &mut self,
2060 results: &mut [(usize, SeatState, AgentOutcome)],
2061 sent: &[SeatJob],
2062 prompts: &Prompts,
2063 run_id: &str,
2064 ) {
2065 for (wi, seat, out) in results.iter_mut() {
2066 let AgentOutcome::Ok(o) = &*out else {
2067 continue;
2068 };
2069 if !has_unconfirmed_command(&o.commands) {
2070 continue;
2071 }
2072 let Some(job) = sent.get(*wi) else { continue };
2073 if !has_context(&job.spec, seat, job.sessions) {
2074 self.state.event(
2075 "implement",
2076 format!(
2077 "{}: the reply named a command whose own CLI never confirmed the exit \
2078 status of, but there is no session left to resume",
2079 seat.key
2080 ),
2081 );
2082 continue;
2083 }
2084 self.state.event(
2085 "implement",
2086 format!(
2087 "{}: the reply named a command whose own CLI never confirmed the exit \
2088 status of; resuming the conversation",
2089 seat.key
2090 ),
2091 );
2092 let mut retry = job.clone();
2093 retry.seat = seat.clone();
2094 retry.prompt = prompt::resume_incomplete(
2095 "a command in your last reply had no confirmed exit status",
2096 );
2097 retry.timeout = retry_budget(job.timeout, true);
2098 retry.stem = format!("{}-confirm", job.stem);
2099 let cache = self.state.config.cache_dir();
2100 let ctx = WaveCtx {
2101 run: run_id,
2102 node: "implement",
2103 prompts,
2104 cache: cache.as_deref(),
2105 round: None,
2106 };
2107 let (resumed_seat, resumed) =
2108 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2109 *seat = resumed_seat;
2110 *out = resumed;
2111 }
2112 }
2113
2114 async fn continue_fix_report(
2135 &mut self,
2136 mut seat: SeatState,
2137 parse_err: String,
2138 job: &SeatJob,
2139 prompts: &Prompts,
2140 run_id: &str,
2141 round: usize,
2142 ) -> (
2143 SeatState,
2144 Option<FixReport>,
2145 Option<String>,
2146 ContinuationRecord,
2147 ) {
2148 let mut last_err = parse_err;
2149 let mut cumulative_wait_ms = 0u64;
2150 let mut attempts = 0usize;
2151 loop {
2152 if !has_context(&job.spec, &seat, job.sessions) {
2153 self.state.event(
2154 "fix",
2155 format!(
2156 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2157 session left to resume into"
2158 ),
2159 );
2160 let outcome = if attempts == 0 {
2161 ContinuationOutcome::NoSession
2162 } else {
2163 ContinuationOutcome::Exhausted
2164 };
2165 return (
2166 seat,
2167 None,
2168 Some(format!("unparsable fix report: {last_err}")),
2169 ContinuationRecord {
2170 attempts,
2171 cumulative_wait_ms,
2172 outcome,
2173 },
2174 );
2175 }
2176 if attempts >= MAX_FIX_CONTINUATIONS {
2177 self.state.event(
2178 "fix",
2179 format!(
2180 "round {round}: fixer's reply still had no adoption report after \
2181 {attempts} continuation(s) ({last_err}); giving up"
2182 ),
2183 );
2184 return (
2185 seat,
2186 None,
2187 Some(format!(
2188 "unparsable fix report after {attempts} continuation(s): {last_err}"
2189 )),
2190 ContinuationRecord {
2191 attempts,
2192 cumulative_wait_ms,
2193 outcome: ContinuationOutcome::Exhausted,
2194 },
2195 );
2196 }
2197 attempts += 1;
2198 self.state.event(
2199 "fix",
2200 format!(
2201 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2202 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2203 ),
2204 );
2205 let mut retry = job.clone();
2206 retry.seat = seat.clone();
2207 retry.prompt = prompt::resume_incomplete(&last_err);
2208 retry.timeout = retry_budget(job.timeout, true);
2209 retry.stem = format!("{}-continue{attempts}", job.stem);
2210 let cache = self.state.config.cache_dir();
2211 let ctx = WaveCtx {
2212 run: run_id,
2213 node: "fix",
2214 prompts,
2215 cache: cache.as_deref(),
2216 round: Some(round),
2217 };
2218 let (resumed_seat, resumed_out) = run_one(
2219 retry,
2220 Arc::clone(&self.sem),
2221 &ctx,
2222 &mut self.state,
2223 attempts,
2224 )
2225 .await;
2226 seat = resumed_seat;
2227 match resumed_out {
2228 AgentOutcome::Ok(o) => {
2229 cumulative_wait_ms += o.duration_ms;
2230 match verdict::extract_json::<FixReport>(&o.text) {
2231 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2232 self.state.event(
2233 "fix",
2234 format!(
2235 "round {round}: fixer's adoption report recovered after \
2236 {attempts} continuation(s)"
2237 ),
2238 );
2239 return (
2240 seat,
2241 Some(report),
2242 None,
2243 ContinuationRecord {
2244 attempts,
2245 cumulative_wait_ms,
2246 outcome: ContinuationOutcome::Resumed,
2247 },
2248 );
2249 }
2250 Ok(_) => {
2258 last_err = "the reply parsed, but it reported a command whose own CLI \
2259 never confirmed an exit status"
2260 .to_owned();
2261 }
2262 Err(e) => last_err = e.to_string(),
2263 }
2264 }
2265 AgentOutcome::Quota(o) => {
2266 cumulative_wait_ms += o.duration_ms;
2267 self.state.quota.push(QuotaLoss {
2268 seat: seat.key.clone(),
2269 node: "fix".to_owned(),
2270 at: Timestamp::now(),
2271 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2272 });
2273 self.state.event(
2274 "fix",
2275 format!(
2276 "round {round}: continuation rate limited (quota); not retrying now"
2277 ),
2278 );
2279 return (
2280 seat,
2281 None,
2282 Some("rate limited (quota) while recovering the fix report".to_owned()),
2283 ContinuationRecord {
2284 attempts,
2285 cumulative_wait_ms,
2286 outcome: ContinuationOutcome::QuotaLost,
2287 },
2288 );
2289 }
2290 AgentOutcome::Dropped(o) => {
2291 cumulative_wait_ms += o.duration_ms;
2292 let why = o
2293 .dropped
2294 .as_ref()
2295 .map(|d| d.why.as_str())
2296 .unwrap_or("the CLI ended the stream without delivering its answer");
2297 last_err = format!("the CLI dropped the stream ({why})");
2298 }
2299 AgentOutcome::Failed(e) => last_err = e,
2300 }
2301 }
2302 }
2303
2304 fn after_implement(&mut self) -> Result<()> {
2305 if self.state.leaks.is_empty() {
2307 let cfg = self.state.config.blind.clone();
2308 let mut leaks = Vec::new();
2309 for c in &self.state.candidates {
2310 let Some(patch) =
2311 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2312 else {
2313 continue;
2314 };
2315 leaks.extend(blind::scan(
2316 &format!("candidate {} patch", c.label),
2317 &patch,
2318 &cfg.vendor_tokens,
2319 ));
2320 }
2321 if !leaks.is_empty() {
2322 let summary = leaks
2323 .iter()
2324 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2325 .collect::<Vec<_>>()
2326 .join(", ");
2327 match cfg.on_leak {
2328 LeakPolicy::Fail => {
2329 self.state.status = RunStatus::Failed;
2330 self.state
2331 .event("blind", format!("vendor text in a patch: {summary}"));
2332 self.state.leaks = leaks;
2333 self.state.save()?;
2334 self.settle_questions();
2335 bail!(
2336 "blind.on_leak = \"fail\" and vendor text reached a \
2337 judged patch: {summary}"
2338 );
2339 }
2340 LeakPolicy::Redact => self.state.event(
2341 "blind",
2342 format!("redacting vendor text for judging: {summary}"),
2343 ),
2344 LeakPolicy::Warn => self.state.event(
2345 "blind",
2346 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2347 ),
2348 }
2349 self.state.leaks = leaks;
2350 }
2351 }
2352
2353 if self.state.viable().is_empty() {
2354 if self.state.all_candidates_verified_noop() {
2355 self.state.status = RunStatus::VerifiedNoop;
2366 self.state.save()?;
2367 self.settle_questions();
2368 return Ok(());
2369 }
2370 self.state.status = RunStatus::Failed;
2371 self.state.save()?;
2372 self.settle_questions();
2373 bail!("no candidate produced a change; nothing to judge");
2374 }
2375 self.state.status = RunStatus::Judging;
2376 self.state.save()?;
2377 Ok(())
2378 }
2379
2380 async fn judge(&mut self) -> Result<()> {
2383 let run_id = self.state.id.clone();
2388 let prompts = self.state.config.prompts.clone();
2389 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2390 return Ok(());
2391 }
2392 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2393 if viable.len() == 1 {
2394 self.state.judge_skipped = true;
2401 self.state.event(
2402 "judge",
2403 format!(
2404 "only candidate {} produced a change; judging skipped",
2405 viable[0].label
2406 ),
2407 );
2408 self.state.save()?;
2409 return Ok(());
2410 }
2411 self.state.status = RunStatus::Judging;
2412
2413 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2414 let language = self.state.config.graph.language.clone();
2415 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2416 let sessions = self.state.config.graph.sessions;
2417 let artifacts = agent::artifacts_dir(&self.state.dir());
2418 let root = self.state.worktree_root();
2419 let base_short = short(&self.state.base_commit);
2420
2421 let mut jobs = Vec::new();
2422 let mut orders = Vec::new();
2423 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2424 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2425 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2426 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2427 let seat_key = format!("judge-{}", j + 1);
2428 let seat = self.seat(&seat_key, &spec.id);
2429 jobs.push(SeatJob {
2430 prompt: prompt::judge(
2431 &self.state.instruction,
2432 &views,
2433 self.roles.judges.len(),
2434 &base_short,
2435 &language,
2436 ),
2437 spec,
2438 seat,
2439 cwd: root.join(format!("judge-{}", j + 1)),
2440 timeout,
2441 allow_write: false,
2442 sessions,
2443 artifacts: artifacts.clone(),
2444 stem: format!("judge-{}", j + 1),
2445 });
2446 }
2447
2448 self.state.event(
2449 "judge",
2450 format!(
2451 "{} judges ranking {} candidates blind",
2452 jobs.len(),
2453 viable.len()
2454 ),
2455 );
2456 let labels_for_check = labels.clone();
2457 let mut quota_losses = Vec::new();
2458 let cache = self.state.config.cache_dir();
2459 let ctx = WaveCtx {
2460 run: &run_id,
2461 node: "judge",
2462 prompts: &prompts,
2463 cache: cache.as_deref(),
2464 round: None,
2465 };
2466 let results = ask_json_wave::<Ranking>(
2467 jobs,
2468 Arc::clone(&self.sem),
2469 self.state.config.graph.retries,
2470 &self.roles.judge_roster,
2471 &ctx,
2472 &mut quota_losses,
2473 &mut self.state,
2474 &move |r: &Ranking| r.validate(&labels_for_check),
2475 )
2476 .await;
2477 self.state.quota.extend(quota_losses);
2478
2479 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2480 let agent_id = seat.agent.clone();
2481 self.state.seats.insert(seat.key.clone(), seat);
2482 let mut record = Judgement {
2483 judge: j + 1,
2484 seat: format!("judge-{}", j + 1),
2485 agent: agent_id,
2486 ranking: Vec::new(),
2487 reasons: BTreeMap::new(),
2488 confidence: None,
2489 order: orders[j].clone(),
2490 failed: None,
2491 duration_ms: 0,
2492 };
2493 match res {
2494 Ok((ranking, out)) => {
2495 record.ranking = ranking.normalized();
2496 record.reasons = ranking.reasons;
2497 record.confidence = ranking.confidence;
2498 record.duration_ms = out.duration_ms;
2499 self.state.event(
2500 "judge",
2501 format!(
2502 "judge {} ranked {}",
2503 j + 1,
2504 record.ranking.iter().collect::<String>()
2505 ),
2506 );
2507 }
2508 Err(e) => {
2509 record.failed = Some(e.to_string());
2510 self.state
2511 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2512 }
2513 }
2514 self.state.judgements.push(record);
2515 self.state.save()?;
2516 }
2517 Ok(())
2518 }
2519
2520 async fn deliberate(&mut self) -> Result<()> {
2523 let run_id = self.state.id.clone();
2528 let prompts = self.state.config.prompts.clone();
2529 if !self.state.deliberation.is_empty() {
2530 return Ok(());
2531 }
2532 let tops: Vec<char> = self
2533 .state
2534 .judgements
2535 .iter()
2536 .filter_map(|j| j.ranking.first().copied())
2537 .collect();
2538 let rounds = self.state.config.graph.deliberate_rounds;
2539 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2540 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2541 self.state.event(
2542 "deliberate",
2543 format!("judges agreed on {} outright; no deliberation", tops[0]),
2544 );
2545 }
2546 self.state.status = RunStatus::Voting;
2547 self.state.save()?;
2548 return Ok(());
2549 }
2550
2551 self.state.status = RunStatus::Deliberating;
2552 self.state.event(
2553 "deliberate",
2554 format!(
2555 "split: first choices were {} — opening {rounds} round(s)",
2556 tops.iter().collect::<String>()
2557 ),
2558 );
2559
2560 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2561 let language = self.state.config.graph.language.clone();
2562 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2563 let sessions = self.state.config.graph.sessions;
2564 let artifacts = agent::artifacts_dir(&self.state.dir());
2565 let root = self.state.worktree_root();
2566 let base_short = short(&self.state.base_commit);
2567
2568 for round in 1..=rounds {
2572 let mut turns: Vec<DeliberationTurn> = Vec::new();
2573 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2574 if self.state.judgements[j].failed.is_some() {
2575 continue;
2576 }
2577 let seat_key = format!("judge-{}", j + 1);
2578 let spec = self.occupant(&seat_key, spec);
2579 let mut seat = self.seat(&seat_key, &spec.id);
2580 let transcript = self.transcript(&turns, j);
2581 let context = if has_context(&spec, &seat, sessions) {
2582 None
2583 } else {
2584 Some(self.candidate_block(&viable, &base_short))
2585 };
2586 let text = prompt::deliberate(
2587 &self.state.instruction,
2588 context.as_deref(),
2589 &transcript,
2590 round,
2591 rounds,
2592 &language,
2593 );
2594 let job = SeatJob {
2595 spec,
2596 seat: seat.clone(),
2597 prompt: text,
2598 cwd: root.join(format!("judge-{}", j + 1)),
2599 timeout,
2600 allow_write: false,
2601 sessions,
2602 artifacts: artifacts.clone(),
2603 stem: format!("delib-{round}-judge-{}", j + 1),
2604 };
2605 let cache = self.state.config.cache_dir();
2606 let ctx = WaveCtx {
2607 run: &run_id,
2608 node: "deliberate",
2609 prompts: &prompts,
2610 cache: cache.as_deref(),
2611 round: None,
2612 };
2613 let (updated, out) =
2614 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2615 seat = updated;
2616 let agent_id = seat.agent.clone();
2617 let seat_key = seat.key.clone();
2618 self.state.seats.insert(seat.key.clone(), seat);
2619 let body = match out {
2620 AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2621 AgentOutcome::Dropped(o) => {
2625 let why =
2626 o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2627 "the CLI ended the stream without delivering its answer",
2628 );
2629 self.state.event(
2630 "deliberate",
2631 format!(
2632 "judge {} skipped: the CLI dropped the stream ({why})",
2633 j + 1
2634 ),
2635 );
2636 continue;
2637 }
2638 AgentOutcome::Quota(o) => {
2639 self.state.quota.push(QuotaLoss {
2640 seat: seat_key,
2641 node: "deliberate".to_owned(),
2642 at: Timestamp::now(),
2643 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2644 });
2645 self.state.event(
2646 "deliberate",
2647 format!("judge {} skipped: rate limited (quota)", j + 1),
2648 );
2649 continue;
2650 }
2651 AgentOutcome::Failed(e) => {
2652 self.state
2653 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2654 continue;
2655 }
2656 };
2657 let tentative = verdict::extract_json::<Position>(&body)
2658 .ok()
2659 .and_then(|p| p.tentative)
2660 .and_then(|s| s.trim().chars().next())
2661 .map(|c| c.to_ascii_uppercase());
2662 self.state.event(
2663 "deliberate",
2664 format!(
2665 "round {round}: judge {} now favours {}",
2666 j + 1,
2667 tentative.map_or("—".to_owned(), |c| c.to_string())
2668 ),
2669 );
2670 turns.push(DeliberationTurn {
2671 judge: j + 1,
2672 agent: agent_id,
2673 body: blind::sanitize_prose(&body, &self.state.config.blind),
2674 tentative,
2675 });
2676 }
2677 self.state
2678 .deliberation
2679 .push(DeliberationRound { round, turns });
2680 self.state.save()?;
2681 }
2682
2683 self.state.status = RunStatus::Voting;
2684 self.state.save()?;
2685 Ok(())
2686 }
2687
2688 async fn vote(&mut self) -> Result<()> {
2691 let run_id = self.state.id.clone();
2696 let prompts = self.state.config.prompts.clone();
2697 if !self.state.votes.is_empty() {
2698 return Ok(());
2699 }
2700 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2701 if viable.len() == 1 {
2702 return Ok(());
2703 }
2704 self.state.status = RunStatus::Voting;
2705
2706 let language = self.state.config.graph.language.clone();
2707 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2708 let sessions = self.state.config.graph.sessions;
2709 let artifacts = agent::artifacts_dir(&self.state.dir());
2710 let root = self.state.worktree_root();
2711 let base_short = short(&self.state.base_commit);
2712 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2713
2714 let mut jobs = Vec::new();
2715 let mut seats_at = Vec::new();
2716 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2717 if self
2718 .state
2719 .judgements
2720 .get(j)
2721 .is_some_and(|r| r.failed.is_some())
2722 {
2723 continue;
2724 }
2725 let seat_key = format!("judge-{}", j + 1);
2726 let spec = self.occupant(&seat_key, spec);
2727 let seat = self.seat(&seat_key, &spec.id);
2728 let mut text = prompt::final_vote(&viable, &language);
2729 if !has_context(&spec, &seat, sessions) {
2730 text = format!(
2731 "{}\n\n# Candidates\n\n{}",
2732 text,
2733 self.candidate_block(&candidates, &base_short)
2734 );
2735 }
2736 jobs.push(SeatJob {
2737 spec,
2738 seat,
2739 prompt: text,
2740 cwd: root.join(format!("judge-{}", j + 1)),
2741 timeout,
2742 allow_write: false,
2743 sessions,
2744 artifacts: artifacts.clone(),
2745 stem: format!("vote-judge-{}", j + 1),
2746 });
2747 seats_at.push(j);
2748 }
2749
2750 self.state.event(
2751 "vote",
2752 format!(
2753 "collecting {} final votes one by one, privately",
2754 jobs.len()
2755 ),
2756 );
2757 let allowed = viable.clone();
2758 let mut quota_losses = Vec::new();
2759 let cache = self.state.config.cache_dir();
2760 let ctx = WaveCtx {
2761 run: &run_id,
2762 node: "vote",
2763 prompts: &prompts,
2764 cache: cache.as_deref(),
2765 round: None,
2766 };
2767 let results = ask_json_wave::<FinalVote>(
2768 jobs,
2769 Arc::clone(&self.sem),
2770 self.state.config.graph.retries,
2771 &[],
2772 &ctx,
2773 &mut quota_losses,
2774 &mut self.state,
2775 &move |v: &FinalVote| match v.label() {
2776 Some(c) if allowed.contains(&c) => Ok(()),
2777 other => bail!("vote {other:?} is not one of {allowed:?}"),
2778 },
2779 )
2780 .await;
2781 self.state.quota.extend(quota_losses);
2782
2783 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2784 let agent_id = seat.agent.clone();
2785 self.state.seats.insert(seat.key.clone(), seat);
2786 let initial = self
2787 .state
2788 .judgements
2789 .get(j)
2790 .and_then(|r| r.ranking.first().copied());
2791 let mut record = VoteRecord {
2792 judge: j + 1,
2793 agent: agent_id,
2794 vote: None,
2795 reason: String::new(),
2796 changed: false,
2797 };
2798 match res {
2799 Ok((v, _)) => {
2800 record.vote = v.label();
2801 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2802 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2803 self.state.event(
2804 "vote",
2805 format!(
2806 "judge {} voted {}{}",
2807 j + 1,
2808 record.vote.unwrap_or('?'),
2809 if record.changed { " (changed)" } else { "" }
2810 ),
2811 );
2812 }
2813 Err(e) => {
2814 self.state
2815 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2816 }
2817 }
2818 self.state.votes.push(record);
2819 self.state.save()?;
2820 }
2821 Ok(())
2822 }
2823
2824 fn tally(&mut self) -> Result<()> {
2827 if self.state.tally.is_some() {
2828 return Ok(());
2829 }
2830 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2831 let tops: Vec<char> = self
2832 .state
2833 .judgements
2834 .iter()
2835 .filter_map(|j| j.ranking.first().copied())
2836 .collect();
2837 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2838
2839 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2842 let mut cast: Vec<char> = Vec::new();
2843 for (i, j) in self.state.judgements.iter().enumerate() {
2844 let vote = self
2845 .state
2846 .votes
2847 .iter()
2848 .find(|v| v.judge == i + 1)
2849 .and_then(|v| v.vote)
2850 .or_else(|| j.ranking.first().copied());
2851 if let Some(v) = vote {
2852 *first_choice.entry(v).or_insert(0) += 1;
2853 cast.push(v);
2854 }
2855 }
2856
2857 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2858 for j in &self.state.judgements {
2859 let n = j.ranking.len();
2860 for (pos, label) in j.ranking.iter().enumerate() {
2861 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2862 }
2863 }
2864
2865 let best = first_choice.values().copied().max().unwrap_or(0);
2866 let mut leaders: Vec<char> = first_choice
2867 .iter()
2868 .filter(|(_, v)| **v == best)
2869 .map(|(k, _)| *k)
2870 .collect();
2871 let mut tie_break = None;
2872 if leaders.len() > 1 {
2873 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2874 let borda_leaders: Vec<char> = leaders
2875 .iter()
2876 .copied()
2877 .filter(|l| borda[l] == top_borda)
2878 .collect();
2879 tie_break = Some(if borda_leaders.len() == 1 {
2880 format!(
2881 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2882 leaders.len()
2883 )
2884 } else {
2885 format!(
2886 "{} way tie on both first-choice votes and Borda points, broken by label order",
2887 leaders.len()
2888 )
2889 });
2890 leaders = borda_leaders;
2891 leaders.sort_unstable();
2892 }
2893 let winner = *leaders
2894 .first()
2895 .or(viable.first())
2896 .context("no candidate to declare a winner from")?;
2897
2898 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2899 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2900 let deliberated = !self.state.deliberation.is_empty();
2901
2902 let quota_seats: std::collections::BTreeSet<&str> =
2906 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2907 let mut present = 0usize;
2908 for (i, j) in self.state.judgements.iter().enumerate() {
2909 if quota_seats.contains(j.seat.as_str()) {
2910 continue;
2911 }
2912 let ranked = !j.ranking.is_empty() && j.failed.is_none();
2913 let voted = self
2914 .state
2915 .votes
2916 .iter()
2917 .any(|v| v.judge == i + 1 && v.vote.is_some());
2918 if ranked || voted {
2919 present += 1;
2920 }
2921 }
2922 let needs_quorum = viable.len() > 1;
2928 let judges_total = if needs_quorum {
2929 self.roles.judges.len()
2930 } else {
2931 0
2932 };
2933 let quorum = if needs_quorum {
2934 judges_total / 2 + 1
2935 } else {
2936 0
2937 };
2938 let met_quorum = !needs_quorum || present >= quorum;
2939 let uncontested = (!needs_quorum).then(|| {
2940 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2941 });
2942
2943 self.state.event(
2944 "tally",
2945 match &uncontested {
2946 Some(reason) => format!("winner {winner} — {reason}"),
2947 None => format!(
2948 "winner {winner} — votes {} | initial {} | {} changed | \
2949 {present}/{judges_total} judges{}",
2950 first_choice
2951 .iter()
2952 .map(|(k, v)| format!("{k}:{v}"))
2953 .collect::<Vec<_>>()
2954 .join(" "),
2955 if unanimous_initial {
2956 "unanimous"
2957 } else {
2958 "split"
2959 },
2960 changed_votes,
2961 if met_quorum {
2962 String::new()
2963 } else {
2964 format!(" — below quorum ({quorum} required)")
2965 },
2966 ),
2967 },
2968 );
2969 if !met_quorum {
2970 self.state.event(
2971 "stall",
2972 format!(
2973 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2974 the run stops here, resumable"
2975 ),
2976 );
2977 }
2978 self.state.tally = Some(Tally {
2979 first_choice,
2980 borda,
2981 winner,
2982 rankings: tops.len(),
2983 unanimous_initial,
2984 deliberated,
2985 changed_votes,
2986 unanimous_final,
2987 tie_break,
2988 judges: judges_total,
2989 present,
2990 quorum,
2991 met_quorum,
2992 uncontested,
2993 });
2994 self.state.status = if met_quorum {
2995 RunStatus::Reviewing
2996 } else {
2997 RunStatus::Stalled
2998 };
2999 self.state.save()?;
3000 Ok(())
3001 }
3002
3003 #[allow(clippy::too_many_lines)]
3024 async fn recover_stall(&mut self) -> Result<bool> {
3025 let run_id = self.state.id.clone();
3030 let prompts = self.state.config.prompts.clone();
3031 let quota_seats: BTreeSet<&str> =
3036 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3037 let absent: Vec<String> = self
3038 .state
3039 .judgements
3040 .iter()
3041 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3042 .map(|j| j.seat.clone())
3043 .collect();
3044 if absent.is_empty() {
3045 return Ok(false);
3046 }
3047 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3048 if viable.len() <= 1 {
3049 return Ok(false);
3050 }
3051 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3052 let language = self.state.config.graph.language.clone();
3053 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3054 let sessions = self.state.config.graph.sessions;
3055 let artifacts = agent::artifacts_dir(&self.state.dir());
3056 let root = self.state.worktree_root();
3057 let base_short = short(&self.state.base_commit);
3058 let candidates: Vec<Candidate> = viable.clone();
3059
3060 let mut positions: Vec<usize> = absent
3062 .iter()
3063 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3064 .collect();
3065 if positions.is_empty() {
3066 return Ok(false);
3067 }
3068 positions.sort_unstable();
3069 positions.dedup();
3070
3071 let mut judge_jobs = Vec::new();
3073 for &j in &positions {
3074 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3075 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3076 let seat_key = format!("judge-{}", j + 1);
3077 let spec = self.roles.judges[j].clone();
3078 let seat = self.seat(&seat_key, &spec.id);
3079 judge_jobs.push(SeatJob {
3080 spec,
3081 seat,
3082 prompt: prompt::judge(
3083 &self.state.instruction,
3084 &views,
3085 self.roles.judges.len(),
3086 &base_short,
3087 &language,
3088 ),
3089 cwd: root.join(seat_key),
3090 timeout,
3091 allow_write: false,
3092 sessions,
3093 artifacts: artifacts.clone(),
3094 stem: format!("judge-{}-recover", j + 1),
3095 });
3096 }
3097
3098 let labels_for_check = labels.clone();
3099 let mut judge_losses = Vec::new();
3100 let retries = self.state.config.graph.retries;
3101 let cache = self.state.config.cache_dir();
3102 let ctx = WaveCtx {
3103 run: &run_id,
3104 node: "judge",
3105 prompts: &prompts,
3106 cache: cache.as_deref(),
3107 round: None,
3108 };
3109 let results = ask_json_wave::<Ranking>(
3110 judge_jobs,
3111 Arc::clone(&self.sem),
3112 retries,
3113 &[],
3114 &ctx,
3115 &mut judge_losses,
3116 &mut self.state,
3117 &move |r: &Ranking| r.validate(&labels_for_check),
3118 )
3119 .await;
3120
3121 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3123 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3124 self.state.seats.insert(seat.key.clone(), seat);
3125 let record = &mut self.state.judgements[j];
3126 match res {
3127 Ok((ranking, out)) => {
3128 record.ranking = ranking.normalized();
3129 record.reasons = ranking.reasons;
3130 record.confidence = ranking.confidence;
3131 record.failed = None;
3132 record.duration_ms = out.duration_ms;
3133 recovered.insert(j);
3134 self.state.event(
3135 "recover",
3136 format!("judge {} ranked again after the limit", j + 1),
3137 );
3138 }
3139 Err(e) => {
3140 self.state
3141 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3142 }
3143 }
3144 }
3145
3146 let mut vote_jobs = Vec::new();
3148 let mut vote_pos: Vec<usize> = Vec::new();
3149 for &j in &recovered {
3150 let seat_key = format!("judge-{}", j + 1);
3151 let spec = self.roles.judges[j].clone();
3152 let seat = self.seat(&seat_key, &spec.id);
3153 let mut text = prompt::final_vote(&labels, &language);
3154 if !has_context(&spec, &seat, sessions) {
3155 text = format!(
3156 "{}\n\n# Candidates\n\n{}",
3157 text,
3158 self.candidate_block(&candidates, &base_short)
3159 );
3160 }
3161 vote_jobs.push(SeatJob {
3162 spec,
3163 seat,
3164 prompt: text,
3165 cwd: root.join(seat_key),
3166 timeout,
3167 allow_write: false,
3168 sessions,
3169 artifacts: artifacts.clone(),
3170 stem: format!("vote-judge-{}-recover", j + 1),
3171 });
3172 vote_pos.push(j);
3173 }
3174 let allowed = labels.clone();
3175 let mut vote_losses = Vec::new();
3176 let vote_retries = self.state.config.graph.retries;
3177 let vote_cache = self.state.config.cache_dir();
3178 let ctx = WaveCtx {
3179 run: &run_id,
3180 node: "vote",
3181 prompts: &prompts,
3182 cache: vote_cache.as_deref(),
3183 round: None,
3184 };
3185 let votes = ask_json_wave::<FinalVote>(
3186 vote_jobs,
3187 Arc::clone(&self.sem),
3188 vote_retries,
3189 &[],
3190 &ctx,
3191 &mut vote_losses,
3192 &mut self.state,
3193 &move |v: &FinalVote| match v.label() {
3194 Some(c) if allowed.contains(&c) => Ok(()),
3195 other => bail!("vote {other:?} is not one of {allowed:?}"),
3196 },
3197 )
3198 .await;
3199 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3200 let agent_id = seat.agent.clone();
3201 self.state.seats.insert(seat.key.clone(), seat);
3202 match res {
3203 Ok((v, _)) => {
3204 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3205 rec.vote = v.label();
3206 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3207 } else {
3208 self.state.votes.push(VoteRecord {
3209 judge: j + 1,
3210 agent: agent_id,
3211 vote: v.label(),
3212 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3213 changed: false,
3214 });
3215 }
3216 self.state.event(
3217 "recover",
3218 format!("judge {} voted again after the limit", j + 1),
3219 );
3220 }
3221 Err(e) => {
3222 self.state
3223 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3224 }
3225 }
3226 }
3227
3228 let recovered_keys: BTreeSet<String> = recovered
3232 .iter()
3233 .map(|&j| format!("judge-{}", j + 1))
3234 .collect();
3235 self.state
3236 .quota
3237 .retain(|q| !recovered_keys.contains(&q.seat));
3238 for loss in judge_losses.into_iter().chain(vote_losses) {
3242 if recovered_keys.contains(&loss.seat) {
3243 continue;
3244 }
3245 self.state.quota.retain(|q| q.seat != loss.seat);
3246 self.state.quota.push(loss);
3247 }
3248
3249 self.state.tally = None;
3251 self.tally()?;
3252 Ok(self
3253 .state
3254 .tally
3255 .as_ref()
3256 .map(|t| t.met_quorum)
3257 .unwrap_or(false))
3258 }
3259
3260 async fn fold_losers(&mut self) -> Result<()> {
3263 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3264 return Ok(());
3265 };
3266 let repo = self.state.repo.clone();
3267 let mut folded = Vec::new();
3268 for i in 0..self.state.candidates.len() {
3269 let c = &self.state.candidates[i];
3270 if c.label == winner || c.folded {
3271 continue;
3272 }
3273 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3274 git::worktree_remove(&repo, &wt).await.ok();
3275 git::branch_delete(&repo, &branch).await.ok();
3276 self.state.candidates[i].folded = true;
3277 folded.push(label.to_string());
3278 }
3279 let root = self.state.worktree_root();
3281 for j in 1..=self.roles.judges.len() {
3282 let wt = root.join(format!("judge-{j}"));
3283 if wt.exists() {
3284 git::worktree_remove(&repo, &wt).await.ok();
3285 }
3286 }
3287 if self.state.config.graph.advise {
3290 for k in 1..=self.state.config.graph.advisors {
3291 let wt = root.join(format!("advisor-{k}"));
3292 if wt.exists() {
3293 git::worktree_remove(&repo, &wt).await.ok();
3294 }
3295 }
3296 }
3297 if !folded.is_empty() {
3298 self.state
3299 .event("fold", format!("folded candidates {}", folded.join(", ")));
3300 self.state.save()?;
3301 }
3302 Ok(())
3303 }
3304
3305 async fn sync_to_base(&mut self) -> Result<()> {
3345 if self.state.status == RunStatus::AlreadyInBase {
3346 return Ok(());
3347 }
3348 let conflicted = self
3349 .state
3350 .base_sync
3351 .as_ref()
3352 .is_some_and(|s| s.conflict.is_some());
3353 let Some(winner) = self.state.winner().cloned() else {
3354 return Ok(());
3355 };
3356
3357 let repo = self.state.repo.clone();
3358 let remote = self.state.config.merge.remote.clone();
3359 let base_branch = self.state.base_branch.clone();
3360 let tracking = format!("{remote}/{base_branch}");
3361
3362 git::fetch(&repo, &remote, &base_branch).await.ok();
3363 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3367 return Ok(());
3368 };
3369
3370 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3371 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3372 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3373
3374 if (behind > 0 || conflicted || head == tip)
3383 && self
3384 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3385 .await?
3386 {
3387 return Ok(());
3388 }
3389 if conflicted {
3390 return Ok(());
3391 }
3392
3393 if behind == 0 {
3394 if let Some(from) = self
3403 .state
3404 .rebase_fixes
3405 .iter()
3406 .rev()
3407 .find_map(|r| r.from.clone())
3408 && from != head
3409 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3410 .await
3411 .is_ok_and(|o| o.ok())
3412 {
3413 git::sync_to_head(&winner.worktree).await?;
3414 }
3415 self.state.base_sync = Some(BaseSync {
3416 tip,
3417 behind: 0,
3418 attempts,
3419 conflict: None,
3420 already_in: None,
3421 });
3422 self.state.save()?;
3423 return Ok(());
3424 }
3425
3426 if attempts >= BASE_SYNC_ROUNDS {
3427 let why = format!(
3428 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3429 rebase(s); rebasing again would only race it",
3430 winner.branch
3431 );
3432 self.state.status = RunStatus::Blocked;
3433 self.state.base_sync = Some(BaseSync {
3434 tip,
3435 behind,
3436 attempts,
3437 conflict: Some(why.clone()),
3438 already_in: None,
3439 });
3440 self.state.event("land", why);
3441 self.state.save()?;
3442 return Ok(());
3443 }
3444
3445 self.state.event(
3446 "land",
3447 format!(
3448 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3449 winner.branch
3450 ),
3451 );
3452 self.state.save()?;
3453
3454 let branch_tracking = format!("{remote}/{}", winner.branch);
3460 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3461 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3462 git::rev_parse(&repo, &branch_tracking).await.ok()
3463 } else {
3464 None
3465 };
3466 if let Some(theirs) = &remote_tip
3470 && !git::is_ancestor(&repo, theirs, &head).await
3471 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3472 .await
3473 .is_ok_and(|missing| missing.is_empty())
3474 {
3475 let why = format!(
3476 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3477 them",
3478 short(theirs),
3479 winner.branch
3480 );
3481 self.state.status = RunStatus::Blocked;
3482 self.state.base_sync = Some(BaseSync {
3483 tip,
3484 behind,
3485 attempts,
3486 conflict: Some(why.clone()),
3487 already_in: None,
3488 });
3489 self.state.event("land", why);
3490 self.state.save()?;
3491 return Ok(());
3492 }
3493
3494 let scratch = self.state.dir().join("base-sync");
3495 let rebased = match crate::rebase::rebase_with_fixer(
3496 &mut self.state,
3497 &scratch,
3498 &winner.branch,
3499 &tracking,
3500 )
3501 .await
3502 {
3503 Ok(crate::rebase::Rebased::Applied) => Ok(None),
3504 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3505 Err(e) => Err(e),
3506 };
3507 let attempts = attempts + 1;
3508 match rebased {
3509 Ok(None) => {
3510 git::sync_to_head(&winner.worktree).await?;
3514 let mut conflict = None;
3515 if let Some(pinned) = &remote_tip {
3516 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3517 match pushed {
3518 Ok(o) if o.ok() => self.state.event(
3519 "land",
3520 format!("pushed rebased {} to {remote}", winner.branch),
3521 ),
3522 Ok(o) => {
3523 conflict = Some(format!(
3524 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
3525 winner.branch,
3526 short(pinned),
3527 o.stderr.chars().take(600).collect::<String>()
3528 ));
3529 }
3530 Err(e) => {
3531 conflict = Some(format!(
3532 "rebased {} locally but could not push it: {e:#}",
3533 winner.branch
3534 ));
3535 }
3536 }
3537 }
3538 if let Some(why) = &conflict {
3539 self.state.status = RunStatus::Blocked;
3540 self.state.event("land", why.clone());
3541 }
3542 self.state.base_sync = Some(BaseSync {
3543 tip: tip.clone(),
3544 behind: 0,
3545 attempts,
3546 conflict,
3547 already_in: None,
3548 });
3549 self.state
3550 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3551 }
3552 Ok(Some(conflict)) => {
3553 let why = format!(
3554 "{} conflicts with {tracking} and did not rebase: {}",
3555 winner.branch,
3556 conflict.chars().take(600).collect::<String>()
3557 );
3558 self.state.status = RunStatus::Blocked;
3559 self.state.base_sync = Some(BaseSync {
3560 tip,
3561 behind,
3562 attempts,
3563 conflict: Some(why.clone()),
3564 already_in: None,
3565 });
3566 self.state.event("land", why);
3567 }
3568 Err(e) => {
3569 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3570 self.state.status = RunStatus::Blocked;
3571 self.state.base_sync = Some(BaseSync {
3572 tip,
3573 behind,
3574 attempts,
3575 conflict: Some(why.clone()),
3576 already_in: None,
3577 });
3578 self.state.event("land", why);
3579 }
3580 }
3581 self.state.save()?;
3582 Ok(())
3583 }
3584
3585 async fn settle_already_in(
3602 &mut self,
3603 branch: &str,
3604 tip: &str,
3605 head: &str,
3606 attempts: usize,
3607 behind: usize,
3608 ) -> Result<bool> {
3609 let repo = self.state.repo.clone();
3610 let remote = self.state.config.merge.remote.clone();
3611 let start = self.state.base_commit.clone();
3612 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3613 Ok(Some(e)) => e,
3614 Ok(None) => return Ok(false),
3615 Err(e) => {
3616 tracing::warn!("already-in-base check for {branch}: {e:#}");
3617 return Ok(false);
3618 }
3619 };
3620 let mut verified = vec![head.to_owned()];
3621 let fetched = git::fetch(&repo, &remote, branch).await;
3622 if matches!(&fetched, Ok(o) if o.ok())
3623 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3624 && theirs != head
3625 {
3626 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3627 Ok(Some(_)) => verified.push(theirs),
3628 _ => return Ok(false),
3629 }
3630 }
3631 let base_branch = self.state.base_branch.clone();
3632 let message = format!(
3633 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3634 land",
3635 evidence.names(),
3636 evidence.proof.as_str()
3637 );
3638 let closed =
3639 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3640 match closed {
3641 Ok(Ok(url)) => self
3642 .state
3643 .event("land", format!("closed {url}: superseded on {base_branch}")),
3644 Ok(Err(why)) => self
3645 .state
3646 .event("land", format!("did not close a pull request: {why}")),
3647 Err(e) => {
3648 let why = format!(
3649 "{branch} is already in {remote}/{base_branch}, but its pull request could \
3650 not be closed ({e:#}); resume to retry"
3651 );
3652 self.state.status = RunStatus::Blocked;
3653 self.state.base_sync = Some(BaseSync {
3654 tip: tip.to_owned(),
3655 behind,
3656 attempts,
3657 conflict: Some(why.clone()),
3658 already_in: None,
3659 });
3660 self.state.event("land", why);
3661 self.state.save()?;
3662 return Ok(true);
3663 }
3664 }
3665 self.state.status = RunStatus::AlreadyInBase;
3666 self.state.base_sync = Some(BaseSync {
3667 tip: tip.to_owned(),
3668 behind,
3669 attempts,
3670 conflict: None,
3671 already_in: Some(evidence),
3672 });
3673 self.state.event("land", message);
3674 self.state.save()?;
3675 self.settle_questions();
3676 Ok(true)
3677 }
3678
3679 fn landing_base(&self) -> String {
3689 self.state
3690 .base_sync
3691 .as_ref()
3692 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3693 }
3694
3695 pub async fn fix_selected(
3728 &mut self,
3729 ids: &[String],
3730 reason: &str,
3731 allow_stale: bool,
3732 ) -> Result<()> {
3733 let reason = reason.trim();
3734 if reason.is_empty() {
3735 bail!("a fix request needs a reason — that is the operator's own record of why");
3736 }
3737 if ids.is_empty() {
3738 bail!("no finding id given");
3739 }
3740 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3741 bail!(
3742 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3743 has already concluded — can be given a targeted fix. A run still \
3744 in progress should simply be resumed; a `merged` run's branch has \
3745 already landed, so its answer is a fresh `magi review <branch>`, \
3746 not reopening this run's own record",
3747 self.state.id,
3748 self.state.status.as_str()
3749 );
3750 }
3751 let Some(winner) = self.state.winner().cloned() else {
3752 bail!("run {} has no winning candidate to fix", self.state.id);
3753 };
3754 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3755 bail!(
3756 "branch `{}` no longer exists; this run cannot be extended",
3757 winner.branch
3758 );
3759 }
3760 let home = crate::run::home();
3761 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3762 bail!(
3763 "run {} is currently being worked on by another magi process",
3764 self.state.id
3765 );
3766 }
3767 let _claim = FixClaim::acquire(&self.state.dir())?;
3773
3774 let mut seen = BTreeSet::new();
3778 let mut findings = Vec::new();
3779 let mut missing = Vec::new();
3780 for id in ids {
3781 if !seen.insert(id.clone()) {
3782 continue;
3783 }
3784 match self.state.finding(id) {
3785 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3786 id: f.id.clone(),
3787 severity: f.severity,
3788 reviewer_vote: rec.vote,
3789 round: round.round,
3790 round_head: round.head.clone(),
3791 reviewer: rec.reviewer,
3792 agent: rec.agent.clone(),
3793 file: f.file.clone(),
3794 line: f.line,
3795 title: f.title.clone(),
3796 detail: f.detail.clone(),
3797 outcome: OperatorFixOutcome::Pending,
3798 }),
3799 None => missing.push(id.clone()),
3800 }
3801 }
3802 if !missing.is_empty() {
3803 bail!(
3804 "unknown finding id(s): {}; nothing was changed",
3805 missing.join(", ")
3806 );
3807 }
3808
3809 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3810 let stale_details: Vec<(String, String)> = findings
3811 .iter()
3812 .filter(|f| f.round_head != head_at_request)
3813 .map(|f| (f.id.clone(), f.round_head.clone()))
3814 .collect();
3815 let stale = !stale_details.is_empty();
3816 if stale && !allow_stale {
3817 bail!(
3818 "the branch has moved since some finding(s) were raised — {} — now \
3819 at {}; pass --allow-stale to fix anyway, or re-run review first",
3820 stale_details
3821 .iter()
3822 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3823 .collect::<Vec<_>>()
3824 .join(", "),
3825 short(&head_at_request)
3826 );
3827 }
3828
3829 let request = OperatorFixRequest {
3830 requested_at: Timestamp::now(),
3831 reason: reason.to_owned(),
3832 findings,
3833 head_at_request: head_at_request.clone(),
3834 allow_stale,
3835 stale,
3836 fix: None,
3837 result_head: None,
3838 follow_up_review_run: None,
3839 };
3840 self.state.event(
3841 "fix",
3842 format!(
3843 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3844 request.findings.len(),
3845 request
3846 .findings
3847 .iter()
3848 .map(|f| f.id.as_str())
3849 .collect::<Vec<_>>()
3850 .join(", "),
3851 ),
3852 );
3853 self.state.operator_fixes.push(request);
3860 self.state.save()?;
3861 let request_index = self.state.operator_fixes.len() - 1;
3862
3863 if winner.worktree.exists() {
3872 let dirty = git::git(
3875 &winner.worktree,
3876 &["status", "--porcelain", "--untracked-files=all"],
3877 )
3878 .await?;
3879 let only_withheld = dirty.lines().all(|l| {
3880 l.strip_prefix("?? ")
3881 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3882 });
3883 if !only_withheld {
3884 bail!(
3885 "`{}` has uncommitted changes; refusing to touch it — commit or \
3886 discard them first",
3887 winner.worktree.display()
3888 );
3889 }
3890 git::worktree_remove(&self.state.repo, &winner.worktree)
3891 .await
3892 .ok();
3893 }
3894 let fix_worktree = self.state.worktree_root().join("operator-fix");
3895 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3896 git::git(
3897 &self.state.repo,
3898 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3899 )
3900 .await
3901 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3902 if !git::is_clean(&fix_worktree).await? {
3903 git::worktree_remove(&self.state.repo, &fix_worktree)
3904 .await
3905 .ok();
3906 bail!(
3907 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3908 winner.branch
3909 );
3910 }
3911
3912 let run_id = self.state.id.clone();
3913 let prompts = self.state.config.prompts.clone();
3914 let language = self.state.config.graph.language.clone();
3915 let sessions = self.state.config.graph.sessions;
3916 let artifacts = agent::artifacts_dir(&self.state.dir());
3917 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3918 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3919 _ => (
3920 self.state
3921 .config
3922 .agent(&winner.agent)
3923 .cloned()
3924 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3925 format!("impl-{}", winner.label),
3926 ),
3927 };
3928 let seat = self.seat(&fix_seat_key, &fix_spec.id);
3929 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3930 .findings
3931 .iter()
3932 .map(|f| Finding {
3933 id: f.id.clone(),
3934 severity: f.severity,
3935 file: f.file.clone(),
3936 line: f.line,
3937 title: f.title.clone(),
3938 detail: f.detail.clone(),
3939 })
3940 .collect();
3941 let job = SeatJob {
3942 prompt: prompt::operator_fix(
3943 &self.state.instruction,
3944 &finding_list,
3945 reason,
3946 &stale_details,
3947 &head_at_request,
3948 &language,
3949 ),
3950 spec: fix_spec.clone(),
3951 seat,
3952 cwd: fix_worktree.clone(),
3953 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3954 allow_write: true,
3955 sessions,
3956 artifacts: artifacts.clone(),
3957 stem: "operator-fix".to_owned(),
3958 };
3959 let cache = self.state.config.cache_dir();
3960 let ctx = WaveCtx {
3961 run: &run_id,
3962 node: "fix",
3963 prompts: &prompts,
3964 cache: cache.as_deref(),
3965 round: None,
3966 };
3967 let (seat, out) =
3968 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3969 let agent_id = seat.agent.clone();
3970
3971 let mut fix = FixRecord {
3972 agent: agent_id,
3973 addressed: Vec::new(),
3974 rejected: Vec::new(),
3975 notes: String::new(),
3976 committed: false,
3977 failed: None,
3978 duration_ms: 0,
3979 continuation: None,
3980 };
3981 let mut final_seat = seat.clone();
3982 match out {
3983 AgentOutcome::Ok(o) => {
3984 fix.duration_ms = o.duration_ms;
3985 let parsed = verdict::extract_json::<FixReport>(&o.text);
3986 let incomplete_reason = match &parsed {
3987 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3988 "the reply parsed, but it reported a command whose own CLI \
3989 never confirmed an exit status"
3990 .to_owned(),
3991 ),
3992 Ok(_) => None,
3993 Err(e) => Some(e.to_string()),
3994 };
3995 match incomplete_reason {
3996 None => {
3997 let report = parsed.expect("checked Ok above");
3998 fix.addressed = report.addressed;
3999 fix.rejected = report.rejected;
4000 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4001 }
4002 Some(reason) => {
4003 let (resumed_seat, resolved, failure, cont) = self
4004 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4005 .await;
4006 fix.duration_ms += cont.cumulative_wait_ms;
4007 fix.continuation = Some(cont);
4008 final_seat = resumed_seat;
4009 match resolved {
4010 Some(report) => {
4011 fix.addressed = report.addressed;
4012 fix.rejected = report.rejected;
4013 fix.notes =
4014 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4015 }
4016 None => fix.failed = failure,
4017 }
4018 }
4019 }
4020 }
4021 AgentOutcome::Dropped(o) => {
4022 fix.duration_ms = o.duration_ms;
4023 let why = o
4024 .dropped
4025 .as_ref()
4026 .map(|d| d.why.as_str())
4027 .unwrap_or("the CLI ended the stream without delivering its answer");
4028 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4029 }
4030 AgentOutcome::Quota(o) => {
4031 self.state.quota.push(QuotaLoss {
4032 seat: final_seat.key.clone(),
4033 node: "fix".to_owned(),
4034 at: Timestamp::now(),
4035 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4036 });
4037 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4038 }
4039 AgentOutcome::Failed(e) => fix.failed = Some(e),
4040 }
4041 if fix.continuation.is_none() {
4042 fix.continuation = Some(ContinuationRecord::not_needed());
4043 }
4044 self.state.seats.insert(final_seat.key.clone(), final_seat);
4045
4046 let rescue_message = format!(
4047 "magi: operator-selected fix ({}) (uncommitted work)",
4048 self.state.operator_fixes[request_index]
4049 .findings
4050 .iter()
4051 .map(|f| f.id.as_str())
4052 .collect::<Vec<_>>()
4053 .join(", ")
4054 );
4055 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4056 self.state.note_withheld("fix", &r.withheld);
4057 }
4058 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4059 fix.committed = after != head_at_request;
4060 git::worktree_remove(&self.state.repo, &fix_worktree)
4061 .await
4062 .ok();
4063
4064 self.state.event(
4065 "fix",
4066 match &fix.failed {
4067 Some(reason) => format!(
4068 "operator fix: adoption report was lost ({reason}); {}",
4069 if fix.committed {
4070 "committed"
4071 } else {
4072 "NO new commit"
4073 }
4074 ),
4075 None => format!(
4076 "operator fix: {} addressed, {} rejected, {}",
4077 fix.addressed.len(),
4078 fix.rejected.len(),
4079 if fix.committed {
4080 "committed"
4081 } else {
4082 "NO new commit"
4083 }
4084 ),
4085 },
4086 );
4087
4088 for f in &mut self.state.operator_fixes[request_index].findings {
4095 f.outcome = if fix.failed.is_some() {
4096 OperatorFixOutcome::Unreported
4097 } else if fix.addressed.contains(&f.id) {
4098 OperatorFixOutcome::Addressed
4099 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4100 OperatorFixOutcome::Rejected { why: r.why.clone() }
4101 } else {
4102 OperatorFixOutcome::Unreported
4103 };
4104 }
4105
4106 let committed = fix.committed;
4107 if committed {
4108 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4109 }
4110 self.state.operator_fixes[request_index].fix = Some(fix);
4111 self.state.save()?;
4114
4115 if committed {
4116 self.state.event(
4117 "fix",
4118 format!(
4119 "operator fix committed {}; opening a follow-up review-only run",
4120 short(&after)
4121 ),
4122 );
4123 let origin =
4126 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4127 match Self::review(
4128 &self.state.repo,
4129 &winner.branch,
4130 self.state.config.clone(),
4131 origin,
4132 )
4133 .await
4134 {
4135 Ok(mut follow_up) => {
4136 follow_up.state.event(
4137 "start",
4138 format!(
4139 "requested by an operator fix on run {} for finding(s) {}",
4140 self.state.id,
4141 self.state.operator_fixes[request_index]
4142 .findings
4143 .iter()
4144 .map(|f| f.id.as_str())
4145 .collect::<Vec<_>>()
4146 .join(", "),
4147 ),
4148 );
4149 follow_up.state.save()?;
4150 let follow_up_id = follow_up.state.id.clone();
4151 if let Err(e) = follow_up.execute().await {
4152 self.state.event(
4153 "fix",
4154 format!(
4155 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4156 ),
4157 );
4158 }
4159 self.state.operator_fixes[request_index].follow_up_review_run =
4160 Some(follow_up_id);
4161 }
4162 Err(e) => {
4163 self.state.event(
4164 "fix",
4165 format!("committed the fix but could not open a follow-up review: {e:#}"),
4166 );
4167 }
4168 }
4169 self.state.save()?;
4170 }
4171
4172 Ok(())
4173 }
4174
4175 fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
4182 match &self.roles.fixer {
4183 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4184 _ => (
4185 self.state
4186 .config
4187 .agent(&winner.agent)
4188 .cloned()
4189 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4190 format!("impl-{}", winner.label),
4191 ),
4192 }
4193 }
4194
4195 async fn review_loop(&mut self) -> Result<()> {
4196 if self
4201 .state
4202 .base_sync
4203 .as_ref()
4204 .is_some_and(|s| s.conflict.is_some())
4205 {
4206 return Ok(());
4207 }
4208 let run_id = self.state.id.clone();
4213 let prompts = self.state.config.prompts.clone();
4214 let Some(winner) = self.state.winner().cloned() else {
4215 return Ok(());
4216 };
4217 let max_rounds = self.state.config.graph.review_rounds;
4218 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4228 if status == RunStatus::Gating {
4231 self.record_contested_handoff();
4232 }
4233 self.state.status = status;
4234 self.state.save()?;
4235 return Ok(());
4236 }
4237 self.state.status = RunStatus::Reviewing;
4238 if self
4248 .state
4249 .reviews
4250 .last()
4251 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4252 {
4253 let shell = self.state.config.shell();
4254 return self
4255 .stop_reviewing(
4256 "the last round's own verification never resolved",
4257 &shell,
4258 &winner.worktree,
4259 )
4260 .await;
4261 }
4262
4263 let repo = self.state.repo.clone();
4264 let root = self.state.worktree_root();
4265 let language = self.state.config.graph.language.clone();
4266 let sessions = self.state.config.graph.sessions;
4267 let artifacts = agent::artifacts_dir(&self.state.dir());
4268 let base = self.landing_base();
4269 let base_short = short(&base);
4270 let reviewers = self.roles.reviewers.clone();
4271 let shell = self.state.config.shell();
4272
4273 for round in (self.state.reviews.len() + 1)..=max_rounds {
4274 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4275 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4276 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4277 let prev_verification = self
4286 .state
4287 .reviews
4288 .last()
4289 .and_then(|r| r.verification_summary(&head));
4290
4291 let mut jobs = Vec::new();
4295 for (r, spec) in reviewers.iter().cloned().enumerate() {
4296 let wt = root.join(format!("review-{}", r + 1));
4297 if wt.exists() {
4298 git::reset_detached(&wt, &head).await?;
4299 } else {
4300 git::worktree_add_detached(&repo, &wt, &head).await?;
4301 }
4302 let seat_key = format!("review-{}", r + 1);
4303 let seat = self.seat(&seat_key, &spec.id);
4304 jobs.push(SeatJob {
4305 prompt: prompt::review(&prompt::ReviewCtx {
4306 instruction: &self.state.instruction,
4307 branch: &winner.branch,
4308 base_short: &base_short,
4309 stat: &stat,
4310 patch: &patch,
4311 verification: prev_verification.as_ref(),
4312 reviewers: reviewers.len(),
4313 round,
4314 rounds: max_rounds,
4315 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4318 lens: Lens::for_seat(r),
4319 language: &language,
4320 }),
4321 spec,
4322 seat,
4323 cwd: wt,
4324 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4325 allow_write: false,
4326 sessions,
4327 artifacts: artifacts.clone(),
4328 stem: format!("review-{round}-{}", r + 1),
4329 });
4330 }
4331
4332 self.state.event(
4333 "review",
4334 format!(
4335 "round {round}: {} reviewers on {}",
4336 jobs.len(),
4337 short(&head)
4338 ),
4339 );
4340 let mut quota_losses = Vec::new();
4341 let review_retries = self.state.config.graph.retries;
4342 let review_cache = self.state.config.cache_dir();
4343 let ctx = WaveCtx {
4344 run: &run_id,
4345 node: "review",
4346 prompts: &prompts,
4347 cache: review_cache.as_deref(),
4348 round: Some(round),
4349 };
4350 let results = ask_json_wave::<Review>(
4351 jobs,
4352 Arc::clone(&self.sem),
4353 review_retries,
4354 &self.roles.reviewer_roster,
4355 &ctx,
4356 &mut quota_losses,
4357 &mut self.state,
4358 &|_: &Review| Ok(()),
4359 )
4360 .await;
4361 let round_quota_missing = quota_losses.len();
4365 self.state.quota.extend(quota_losses);
4366
4367 let mut records = Vec::new();
4368 let mut all_findings = Vec::new();
4369 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4370 let agent_id = seat.agent.clone();
4371 self.state.seats.insert(seat.key.clone(), seat);
4372 let mut record = ReviewRecord {
4373 reviewer: r + 1,
4374 agent: agent_id,
4375 summary: String::new(),
4376 findings: Vec::new(),
4377 vote: None,
4378 failed: None,
4379 duration_ms: 0,
4380 attempts,
4386 };
4387 match res {
4388 Ok((review, out)) => {
4389 record.summary =
4397 blind::sanitize_prose(&review.summary, &self.state.config.blind);
4398 record.vote = Some(review.vote);
4399 record.duration_ms = out.duration_ms;
4400 for (n, mut f) in review.findings.into_iter().enumerate() {
4401 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4404 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4405 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4406 f.file = f
4412 .file
4413 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4414 all_findings.push(f.clone());
4415 record.findings.push(f);
4416 }
4417 self.state.event(
4418 "review",
4419 format!(
4420 "round {round}: reviewer {} voted {} with {} finding(s)",
4421 r + 1,
4422 review.vote.label(),
4423 record.findings.len()
4424 ),
4425 );
4426 }
4427 Err(e) => {
4428 record.failed = Some(e.to_string());
4429 self.state.event(
4430 "review",
4431 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4432 );
4433 }
4434 }
4435 records.push(record);
4436 }
4437
4438 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4445 let vote_split =
4446 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4447 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4448 if vote_split {
4449 self.state.event(
4450 "review",
4451 format!(
4452 "round {round}: votes split ({}) — one round of reconsideration",
4453 initial_votes
4454 .iter()
4455 .map(|v| v.label())
4456 .collect::<Vec<_>>()
4457 .join(", ")
4458 ),
4459 );
4460 let panel: Vec<ReviewSeatReport<'_>> = records
4463 .iter()
4464 .filter_map(|r| {
4465 r.vote.map(|vote| ReviewSeatReport {
4466 reviewer: r.reviewer,
4467 vote,
4468 summary: &r.summary,
4469 findings: &r.findings,
4470 })
4471 })
4472 .collect();
4473
4474 let mut jobs = Vec::new();
4475 let mut seats_at = Vec::new();
4476 for (r, spec) in reviewers.iter().cloned().enumerate() {
4477 if records[r].vote.is_none() {
4481 continue;
4482 }
4483 let wt = root.join(format!("review-{}", r + 1));
4484 let seat_key = format!("review-{}", r + 1);
4485 let spec = self.occupant(&seat_key, spec);
4486 let seat = self.seat(&seat_key, &spec.id);
4487 let patch_ctx = if has_context(&spec, &seat, sessions) {
4492 None
4493 } else {
4494 Some(ReviewPatch {
4495 branch: &winner.branch,
4496 base_short: &base_short,
4497 stat: &stat,
4498 patch: &patch,
4499 })
4500 };
4501 let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
4502 instruction: &self.state.instruction,
4503 reviewer: r + 1,
4504 lens: Lens::for_seat(r),
4505 panel: &panel,
4506 patch: patch_ctx,
4507 round,
4508 rounds: max_rounds,
4509 language: &language,
4510 });
4511 jobs.push(SeatJob {
4512 prompt,
4513 spec,
4514 seat,
4515 cwd: wt,
4516 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4517 allow_write: false,
4518 sessions,
4519 artifacts: artifacts.clone(),
4520 stem: format!("review-{round}-reconsider-{}", r + 1),
4521 });
4522 seats_at.push(r);
4523 }
4524
4525 let mut recon_quota_losses = Vec::new();
4526 let recon_cache = self.state.config.cache_dir();
4527 let recon_ctx = WaveCtx {
4528 run: &run_id,
4529 node: "review",
4530 prompts: &prompts,
4531 cache: recon_cache.as_deref(),
4532 round: Some(round),
4533 };
4534 let recon_results = ask_json_wave::<ReviewRevote>(
4535 jobs,
4536 Arc::clone(&self.sem),
4537 review_retries,
4538 &[],
4539 &recon_ctx,
4540 &mut recon_quota_losses,
4541 &mut self.state,
4542 &|_: &ReviewRevote| Ok(()),
4543 )
4544 .await;
4545 self.state.quota.extend(recon_quota_losses);
4546
4547 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4548 let agent_id = seat.agent.clone();
4549 self.state.seats.insert(seat.key.clone(), seat);
4550 let mut rec = ReviewRevoteRecord {
4551 reviewer: r + 1,
4552 agent: agent_id,
4553 vote: None,
4554 reason: String::new(),
4555 failed: None,
4556 };
4557 match res {
4558 Ok((rv, _)) => {
4559 rec.vote = Some(rv.vote);
4560 rec.reason =
4561 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4562 self.state.event(
4563 "review",
4564 format!(
4565 "round {round}: reviewer {} revoted {}",
4566 r + 1,
4567 rv.vote.label()
4568 ),
4569 );
4570 }
4571 Err(e) => {
4572 rec.failed = Some(e.to_string());
4573 self.state.event(
4574 "review",
4575 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4576 );
4577 }
4578 }
4579 reconsideration.push(rec);
4580 }
4581 } else if initial_votes.len() > 1 {
4582 self.state.event(
4583 "review",
4584 format!(
4585 "round {round}: votes agreed ({}) — no reconsideration",
4586 initial_votes[0].label()
4587 ),
4588 );
4589 }
4590
4591 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4592 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4593 let defer_e2e =
4604 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4605 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4606 let reason =
4607 format!("{blocking} blocking finding(s) already required a fix this round");
4608 self.state.event(
4609 "verify",
4610 format!(
4611 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4612 {}); it will run once a round has none left",
4613 short(&head)
4614 ),
4615 );
4616 (Vec::new(), false, true, Some(reason))
4617 } else {
4618 let e2e_commands = self.state.config.verify.e2e.clone();
4619 let cache_dir = self.state.config.cache_dir();
4620 let context = format!("round {round}");
4621 let (e2e, verify_retried) = with_cache_lease(
4622 &mut self.state,
4623 cache_dir.as_deref(),
4624 "e2e",
4625 "e2e",
4626 &winner.worktree,
4627 &head,
4628 verify_timeout,
4629 &context,
4630 |state, budget| {
4631 let shell = shell.clone();
4632 let e2e_commands = e2e_commands.clone();
4633 let worktree = winner.worktree.clone();
4634 let context = context.clone();
4635 async move {
4636 run_e2e_with_retry(
4637 state,
4638 &shell,
4639 &e2e_commands,
4640 &worktree,
4641 budget,
4642 &context,
4643 )
4644 .await
4645 }
4646 },
4647 )
4648 .await;
4649 (e2e, verify_retried, false, None)
4650 };
4651
4652 let expected = records.len();
4653 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4654 let incomplete = answered < expected;
4655 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4656 let policy = self.state.config.graph.incomplete_review;
4657 let clean = round_is_clean(
4658 blocking,
4659 e2e_ok,
4660 answered,
4661 expected,
4662 round_quota_missing,
4663 policy,
4664 );
4665
4666 let mut round_record = ReviewRound {
4667 round,
4668 head: head.clone(),
4669 verified_head: None,
4670 verified_at: None,
4671 reviews: records,
4672 e2e,
4673 verify_retried,
4674 e2e_deferred,
4675 e2e_defer_reason,
4676 fix: None,
4677 blocking,
4678 answered,
4679 expected,
4680 clean,
4681 progressed: false,
4682 vote_split,
4683 reconsideration,
4684 verdict: None,
4685 };
4686 round_record.verdict = ReviewVote::worst(
4690 round_record
4691 .final_votes()
4692 .into_iter()
4693 .map(|(_, _, vote)| vote),
4694 );
4695 if !matches!(
4706 round_record.e2e_status(),
4707 E2eStatus::Deferred | E2eStatus::NotConfigured
4708 ) {
4709 round_record.verified_head = Some(head.clone());
4710 round_record.verified_at = Some(Timestamp::now());
4711 }
4712 let this_round_verification = round_record.verification_summary(&head);
4713
4714 if incomplete {
4715 let missing: Vec<String> = round_record
4716 .reviews
4717 .iter()
4718 .filter(|r| r.failed.is_some())
4719 .map(|r| format!("review-{}", r.reviewer))
4720 .collect();
4721 self.state.event(
4722 "review",
4723 format!(
4724 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4725 missing.join(", ")
4726 ),
4727 );
4728 }
4729
4730 if clean {
4731 self.state.event(
4732 "review",
4733 if incomplete && policy == IncompleteReviewPolicy::Warn {
4734 format!(
4735 "round {round}: clean (warn policy, incomplete panel) — no \
4736 blocking findings from the seats that answered, verification green"
4737 )
4738 } else if incomplete {
4739 format!(
4740 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4741 quorum) — no blocking findings from the seats that answered, \
4742 verification green",
4743 expected - answered
4744 )
4745 } else {
4746 format!("round {round}: clean — no blocking findings, verification green")
4747 },
4748 );
4749 self.state.reviews.push(round_record);
4750 self.state.status = RunStatus::Gating;
4751 self.state.save()?;
4752 return Ok(());
4753 }
4754
4755 if incomplete && blocking == 0 && e2e_ok {
4763 self.state.reviews.push(round_record);
4764 self.state.save()?;
4765 if round == max_rounds {
4766 self.state.status = RunStatus::Blocked;
4767 self.state.event(
4768 "review",
4769 format!(
4770 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4771 refusing to call it clean",
4772 expected - answered
4773 ),
4774 );
4775 return Ok(());
4776 }
4777 continue;
4778 }
4779
4780 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4792 self.state.reviews.push(round_record);
4793 return self
4794 .stop_reviewing(
4795 "the round's own verification could not run",
4796 &shell,
4797 &winner.worktree,
4798 )
4799 .await;
4800 }
4801
4802 if round == max_rounds {
4803 self.state.reviews.push(round_record);
4804 return self
4805 .stop_reviewing(
4806 &format!(
4807 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4808 ),
4809 &shell,
4810 &winner.worktree,
4811 )
4812 .await;
4813 }
4814
4815 let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4818 let seat = self.seat(&fix_seat_key, &fix_spec.id);
4819 let blocking_findings: Vec<_> = all_findings
4820 .iter()
4821 .filter(|f| f.severity.blocks())
4822 .cloned()
4823 .collect();
4824 let job = SeatJob {
4825 prompt: prompt::fix(
4826 &self.state.instruction,
4827 &blocking_findings,
4828 this_round_verification.as_ref(),
4829 round,
4830 max_rounds,
4831 &language,
4832 ),
4833 spec: fix_spec.clone(),
4834 seat,
4835 cwd: winner.worktree.clone(),
4836 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4837 allow_write: true,
4838 sessions,
4839 artifacts: artifacts.clone(),
4840 stem: format!("fix-{round}"),
4841 };
4842 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4843 let cache = self.state.config.cache_dir();
4844 let ctx = WaveCtx {
4845 run: &run_id,
4846 node: "fix",
4847 prompts: &prompts,
4848 cache: cache.as_deref(),
4849 round: Some(round),
4850 };
4851 let (seat, out) =
4852 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4853 let agent_id = seat.agent.clone();
4854
4855 let mut fix = FixRecord {
4856 agent: agent_id,
4857 addressed: Vec::new(),
4858 rejected: Vec::new(),
4859 notes: String::new(),
4860 committed: false,
4861 failed: None,
4862 duration_ms: 0,
4863 continuation: None,
4864 };
4865 let mut continuation = ContinuationRecord::not_needed();
4866 let mut final_seat = seat.clone();
4867 match out {
4868 AgentOutcome::Ok(o) => {
4869 fix.duration_ms = o.duration_ms;
4870 let parsed = verdict::extract_json::<FixReport>(&o.text);
4871 let incomplete_reason = match &parsed {
4878 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4879 "the reply parsed, but it reported a command whose own CLI never \
4880 confirmed an exit status"
4881 .to_owned(),
4882 ),
4883 Ok(_) => None,
4884 Err(e) => Some(e.to_string()),
4885 };
4886 match incomplete_reason {
4887 None => {
4888 let report = parsed.expect("checked Ok above");
4889 fix.addressed = report.addressed;
4890 fix.rejected = report.rejected;
4891 fix.notes =
4892 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4893 }
4894 Some(reason) => {
4895 let (resumed_seat, resolved, failure, cont) = self
4896 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4897 .await;
4898 fix.duration_ms += cont.cumulative_wait_ms;
4899 continuation = cont;
4900 final_seat = resumed_seat;
4901 match resolved {
4902 Some(report) => {
4903 fix.addressed = report.addressed;
4904 fix.rejected = report.rejected;
4905 fix.notes = blind::sanitize_prose(
4906 &report.notes,
4907 &self.state.config.blind,
4908 );
4909 }
4910 None => fix.failed = failure,
4911 }
4912 }
4913 }
4914 }
4915 AgentOutcome::Dropped(o) => {
4917 fix.duration_ms = o.duration_ms;
4918 let why = o
4919 .dropped
4920 .as_ref()
4921 .map(|d| d.why.as_str())
4922 .unwrap_or("the CLI ended the stream without delivering its answer");
4923 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4924 }
4925 AgentOutcome::Quota(o) => {
4926 self.state.quota.push(QuotaLoss {
4927 seat: final_seat.key.clone(),
4928 node: "fix".to_owned(),
4929 at: Timestamp::now(),
4930 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4931 });
4932 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4933 }
4934 AgentOutcome::Failed(e) => fix.failed = Some(e),
4935 }
4936 fix.continuation = Some(continuation);
4937 self.state.seats.insert(final_seat.key.clone(), final_seat);
4938 if let Ok(r) = git::rescue_commit(
4939 &winner.worktree,
4940 &format!("magi: review round {round} fixes (uncommitted work)"),
4941 )
4942 .await
4943 {
4944 self.state.note_withheld("fix", &r.withheld);
4945 }
4946 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4947 fix.committed = after != before;
4948 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4956 let progressed = diff_after != patch;
4957 let commit_note = if fix.committed {
4958 "committed"
4959 } else {
4960 "NO new commit"
4961 };
4962 let tree_note = if progressed {
4963 "changed vs base"
4964 } else {
4965 "unchanged vs base"
4966 };
4967 self.state.event(
4968 "fix",
4969 match &fix.failed {
4970 Some(reason) => {
4976 format!(
4977 "round {round}: fixer's adoption report was lost ({reason}); \
4978 {commit_note}, tree {tree_note}"
4979 )
4980 }
4981 None => format!(
4982 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4983 {tree_note}{}",
4984 fix.addressed.len(),
4985 fix.rejected.len(),
4986 if continuation.outcome == ContinuationOutcome::Resumed {
4987 format!(
4988 " (adoption report recovered after {} continuation(s))",
4989 continuation.attempts
4990 )
4991 } else {
4992 String::new()
4993 },
4994 ),
4995 },
4996 );
4997 round_record.fix = Some(fix);
4998 round_record.progressed = progressed;
4999 self.state.reviews.push(round_record);
5000 self.state.save()?;
5001
5002 if matches!(
5015 continuation.outcome,
5016 ContinuationOutcome::Exhausted
5017 | ContinuationOutcome::QuotaLost
5018 | ContinuationOutcome::NoSession
5019 ) {
5020 return self
5021 .stop_reviewing(
5022 "the fixer's adoption report never came back, even after resuming its \
5023 own seat; refusing to start another round against the same worktree \
5024 while that is unresolved",
5025 &shell,
5026 &winner.worktree,
5027 )
5028 .await;
5029 }
5030
5031 let streak = self
5032 .state
5033 .reviews
5034 .iter()
5035 .rev()
5036 .take_while(|r| !r.progressed)
5037 .count();
5038 if streak >= STAGNANT_LIMIT {
5039 return self
5040 .stop_reviewing(
5041 &format!(
5042 "the tree has not moved against base for {streak} round(s) in a row"
5043 ),
5044 &shell,
5045 &winner.worktree,
5046 )
5047 .await;
5048 }
5049 }
5050 Ok(())
5051 }
5052
5053 fn record_contested_handoff(&mut self) {
5087 if self.state.contested_handoff.is_some() {
5088 return;
5089 }
5090 let Some(contested) = self
5091 .state
5092 .reviews
5093 .last()
5094 .and_then(ReviewRound::contested_handoff)
5095 else {
5096 return;
5097 };
5098 self.state.event(
5099 "review",
5100 format!(
5101 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5102 wait for the owner's approval",
5103 contested.findings.len(),
5104 contested.rejecters.len()
5105 ),
5106 );
5107 self.state.contested_handoff = Some(contested);
5108 }
5109
5110 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5111 let round_idx = self.state.reviews.len() - 1;
5112 let needs_catchup_run = matches!(
5120 self.state.reviews[round_idx].e2e_status(),
5121 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5122 );
5123 if needs_catchup_run {
5124 let round = self.state.reviews[round_idx].round;
5125 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5126 let commands = self.state.config.verify.e2e.clone();
5127 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5128 let cache_dir = self.state.config.cache_dir();
5129 let context = format!(
5130 "round {round}: verification unresolved, catching up before the final decision"
5131 );
5132 let (outcomes, verify_retried) = with_cache_lease(
5133 &mut self.state,
5134 cache_dir.as_deref(),
5135 "e2e",
5136 "e2e",
5137 worktree,
5138 &attempted_head,
5139 timeout,
5140 &context,
5141 |state, budget| {
5142 let shell = shell.to_vec();
5143 let commands = commands.clone();
5144 let context = context.clone();
5145 async move {
5146 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5147 .await
5148 }
5149 },
5150 )
5151 .await;
5152 let last = &mut self.state.reviews[round_idx];
5153 last.e2e = outcomes;
5154 last.verify_retried = verify_retried;
5155 last.verified_head = Some(attempted_head);
5162 last.verified_at = Some(Timestamp::now());
5163 if verify_inconclusive(&last.e2e) {
5164 self.state.save()?;
5171 return Ok(());
5172 }
5173 last.e2e_deferred = false;
5174 }
5175 let last = &self.state.reviews[round_idx];
5176 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5177
5178 match last.e2e_status() {
5179 E2eStatus::Failed => {
5180 let red: Vec<String> = last
5181 .e2e
5182 .iter()
5183 .filter(|o| !o.ok())
5184 .map(|o| {
5185 format!(
5186 "`{}` -> {:?}\n{}",
5187 o.command,
5188 o.code,
5189 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5190 )
5191 })
5192 .collect();
5193 self.state
5194 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5195 self.state.status = RunStatus::Blocked;
5196 }
5197 E2eStatus::ResourceBlocked => {
5202 self.state.event(
5203 "review",
5204 format!(
5205 "{why}; e2e could not run (shared build cache unavailable); not \
5206 deciding yet"
5207 ),
5208 );
5209 }
5210 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5211 self.state.event(
5212 "review",
5213 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5214 );
5215 self.record_contested_handoff();
5216 self.state.status = RunStatus::Gating;
5217 }
5218 }
5219 self.state.save()?;
5220 Ok(())
5221 }
5222
5223 async fn gate(&mut self) -> Result<()> {
5226 if self.state.status == RunStatus::Failed
5238 || self
5239 .state
5240 .base_sync
5241 .as_ref()
5242 .is_some_and(|s| s.conflict.is_some())
5243 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5244 != Some(RunStatus::Gating)
5245 {
5246 return Ok(());
5247 }
5248 if self.state.gate_ran {
5249 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5260 self.state.status = RunStatus::Blocked;
5261 self.state.save()?;
5262 }
5263 return Ok(());
5264 }
5265 let Some(winner) = self.state.winner().cloned() else {
5266 return Ok(());
5267 };
5268 self.state.status = RunStatus::Gating;
5269 let mut outcomes = self.run_gate(&winner).await?;
5270 loop {
5271 if verify_inconclusive(&outcomes) {
5282 self.state.save()?;
5283 return Ok(());
5284 }
5285 if outcomes.iter().all(CommandOutcome::ok) {
5286 break;
5287 }
5288 match self.gate_fix_round(&winner, &outcomes).await? {
5289 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5290 GateFix::Stop => break,
5291 GateFix::Defer => {
5292 self.state.save()?;
5293 return Ok(());
5294 }
5295 }
5296 }
5297 let passed = outcomes.iter().all(CommandOutcome::ok);
5298 self.state.gate = outcomes;
5299 self.state.gate_ran = true;
5300 if !passed {
5301 self.state.status = RunStatus::Blocked;
5302 let spent = self.state.gate_fixes.len();
5303 self.state.event(
5304 "gate",
5305 if spent == 0 {
5306 "gate failed; not merging".to_owned()
5307 } else {
5308 format!("gate failed after {spent} gate-fix round(s); not merging")
5309 },
5310 );
5311 }
5312 self.state.save()?;
5313 Ok(())
5314 }
5315
5316 async fn run_pre_gate(&mut self, winner: &Candidate) {
5326 let commands = self.state.config.verify.pre_gate.clone();
5327 if commands.is_empty() {
5328 return;
5329 }
5330 let shell = self.state.config.shell();
5331 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5332 let (outcomes, _) = run_commands(
5333 &mut self.state,
5334 "pre_gate",
5335 "pre_gate",
5336 0,
5337 &shell,
5338 &commands,
5339 &winner.worktree,
5340 timeout,
5341 )
5342 .await;
5343 for o in &outcomes {
5344 if !o.ok() {
5345 tracing::warn!(
5346 "pre_gate `{}` failed ({:?}); the gate decides",
5347 o.command,
5348 o.code
5349 );
5350 }
5351 self.state.event(
5352 "pre_gate",
5353 format!(
5354 "`{}` -> {}",
5355 o.command,
5356 if o.ok() {
5357 "pass".to_owned()
5358 } else {
5359 format!(
5360 "FAIL ({:?})\n{}",
5361 o.code,
5362 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5363 )
5364 }
5365 ),
5366 );
5367 }
5368 self.state.pre_gate = outcomes;
5369 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5370 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5371 Ok(head) => {
5372 self.state
5373 .event("pre_gate", format!("committed mechanical fixes ({head})"));
5374 self.state.pre_gate_commit = Some(head);
5375 }
5376 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5377 },
5378 Ok(false) => {}
5379 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5380 }
5381 if let Err(e) = self.state.save() {
5382 tracing::warn!("could not persist the pre_gate record: {e:#}");
5383 }
5384 }
5385
5386 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5389 self.run_pre_gate(winner).await;
5390 let shell = self.state.config.shell();
5391 let gate_commands = self.state.config.verify.gate.clone();
5392 let outcomes = if gate_commands.is_empty() {
5401 Vec::new()
5402 } else {
5403 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5404 let cache_dir = self.state.config.cache_dir();
5405 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5406 let (outcomes, _) = with_cache_lease(
5407 &mut self.state,
5408 cache_dir.as_deref(),
5409 "gate",
5410 "gate",
5411 &winner.worktree,
5412 &head,
5413 timeout,
5414 "final gate",
5415 |state, budget| {
5416 let shell = shell.clone();
5417 let gate_commands = gate_commands.clone();
5418 let worktree = winner.worktree.clone();
5419 async move {
5420 let (outcomes, timed_out_pids) = run_commands(
5421 state,
5422 "gate",
5423 "gate",
5424 0,
5425 &shell,
5426 &gate_commands,
5427 &worktree,
5428 budget,
5429 )
5430 .await;
5431 (outcomes, false, timed_out_pids)
5432 }
5433 },
5434 )
5435 .await;
5436 outcomes
5437 };
5438 if outcomes.is_empty() {
5439 self.state.event(
5444 "gate",
5445 "no gate commands configured; nothing to check, passing",
5446 );
5447 }
5448 for o in &outcomes {
5449 self.state.event(
5450 "gate",
5451 format!(
5452 "`{}` -> {}",
5453 o.command,
5454 if o.ok() {
5455 "pass".to_owned()
5456 } else {
5457 format!(
5458 "FAIL ({:?})\n{}",
5459 o.code,
5460 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5461 )
5462 }
5463 ),
5464 );
5465 }
5466 Ok(outcomes)
5467 }
5468
5469 async fn gate_fix_round(
5482 &mut self,
5483 winner: &Candidate,
5484 outcomes: &[CommandOutcome],
5485 ) -> Result<GateFix> {
5486 let cap = self.state.config.graph.gate_fix_rounds;
5487 let spent = self.state.gate_fixes.len();
5488 if spent >= cap {
5489 if cap > 0 {
5490 self.state.event(
5491 "gate",
5492 format!("{spent} gate-fix round(s) spent and the gate still fails"),
5493 );
5494 }
5495 return Ok(GateFix::Stop);
5496 }
5497 if !gate_fixable(outcomes) {
5498 self.state.event(
5499 "gate",
5500 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5501 command or similar); not spending a fix round on it",
5502 );
5503 return Ok(GateFix::Stop);
5504 }
5505 let min_free = self.state.config.disk.min_free_bytes;
5506 if min_free > 0 {
5507 match crate::disk::free_bytes(&winner.worktree) {
5508 Ok(free) if crate::disk::enough_space(free, min_free) => {}
5509 Ok(free) => {
5510 self.state.event(
5511 "gate",
5512 format!(
5513 "only {free} bytes free ({min_free} required by `[disk] \
5514 min_free_bytes`); not spending a fix round on a failure the disk \
5515 may explain"
5516 ),
5517 );
5518 return Ok(GateFix::Stop);
5519 }
5520 Err(e) => {
5521 self.state.event(
5522 "gate",
5523 format!("free disk space could not be measured ({e:#}); no fix round"),
5524 );
5525 return Ok(GateFix::Stop);
5526 }
5527 }
5528 }
5529
5530 let attempt = spent + 1;
5531 let run_id = self.state.id.clone();
5532 let prompts = self.state.config.prompts.clone();
5533 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5534 let base = self.landing_base();
5535 let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5536 let seat = self.seat(&fix_seat_key, &fix_spec.id);
5537 let job = SeatJob {
5538 prompt: prompt::gate_fix(
5539 &self.state.instruction,
5540 &failed,
5541 attempt,
5542 cap,
5543 &self.state.config.graph.language,
5544 ),
5545 spec: fix_spec,
5546 seat,
5547 cwd: winner.worktree.clone(),
5548 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5549 allow_write: true,
5550 sessions: self.state.config.graph.sessions,
5551 artifacts: agent::artifacts_dir(&self.state.dir()),
5552 stem: format!("gate-fix-{attempt}"),
5553 };
5554 self.state.event(
5555 "gate",
5556 format!("gate failed; gate-fix round {attempt} of {cap}"),
5557 );
5558 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5559 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5560 let cache = self.state.config.cache_dir();
5561 let ctx = WaveCtx {
5562 run: &run_id,
5563 node: "gate-fix",
5564 prompts: &prompts,
5565 cache: cache.as_deref(),
5566 round: None,
5567 };
5568 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5569 let mut record = GateFixRecord {
5570 agent: seat.agent.clone(),
5571 failed,
5572 notes: String::new(),
5573 committed: false,
5574 error: None,
5575 };
5576 match out {
5577 AgentOutcome::Ok(o) => {
5578 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5581 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5582 }
5583 }
5584 AgentOutcome::Dropped(_) => {
5585 record.error = Some("the CLI dropped the stream".to_owned());
5586 }
5587 AgentOutcome::Quota(o) => {
5588 self.state.quota.push(QuotaLoss {
5589 seat: seat.key.clone(),
5590 node: "gate-fix".to_owned(),
5591 at: Timestamp::now(),
5592 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5593 });
5594 record.error = Some("rate limited (quota); fixer could not run".to_owned());
5595 }
5596 AgentOutcome::Failed(e) => record.error = Some(e),
5597 }
5598 self.state.seats.insert(seat.key.clone(), seat);
5599 if let Ok(r) = git::rescue_commit(
5600 &winner.worktree,
5601 &format!("magi: gate fix {attempt} (uncommitted work)"),
5602 )
5603 .await
5604 {
5605 self.state.note_withheld("gate-fix", &r.withheld);
5606 }
5607 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5608 record.committed = after != before;
5609 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5610 let note = record.error.clone();
5611 self.state.gate_fixes.push(record);
5612 self.state.save()?;
5613 if !changed {
5614 self.state.event(
5615 "gate",
5616 match note {
5617 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5618 None => format!("gate-fix round {attempt}: the tree did not change"),
5619 },
5620 );
5621 return Ok(GateFix::Stop);
5622 }
5623 self.state.event(
5624 "gate",
5625 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5626 );
5627
5628 let commands = self.state.config.verify.e2e.clone();
5629 if !commands.is_empty() {
5630 let shell = self.state.config.shell();
5631 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5632 let cache_dir = self.state.config.cache_dir();
5633 let context = format!("gate-fix round {attempt}");
5634 let (e2e, _) = with_cache_lease(
5635 &mut self.state,
5636 cache_dir.as_deref(),
5637 "e2e",
5638 "e2e",
5639 &winner.worktree,
5640 &after,
5641 timeout,
5642 &context,
5643 |state, budget| {
5644 let shell = shell.clone();
5645 let commands = commands.clone();
5646 let context = context.clone();
5647 let worktree = winner.worktree.clone();
5648 async move {
5649 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5650 .await
5651 }
5652 },
5653 )
5654 .await;
5655 if verify_inconclusive(&e2e) {
5656 return Ok(GateFix::Defer);
5657 }
5658 if e2e.iter().any(|o| !o.ok()) {
5659 self.state.event(
5660 "gate",
5661 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5662 );
5663 return Ok(GateFix::Stop);
5664 }
5665 }
5666 Ok(GateFix::Retry)
5667 }
5668
5669 async fn merge(&mut self) -> Result<()> {
5672 if self
5687 .state
5688 .base_sync
5689 .as_ref()
5690 .is_some_and(|s| s.conflict.is_some())
5691 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5692 != Some(RunStatus::Gating)
5693 || !self.state.gate_status().ok()
5702 {
5703 return Ok(());
5704 }
5705 if self.state.merge.is_some() {
5714 return Ok(());
5715 }
5716 let Some(winner) = self.state.winner().cloned() else {
5717 return Ok(());
5718 };
5719 let repo = self.state.repo.clone();
5720 let base = self.state.base_branch.clone();
5721 let mode = self.state.config.merge.mode;
5722 let style = self.state.config.merge.style;
5723 let pr = pr_message(&self.state, winner.label);
5724 let message = pr.commit_message();
5725
5726 let outcome = match mode {
5727 MergeMode::None => MergeOutcome {
5728 mode,
5729 ok: true,
5730 detail: manual_merge_command(style, &repo, &winner.branch, &message),
5731 empty: false,
5732 },
5733 MergeMode::Pr | MergeMode::Local
5734 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
5735 {
5736 MergeOutcome {
5737 mode,
5738 ok: false,
5739 detail: empty_candidate_detail(&self.state, &base),
5740 empty: true,
5741 }
5742 }
5743 MergeMode::Local => {
5744 let on = git::current_branch(&repo).await?;
5745 if on.as_deref() != Some(base.as_str()) {
5746 MergeOutcome {
5747 mode,
5748 ok: false,
5749 detail: format!(
5750 "{} has {} checked out, not the base branch {base}",
5751 repo.display(),
5752 on.unwrap_or_else(|| "a detached HEAD".to_owned())
5753 ),
5754 empty: false,
5755 }
5756 } else if !git::is_clean(&repo).await? {
5757 MergeOutcome {
5758 mode,
5759 ok: false,
5760 detail: format!("{} is dirty; refusing to merge", repo.display()),
5761 empty: false,
5762 }
5763 } else {
5764 let out = match style {
5765 MergeStyle::Merge => {
5766 git::merge_no_ff(&repo, &winner.branch, &message).await?
5767 }
5768 MergeStyle::Squash => {
5769 git::merge_squash(&repo, &winner.branch, &message).await?
5770 }
5771 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5772 };
5773 MergeOutcome {
5774 mode,
5775 ok: out.ok(),
5776 detail: if out.ok() { out.stdout } else { out.stderr },
5777 empty: false,
5778 }
5779 }
5780 }
5781 MergeMode::Pr => {
5782 let remote = self.state.config.merge.remote.clone();
5783 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5784 if !pushed.ok() {
5785 MergeOutcome {
5786 mode,
5787 ok: false,
5788 detail: pushed.stderr,
5789 empty: false,
5790 }
5791 } else {
5792 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
5798 let out = match pr_merge_plan(found) {
5799 PrPlan::Create => {
5800 gh_pr_create(
5801 &winner.worktree,
5802 &base,
5803 &winner.branch,
5804 &pr.title,
5805 &pr.body,
5806 )
5807 .await
5808 }
5809 PrPlan::Adopt { url, title } => {
5810 self.state
5811 .event("merge", format!("Pr: adopted open pull request {url}"));
5812 if title != pr.title
5813 && let Err(e) =
5814 land::set_pr_title(&winner.worktree, &url, &pr.title).await
5815 {
5816 tracing::warn!("could not refresh title of {url}: {e:#}");
5817 self.state
5818 .event("merge", format!("Pr: title refresh failed: {e:#}"));
5819 }
5820 Ok(url)
5821 }
5822 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
5823 };
5824 match out {
5825 Ok(url) => MergeOutcome {
5826 mode,
5827 ok: true,
5828 detail: url,
5829 empty: false,
5830 },
5831 Err(e) => MergeOutcome {
5832 mode,
5833 ok: false,
5834 detail: e.to_string(),
5835 empty: false,
5836 },
5837 }
5838 }
5839 }
5840 };
5841
5842 self.state.status = match (mode, outcome.ok) {
5843 (MergeMode::None, _) => RunStatus::Ready,
5844 (_, true) => RunStatus::Merged,
5845 (_, false) => RunStatus::Blocked,
5846 };
5847 self.state.event(
5848 "merge",
5849 format!(
5850 "{:?}: {}",
5851 mode,
5852 outcome.detail.lines().next().unwrap_or("")
5853 ),
5854 );
5855 self.state.merge = Some(outcome);
5856 self.state.save()?;
5857
5858 if self.state.config.graph.land
5864 && mode == MergeMode::Pr
5865 && self.state.status == RunStatus::Merged
5866 {
5867 self.run_land().await?;
5868 }
5869 self.settle_questions();
5874 Ok(())
5875 }
5876
5877 async fn run_land(&mut self) -> Result<()> {
5888 let url = self
5889 .state
5890 .merge
5891 .as_ref()
5892 .map(|m| m.detail.clone())
5893 .unwrap_or_default();
5894 let url = url.lines().next().unwrap_or("").trim().to_owned();
5895 if !url.starts_with("http") {
5896 return Ok(());
5897 }
5898 match land::land(&mut self.state, &url).await {
5901 Ok(pr) if self.state.parked => {
5902 let _ = pr;
5906 }
5907 Ok(pr) => {
5908 self.state.status = match pr.state {
5909 land::PrLifecycle::Merged => RunStatus::Merged,
5910 _ => RunStatus::Blocked,
5911 };
5912 if bump::should_release_bump(self.state.status)
5919 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
5920 {
5921 self.state
5927 .event("bump", format!("release bump skipped: {e:#}"));
5928 }
5929 if self.state.status == RunStatus::Merged {
5932 crate::followup::after_merge(&mut self.state, &pr.url).await;
5933 }
5934 self.state.save()?;
5935 }
5936 Err(e) => {
5937 self.state.status = RunStatus::Blocked;
5938 self.state.event("land", format!("gave up: {e}"));
5939 self.state.save()?;
5940 }
5941 }
5942 Ok(())
5943 }
5944
5945 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
5949 if let Some(existing) = self.state.seats.get(key)
5950 && existing.agent == agent
5951 {
5952 return existing.clone();
5953 }
5954 let fresh = SeatState::new(key, agent, self.state.seed);
5955 self.state.seats.insert(key.to_owned(), fresh.clone());
5956 fresh
5957 }
5958
5959 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
5965 match self.state.seats.get(key) {
5966 Some(s) if s.agent != spec.id => {
5967 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
5968 }
5969 _ => spec,
5970 }
5971 }
5972
5973 fn view(&self, c: &Candidate) -> CandidateView {
5975 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
5976 .unwrap_or_default();
5977 let (patch, _) = blind::sanitize_patch(
5978 &format!("candidate {} patch", c.label),
5979 &raw,
5980 &self.state.config.blind,
5981 );
5982 CandidateView {
5983 label: c.label,
5984 branch: c.branch.clone(),
5985 summary: c.summary.clone(),
5986 stat: c.stat.clone(),
5987 patch,
5988 }
5989 }
5990
5991 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
5993 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
5994 prompt::judge(
5995 "(see above)",
5996 &views,
5997 self.roles.judges.len(),
5998 base_short,
5999 "en",
6000 )
6001 }
6002
6003 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6010 let mut turns = Vec::new();
6011 for j in &self.state.judgements {
6012 if j.ranking.is_empty() {
6013 continue;
6014 }
6015 let reasons = j
6016 .reasons
6017 .iter()
6018 .map(|(k, v)| format!("- {k}: {v}"))
6019 .collect::<Vec<_>>()
6020 .join("\n");
6021 turns.push(Turn {
6022 who: format!("Judge {} (opening ranking)", j.judge),
6023 is_self: j.judge == self_idx + 1,
6024 body: format!(
6025 "Ranked {}{}{reasons}",
6026 j.ranking.iter().collect::<String>(),
6027 if reasons.is_empty() {
6028 ""
6029 } else {
6030 ", because:\n"
6031 }
6032 ),
6033 });
6034 }
6035 for t in self
6036 .state
6037 .deliberation
6038 .iter()
6039 .flat_map(|r| r.turns.iter())
6040 .chain(current)
6041 {
6042 turns.push(Turn {
6043 who: format!("Judge {}", t.judge),
6044 is_self: t.judge == self_idx + 1,
6045 body: t.body.clone(),
6046 });
6047 }
6048 turns
6049 }
6050}
6051
6052fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6054 agent::has_session(spec.kind, seat, sessions)
6055}
6056
6057fn next_untried_in_roster<'a>(
6078 roster: &'a [AgentSpec],
6079 start: usize,
6080 tried: &BTreeSet<String>,
6081) -> Option<&'a AgentSpec> {
6082 roster
6083 .get(start + 1..)?
6084 .iter()
6085 .find(|s| !tried.contains(&s.id))
6086}
6087
6088fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6092 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6093}
6094
6095const TIMED_OUT: &str = "timed out";
6098
6099#[derive(Debug, Clone, PartialEq, Eq)]
6102enum FailClass {
6103 Quota,
6104 Timeout,
6105 Other(String),
6107}
6108
6109impl FailClass {
6110 fn of(out: &AgentOutcome) -> Option<Self> {
6112 match out {
6113 AgentOutcome::Ok(_) => None,
6114 AgentOutcome::Quota(_) => Some(Self::Quota),
6115 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6116 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6117 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6118 }
6119 }
6120
6121 fn stem_word(&self) -> &'static str {
6123 match self {
6124 Self::Quota => "quota",
6125 _ => "handover",
6126 }
6127 }
6128}
6129
6130fn failure_signature(msg: &str) -> String {
6134 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6135 let mut out = Vec::new();
6136 for word in line.split_whitespace() {
6137 if word.contains('/') || word.contains('\\') {
6138 out.push("<path>".to_owned());
6139 continue;
6140 }
6141 let mut w = String::new();
6142 let mut in_digits = false;
6143 for c in word.chars() {
6144 if c.is_ascii_digit() {
6145 if !in_digits {
6146 w.push('#');
6147 }
6148 in_digits = true;
6149 } else {
6150 in_digits = false;
6151 w.push(c);
6152 }
6153 }
6154 out.push(w);
6155 }
6156 out.join(" ").chars().take(120).collect()
6157}
6158
6159fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6166 match cur {
6167 FailClass::Quota | FailClass::Timeout => true,
6168 FailClass::Other(_) => prev != Some(cur),
6169 }
6170}
6171
6172fn fail_reason(out: &AgentOutcome) -> String {
6174 match out {
6175 AgentOutcome::Ok(_) => String::new(),
6176 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6177 AgentOutcome::Dropped(o) => format!(
6178 "the CLI dropped the stream ({})",
6179 o.dropped
6180 .as_ref()
6181 .map(|d| d.why.as_str())
6182 .unwrap_or("it ended without delivering its answer")
6183 ),
6184 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6185 }
6186}
6187
6188fn record_handover(
6191 state: &mut RunState,
6192 node: &str,
6193 seat: &str,
6194 from: &str,
6195 to: &str,
6196 class: &FailClass,
6197 reason: &str,
6198) {
6199 let message = if *class == FailClass::Quota {
6200 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6201 } else {
6202 format!("{seat}: handed over {from} -> {to} ({reason})")
6203 };
6204 state.event(node, message);
6205 state.handovers.push(Handover {
6206 at: Timestamp::now(),
6207 node: node.to_owned(),
6208 seat: seat.to_owned(),
6209 from: from.to_owned(),
6210 to: to.to_owned(),
6211 reason: reason.to_owned(),
6212 });
6213}
6214
6215fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6229 commands.iter().any(|c| c.exit_code.is_none())
6230}
6231
6232fn verified_noop_claim(
6245 usable: bool,
6246 commands: &[agent::CommandEvidence],
6247 text: &str,
6248) -> Option<String> {
6249 (usable && !has_unconfirmed_command(commands))
6250 .then(|| verdict::verified_noop(text))
6251 .flatten()
6252}
6253
6254fn short(commit: &str) -> String {
6255 commit.chars().take(7).collect()
6256}
6257
6258fn make_executable(path: &Path) -> Result<()> {
6259 #[cfg(unix)]
6260 {
6261 use std::os::unix::fs::PermissionsExt as _;
6262 let mut perms = std::fs::metadata(path)?.permissions();
6263 perms.set_mode(0o755);
6264 std::fs::set_permissions(path, perms)?;
6265 }
6266 #[cfg(not(unix))]
6267 {
6268 let _ = path;
6269 }
6270 Ok(())
6271}
6272
6273struct WaveCtx<'a> {
6280 run: &'a str,
6283 node: &'a str,
6285 prompts: &'a Prompts,
6286 cache: Option<&'a Path>,
6288 round: Option<usize>,
6291}
6292
6293async fn run_one(
6295 job: SeatJob,
6296 sem: Arc<Semaphore>,
6297 ctx: &WaveCtx<'_>,
6298 state: &mut RunState,
6299 attempt: usize,
6300) -> (SeatState, AgentOutcome) {
6301 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6302 .await
6303 .pop()
6304 .expect("one job in, one result out");
6305 (seat, out)
6306}
6307
6308async fn wave(
6314 jobs: Vec<SeatJob>,
6315 sem: Arc<Semaphore>,
6316 ctx: &WaveCtx<'_>,
6317 state: &mut RunState,
6318 attempt: usize,
6319) -> Vec<(usize, SeatState, AgentOutcome)> {
6320 let WaveCtx {
6321 run,
6322 node,
6323 prompts,
6324 cache,
6325 round,
6326 } = *ctx;
6327 for job in &jobs {
6328 state.seat_started(node, &job.seat.key, job.timeout, attempt);
6329 }
6330 if let Err(e) = state.save() {
6331 tracing::warn!("could not persist in-progress seats: {e:#}");
6336 }
6337 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6353 let wait_started = Instant::now();
6354 let cache_guard = if let Some(cache_dir) = cache {
6355 if jobs_had_a_writer {
6356 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6357 let budget = jobs
6358 .iter()
6359 .map(|j| j.timeout)
6360 .max()
6361 .unwrap_or(Duration::from_secs(60));
6362 acquire_cache_lease(state, cache_dir, &owner, budget, node)
6363 .await
6364 .ok()
6365 } else {
6366 None
6367 }
6368 } else {
6369 None
6370 };
6371 let waited_for_lease = wait_started.elapsed();
6378 let mut set = tokio::task::JoinSet::new();
6379 let overlay = prompts.overlay(node);
6380 for (i, mut job) in jobs.into_iter().enumerate() {
6381 job.timeout = job.timeout.saturating_sub(waited_for_lease);
6382 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6383 if cache.is_some() {
6384 job.prompt.push('\n');
6385 job.prompt
6386 .push_str(&prompt::build_cache_note(node, job.allow_write));
6387 }
6388 let sem = Arc::clone(&sem);
6389 let run = run.to_owned();
6390 let node = node.to_owned();
6391 let attachments = if node == "implement" {
6394 state.attachments.clone()
6395 } else {
6396 Vec::new()
6397 };
6398 let cache = cache
6409 .filter(|_| job.allow_write && cache_guard.is_some())
6410 .map(Path::to_path_buf);
6411 set.spawn(async move {
6412 let _permit = sem.acquire().await;
6413 let mut seat = job.seat;
6414 let out = agent::invoke(
6415 &job.spec,
6416 &mut seat,
6417 &Invocation {
6418 cwd: &job.cwd,
6419 prompt: &job.prompt,
6420 timeout: job.timeout,
6421 allow_write: job.allow_write,
6422 sessions: job.sessions,
6423 artifacts: &job.artifacts,
6424 stem: &job.stem,
6425 run: &run,
6426 node: &node,
6427 cache_dir: cache.as_deref(),
6428 attachments: &attachments,
6429 writable: &[],
6430 },
6431 )
6432 .await;
6433 let out = match out {
6434 Ok(o) if o.usable() => AgentOutcome::Ok(o),
6435 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
6436 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
6444 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
6445 Ok(o) => AgentOutcome::Failed(format!(
6446 "exited with {:?} and no usable output",
6447 o.exit_code
6448 )),
6449 Err(e) => AgentOutcome::Failed(e.to_string()),
6450 };
6451 (i, seat, out)
6452 });
6453 }
6454 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
6455 while let Some(joined) = set.join_next().await {
6456 let (i, seat, out) = match joined {
6457 Ok(v) => v,
6458 Err(e) => {
6462 tracing::error!("agent task panicked: {e}");
6463 continue;
6464 }
6465 };
6466 state.seat_finished(&seat.key);
6467 record_jobs(state, node, round, &seat.key, &out);
6468 if let Err(e) = state.save() {
6469 tracing::warn!("could not persist a seat's completion: {e:#}");
6470 }
6471 if collected.len() <= i {
6472 collected.resize_with(i + 1, || None);
6473 }
6474 collected[i] = Some((i, seat, out));
6475 }
6476 if state
6482 .active
6483 .values()
6484 .any(|a| a.node == node && a.attempt == attempt)
6485 {
6486 state
6487 .active
6488 .retain(|_, a| !(a.node == node && a.attempt == attempt));
6489 if let Err(e) = state.save() {
6490 tracing::warn!("could not persist the end of a wave: {e:#}");
6491 }
6492 }
6493 if let Some(cache_dir) = cache
6500 && jobs_had_a_writer
6501 {
6502 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
6503 }
6504 if let Some(guard) = cache_guard {
6505 guard.release();
6506 }
6507 collected.into_iter().flatten().collect()
6508}
6509
6510fn record_jobs(
6521 state: &mut RunState,
6522 node: &str,
6523 round: Option<usize>,
6524 seat: &str,
6525 out: &AgentOutcome,
6526) {
6527 let commands: &[agent::CommandEvidence] = match out {
6528 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
6529 AgentOutcome::Failed(_) => &[],
6530 };
6531 let checked_at = Timestamp::now();
6532 for c in commands {
6533 state.jobs.push(JobRecord {
6534 node: node.to_owned(),
6535 round,
6536 seat: seat.to_owned(),
6537 id: c.id.clone(),
6538 description: c.description.clone(),
6539 checked_at,
6540 status: match c.exit_code {
6541 Some(0) => JobStatus::Completed,
6542 Some(_) => JobStatus::Failed,
6543 None => JobStatus::Unknown,
6544 },
6545 exit_code: c.exit_code,
6546 result_summary: c.result_summary.clone(),
6547 source: c.source.clone(),
6548 });
6549 }
6550}
6551
6552fn round_is_clean(
6573 blocking: usize,
6574 e2e_ok: bool,
6575 answered: usize,
6576 expected: usize,
6577 quota_missing: usize,
6578 policy: IncompleteReviewPolicy,
6579) -> bool {
6580 if blocking != 0 || !e2e_ok {
6581 return false;
6582 }
6583 if answered == expected || policy == IncompleteReviewPolicy::Warn {
6584 return true;
6585 }
6586 answered > 0 && expected - answered <= quota_missing
6587}
6588
6589fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6613 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6614 return Some(RunStatus::Gating);
6615 }
6616 let last = reviews.last()?;
6617 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6618 if reviews.len() < max_rounds && !stagnant {
6619 return None;
6620 }
6621 if last.incomplete() && last.blocking == 0 {
6622 return Some(RunStatus::Blocked);
6623 }
6624 if last.e2e_status() == E2eStatus::ResourceBlocked {
6625 return None;
6626 }
6627 Some(if last.e2e.iter().all(CommandOutcome::ok) {
6628 RunStatus::Gating
6629 } else {
6630 RunStatus::Blocked
6631 })
6632}
6633
6634fn retry_budget(full: Duration, nudged: bool) -> Duration {
6649 if nudged {
6650 (full / 4).max(Duration::from_secs(120)).min(full)
6651 } else {
6652 full
6653 }
6654}
6655
6656#[allow(clippy::too_many_arguments)]
6680async fn ask_json_wave<T>(
6681 jobs: Vec<SeatJob>,
6682 sem: Arc<Semaphore>,
6683 retries: usize,
6684 roster: &[AgentSpec],
6685 ctx: &WaveCtx<'_>,
6686 losses: &mut Vec<QuotaLoss>,
6687 state: &mut RunState,
6688 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
6689) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6690where
6691 T: serde::de::DeserializeOwned + Send + 'static,
6692{
6693 let n = jobs.len();
6694 let originals: Vec<SeatJob> = jobs;
6695 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
6696 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
6697 let mut nudges: Vec<usize> = vec![0; n];
6704 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
6708 let mut tried: Vec<BTreeSet<String>> = specs
6709 .iter()
6710 .map(|s| BTreeSet::from([s.id.clone()]))
6711 .collect();
6712 let starts: Vec<usize> = specs
6713 .iter()
6714 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
6715 .collect();
6716 let mut prev: Vec<Option<FailClass>> = vec![None; n];
6717 let mut fresh: Vec<Option<String>> = vec![None; n];
6718 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
6719 let mut pending: Vec<usize> = (0..n).collect();
6720
6721 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
6724 for round in 0..max_rounds {
6725 if pending.is_empty() {
6726 break;
6727 }
6728 let mut batch = Vec::with_capacity(pending.len());
6729 let mut renudged: Vec<&str> = Vec::new();
6730 for &i in &pending {
6731 let src = &originals[i];
6732 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
6735 (
6736 src.prompt.clone(),
6737 src.timeout,
6738 format!("{}-{word}-{}", src.stem, specs[i].id),
6739 )
6740 } else if nudges[i] == 0 {
6741 (src.prompt.clone(), src.timeout, src.stem.clone())
6742 } else {
6743 renudged.push(src.seat.key.as_str());
6744 let why = done[i]
6745 .as_ref()
6746 .and_then(|r| r.as_ref().err().map(ToString::to_string))
6747 .unwrap_or_else(|| "no parsable answer".to_owned());
6748 let nudge = prompt::nudge(&why);
6749 let nudged = has_context(&specs[i], &seats[i], src.sessions);
6750 let prompt = if nudged {
6751 nudge
6752 } else {
6753 format!("{}\n\n---\n\n{}", src.prompt, nudge)
6754 };
6755 (
6756 prompt,
6757 retry_budget(src.timeout, nudged),
6758 format!("{}-retry{}", src.stem, nudges[i]),
6759 )
6760 };
6761 batch.push(SeatJob {
6762 spec: specs[i].clone(),
6763 seat: seats[i].clone(),
6764 cwd: src.cwd.clone(),
6765 prompt,
6766 timeout,
6767 allow_write: src.allow_write,
6768 sessions: src.sessions,
6769 artifacts: src.artifacts.clone(),
6770 stem,
6771 });
6772 }
6773
6774 if !renudged.is_empty() {
6775 state.event(
6776 ctx.node,
6777 format!("retry {round}: re-asking {}", renudged.join(", ")),
6778 );
6779 }
6780 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
6781 let mut still = Vec::new();
6782 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
6783 seats[i] = seat;
6784 let class = FailClass::of(&out);
6785 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
6788 && nudges[i] < retries
6789 && !roster.is_empty();
6790 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
6791 let next = should_hand_over(prev[i].as_ref(), &cur)
6792 .then(|| next_untried_in_roster(roster, starts[i], &tried[i]).cloned())
6793 .flatten();
6794 if let Some(next) = next {
6795 record_handover(
6796 state,
6797 ctx.node,
6798 &originals[i].seat.key,
6799 &specs[i].id,
6800 &next.id,
6801 &cur,
6802 &fail_reason(&out),
6803 );
6804 tried[i].insert(next.id.clone());
6805 prev[i] = Some(cur.clone());
6806 seats[i] = handover_seat(&originals[i].seat.key, &next.id, state.seed);
6807 specs[i] = next;
6808 fresh[i] = Some(cur.stem_word().to_owned());
6809 nudges[i] = 0;
6810 done[i] = Some(Err(anyhow::anyhow!(
6811 "handed over after: {}",
6812 fail_reason(&out)
6813 )));
6814 still.push(i);
6815 continue;
6816 }
6817 }
6818 let parsed = match out {
6819 AgentOutcome::Ok(o) => match verdict::extract_json::<T>(&o.text) {
6820 Ok(v) => match validate(&v) {
6821 Ok(()) => Ok((v, o)),
6822 Err(e) => Err(e),
6823 },
6824 Err(e) => Err(e),
6825 },
6826 AgentOutcome::Quota(o) => {
6827 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
6828 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
6829 }
6830 AgentOutcome::Dropped(o) => {
6834 let why = o
6835 .dropped
6836 .as_ref()
6837 .map(|d| d.why.as_str())
6838 .unwrap_or("the CLI ended the stream without delivering its answer");
6839 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
6840 }
6841 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
6842 };
6843 let quota = class == Some(FailClass::Quota);
6844 let failed = parsed.is_err();
6845 done[i] = Some(parsed);
6846 let agent_failure = class.is_some() && !roster.is_empty() && !nudge_first;
6855 if failed && !quota && !agent_failure && nudges[i] < retries {
6856 nudges[i] += 1;
6857 still.push(i);
6858 }
6859 }
6860 pending = still;
6861 }
6862
6863 for (i, q) in last_quota.into_iter().enumerate() {
6866 if let Some(reset) = q {
6867 losses.push(QuotaLoss {
6868 seat: originals[i].seat.key.clone(),
6869 node: ctx.node.to_owned(),
6870 at: Timestamp::now(),
6871 reset,
6872 });
6873 }
6874 }
6875
6876 seats
6877 .into_iter()
6878 .zip(done)
6879 .zip(nudges)
6880 .map(|((seat, res), attempts)| {
6881 (
6882 seat,
6883 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
6884 attempts,
6885 )
6886 })
6887 .collect()
6888}
6889
6890async fn acquire_cache_lease(
6903 state: &mut RunState,
6904 cache_dir: &Path,
6905 owner: &crate::cache::Owner,
6906 budget: Duration,
6907 context: &str,
6908) -> Result<crate::cache::Guard> {
6909 let home = crate::run::home();
6910 let started = Instant::now();
6911 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
6912 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
6913 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
6914 Err(e) => {
6915 state.event(
6916 "verify",
6917 format!("{context}: could not check the shared build cache: {e:#}"),
6918 );
6919 if let Err(e2) = state.save() {
6920 tracing::warn!("could not persist a cache-check failure: {e2:#}");
6921 }
6922 return Err(e);
6923 }
6924 };
6925 state.event(
6926 "verify",
6927 format!(
6928 "{context}: waiting for the shared build cache at {} ({})",
6929 cache_dir.display(),
6930 busy.describe()
6931 ),
6932 );
6933 if let Err(e) = state.save() {
6934 tracing::warn!("could not persist a cache wait: {e:#}");
6935 }
6936 let remaining = budget.saturating_sub(started.elapsed());
6937 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
6938 Ok(g) => Ok(g),
6939 Err(e) => {
6940 state.event("verify", format!("{context}: {e:#}"));
6941 if let Err(e2) = state.save() {
6942 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
6943 }
6944 Err(e)
6945 }
6946 }
6947}
6948
6949#[allow(clippy::too_many_arguments)]
6970async fn with_cache_lease<'s, F, Fut>(
6971 state: &'s mut RunState,
6972 cache_dir: Option<&Path>,
6973 node: &str,
6974 seat: &str,
6975 worktree: &Path,
6976 head: &str,
6977 budget: Duration,
6978 context: &str,
6979 body: F,
6980) -> (Vec<CommandOutcome>, bool)
6981where
6982 F: FnOnce(&'s mut RunState, Duration) -> Fut,
6983 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
6984{
6985 let Some(cache_dir) = cache_dir else {
6986 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
6987 return (outcomes, retried);
6988 };
6989 let home = crate::run::home();
6990 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
6991 let started = Instant::now();
6992 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
6993 Ok(g) => g,
6994 Err(e) => {
6995 return (
6996 vec![CommandOutcome {
6997 command: "(waiting for the shared build cache)".to_owned(),
6998 code: None,
6999 output_tail: e.to_string(),
7000 duration_ms: started.elapsed().as_millis() as u64,
7001 resource_blocked: true,
7002 }],
7003 false,
7004 );
7005 }
7006 };
7007 let identity = crate::cache::Identity::new(worktree, head);
7008 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7009 state.event(
7017 "verify",
7018 format!(
7019 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7020 worktree.display(),
7021 short(head)
7022 ),
7023 );
7024 guard.release();
7025 return (
7026 vec![CommandOutcome {
7027 command: "(confirming the shared build cache is fresh)".to_owned(),
7028 code: None,
7029 output_tail: e.to_string(),
7030 duration_ms: started.elapsed().as_millis() as u64,
7031 resource_blocked: true,
7032 }],
7033 false,
7034 );
7035 }
7036 let remaining = budget.saturating_sub(started.elapsed());
7037 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7038 if !timed_out_pids.is_empty() {
7043 wait_for_timed_out_children_to_die(&timed_out_pids).await;
7044 }
7045 guard.release();
7046 (outcomes, retried)
7047}
7048
7049async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7061 wait_for_pids_with(
7062 pids,
7063 crate::proc::pid_alive,
7064 LEASE_RELEASE_POLL,
7065 LEASE_RELEASE_MAX_WAIT,
7066 )
7067 .await;
7068}
7069
7070async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7076 pids: &[u32],
7077 alive: F,
7078 poll: Duration,
7079 max_wait: Duration,
7080) {
7081 let deadline = Instant::now() + max_wait;
7082 loop {
7083 if pids.iter().all(|&pid| !alive(pid)) {
7084 return;
7085 }
7086 if Instant::now() >= deadline {
7087 return;
7088 }
7089 tokio::time::sleep(poll).await;
7090 }
7091}
7092
7093fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7100 outcomes.iter().any(|o| o.resource_blocked)
7101}
7102
7103enum GateFix {
7105 Retry,
7107 Stop,
7110 Defer,
7113}
7114
7115fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7123 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7124 red.peek().is_some()
7125 && red.all(|o| {
7126 !o.resource_blocked
7127 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7128 && !o.output_tail.trim().is_empty()
7129 })
7130}
7131
7132fn e2e_outcome_label(o: &CommandOutcome) -> String {
7136 if o.ok() {
7137 return "pass".to_owned();
7138 }
7139 let reason = if o.build_failed() {
7140 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7141 } else {
7142 format!("FAIL ({:?})", o.code)
7143 };
7144 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7145}
7146
7147async fn run_e2e_with_retry(
7155 state: &mut RunState,
7156 shell: &[String],
7157 commands: &[String],
7158 worktree: &Path,
7159 timeout: Duration,
7160 context: &str,
7161) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7162 let (mut e2e, mut timed_out_pids) = run_commands(
7163 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7164 )
7165 .await;
7166 for o in &e2e {
7167 state.event(
7168 "verify",
7169 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7170 );
7171 }
7172 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7176 if verify_retried {
7177 state.event(
7178 "verify",
7179 format!(
7180 "{context}: verify could not build/link, not a test result — retrying once \
7181 before concluding"
7182 ),
7183 );
7184 let retried = run_commands(
7185 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7186 )
7187 .await;
7188 e2e = retried.0;
7189 timed_out_pids.extend(retried.1);
7192 for o in &e2e {
7193 state.event(
7194 "verify",
7195 format!(
7196 "{context}: retry `{}` -> {}",
7197 o.command,
7198 e2e_outcome_label(o)
7199 ),
7200 );
7201 }
7202 }
7203 (e2e, verify_retried, timed_out_pids)
7204}
7205
7206#[allow(clippy::too_many_arguments)]
7222async fn run_commands(
7223 state: &mut RunState,
7224 node: &str,
7225 task: &str,
7226 attempt: usize,
7227 shell: &[String],
7228 commands: &[String],
7229 cwd: &Path,
7230 timeout: Duration,
7231) -> (Vec<CommandOutcome>, Vec<u32>) {
7232 if commands.is_empty() {
7233 return (Vec::new(), Vec::new());
7238 }
7239 let mut out = Vec::new();
7240 let mut timed_out_pids = Vec::new();
7241 let total = commands.len();
7242 for (idx, command) in commands.iter().enumerate() {
7243 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7244 if let Err(e) = state.save() {
7245 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7246 }
7247 let started = Instant::now();
7248 let mut cmd = tokio::process::Command::new(&shell[0]);
7249 cmd.quiet();
7250 cmd.args(&shell[1..])
7251 .arg(command)
7252 .current_dir(cwd)
7253 .stdin(std::process::Stdio::null())
7254 .stdout(std::process::Stdio::piped())
7255 .stderr(std::process::Stdio::piped())
7256 .kill_on_drop(true);
7257 let spawned = cmd.spawn();
7258 let (code, body) = match spawned {
7259 Ok(child) => {
7260 let pid = child.id();
7265 match tokio::time::timeout(timeout, child.wait_with_output()).await {
7266 Ok(Ok(o)) => {
7267 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7268 body.push_str(&String::from_utf8_lossy(&o.stderr));
7269 (o.status.code(), body)
7270 }
7271 Ok(Err(e)) => (None, format!("failed to run: {e}")),
7272 Err(_) => {
7273 if let Some(pid) = pid {
7274 timed_out_pids.push(pid);
7275 }
7276 (None, format!("timed out after {}s", timeout.as_secs()))
7277 }
7278 }
7279 }
7280 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7281 };
7282 out.push(CommandOutcome {
7283 command: command.clone(),
7284 code,
7285 output_tail: tail(&body, OUTPUT_TAIL),
7286 duration_ms: started.elapsed().as_millis() as u64,
7287 resource_blocked: false,
7288 });
7289 }
7290 state.task_finished(task);
7291 if let Err(e) = state.save() {
7292 tracing::warn!("could not persist the end of {task}: {e:#}");
7293 }
7294 (out, timed_out_pids)
7295}
7296
7297fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
7309 let repo = repo.display();
7310 match style {
7311 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
7312 MergeStyle::Squash => {
7313 let subject = message
7316 .lines()
7317 .next()
7318 .unwrap_or(branch)
7319 .replace(['\\', '"', '$', '`'], "");
7320 format!(
7321 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
7322 )
7323 }
7324 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
7325 }
7326}
7327
7328const PR_TITLE_MAX: usize = 240;
7339
7340struct PrMessage {
7345 title: String,
7346 body: String,
7347}
7348
7349impl PrMessage {
7350 fn commit_message(&self) -> String {
7354 format!("{}\n\n{}", self.title, self.body)
7355 }
7356}
7357
7358fn title_marker(line: &str) -> Option<&str> {
7360 let line = line.trim();
7361 let head = line.get(..6)?;
7362 head.eq_ignore_ascii_case("title:")
7363 .then(|| line[6..].trim())
7364}
7365
7366fn summary_title(summary: &str) -> Option<String> {
7371 let first = summary.lines().find(|l| !l.trim().is_empty())?;
7372 let raw = title_marker(first)?;
7373 if raw.is_empty() {
7374 return None;
7375 }
7376 let title = queue::title_from(raw, PR_TITLE_MAX);
7377 let lower = title.to_ascii_lowercase();
7378 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7379 return None;
7380 }
7381 Some(title)
7382}
7383
7384const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
7386
7387const EXISTING_BRANCH: &str = "(existing branch)";
7389
7390fn is_review_run(state: &RunState) -> bool {
7394 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
7395}
7396
7397fn review_title(state: &RunState) -> Option<String> {
7402 let raw = state.reviewed_commits.as_ref()?.first()?.trim();
7403 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
7404 return None;
7405 }
7406 let title = queue::title_from(raw, PR_TITLE_MAX);
7407 let lower = title.to_ascii_lowercase();
7408 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7409 return None;
7410 }
7411 Some(title)
7412}
7413
7414fn neutral_title(state: &RunState, winner: char) -> String {
7415 format!(
7416 "chore: land candidate {} of run {}",
7417 winner.to_ascii_uppercase(),
7418 state.id
7419 )
7420}
7421
7422pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
7428 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
7429 ""
7430 } else {
7431 pr_title
7432 }
7433}
7434
7435pub fn landing_subject_source(state: &RunState) -> String {
7439 if is_review_run(state) {
7440 let winner = state.candidates.first().map_or('A', |c| c.label);
7441 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
7442 }
7443 state.instruction.clone()
7444}
7445
7446fn summary_without_title(summary: &str) -> String {
7449 let mut lines = summary.trim().lines().peekable();
7450 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
7451 lines.next();
7452 }
7453 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
7454}
7455
7456fn pr_message(state: &RunState, winner: char) -> PrMessage {
7470 let summary = state
7471 .candidates
7472 .iter()
7473 .find(|c| c.label == winner)
7474 .map(|c| c.summary.as_str())
7475 .unwrap_or_default();
7476 let review = is_review_run(state);
7479 let title = if review {
7480 review_title(state).unwrap_or_else(|| neutral_title(state, winner))
7481 } else {
7482 summary_title(summary).unwrap_or_else(|| {
7483 let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
7484 if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
7485 t
7486 } else {
7487 neutral_title(state, winner)
7488 }
7489 })
7490 };
7491
7492 let mut body = String::new();
7493 let what = summary_without_title(summary);
7494 if !what.is_empty() {
7495 body.push_str("## Summary\n\n");
7496 body.push_str(&what);
7497 body.push_str("\n\n");
7498 }
7499
7500 let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
7501 if let Some(fix) = fix
7502 && !fix.notes.trim().is_empty()
7503 {
7504 body.push_str("## Review fixes\n\n");
7505 body.push_str(fix.notes.trim());
7506 body.push_str("\n\n");
7507 }
7508
7509 let open = state.open_findings();
7510 if !open.is_empty() {
7511 body.push_str("## Open review findings\n\n");
7512 for f in &open {
7513 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
7514 }
7515 body.push('\n');
7516 }
7517
7518 if let Some(fix) = fix
7519 && !fix.rejected.is_empty()
7520 {
7521 body.push_str("## Declined by the fixer\n\n");
7522 for r in &fix.rejected {
7523 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
7524 }
7525 body.push('\n');
7526 }
7527
7528 if review {
7529 body.push_str("## Commits under review\n\n");
7531 match &state.reviewed_commits {
7532 Some(subjects) => {
7533 for s in subjects {
7534 body.push_str(&format!("- {}\n", s.trim()));
7535 }
7536 }
7537 None => {
7538 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
7541 body.push_str(rest.trim());
7542 body.push('\n');
7543 }
7544 }
7545 } else {
7546 let task = state.instruction.trim();
7547 let task = if task.is_empty() {
7548 "(empty task)"
7549 } else {
7550 task
7551 };
7552 body.push_str(&format!(
7553 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
7554 task.replace("</details>", "</details>")
7555 ));
7556 }
7557
7558 body.push_str(&format!(
7559 "\n---\nmagi:run/{} magi:candidate-{}\n",
7560 state.id,
7561 winner.to_ascii_lowercase()
7562 ));
7563
7564 let id = crate::scrub::Identity::current();
7567 PrMessage {
7568 title: crate::scrub::scrub(&title, &id),
7569 body: crate::scrub::scrub(&body, &id),
7570 }
7571}
7572
7573fn seeded_instruction(state: &RunState) -> String {
7577 match refs::describe(&state.seeds) {
7578 Some(facts) => format!(
7579 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
7580 Candidates start from the unmerged branch named above, when there \
7581 is one, and carry any unmerged commit named by sha as a \
7582 cherry-pick. Check that this is what the task meant before \
7583 building on it.",
7584 state.instruction
7585 ),
7586 None => state.instruction.clone(),
7587 }
7588}
7589
7590async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
7594 let base = &state.base_branch;
7595 let mut against = base.clone();
7596 if mode == MergeMode::Pr {
7597 let remote = &state.config.merge.remote;
7598 let tracking = format!("{remote}/{base}");
7599 let fetched = git::fetch(repo, remote, base).await;
7600 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
7601 against = tracking;
7602 }
7603 }
7604 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
7605}
7606
7607fn empty_candidate_detail(state: &RunState, base: &str) -> String {
7610 let mut detail = format!(
7611 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
7612 nothing to open a pull request for"
7613 );
7614 match refs::describe(&state.seeds) {
7615 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
7616 None => detail.push_str(
7617 "\nThe task names no existing branch or commit; if it means to land work \
7618 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
7619 ),
7620 }
7621 detail
7622}
7623
7624#[derive(Debug, PartialEq, Eq)]
7627enum PrPlan {
7628 Create,
7629 Adopt { url: String, title: String },
7630 Stop(String),
7631}
7632
7633fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
7636 match found {
7637 Ok(land::OpenPr::None) => PrPlan::Create,
7638 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
7639 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
7640 "several open pull requests exist for this branch, not picking one: {}",
7641 urls.join(" ")
7642 )),
7643 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
7644 }
7645}
7646
7647async fn gh_pr_create(
7649 cwd: &Path,
7650 base: &str,
7651 head: &str,
7652 title: &str,
7653 body: &str,
7654) -> Result<String> {
7655 let out = tokio::process::Command::new("gh")
7656 .args([
7657 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
7658 ])
7659 .current_dir(cwd)
7660 .quiet()
7661 .stdin(std::process::Stdio::null())
7662 .output()
7663 .await
7664 .context("spawn gh")?;
7665 if out.status.success() {
7666 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
7667 } else {
7668 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
7669 }
7670}
7671
7672pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
7681 let repo = state.repo.clone();
7682 let root = state.worktree_root();
7683 let winner = state.tally.as_ref().map(|t| t.winner);
7684 let mut removed = Vec::new();
7685
7686 for i in 0..state.candidates.len() {
7687 let c = state.candidates[i].clone();
7688 let is_winner = Some(c.label) == winner;
7689 if is_winner && !drop_winner {
7690 continue;
7691 }
7692 if c.worktree.exists() {
7693 git::worktree_remove(&repo, &c.worktree).await.ok();
7694 removed.push(c.worktree.to_string_lossy().into_owned());
7695 }
7696 let handed_over = state.released_branches.contains(&c.branch);
7699 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
7700 git::branch_delete(&repo, &c.branch).await.ok();
7701 removed.push(c.branch.clone());
7702 }
7703 state.candidates[i].folded = true;
7704 }
7705
7706 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
7707 let path = name.path();
7708 let keep = !drop_winner
7709 && winner.is_some_and(|w| {
7710 path.file_name()
7711 .is_some_and(|n| n == format!("cand-{w}").as_str())
7712 });
7713 if keep {
7714 continue;
7715 }
7716 git::worktree_remove(&repo, &path).await.ok();
7717 removed.push(path.to_string_lossy().into_owned());
7718 }
7719
7720 remove_if_empty(&root);
7729
7730 if state.enabled_worktree_config && drop_winner {
7731 git::release_worktree_config(&repo).await.ok();
7735 state.enabled_worktree_config = false;
7736 }
7737 state.save_under(home)?;
7738 Ok(removed)
7739}
7740
7741fn remove_if_empty(dir: &Path) {
7752 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
7753 std::fs::remove_dir(dir).ok();
7754 }
7755}
7756
7757pub fn worst_open(state: &RunState) -> Option<Severity> {
7759 state
7760 .reviews
7761 .last()?
7762 .reviews
7763 .iter()
7764 .flat_map(|r| r.findings.iter())
7765 .map(|f| f.severity)
7766 .max()
7767}
7768
7769#[cfg(test)]
7770mod tests {
7771 #[test]
7772 fn pr_merge_plan_creates_adopts_or_stops() {
7773 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
7774 assert_eq!(
7775 pr_merge_plan(Ok(land::OpenPr::One {
7776 url: "u".into(),
7777 title: "t".into()
7778 })),
7779 PrPlan::Adopt {
7780 url: "u".into(),
7781 title: "t".into()
7782 }
7783 );
7784 let PrPlan::Stop(many) =
7785 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
7786 else {
7787 panic!("many must stop");
7788 };
7789 assert!(many.contains('a') && many.contains('b'));
7790 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
7791 panic!("a failed lookup must stop");
7792 };
7793 assert!(err.contains("bad token"));
7794 }
7795
7796 use super::*;
7797 use crate::run::GateStatus;
7798 use std::collections::BTreeMap;
7799 use std::time::Duration;
7800
7801 fn conductor() -> AgentSpec {
7802 AgentSpec {
7803 id: "conductor".to_owned(),
7804 kind: crate::config::AgentKind::Command,
7805 model: None,
7806 command: vec!["true".to_owned()],
7807 extra_args: Vec::new(),
7808 env: BTreeMap::new(),
7809 prompt_delivery: None,
7810 }
7811 }
7812
7813 fn spec(id: &str) -> AgentSpec {
7814 AgentSpec {
7815 id: id.to_owned(),
7816 kind: crate::config::AgentKind::Command,
7817 model: None,
7818 command: vec!["true".to_owned()],
7819 extra_args: Vec::new(),
7820 env: BTreeMap::new(),
7821 prompt_delivery: None,
7822 }
7823 }
7824
7825 #[test]
7832 fn failure_signature_ignores_numbers_and_paths() {
7833 assert_eq!(
7834 failure_signature("exited with Some(2) and no usable output"),
7835 failure_signature("exited with Some(137) and no usable output")
7836 );
7837 assert_eq!(
7838 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
7839 failure_signature("cannot open /var/x.txt: denied")
7840 );
7841 assert_ne!(failure_signature("boom"), failure_signature("bang"));
7842 }
7843
7844 #[test]
7845 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
7846 let other = FailClass::Other("x".into());
7847 assert!(should_hand_over(None, &FailClass::Quota));
7848 assert!(should_hand_over(Some(&other), &FailClass::Quota));
7849 assert!(should_hand_over(
7850 Some(&FailClass::Timeout),
7851 &FailClass::Timeout
7852 ));
7853 assert!(should_hand_over(None, &other));
7854 assert!(!should_hand_over(Some(&other), &other));
7855 assert!(should_hand_over(
7856 Some(&other),
7857 &FailClass::Other("y".into())
7858 ));
7859 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
7861 assert!(should_hand_over(Some(&FailClass::Quota), &other));
7862 }
7863
7864 #[test]
7865 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
7866 let a = SeatState::new("judge-1", "alpha", 7);
7867 let b = handover_seat("judge-1", "beta", 7);
7868 assert_ne!(a.claude_session, b.claude_session);
7869 assert_eq!(b.turns, 0);
7870 }
7871
7872 #[test]
7873 fn a_timeout_is_classified_apart_from_other_failures() {
7874 assert_eq!(
7875 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
7876 Some(FailClass::Timeout)
7877 );
7878 assert!(matches!(
7879 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
7880 Some(FailClass::Other(_))
7881 ));
7882 }
7883
7884 #[test]
7885 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
7886 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
7887 let tried = BTreeSet::from(["beta".to_owned()]);
7888 let next = next_untried_in_roster(&roster, 1, &tried);
7891 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
7892 }
7893
7894 #[test]
7895 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
7896 let roster = vec![spec("alpha"), spec("beta")];
7897 let tried = BTreeSet::from(["beta".to_owned()]);
7898 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
7902 }
7903
7904 #[test]
7905 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
7906 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
7907 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
7908 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
7912 }
7913
7914 #[test]
7915 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
7916 let roster = vec![spec("a"), spec("a"), spec("b")];
7917 let tried = BTreeSet::from(["a".to_owned()]);
7918 let next = next_untried_in_roster(&roster, 0, &tried);
7919 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
7920 }
7921
7922 #[test]
7923 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
7924 let roster = vec![spec("a"), spec("b")];
7925 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
7926 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
7927 }
7928
7929 #[test]
7930 fn remove_if_empty_only_ever_takes_a_bare_directory() {
7931 let dir = tempfile::tempdir().unwrap();
7932 let bay = dir.path().join("ffff");
7933
7934 remove_if_empty(&bay);
7936 assert!(!bay.exists());
7937
7938 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
7941 remove_if_empty(&bay);
7942 assert!(bay.exists(), "non-empty directory must survive");
7943
7944 std::fs::remove_dir(bay.join("cand-A")).unwrap();
7946 remove_if_empty(&bay);
7947 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
7948 }
7949
7950 #[test]
7959 fn a_full_panel_that_found_nothing_is_clean() {
7960 assert!(round_is_clean(
7961 0,
7962 true,
7963 2,
7964 2,
7965 0,
7966 IncompleteReviewPolicy::Block
7967 ));
7968 }
7969
7970 #[test]
7971 fn a_missing_seat_is_never_clean_under_the_default_policy() {
7972 assert!(!round_is_clean(
7973 0,
7974 true,
7975 1,
7976 2,
7977 0,
7978 IncompleteReviewPolicy::Block
7979 ));
7980 }
7981
7982 #[test]
7983 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
7984 assert!(!round_is_clean(
7985 1,
7986 true,
7987 1,
7988 2,
7989 0,
7990 IncompleteReviewPolicy::Warn
7991 ));
7992 }
7993
7994 #[test]
7995 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
7996 assert!(round_is_clean(
7997 0,
7998 true,
7999 1,
8000 2,
8001 0,
8002 IncompleteReviewPolicy::Warn
8003 ));
8004 }
8005
8006 #[test]
8007 fn a_full_panel_with_an_open_finding_is_not_clean() {
8008 assert!(!round_is_clean(
8009 1,
8010 true,
8011 2,
8012 2,
8013 0,
8014 IncompleteReviewPolicy::Block
8015 ));
8016 }
8017
8018 #[test]
8019 fn a_full_panel_with_a_red_e2e_is_not_clean() {
8020 assert!(!round_is_clean(
8021 0,
8022 false,
8023 2,
8024 2,
8025 0,
8026 IncompleteReviewPolicy::Block
8027 ));
8028 }
8029
8030 #[test]
8037 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
8038 assert!(round_is_clean(
8041 0,
8042 true,
8043 1,
8044 2,
8045 1,
8046 IncompleteReviewPolicy::Block
8047 ));
8048 }
8049
8050 #[test]
8051 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
8052 assert!(!round_is_clean(
8055 0,
8056 true,
8057 1,
8058 2,
8059 0,
8060 IncompleteReviewPolicy::Block
8061 ));
8062 }
8063
8064 #[test]
8065 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
8066 assert!(!round_is_clean(
8067 1,
8068 true,
8069 1,
8070 2,
8071 1,
8072 IncompleteReviewPolicy::Block
8073 ));
8074 assert!(!round_is_clean(
8075 0,
8076 false,
8077 1,
8078 2,
8079 1,
8080 IncompleteReviewPolicy::Block
8081 ));
8082 }
8083
8084 #[test]
8085 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
8086 assert!(!round_is_clean(
8090 0,
8091 true,
8092 0,
8093 2,
8094 2,
8095 IncompleteReviewPolicy::Block
8096 ));
8097 }
8098
8099 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
8100 CommandOutcome {
8101 command: "test".to_owned(),
8102 code,
8103 output_tail: String::new(),
8104 duration_ms: 0,
8105 resource_blocked,
8106 }
8107 }
8108
8109 #[test]
8110 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
8111 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
8112 assert!(
8113 !verify_inconclusive(&[outcome(Some(1), false)]),
8114 "an ordinary failure is still evidence about the patch"
8115 );
8116 assert!(verify_inconclusive(&[outcome(None, true)]));
8117 assert!(
8118 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
8119 "one inconclusive outcome taints the whole batch"
8120 );
8121 assert!(!verify_inconclusive(&[]));
8122 }
8123
8124 #[tokio::test]
8125 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
8126 let calls = std::sync::atomic::AtomicUsize::new(0);
8130 let started = Instant::now();
8131 wait_for_pids_with(
8132 &[123],
8133 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
8134 Duration::from_millis(5),
8135 Duration::from_secs(5),
8136 )
8137 .await;
8138 assert!(
8139 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
8140 "must keep checking rather than deciding on the first answer"
8141 );
8142 assert!(
8143 started.elapsed() < Duration::from_secs(1),
8144 "must return the moment it is confirmed dead, not wait out the ceiling"
8145 );
8146 }
8147
8148 #[tokio::test]
8149 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
8150 let started = Instant::now();
8151 wait_for_pids_with(
8152 &[123],
8153 |_| true, Duration::from_millis(5),
8155 Duration::from_millis(30),
8156 )
8157 .await;
8158 let elapsed = started.elapsed();
8159 assert!(
8160 elapsed >= Duration::from_millis(30),
8161 "must not give up before its own ceiling: {elapsed:?}"
8162 );
8163 assert!(
8164 elapsed < Duration::from_secs(1),
8165 "must not wait past its own ceiling either: {elapsed:?}"
8166 );
8167 }
8168
8169 #[tokio::test]
8170 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
8171 let started = Instant::now();
8172 wait_for_pids_with(
8173 &[],
8174 |_| true,
8175 Duration::from_secs(5),
8176 Duration::from_secs(5),
8177 )
8178 .await;
8179 assert!(
8180 started.elapsed() < Duration::from_millis(200),
8181 "an empty pid list has nothing to confirm"
8182 );
8183 }
8184
8185 fn review_round(
8191 clean: bool,
8192 blocking: usize,
8193 answered: usize,
8194 expected: usize,
8195 progressed: bool,
8196 e2e_ok: bool,
8197 ) -> ReviewRound {
8198 ReviewRound {
8199 round: 1,
8200 head: "h".to_owned(),
8201 verified_head: None,
8202 verified_at: None,
8203 reviews: Vec::new(),
8204 e2e: vec![CommandOutcome {
8205 command: "test".to_owned(),
8206 code: Some(if e2e_ok { 0 } else { 1 }),
8207 output_tail: String::new(),
8208 duration_ms: 0,
8209 resource_blocked: false,
8210 }],
8211 verify_retried: false,
8212 e2e_deferred: false,
8213 e2e_defer_reason: None,
8214 fix: None,
8215 blocking,
8216 answered,
8217 expected,
8218 clean,
8219 progressed,
8220 vote_split: false,
8221 reconsideration: Vec::new(),
8222 verdict: None,
8223 }
8224 }
8225
8226 #[test]
8227 fn review_conclusion_is_none_when_nothing_has_run() {
8228 assert_eq!(review_conclusion(&[], 3), None);
8229 }
8230
8231 #[test]
8232 fn review_conclusion_is_none_while_rounds_remain() {
8233 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
8234 assert_eq!(review_conclusion(&rounds, 3), None);
8235 }
8236
8237 #[test]
8238 fn review_conclusion_is_gating_once_a_round_is_clean() {
8239 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
8240 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
8241 }
8242
8243 #[test]
8244 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
8245 let rounds = vec![
8246 review_round(false, 1, 2, 2, true, true),
8247 review_round(false, 1, 2, 2, true, true),
8248 ];
8249 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
8250 }
8251
8252 #[test]
8253 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
8254 let rounds = vec![
8255 review_round(false, 1, 2, 2, true, true),
8256 review_round(false, 1, 2, 2, true, false),
8257 ];
8258 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
8259 }
8260
8261 #[test]
8262 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
8263 let mut blocked = review_round(false, 1, 2, 2, true, false);
8270 blocked.e2e[0].resource_blocked = true;
8271 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
8272 assert_eq!(review_conclusion(&rounds, 2), None);
8273 }
8274
8275 #[test]
8276 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
8277 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
8279 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
8280 }
8281
8282 #[test]
8283 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
8284 let rounds = vec![
8285 review_round(false, 1, 2, 2, false, true),
8286 review_round(false, 1, 2, 2, false, true),
8287 ];
8288 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
8289 }
8290
8291 fn secs(n: u64) -> Duration {
8292 Duration::from_secs(n)
8293 }
8294
8295 fn init_repo(dir: &Path) {
8298 let run = |args: &[&str]| {
8299 let out = std::process::Command::new("git")
8300 .args(args)
8301 .current_dir(dir)
8302 .quiet()
8303 .output()
8304 .expect("spawn git");
8305 assert!(
8306 out.status.success(),
8307 "git {args:?} failed: {}",
8308 String::from_utf8_lossy(&out.stderr)
8309 );
8310 };
8311 run(&["init", "-b", "main"]);
8312 run(&["config", "user.name", "magi test"]);
8313 run(&["config", "user.email", "magi@example.com"]);
8314 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
8315 run(&["add", "-A"]);
8316 run(&["commit", "-m", "init"]);
8317 }
8318
8319 fn ask_test_home() {
8327 crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
8328 }
8329
8330 fn runner_at(status: RunStatus) -> Runner {
8333 let mut state = RunState::new(
8334 PathBuf::from("/nonexistent/repo"),
8335 "main".to_owned(),
8336 "deadbeef".to_owned(),
8337 "task".to_owned(),
8338 Config::default(),
8339 );
8340 state.status = status;
8341 Runner {
8342 state,
8343 roles: ResolvedRoles {
8344 implementers: Vec::new(),
8345 judges: Vec::new(),
8346 reviewers: Vec::new(),
8347 fixer: None,
8348 conductor: conductor(),
8349 implementer_roster: Vec::new(),
8350 judge_roster: Vec::new(),
8351 reviewer_roster: Vec::new(),
8352 },
8353 sem: Arc::new(Semaphore::new(1)),
8354 pause: Pause::new(),
8355 interrupt: Pause::new(),
8356 }
8357 }
8358
8359 #[test]
8363 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
8364 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8365 let mut runner = runner_at(RunStatus::Implementing);
8366 let interrupt = Pause::new();
8367 runner.watch_interrupt(interrupt.clone());
8368
8369 interrupt.park_because("task a1b2 asked to run first");
8370
8371 assert!(runner.park_here().expect("park_here"));
8372 assert!(runner.state.parked);
8373 let last = runner.state.events.last().expect("a park event");
8374 assert_eq!(last.node, "park");
8375 assert!(
8376 last.message.contains("task a1b2 asked to run first"),
8377 "expected the interrupt reason in {:?}",
8378 last.message
8379 );
8380 }
8381
8382 #[test]
8390 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
8391 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8392 let mut runner = runner_at(RunStatus::Implementing);
8393 let shutdown = Pause::new();
8394 runner.on_pause(shutdown.clone());
8395 let interrupt = Pause::new();
8396 runner.watch_interrupt(interrupt.clone());
8397
8398 assert!(!runner.park_here().expect("park_here"));
8400 assert!(!runner.state.parked);
8401
8402 interrupt.park_because("test");
8404 assert!(!shutdown.parked());
8405 assert!(runner.park_here().expect("park_here"));
8406 }
8407
8408 #[tokio::test]
8422 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
8423 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8424 let mut runner = runner_at(RunStatus::Implementing);
8425 let interrupt = Pause::new();
8426 runner.watch_interrupt(interrupt.clone());
8427
8428 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
8429 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
8430
8431 let node = async move {
8435 started_tx.send(()).expect("send started");
8436 finish_rx.await.expect("recv finish");
8437 "node finished"
8438 };
8439
8440 let interrupter = async move {
8441 started_rx.await.expect("recv started");
8442 interrupt.park_because("higher-priority task waiting");
8444 tokio::task::yield_now().await;
8448 finish_tx.send(()).expect("send finish");
8449 };
8450
8451 let (node_result, ()) = tokio::join!(node, interrupter);
8452 assert_eq!(
8453 node_result, "node finished",
8454 "the in-flight call ran to completion"
8455 );
8456
8457 assert!(runner.park_here().expect("park_here"));
8460 assert!(runner.state.parked);
8461 }
8462
8463 #[test]
8469 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
8470 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8471 let mut runner = runner_at(RunStatus::Judging);
8472 runner.state.config.agents = vec![conductor()];
8476 runner.state.candidates = vec![Candidate {
8477 index: 0,
8478 label: 'A',
8479 agent: "alpha".to_owned(),
8480 branch: "magi/x/A".to_owned(),
8481 worktree: PathBuf::from("/nonexistent/worktree"),
8482 summary: "did the thing".to_owned(),
8483 stat: "1 file changed".to_owned(),
8484 files: 1,
8485 commits: 1,
8486 empty: false,
8487 failed: None,
8488 verified_noop: None,
8489 duration_ms: 1234,
8490 folded: false,
8491 }];
8492 let run_id = runner.state.id.clone();
8493
8494 let interrupt = Pause::new();
8495 runner.watch_interrupt(interrupt.clone());
8496 interrupt.park_because("task c3d4 asked to run first");
8497 assert!(runner.park_here().expect("park_here"));
8498
8499 let resumed = Runner::resume(&run_id).expect("resume");
8500 assert_eq!(resumed.state.candidates.len(), 1);
8501 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
8502 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
8503 assert_eq!(resumed.state.status, runner.state.status);
8504 assert!(
8505 resumed.state.parked,
8506 "still parked until `execute` actually walks the graph again"
8507 );
8508 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
8509 }
8510
8511 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
8513 let mut q = ask::Question::new(
8514 run.to_owned(),
8515 "implement".to_owned(),
8516 "impl-A".to_owned(),
8517 "Which storage backend should the cache use?".to_owned(),
8518 String::new(),
8519 vec!["SQLite".to_owned(), "Redis".to_owned()],
8520 );
8521 store.put(&mut q).unwrap();
8522 q
8523 }
8524
8525 #[test]
8526 fn a_failed_runs_open_question_is_abandoned() {
8527 ask_test_home();
8528 let store = ask::Questions::open();
8529 let mut runner = runner_at(RunStatus::Failed);
8530 let run = runner.state.id.clone();
8531 let q = ask_open_question(&store, &run);
8532
8533 runner.settle_questions();
8534
8535 let back = store.get(&q.id).unwrap();
8536 assert!(
8537 !back.status.open(),
8538 "the seat that asked died with the run; nobody is left to read an answer"
8539 );
8540 assert!(
8541 back.detail.contains(&run) && back.detail.contains("failed"),
8542 "the reason names what the run became, not just that it is gone: {}",
8543 back.detail
8544 );
8545 }
8546
8547 #[test]
8548 fn a_merged_runs_open_question_is_abandoned_too() {
8549 ask_test_home();
8550 let store = ask::Questions::open();
8551 for status in [RunStatus::Merged, RunStatus::Ready] {
8554 let mut runner = runner_at(status);
8555 let run = runner.state.id.clone();
8556 let q = ask_open_question(&store, &run);
8557
8558 runner.settle_questions();
8559
8560 let back = store.get(&q.id).unwrap();
8561 assert!(
8562 !back.status.open(),
8563 "{status:?} run's question must not outlive the run"
8564 );
8565 }
8566 }
8567
8568 #[test]
8569 fn a_still_resumable_runs_open_question_is_left_alone() {
8570 ask_test_home();
8571 let store = ask::Questions::open();
8572 for status in [RunStatus::Blocked, RunStatus::Stalled] {
8578 let mut runner = runner_at(status);
8579 let run = runner.state.id.clone();
8580 let q = ask_open_question(&store, &run);
8581
8582 runner.settle_questions();
8583
8584 let back = store.get(&q.id).unwrap();
8585 assert!(
8586 back.status.open(),
8587 "{status:?} is still alive; the question must still be waiting"
8588 );
8589 }
8590 }
8591
8592 #[test]
8593 fn settle_questions_never_touches_an_already_answered_question() {
8594 ask_test_home();
8595 let store = ask::Questions::open();
8596 let mut runner = runner_at(RunStatus::Failed);
8597 let run = runner.state.id.clone();
8598 let mut q = ask_open_question(&store, &run);
8599 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
8600 .unwrap();
8601 store.put(&mut q).unwrap();
8602
8603 runner.settle_questions();
8608 runner.settle_questions();
8609
8610 let back = store.get(&q.id).unwrap();
8611 assert_eq!(
8612 back.status,
8613 ask::QuestionStatus::Answered,
8614 "a real answer is a decision on record, never overwritten by a sweep"
8615 );
8616 }
8617
8618 #[tokio::test]
8629 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
8630 crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
8631 let tmp = tempfile::tempdir().expect("tempdir");
8632 let repo = tmp.path().join("repo");
8633 std::fs::create_dir_all(&repo).unwrap();
8634 init_repo(&repo);
8635
8636 let mut config = Config::default();
8637 config.graph.worktree_root = Some(tmp.path().join("wt"));
8638
8639 let mut state = RunState::new(
8640 repo.clone(),
8641 "main".to_owned(),
8642 "deadbeef".to_owned(),
8643 "task".to_owned(),
8644 config,
8645 );
8646 let root = state.worktree_root();
8647 let wt_a = root.join("cand-A");
8648 let wt_b = root.join("cand-B");
8649 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
8650 .await
8651 .expect("worktree A");
8652 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
8653 .await
8654 .expect("worktree B");
8655
8656 state.candidates = vec![
8657 Candidate {
8658 index: 0,
8659 label: 'A',
8660 agent: "alpha".to_owned(),
8661 branch: "magi/x/A".to_owned(),
8662 worktree: wt_a.clone(),
8663 summary: String::new(),
8664 stat: String::new(),
8665 files: 0,
8666 commits: 0,
8667 empty: false,
8668 failed: None,
8669 verified_noop: None,
8670 duration_ms: 0,
8671 folded: false,
8672 },
8673 Candidate {
8674 index: 1,
8675 label: 'B',
8676 agent: "beta".to_owned(),
8677 branch: "magi/x/B".to_owned(),
8678 worktree: wt_b.clone(),
8679 summary: String::new(),
8680 stat: String::new(),
8681 files: 0,
8682 commits: 0,
8683 empty: false,
8684 failed: None,
8685 verified_noop: None,
8686 duration_ms: 0,
8687 folded: false,
8688 },
8689 ];
8690 state.tally = Some(Tally {
8691 first_choice: BTreeMap::from([('A', 1)]),
8692 borda: BTreeMap::new(),
8693 winner: 'A',
8694 rankings: 1,
8695 unanimous_initial: true,
8696 deliberated: false,
8697 changed_votes: 0,
8698 unanimous_final: true,
8699 tie_break: None,
8700 judges: 1,
8701 present: 1,
8702 quorum: 1,
8703 met_quorum: true,
8704 uncontested: None,
8705 });
8706 state.status = RunStatus::Ready;
8707
8708 fold_run(&mut state, false, &crate::run::home())
8709 .await
8710 .expect("fold_run");
8711
8712 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
8713 assert!(
8714 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
8715 "the unmerged winner's branch survives"
8716 );
8717 assert!(
8718 !state.candidates[0].folded,
8719 "the winner is not marked folded"
8720 );
8721
8722 assert!(!wt_b.exists(), "the loser's worktree is removed");
8723 assert!(
8724 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
8725 "the loser's branch is removed"
8726 );
8727 assert!(state.candidates[1].folded, "the loser is marked folded");
8728 }
8729
8730 #[tokio::test]
8733 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
8734 let tmp = tempfile::tempdir().expect("tempdir");
8735 let repo = tmp.path().join("repo");
8736 std::fs::create_dir_all(&repo).unwrap();
8737 init_repo(&repo);
8738 let home = tmp.path().join("home");
8739
8740 let mut config = Config::default();
8741 config.graph.worktree_root = Some(tmp.path().join("wt"));
8742 let mut state = RunState::new(
8743 repo.clone(),
8744 "main".to_owned(),
8745 "deadbeef".to_owned(),
8746 "task".to_owned(),
8747 config,
8748 );
8749 git::git(&repo, &["branch", "magi/x/A", "main"])
8751 .await
8752 .expect("branch");
8753 state.candidates = vec![Candidate {
8754 index: 0,
8755 label: 'A',
8756 agent: "alpha".to_owned(),
8757 branch: "magi/x/A".to_owned(),
8758 worktree: state.worktree_root().join("cand-A"),
8759 summary: String::new(),
8760 stat: String::new(),
8761 files: 0,
8762 commits: 0,
8763 empty: false,
8764 failed: None,
8765 verified_noop: None,
8766 duration_ms: 0,
8767 folded: true,
8768 }];
8769 state.released_to = Some("20260901-000000-new1".to_owned());
8770 state.released_branches = vec!["magi/x/A".to_owned()];
8771
8772 fold_run(&mut state, true, &home).await.expect("fold_run");
8773
8774 assert!(
8775 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
8776 "the handed-over branch survives a fold"
8777 );
8778 }
8779
8780 #[tokio::test]
8784 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
8785 let tmp = tempfile::tempdir().expect("tempdir");
8786 let repo = tmp.path().join("repo");
8787 std::fs::create_dir_all(&repo).unwrap();
8788 init_repo(&repo);
8789 let run = |args: &[&str]| {
8790 let out = std::process::Command::new("git")
8791 .quiet()
8792 .args(args)
8793 .current_dir(&repo)
8794 .output()
8795 .expect("spawn git");
8796 assert!(out.status.success(), "git {args:?}");
8797 };
8798 run(&["branch", "magi/x/A"]);
8799 run(&["checkout", "-q", "-b", "magi/x/B"]);
8800 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
8801 run(&["add", "-A"]);
8802 run(&["commit", "-q", "-m", "work"]);
8803 run(&["checkout", "-q", "main"]);
8804
8805 let mut state = RunState::new(
8806 repo.clone(),
8807 "main".to_owned(),
8808 "deadbeef".to_owned(),
8809 "task".to_owned(),
8810 Config::default(),
8811 );
8812 state.seeds = vec![refs::Seed {
8813 token: "magi/27b2/A".to_owned(),
8814 kind: refs::SeedKind::Unresolved,
8815 sha: String::new(),
8816 branch: true,
8817 detail: "no branch or commit named magi/27b2/A".to_owned(),
8818 }];
8819
8820 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
8821 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
8822 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
8823 let detail = empty_candidate_detail(&state, "main");
8824 assert!(detail.starts_with("empty candidate"), "{detail}");
8825 assert!(detail.contains("magi/27b2/A"), "{detail}");
8826 }
8827
8828 #[tokio::test]
8837 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
8838 let tmp = tempfile::tempdir().expect("tempdir");
8839 let repo = tmp.path().join("repo");
8840 std::fs::create_dir_all(&repo).unwrap();
8841 init_repo(&repo);
8842
8843 let mut config = Config::default();
8844 config.merge.mode = MergeMode::Local;
8845
8846 let mut state = RunState::new(
8847 repo.clone(),
8848 "main".to_owned(),
8849 "deadbeef".to_owned(),
8850 "task".to_owned(),
8851 config,
8852 );
8853 state.candidates = vec![Candidate {
8854 index: 0,
8855 label: 'A',
8856 agent: "alpha".to_owned(),
8857 branch: "does-not-exist".to_owned(),
8858 worktree: repo.clone(),
8859 summary: String::new(),
8860 stat: String::new(),
8861 files: 0,
8862 commits: 0,
8863 empty: false,
8864 failed: None,
8865 verified_noop: None,
8866 duration_ms: 0,
8867 folded: false,
8868 }];
8869 state.tally = Some(Tally {
8870 first_choice: BTreeMap::from([('A', 1)]),
8871 borda: BTreeMap::new(),
8872 winner: 'A',
8873 rankings: 1,
8874 unanimous_initial: true,
8875 deliberated: false,
8876 changed_votes: 0,
8877 unanimous_final: true,
8878 tie_break: None,
8879 judges: 0,
8880 present: 0,
8881 quorum: 0,
8882 met_quorum: true,
8883 uncontested: Some("only candidate A produced a change".to_owned()),
8884 });
8885 state.reviews = vec![ReviewRound {
8886 round: 1,
8887 head: "deadbeef".to_owned(),
8888 verified_head: None,
8889 verified_at: None,
8890 reviews: Vec::new(),
8891 e2e: Vec::new(),
8892 fix: None,
8893 blocking: 0,
8894 answered: 0,
8895 expected: 0,
8896 clean: true,
8897 verify_retried: false,
8898 e2e_deferred: false,
8899 e2e_defer_reason: None,
8900 progressed: false,
8901 vote_split: false,
8902 reconsideration: Vec::new(),
8903 verdict: None,
8904 }];
8905 state.gate = vec![CommandOutcome {
8906 command: "test".to_owned(),
8907 code: Some(0),
8908 output_tail: String::new(),
8909 duration_ms: 0,
8910 resource_blocked: false,
8911 }];
8912 state.gate_ran = true;
8913 state.status = RunStatus::Ready;
8918 state.merge = Some(MergeOutcome {
8919 mode: MergeMode::Local,
8920 ok: false,
8921 detail: "already concluded".to_owned(),
8922 empty: false,
8923 });
8924
8925 let mut runner = Runner {
8926 state,
8927 roles: ResolvedRoles {
8928 implementers: Vec::new(),
8929 judges: Vec::new(),
8930 reviewers: Vec::new(),
8931 fixer: None,
8932 conductor: conductor(),
8933 implementer_roster: Vec::new(),
8934 judge_roster: Vec::new(),
8935 reviewer_roster: Vec::new(),
8936 },
8937 sem: Arc::new(Semaphore::new(1)),
8938 pause: Pause::new(),
8939 interrupt: Pause::new(),
8940 };
8941
8942 runner.merge().await.expect("merge");
8943
8944 assert_eq!(
8945 runner.state.status,
8946 RunStatus::Ready,
8947 "a concluded run's status must not change on reentry"
8948 );
8949 assert_eq!(
8950 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8951 Some("already concluded"),
8952 "merge must not run again once the node already recorded an outcome"
8953 );
8954 }
8955
8956 #[tokio::test]
8965 async fn merge_refuses_a_gate_that_has_not_actually_run() {
8966 let tmp = tempfile::tempdir().expect("tempdir");
8967 let repo = tmp.path().join("repo");
8968 std::fs::create_dir_all(&repo).unwrap();
8969 init_repo(&repo);
8970
8971 let mut config = Config::default();
8972 config.merge.mode = MergeMode::Local;
8973
8974 let mut state = RunState::new(
8975 repo.clone(),
8976 "main".to_owned(),
8977 "deadbeef".to_owned(),
8978 "task".to_owned(),
8979 config,
8980 );
8981 state.candidates = vec![Candidate {
8982 index: 0,
8983 label: 'A',
8984 agent: "alpha".to_owned(),
8985 branch: "does-not-exist".to_owned(),
8986 worktree: repo.clone(),
8987 summary: String::new(),
8988 stat: String::new(),
8989 files: 0,
8990 commits: 0,
8991 empty: false,
8992 failed: None,
8993 verified_noop: None,
8994 duration_ms: 0,
8995 folded: false,
8996 }];
8997 state.tally = Some(Tally {
8998 first_choice: BTreeMap::from([('A', 1)]),
8999 borda: BTreeMap::new(),
9000 winner: 'A',
9001 rankings: 1,
9002 unanimous_initial: true,
9003 deliberated: false,
9004 changed_votes: 0,
9005 unanimous_final: true,
9006 tie_break: None,
9007 judges: 0,
9008 present: 0,
9009 quorum: 0,
9010 met_quorum: true,
9011 uncontested: Some("only candidate A produced a change".to_owned()),
9012 });
9013 state.reviews = vec![ReviewRound {
9014 round: 1,
9015 head: "deadbeef".to_owned(),
9016 verified_head: None,
9017 verified_at: None,
9018 reviews: Vec::new(),
9019 e2e: Vec::new(),
9020 fix: None,
9021 blocking: 0,
9022 answered: 0,
9023 expected: 0,
9024 clean: true,
9025 verify_retried: false,
9026 e2e_deferred: false,
9027 e2e_defer_reason: None,
9028 progressed: false,
9029 vote_split: false,
9030 reconsideration: Vec::new(),
9031 verdict: None,
9032 }];
9033 state.gate = Vec::new();
9035 state.gate_ran = false;
9036 state.status = RunStatus::Gating;
9037
9038 let mut runner = Runner {
9039 state,
9040 roles: ResolvedRoles {
9041 implementers: Vec::new(),
9042 judges: Vec::new(),
9043 reviewers: Vec::new(),
9044 fixer: None,
9045 conductor: conductor(),
9046 implementer_roster: Vec::new(),
9047 judge_roster: Vec::new(),
9048 reviewer_roster: Vec::new(),
9049 },
9050 sem: Arc::new(Semaphore::new(1)),
9051 pause: Pause::new(),
9052 interrupt: Pause::new(),
9053 };
9054
9055 runner.merge().await.expect("merge");
9056
9057 assert!(
9058 runner.state.merge.is_none(),
9059 "an empty gate must never be read as a passing one: {:?}",
9060 runner.state.merge
9061 );
9062 }
9063
9064 #[tokio::test]
9071 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
9072 let tmp = tempfile::tempdir().expect("tempdir");
9073 let repo = tmp.path().join("repo");
9074 std::fs::create_dir_all(&repo).unwrap();
9075 init_repo(&repo);
9076
9077 let config = Config::default();
9079
9080 let mut state = RunState::new(
9081 repo.clone(),
9082 "main".to_owned(),
9083 "deadbeef".to_owned(),
9084 "task".to_owned(),
9085 config,
9086 );
9087 state.candidates = vec![Candidate {
9088 index: 0,
9089 label: 'A',
9090 agent: "alpha".to_owned(),
9091 branch: "does-not-exist".to_owned(),
9092 worktree: repo.clone(),
9093 summary: String::new(),
9094 stat: String::new(),
9095 files: 0,
9096 commits: 0,
9097 empty: false,
9098 failed: None,
9099 verified_noop: None,
9100 duration_ms: 0,
9101 folded: false,
9102 }];
9103 state.tally = Some(Tally {
9104 first_choice: BTreeMap::from([('A', 1)]),
9105 borda: BTreeMap::new(),
9106 winner: 'A',
9107 rankings: 1,
9108 unanimous_initial: true,
9109 deliberated: false,
9110 changed_votes: 0,
9111 unanimous_final: true,
9112 tie_break: None,
9113 judges: 0,
9114 present: 0,
9115 quorum: 0,
9116 met_quorum: true,
9117 uncontested: Some("only candidate A produced a change".to_owned()),
9118 });
9119 state.reviews = vec![ReviewRound {
9120 round: 1,
9121 head: "deadbeef".to_owned(),
9122 verified_head: None,
9123 verified_at: None,
9124 reviews: Vec::new(),
9125 e2e: Vec::new(),
9126 fix: None,
9127 blocking: 0,
9128 answered: 0,
9129 expected: 0,
9130 clean: true,
9131 verify_retried: false,
9132 e2e_deferred: false,
9133 e2e_defer_reason: None,
9134 progressed: false,
9135 vote_split: false,
9136 reconsideration: Vec::new(),
9137 verdict: None,
9138 }];
9139
9140 let mut runner = Runner {
9141 state,
9142 roles: ResolvedRoles {
9143 implementers: Vec::new(),
9144 judges: Vec::new(),
9145 reviewers: Vec::new(),
9146 fixer: None,
9147 conductor: conductor(),
9148 implementer_roster: Vec::new(),
9149 judge_roster: Vec::new(),
9150 reviewer_roster: Vec::new(),
9151 },
9152 sem: Arc::new(Semaphore::new(1)),
9153 pause: Pause::new(),
9154 interrupt: Pause::new(),
9155 };
9156
9157 runner.gate().await.expect("gate");
9158 assert!(
9159 runner.state.gate_ran,
9160 "zero configured commands is still a real attempt, not an unrun gate"
9161 );
9162 assert!(runner.state.gate.is_empty());
9163 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
9164 assert_ne!(
9165 runner.state.status,
9166 RunStatus::Blocked,
9167 "a gate with nothing to check must not read as failed"
9168 );
9169
9170 runner.merge().await.expect("merge");
9171 assert_eq!(
9172 runner.state.status,
9173 RunStatus::Ready,
9174 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
9175 );
9176 }
9177
9178 #[tokio::test]
9189 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
9190 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9191 let home = crate::run::home();
9192
9193 let tmp = tempfile::tempdir().expect("tempdir");
9194 let repo = tmp.path().join("repo");
9195 std::fs::create_dir_all(&repo).unwrap();
9196 init_repo(&repo);
9197 let cache_dir = tmp.path().join("target");
9200
9201 let mut config = Config::default();
9202 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
9203 config.graph.timeout_verify = Some(2);
9206
9207 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9208 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9209 .expect("no io error acquiring directly")
9210 {
9211 crate::cache::AcquireOutcome::Acquired(g) => g,
9212 crate::cache::AcquireOutcome::Busy(b) => {
9213 panic!("expected the direct acquire to win the lease first: {b:?}")
9214 }
9215 };
9216
9217 let mut state = RunState::new(
9218 repo.clone(),
9219 "main".to_owned(),
9220 "deadbeef".to_owned(),
9221 "task".to_owned(),
9222 config,
9223 );
9224 state.candidates = vec![Candidate {
9225 index: 0,
9226 label: 'A',
9227 agent: "alpha".to_owned(),
9228 branch: "does-not-exist".to_owned(),
9229 worktree: repo.clone(),
9230 summary: String::new(),
9231 stat: String::new(),
9232 files: 0,
9233 commits: 0,
9234 empty: false,
9235 failed: None,
9236 verified_noop: None,
9237 duration_ms: 0,
9238 folded: false,
9239 }];
9240 state.tally = Some(Tally {
9241 first_choice: BTreeMap::from([('A', 1)]),
9242 borda: BTreeMap::new(),
9243 winner: 'A',
9244 rankings: 1,
9245 unanimous_initial: true,
9246 deliberated: false,
9247 changed_votes: 0,
9248 unanimous_final: true,
9249 tie_break: None,
9250 judges: 0,
9251 present: 0,
9252 quorum: 0,
9253 met_quorum: true,
9254 uncontested: Some("only candidate A produced a change".to_owned()),
9255 });
9256 state.reviews = vec![ReviewRound {
9257 round: 1,
9258 head: "deadbeef".to_owned(),
9259 verified_head: None,
9260 verified_at: None,
9261 reviews: Vec::new(),
9262 e2e: Vec::new(),
9263 fix: None,
9264 blocking: 0,
9265 answered: 0,
9266 expected: 0,
9267 clean: true,
9268 verify_retried: false,
9269 e2e_deferred: false,
9270 e2e_defer_reason: None,
9271 progressed: false,
9272 vote_split: false,
9273 reconsideration: Vec::new(),
9274 verdict: None,
9275 }];
9276
9277 let mut runner = Runner {
9278 state,
9279 roles: ResolvedRoles {
9280 implementers: Vec::new(),
9281 judges: Vec::new(),
9282 reviewers: Vec::new(),
9283 fixer: None,
9284 conductor: conductor(),
9285 implementer_roster: Vec::new(),
9286 judge_roster: Vec::new(),
9287 reviewer_roster: Vec::new(),
9288 },
9289 sem: Arc::new(Semaphore::new(1)),
9290 pause: Pause::new(),
9291 interrupt: Pause::new(),
9292 };
9293
9294 let started = std::time::Instant::now();
9295 runner.gate().await.expect("gate");
9296 assert!(
9297 started.elapsed() < Duration::from_secs(1),
9298 "a gate with nothing to run must never wait on a lease it never needed"
9299 );
9300 assert!(
9301 runner.state.gate_ran,
9302 "zero commands is still a real, immediate attempt"
9303 );
9304 assert!(runner.state.gate.is_empty());
9305 assert_ne!(
9306 runner.state.status,
9307 RunStatus::Blocked,
9308 "must not read as resource-blocked on a lease it never asked for"
9309 );
9310 }
9311
9312 #[tokio::test]
9322 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
9323 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9324
9325 let tmp = tempfile::tempdir().expect("tempdir");
9326 let repo = tmp.path().join("repo");
9327 std::fs::create_dir_all(&repo).unwrap();
9328 init_repo(&repo);
9329
9330 let mut config = Config::default();
9331 config.verify.gate = vec![
9332 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
9333 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
9334 .to_owned(),
9335 ];
9336
9337 let mut state = RunState::new(
9338 repo.clone(),
9339 "main".to_owned(),
9340 "deadbeef".to_owned(),
9341 "task".to_owned(),
9342 config,
9343 );
9344 let run_id = state.id.clone();
9345 state.candidates = vec![Candidate {
9346 index: 0,
9347 label: 'A',
9348 agent: "alpha".to_owned(),
9349 branch: "does-not-exist".to_owned(),
9350 worktree: repo.clone(),
9351 summary: String::new(),
9352 stat: String::new(),
9353 files: 0,
9354 commits: 0,
9355 empty: false,
9356 failed: None,
9357 verified_noop: None,
9358 duration_ms: 0,
9359 folded: false,
9360 }];
9361 state.tally = Some(Tally {
9362 first_choice: BTreeMap::from([('A', 1)]),
9363 borda: BTreeMap::new(),
9364 winner: 'A',
9365 rankings: 1,
9366 unanimous_initial: true,
9367 deliberated: false,
9368 changed_votes: 0,
9369 unanimous_final: true,
9370 tie_break: None,
9371 judges: 0,
9372 present: 0,
9373 quorum: 0,
9374 met_quorum: true,
9375 uncontested: Some("only candidate A produced a change".to_owned()),
9376 });
9377 state.reviews = vec![ReviewRound {
9378 round: 1,
9379 head: "deadbeef".to_owned(),
9380 verified_head: None,
9381 verified_at: None,
9382 reviews: Vec::new(),
9383 e2e: Vec::new(),
9384 fix: None,
9385 blocking: 0,
9386 answered: 0,
9387 expected: 0,
9388 clean: true,
9389 verify_retried: false,
9390 e2e_deferred: false,
9391 e2e_defer_reason: None,
9392 progressed: false,
9393 vote_split: false,
9394 reconsideration: Vec::new(),
9395 verdict: None,
9396 }];
9397
9398 let mut runner = Runner {
9399 state,
9400 roles: ResolvedRoles {
9401 implementers: Vec::new(),
9402 judges: Vec::new(),
9403 reviewers: Vec::new(),
9404 fixer: None,
9405 conductor: conductor(),
9406 implementer_roster: Vec::new(),
9407 judge_roster: Vec::new(),
9408 reviewer_roster: Vec::new(),
9409 },
9410 sem: Arc::new(Semaphore::new(1)),
9411 pause: Pause::new(),
9412 interrupt: Pause::new(),
9413 };
9414
9415 let started_marker = repo.join("started.marker");
9416 let release_marker = repo.join("release.marker");
9417 let poller = tokio::spawn(async move {
9418 for _ in 0..100 {
9423 if started_marker.exists()
9424 && let Ok(s) = crate::run::RunState::load(&run_id)
9425 && let Some(a) = s.active.get("gate")
9426 {
9427 std::fs::write(&release_marker, b"go").expect("release marker");
9428 return Some(a.clone());
9429 }
9430 tokio::time::sleep(Duration::from_millis(50)).await;
9431 }
9432 None
9433 });
9434
9435 runner.gate().await.expect("gate");
9436 let captured = poller.await.expect("poller task");
9437 let captured = captured.expect(
9438 "the poller never saw a `gate` task entry in run.json while the command was \
9439 still blocked on its own release marker",
9440 );
9441
9442 assert_eq!(captured.task.as_deref(), Some("gate"));
9443 assert_eq!(captured.node, "gate");
9444 assert_eq!(captured.index, Some(1));
9445 assert_eq!(captured.total, Some(1));
9446 assert!(
9447 captured
9448 .command
9449 .as_deref()
9450 .is_some_and(|c| c.contains("started.marker")),
9451 "{captured:?}"
9452 );
9453
9454 assert!(
9455 runner.state.active.is_empty(),
9456 "the entry must be cleared once the command actually finished: {:?}",
9457 runner.state.active
9458 );
9459 assert!(runner.state.gate_ran);
9460 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
9461 }
9462
9463 #[tokio::test]
9466 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
9467 use crate::verdict::{Finding, ReviewVote, Severity};
9468 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9469 let tmp = tempfile::tempdir().expect("tempdir");
9470 let repo = tmp.path().join("repo");
9471 std::fs::create_dir_all(&repo).unwrap();
9472 init_repo(&repo);
9473
9474 for (severity, vote, expect) in [
9475 (Severity::Major, ReviewVote::Reject, true),
9476 (Severity::Minor, ReviewVote::Reject, false),
9477 (Severity::Major, ReviewVote::Approve, false),
9478 ] {
9479 let mut round = review_round(false, 1, 1, 1, false, true);
9480 round.reviews = vec![ReviewRecord {
9481 reviewer: 1,
9482 agent: "alpha".to_owned(),
9483 summary: String::new(),
9484 findings: vec![Finding {
9485 id: "R1-1-1".to_owned(),
9486 severity,
9487 file: None,
9488 line: None,
9489 title: "t".to_owned(),
9490 detail: String::new(),
9491 }],
9492 vote: Some(vote),
9493 failed: None,
9494 duration_ms: 0,
9495 attempts: 0,
9496 }];
9497 let mut state = RunState::new(
9498 repo.clone(),
9499 "main".to_owned(),
9500 "deadbeef".to_owned(),
9501 "task".to_owned(),
9502 Config::default(),
9503 );
9504 state.reviews = vec![round];
9505 let mut runner = Runner {
9506 state,
9507 roles: ResolvedRoles {
9508 implementers: Vec::new(),
9509 judges: Vec::new(),
9510 reviewers: Vec::new(),
9511 fixer: None,
9512 conductor: conductor(),
9513 implementer_roster: Vec::new(),
9514 judge_roster: Vec::new(),
9515 reviewer_roster: Vec::new(),
9516 },
9517 sem: Arc::new(Semaphore::new(1)),
9518 pause: Pause::new(),
9519 interrupt: Pause::new(),
9520 };
9521 let shell = runner.state.config.shell();
9522 runner
9523 .stop_reviewing("round budget spent", &shell, &repo)
9524 .await
9525 .expect("stop_reviewing");
9526 assert_eq!(runner.state.status, RunStatus::Gating);
9527 assert_eq!(
9528 runner.state.contested_handoff.is_some(),
9529 expect,
9530 "{severity:?} + {vote:?}"
9531 );
9532 }
9533 }
9534
9535 #[tokio::test]
9548 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
9549 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9550 let home = crate::run::home();
9551
9552 let tmp = tempfile::tempdir().expect("tempdir");
9553 let repo = tmp.path().join("repo");
9554 std::fs::create_dir_all(&repo).unwrap();
9555 init_repo(&repo);
9556 let head = crate::git::rev_parse(&repo, "HEAD")
9557 .await
9558 .expect("rev-parse");
9559 let cache_dir = tmp.path().join("target");
9562
9563 let mut config = Config::default();
9564 config.verify.e2e = vec![format!(
9565 "CARGO_TARGET_DIR='{}' test -f README.md",
9566 cache_dir.display()
9567 )];
9568 config.graph.review_rounds = 1;
9569 config.graph.timeout_verify = Some(2);
9572
9573 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9574 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9575 .expect("no io error acquiring directly")
9576 {
9577 crate::cache::AcquireOutcome::Acquired(g) => g,
9578 crate::cache::AcquireOutcome::Busy(b) => {
9579 panic!("expected the direct acquire to win the lease first: {b:?}")
9580 }
9581 };
9582
9583 let mut state = RunState::new(
9584 repo.clone(),
9585 "main".to_owned(),
9586 head.clone(),
9587 "task".to_owned(),
9588 config,
9589 );
9590 state.candidates = vec![Candidate {
9591 index: 0,
9592 label: 'A',
9593 agent: "alpha".to_owned(),
9594 branch: "does-not-exist".to_owned(),
9595 worktree: repo.clone(),
9596 summary: String::new(),
9597 stat: String::new(),
9598 files: 0,
9599 commits: 0,
9600 empty: false,
9601 failed: None,
9602 verified_noop: None,
9603 duration_ms: 0,
9604 folded: false,
9605 }];
9606 state.tally = Some(Tally {
9607 first_choice: BTreeMap::from([('A', 1)]),
9608 borda: BTreeMap::new(),
9609 winner: 'A',
9610 rankings: 1,
9611 unanimous_initial: true,
9612 deliberated: false,
9613 changed_votes: 0,
9614 unanimous_final: true,
9615 tie_break: None,
9616 judges: 0,
9617 present: 0,
9618 quorum: 0,
9619 met_quorum: true,
9620 uncontested: Some("only candidate A produced a change".to_owned()),
9621 });
9622 state.reviews = vec![ReviewRound {
9626 round: 1,
9627 head: head.clone(),
9628 verified_head: None,
9629 verified_at: None,
9630 reviews: Vec::new(),
9631 e2e: Vec::new(),
9632 fix: None,
9633 blocking: 1,
9634 answered: 1,
9635 expected: 1,
9636 clean: false,
9637 verify_retried: false,
9638 e2e_deferred: true,
9639 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
9640 progressed: false,
9641 vote_split: false,
9642 reconsideration: Vec::new(),
9643 verdict: None,
9644 }];
9645
9646 let mut runner = Runner {
9647 state,
9648 roles: ResolvedRoles {
9649 implementers: Vec::new(),
9650 judges: Vec::new(),
9651 reviewers: Vec::new(),
9652 fixer: None,
9653 conductor: conductor(),
9654 implementer_roster: Vec::new(),
9655 judge_roster: Vec::new(),
9656 reviewer_roster: Vec::new(),
9657 },
9658 sem: Arc::new(Semaphore::new(1)),
9659 pause: Pause::new(),
9660 interrupt: Pause::new(),
9661 };
9662
9663 let shell = runner.state.config.shell();
9664 runner
9665 .stop_reviewing("round budget spent", &shell, &repo)
9666 .await
9667 .expect("stop_reviewing");
9668
9669 let last = runner.state.reviews.last().expect("round record");
9670 assert_eq!(
9671 last.e2e_status(),
9672 E2eStatus::ResourceBlocked,
9673 "the shared cache is still held; the attempt must read as blocked, not deferred or \
9674 failed: {last:?}"
9675 );
9676 assert_eq!(
9677 last.verified_head.as_deref(),
9678 Some(head.as_str()),
9679 "which commit this attempt targeted is known even though nothing finished checking \
9680 it"
9681 );
9682 let first_attempt_at = last
9683 .verified_at
9684 .expect("when this attempt ran is known too");
9685 assert_ne!(
9686 runner.state.status,
9687 RunStatus::Blocked,
9688 "contention is evidence about the machine, not the patch — it must not settle the \
9689 run as blocked: {:?}",
9690 runner.state.status
9691 );
9692 assert!(
9693 !runner
9694 .state
9695 .events
9696 .iter()
9697 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
9698 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
9699 runner.state.events
9700 );
9701
9702 runner
9707 .stop_reviewing("round budget spent", &shell, &repo)
9708 .await
9709 .expect("stop_reviewing retry");
9710 assert_eq!(
9711 runner.state.reviews.len(),
9712 1,
9713 "no new round was started: {:?}",
9714 runner.state.reviews
9715 );
9716 let last = runner.state.reviews.last().expect("round record");
9717 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
9718 assert!(
9719 last.verified_at.expect("still known") > first_attempt_at,
9720 "a second reentry must be a fresh attempt, not a stale copy of the first"
9721 );
9722 assert_ne!(runner.state.status, RunStatus::Blocked);
9723
9724 held.release();
9725 }
9726
9727 #[tokio::test]
9739 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
9740 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9741 let home = crate::run::home();
9742
9743 let tmp = tempfile::tempdir().expect("tempdir");
9744 let repo = tmp.path().join("repo");
9745 std::fs::create_dir_all(&repo).unwrap();
9746 init_repo(&repo);
9747 let head = crate::git::rev_parse(&repo, "HEAD")
9748 .await
9749 .expect("rev-parse");
9750 let cache_dir = tmp.path().join("target");
9751
9752 let mut config = Config::default();
9753 config.verify.e2e = vec![format!(
9754 "CARGO_TARGET_DIR='{}' test -f README.md",
9755 cache_dir.display()
9756 )];
9757 config.graph.review_rounds = 1;
9758 config.graph.timeout_verify = Some(2);
9759
9760 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9761 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9762 .expect("no io error acquiring directly")
9763 {
9764 crate::cache::AcquireOutcome::Acquired(g) => g,
9765 crate::cache::AcquireOutcome::Busy(b) => {
9766 panic!("expected the direct acquire to win the lease first: {b:?}")
9767 }
9768 };
9769
9770 let mut state = RunState::new(
9771 repo.clone(),
9772 "main".to_owned(),
9773 head.clone(),
9774 "task".to_owned(),
9775 config,
9776 );
9777 state.candidates = vec![Candidate {
9778 index: 0,
9779 label: 'A',
9780 agent: "alpha".to_owned(),
9781 branch: "does-not-exist".to_owned(),
9782 worktree: repo.clone(),
9783 summary: String::new(),
9784 stat: String::new(),
9785 files: 0,
9786 commits: 0,
9787 empty: false,
9788 failed: None,
9789 verified_noop: None,
9790 duration_ms: 0,
9791 folded: false,
9792 }];
9793 state.tally = Some(Tally {
9794 first_choice: BTreeMap::from([('A', 1)]),
9795 borda: BTreeMap::new(),
9796 winner: 'A',
9797 rankings: 1,
9798 unanimous_initial: true,
9799 deliberated: false,
9800 changed_votes: 0,
9801 unanimous_final: true,
9802 tie_break: None,
9803 judges: 0,
9804 present: 0,
9805 quorum: 0,
9806 met_quorum: true,
9807 uncontested: Some("only candidate A produced a change".to_owned()),
9808 });
9809 state.reviews = vec![ReviewRound {
9813 round: 1,
9814 head: head.clone(),
9815 verified_head: Some(head.clone()),
9816 verified_at: Some(jiff::Timestamp::now()),
9817 reviews: Vec::new(),
9818 e2e: vec![CommandOutcome {
9819 command: format!(
9820 "CARGO_TARGET_DIR='{}' test -f README.md",
9821 cache_dir.display()
9822 ),
9823 code: None,
9824 output_tail: "waiting for the shared build cache".to_owned(),
9825 duration_ms: 0,
9826 resource_blocked: true,
9827 }],
9828 fix: None,
9829 blocking: 1,
9830 answered: 1,
9831 expected: 1,
9832 clean: false,
9833 verify_retried: false,
9834 e2e_deferred: false,
9835 e2e_defer_reason: None,
9836 progressed: false,
9837 vote_split: false,
9838 reconsideration: Vec::new(),
9839 verdict: None,
9840 }];
9841
9842 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
9843 let mut runner = Runner {
9844 state,
9845 roles: ResolvedRoles {
9846 implementers: Vec::new(),
9847 judges: Vec::new(),
9848 reviewers: Vec::new(),
9849 fixer: None,
9850 conductor: conductor(),
9851 implementer_roster: Vec::new(),
9852 judge_roster: Vec::new(),
9853 reviewer_roster: Vec::new(),
9854 },
9855 sem: Arc::new(Semaphore::new(1)),
9856 pause: Pause::new(),
9857 interrupt: Pause::new(),
9858 };
9859
9860 runner.review_loop().await.expect("review_loop");
9865
9866 assert_eq!(
9867 runner.state.reviews.len(),
9868 1,
9869 "no new round was started on top of the unresolved one: {:?}",
9870 runner.state.reviews
9871 );
9872 let last = &runner.state.reviews[0];
9873 assert_eq!(
9874 last.e2e_status(),
9875 E2eStatus::ResourceBlocked,
9876 "still contended: {last:?}"
9877 );
9878 assert!(
9879 last.verified_at.expect("still known") > first_attempt_at,
9880 "review_loop must have actually retried the check, not left it exactly as found"
9881 );
9882 assert_ne!(
9883 runner.state.status,
9884 RunStatus::Blocked,
9885 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
9886 runner.state.status
9887 );
9888
9889 held.release();
9890 }
9891
9892 #[tokio::test]
9893 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
9894 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9895 let tmp = tempfile::tempdir().expect("tempdir");
9896 let repo = tmp.path().join("repo");
9897 std::fs::create_dir_all(&repo).unwrap();
9898 init_repo(&repo);
9899
9900 let mut config = Config::default();
9901 config.merge.mode = MergeMode::Pr;
9902 config.graph.land = true;
9903 config.graph.land_approval = false;
9904
9905 let mut state = RunState::new(
9906 repo.clone(),
9907 "main".to_owned(),
9908 "deadbeef".to_owned(),
9909 "task".to_owned(),
9910 config,
9911 );
9912 state.candidates = vec![Candidate {
9913 index: 0,
9914 label: 'A',
9915 agent: "alpha".to_owned(),
9916 branch: "does-not-exist".to_owned(),
9917 worktree: repo.clone(),
9918 summary: String::new(),
9919 stat: String::new(),
9920 files: 0,
9921 commits: 0,
9922 empty: false,
9923 failed: None,
9924 verified_noop: None,
9925 duration_ms: 0,
9926 folded: false,
9927 }];
9928 state.tally = Some(Tally {
9929 first_choice: BTreeMap::from([('A', 1)]),
9930 borda: BTreeMap::new(),
9931 winner: 'A',
9932 rankings: 1,
9933 unanimous_initial: true,
9934 deliberated: false,
9935 changed_votes: 0,
9936 unanimous_final: true,
9937 tie_break: None,
9938 judges: 0,
9939 present: 0,
9940 quorum: 0,
9941 met_quorum: true,
9942 uncontested: Some("only candidate A produced a change".to_owned()),
9943 });
9944 state.reviews = vec![ReviewRound {
9945 round: 1,
9946 head: "deadbeef".to_owned(),
9947 verified_head: None,
9948 verified_at: None,
9949 reviews: Vec::new(),
9950 e2e: Vec::new(),
9951 fix: None,
9952 blocking: 0,
9953 answered: 0,
9954 expected: 0,
9955 clean: true,
9956 verify_retried: false,
9957 e2e_deferred: false,
9958 e2e_defer_reason: None,
9959 progressed: false,
9960 vote_split: false,
9961 reconsideration: Vec::new(),
9962 verdict: None,
9963 }];
9964 state.gate = vec![CommandOutcome {
9965 command: "test".to_owned(),
9966 code: Some(0),
9967 output_tail: String::new(),
9968 duration_ms: 0,
9969 resource_blocked: false,
9970 }];
9971 state.gate_ran = true;
9972 state.status = RunStatus::Landing;
9976 state.merge = Some(MergeOutcome {
9977 mode: MergeMode::Pr,
9978 ok: true,
9979 detail: "https://example.invalid/x/y/pull/1".to_owned(),
9980 empty: false,
9981 });
9982
9983 ask_test_home();
9987 let store = ask::Questions::open();
9988 let q = ask_open_question(&store, &state.id);
9989
9990 let mut runner = Runner {
9991 state,
9992 roles: ResolvedRoles {
9993 implementers: Vec::new(),
9994 judges: Vec::new(),
9995 reviewers: Vec::new(),
9996 fixer: None,
9997 conductor: conductor(),
9998 implementer_roster: Vec::new(),
9999 judge_roster: Vec::new(),
10000 reviewer_roster: Vec::new(),
10001 },
10002 sem: Arc::new(Semaphore::new(1)),
10003 pause: Pause::new(),
10004 interrupt: Pause::new(),
10005 };
10006
10007 runner.execute().await.expect("execute");
10012
10013 assert_eq!(
10014 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10015 Some("https://example.invalid/x/y/pull/1"),
10016 "reentry must not push again or open a second pull request over the \
10017 one `land` is already watching"
10018 );
10019 assert_ne!(
10020 runner.state.status,
10021 RunStatus::Landing,
10022 "land could not actually reach the fake pull request, so it must \
10023 have given up rather than left the run silently parked forever"
10024 );
10025 assert_eq!(runner.state.status, RunStatus::Blocked);
10029 assert!(
10030 store.get(&q.id).unwrap().status.open(),
10031 "Blocked is still alive; settle_questions must have been a no-op here"
10032 );
10033 }
10034
10035 fn state_with_round(round: ReviewRound) -> RunState {
10036 let mut s = RunState::new(
10037 PathBuf::from("/repo"),
10038 "main".to_owned(),
10039 "abc1234".to_owned(),
10040 "add retries".to_owned(),
10041 Config::default(),
10042 );
10043 s.reviews = vec![round];
10044 s
10045 }
10046
10047 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
10048 crate::verdict::Finding {
10049 id: id.to_owned(),
10050 severity,
10051 file: None,
10052 line: None,
10053 title: title.to_owned(),
10054 detail: String::new(),
10055 }
10056 }
10057
10058 #[test]
10059 fn pr_body_names_open_findings_and_declined_ones() {
10060 let round = ReviewRound {
10061 round: 2,
10062 head: "deadbee".to_owned(),
10063 verified_head: None,
10064 verified_at: None,
10065 reviews: vec![ReviewRecord {
10066 attempts: 0,
10067 reviewer: 1,
10068 agent: "alpha".to_owned(),
10069 summary: String::new(),
10070 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
10071 vote: None,
10072 failed: None,
10073 duration_ms: 0,
10074 }],
10075 e2e: vec![CommandOutcome {
10076 command: "cargo test".to_owned(),
10077 code: Some(0),
10078 output_tail: String::new(),
10079 duration_ms: 0,
10080 resource_blocked: false,
10081 }],
10082 verify_retried: false,
10083 e2e_deferred: false,
10084 e2e_defer_reason: None,
10085 fix: Some(FixRecord {
10086 agent: "alpha".to_owned(),
10087 addressed: Vec::new(),
10088 rejected: vec![crate::verdict::Rejection {
10089 id: "R1-1-1".to_owned(),
10090 why: "not reachable from any caller".to_owned(),
10091 }],
10092 notes: String::new(),
10093 committed: true,
10094 failed: None,
10095 duration_ms: 0,
10096 continuation: None,
10097 }),
10098 blocking: 0,
10099 answered: 1,
10100 expected: 1,
10101 clean: false,
10102 progressed: true,
10103 vote_split: false,
10104 reconsideration: Vec::new(),
10105 verdict: None,
10106 };
10107 let state = state_with_round(round);
10108 let body = pr_message(&state, 'A').body;
10109
10110 assert!(body.contains("add retries"), "the task must still be there");
10111 assert!(body.contains("R2-1-1"), "{body}");
10112 assert!(body.contains("unused import"), "{body}");
10113 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
10114 assert!(
10115 body.contains("not reachable from any caller"),
10116 "the reason it was declined: {body}"
10117 );
10118 }
10119
10120 #[test]
10121 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
10122 let round = ReviewRound {
10123 round: 1,
10124 head: "deadbee".to_owned(),
10125 verified_head: None,
10126 verified_at: None,
10127 reviews: vec![ReviewRecord {
10128 attempts: 0,
10129 reviewer: 1,
10130 agent: "alpha".to_owned(),
10131 summary: String::new(),
10132 findings: Vec::new(),
10133 vote: None,
10134 failed: None,
10135 duration_ms: 0,
10136 }],
10137 e2e: Vec::new(),
10138 verify_retried: false,
10139 e2e_deferred: false,
10140 e2e_defer_reason: None,
10141 fix: None,
10142 blocking: 0,
10143 answered: 1,
10144 expected: 1,
10145 clean: true,
10146 progressed: false,
10147 vote_split: false,
10148 reconsideration: Vec::new(),
10149 verdict: None,
10150 };
10151 let state = state_with_round(round);
10152 let body = pr_message(&state, 'A').body;
10153 assert!(!body.contains("Open review findings"), "{body}");
10154 assert!(!body.contains("Declined"), "{body}");
10155 }
10156
10157 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
10158 let mut state = RunState::new(
10159 PathBuf::from("/repo"),
10160 "main".to_owned(),
10161 "abc1234".to_owned(),
10162 instruction.to_owned(),
10163 Config::default(),
10164 );
10165 state.candidates.push(Candidate {
10166 index: 0,
10167 label: 'A',
10168 agent: "alpha".to_owned(),
10169 branch: "magi/x/A".to_owned(),
10170 worktree: PathBuf::from("/wt"),
10171 summary: summary.to_owned(),
10172 stat: String::new(),
10173 files: 1,
10174 commits: 1,
10175 empty: false,
10176 failed: None,
10177 verified_noop: None,
10178 folded: false,
10179 duration_ms: 0,
10180 });
10181 state
10182 }
10183
10184 fn review_state(subjects: &[&str]) -> RunState {
10185 let mut state = state_with_summary(
10186 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
10187 "",
10188 );
10189 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
10190 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
10191 state
10192 }
10193
10194 #[test]
10195 fn pr_message_review_single_commit_uses_its_subject() {
10196 let state = review_state(&["feat(nats): per-role user"]);
10197 let m = pr_message(&state, 'A');
10198 assert_eq!(m.title, "feat(nats): per-role user");
10199 assert!(!m.body.contains("Review the work already"), "{}", m.body);
10200 assert!(m.body.contains("## Commits under review"), "{}", m.body);
10201 }
10202
10203 #[test]
10204 fn pr_message_review_multi_commit_takes_the_oldest() {
10205 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
10206 let m = pr_message(&state, 'A');
10207 assert_eq!(m.title, "feat: the change");
10208 for s in ["feat: the change", "fix: typo", "fix: again"] {
10209 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
10210 }
10211 }
10212
10213 #[test]
10214 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
10215 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
10216 let state = review_state(&[first, "fix: later fixup"]);
10217 let m = pr_message(&state, 'A');
10218 assert!(
10219 m.title.starts_with("chore: land candidate A of run"),
10220 "{}",
10221 m.title
10222 );
10223 }
10224 }
10225
10226 #[test]
10227 fn pr_message_review_bounds_a_long_english_subject() {
10228 let long = format!("feat: {}", "word ".repeat(100));
10229 let m = pr_message(&review_state(&[&long]), 'A');
10230 assert!(m.title.starts_with("feat: word"), "{}", m.title);
10231 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10232 }
10233
10234 #[test]
10235 fn pr_message_implementation_run_is_unchanged_by_review_support() {
10236 let state = state_with_summary("add retries\n\ndetails", "- did some things");
10237 let m = pr_message(&state, 'A');
10238 assert_eq!(m.title, "add retries");
10239 assert!(m.body.contains("<summary>Original task</summary>"));
10240 assert!(!m.body.contains("Commits under review"));
10241 assert_eq!(landing_subject_source(&state), state.instruction);
10242 }
10243
10244 #[test]
10245 fn review_run_squash_subject_is_the_change_not_the_prompt() {
10246 let state = review_state(&["feat: the change", "fix: typo"]);
10247 let source = landing_subject_source(&state);
10248 assert_eq!(land::merge_subject("", &source), "feat: the change");
10249 assert_eq!(
10250 land::merge_subject("magi: candidate A (uncommitted work)", &source),
10251 "feat: the change"
10252 );
10253 assert_eq!(
10255 land::merge_subject("feat: renamed by hand", &source),
10256 "feat: renamed by hand"
10257 );
10258 let blank = review_state(&["日本語"]);
10259 assert!(
10260 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
10261 );
10262 }
10263
10264 #[test]
10265 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
10266 let state = review_state(&["feat: the change"]);
10267 let source = landing_subject_source(&state);
10268 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
10269 assert_eq!(
10270 land::merge_subject(landing_title(&state, old), &source),
10271 "feat: the change"
10272 );
10273 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
10274 let task = state_with_summary("add retries", "");
10275 assert_eq!(landing_title(&task, old), old);
10276 }
10277
10278 #[test]
10279 fn pr_message_describes_the_change_not_the_task() {
10280 let state = state_with_summary(
10281 "今回やってほしいこと: results projector を直す",
10282 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
10283 );
10284 let m = pr_message(&state, 'A');
10285 assert_eq!(m.title, "fix(web): batch the runs list reads");
10286 assert!(
10287 m.body.starts_with("## Summary\n\n- reads run.json once"),
10288 "{}",
10289 m.body
10290 );
10291 assert!(!m.body.contains("TITLE:"), "{}", m.body);
10292 let task_at = m.body.find("今回やってほしいこと").unwrap();
10293 let details_at = m.body.find("<details>").unwrap();
10294 assert!(
10295 details_at < task_at,
10296 "the task lives inside <details>: {}",
10297 m.body
10298 );
10299 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
10300 assert!(m.body.contains("magi:candidate-a"));
10301 }
10302
10303 #[test]
10304 fn pr_message_falls_back_to_the_task_without_a_title_line() {
10305 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
10306 let m = pr_message(&state, 'A');
10307 assert_eq!(m.title, "add retries");
10308 assert!(
10309 m.body.contains("## Summary\n\n- did some things"),
10310 "{}",
10311 m.body
10312 );
10313
10314 let none = RunState::new(
10315 PathBuf::from("/repo"),
10316 "main".to_owned(),
10317 "abc1234".to_owned(),
10318 "add retries".to_owned(),
10319 Config::default(),
10320 );
10321 let m = pr_message(&none, 'A');
10322 assert_eq!(m.title, "add retries");
10323 assert!(!m.body.contains("## Summary"), "{}", m.body);
10324 }
10325
10326 #[test]
10327 fn pr_message_refuses_the_candidate_commit_subject() {
10328 for bad in [
10329 "TITLE: magi: candidate A (uncommitted work)",
10330 "TITLE: chore: stuff (uncommitted work)",
10331 "TITLE: ",
10332 ] {
10333 let state = state_with_summary("add retries", bad);
10334 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
10335 }
10336 }
10337
10338 #[test]
10339 fn pr_message_bounds_a_very_long_task_and_title() {
10340 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
10341 let state = state_with_summary(&long, "- nothing");
10342 let m = pr_message(&state, 'A');
10343 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10344 assert!(!m.title.contains('\n'));
10345
10346 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
10347 let m = pr_message(&state, 'A');
10348 assert!(m.title.starts_with("feat: "));
10349 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10350 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
10351 }
10352
10353 #[test]
10354 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
10355 let mut state = state_with_summary(
10359 "add retries",
10360 "TITLE: fix(web): batch reads\n- reads run.json once",
10361 );
10362 state.config.graph.language = "ja".to_owned();
10363 let m = pr_message(&state, 'A');
10364 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
10365
10366 let task = "今回やってほしいこと: results projector を直す";
10369 let mut state = state_with_summary(task, "- no title line");
10370 state.config.graph.language = "ja".to_owned();
10371 let m = pr_message(&state, 'A');
10372 assert_eq!(
10373 m.title,
10374 format!("chore: land candidate A of run {}", state.id)
10375 );
10376 assert!(
10377 m.body.contains(&format!(
10378 "<summary>Original task</summary>\n\n{task}\n\n</details>"
10379 )),
10380 "{}",
10381 m.body
10382 );
10383 }
10384
10385 #[test]
10386 fn pr_message_scrubs_home_paths_and_addresses() {
10387 let state = state_with_summary(
10388 "fix it in /Users/someone/src/x",
10389 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
10390 );
10391 let m = pr_message(&state, 'A');
10392 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
10393 assert!(!m.body.contains(leak), "{}", m.body);
10394 }
10395 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
10396 }
10397
10398 #[test]
10399 fn pr_message_survives_a_task_that_closes_details() {
10400 let state = state_with_summary("a </details> b", "TITLE: fix: x");
10401 let m = pr_message(&state, 'A');
10402 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
10403 }
10404
10405 #[test]
10406 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
10407 let cmd = manual_merge_command(
10408 MergeStyle::Squash,
10409 Path::new("/repo"),
10410 "b",
10411 "fix: \"quoted\" $(x) `y`\n\nbody",
10412 );
10413 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
10414 }
10415
10416 #[test]
10417 fn manual_merge_command_matches_the_configured_style() {
10418 let repo = Path::new("/repo");
10419 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
10420
10421 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
10422 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
10423
10424 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
10425 assert_eq!(
10426 squash,
10427 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
10428 \"Merge magi run 0832 (candidate A)\""
10429 );
10430
10431 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
10432 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
10433 }
10434
10435 #[test]
10436 fn a_nudge_gets_a_quarter_of_the_budget() {
10437 assert_eq!(retry_budget(secs(1200), true), secs(300));
10439 assert_eq!(retry_budget(secs(3600), true), secs(900));
10440 }
10441
10442 #[test]
10443 fn a_resent_prompt_keeps_the_whole_budget() {
10444 assert_eq!(retry_budget(secs(1200), false), secs(1200));
10447 assert_eq!(retry_budget(secs(60), false), secs(60));
10448 }
10449
10450 #[test]
10451 fn the_floor_never_exceeds_the_original_budget() {
10452 assert_eq!(retry_budget(secs(60), true), secs(60));
10456 assert_eq!(retry_budget(secs(480), true), secs(120));
10457 assert_eq!(retry_budget(secs(0), true), secs(0));
10458 }
10459
10460 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
10461 agent::CommandEvidence {
10462 id: "item1".to_owned(),
10463 description: "cargo test".to_owned(),
10464 exit_code,
10465 result_summary: String::new(),
10466 source: "codex".to_owned(),
10467 }
10468 }
10469
10470 #[test]
10471 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
10472 assert!(!has_unconfirmed_command(&[]));
10476 }
10477
10478 #[test]
10479 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
10480 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
10484 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
10485 assert!(!has_unconfirmed_command(&[
10486 evidence(Some(0)),
10487 evidence(Some(101))
10488 ]));
10489 }
10490
10491 #[test]
10492 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
10493 assert!(has_unconfirmed_command(&[
10494 evidence(Some(0)),
10495 evidence(None)
10496 ]));
10497 }
10498
10499 #[test]
10500 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
10501 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
10502 assert_eq!(
10503 verified_noop_claim(true, &[], text).as_deref(),
10504 Some("already fixed by b32cfc4, on main.")
10505 );
10506 }
10507
10508 #[test]
10509 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
10510 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
10513 assert!(verified_noop_claim(false, &[], text).is_none());
10514 }
10515
10516 #[test]
10517 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
10518 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
10519 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
10520 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
10522 }
10523
10524 #[test]
10525 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
10526 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
10527 }
10528
10529 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
10532 runner.state.candidates = shape
10533 .iter()
10534 .enumerate()
10535 .map(|(i, &(empty, verified))| Candidate {
10536 index: i,
10537 label: (b'A' + i as u8) as char,
10538 agent: "sonnet".to_owned(),
10539 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
10540 worktree: PathBuf::from(format!("/wt/{i}")),
10541 summary: String::new(),
10542 stat: String::new(),
10543 files: 0,
10544 commits: 0,
10545 empty,
10546 failed: None,
10547 verified_noop: verified.map(str::to_owned),
10548 duration_ms: 0,
10549 folded: false,
10550 })
10551 .collect();
10552 }
10553
10554 #[test]
10555 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
10556 ask_test_home();
10557 let mut runner = runner_at(RunStatus::Implementing);
10558 set_candidates(
10559 &mut runner,
10560 &[
10561 (true, Some("already on main at b32cfc4")),
10562 (true, Some("same fix, see the existing test")),
10563 ],
10564 );
10565
10566 runner
10567 .after_implement()
10568 .expect("a verified no-op is not an error");
10569
10570 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
10571 }
10572
10573 #[test]
10574 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
10575 ask_test_home();
10576 let mut runner = runner_at(RunStatus::Implementing);
10577 set_candidates(
10581 &mut runner,
10582 &[(true, Some("already on main at b32cfc4")), (true, None)],
10583 );
10584
10585 let err = runner
10586 .after_implement()
10587 .expect_err("an unverified empty candidate must still fail the run");
10588
10589 assert!(
10590 err.to_string().contains("no candidate produced a change"),
10591 "{err}"
10592 );
10593 assert_eq!(runner.state.status, RunStatus::Failed);
10594 }
10595
10596 #[test]
10597 fn after_implement_still_fails_an_ordinary_all_empty_run() {
10598 ask_test_home();
10599 let mut runner = runner_at(RunStatus::Implementing);
10600 set_candidates(&mut runner, &[(true, None), (true, None)]);
10601
10602 let err = runner
10603 .after_implement()
10604 .expect_err("no candidate declared anything; this is an ordinary failure");
10605
10606 assert!(
10607 err.to_string().contains("no candidate produced a change"),
10608 "{err}"
10609 );
10610 assert_eq!(runner.state.status, RunStatus::Failed);
10611 }
10612}