1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::refs;
46use crate::run::{
47 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
48 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord, Handover,
49 JobRecord, JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome,
50 OperatorFixRequest, Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState,
51 RunStatus, SeatHistory, Tally, VoteRecord, tail, write_artifact,
52};
53use crate::verdict::{
54 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
55 ReviewVote, Severity,
56};
57
58const OUTPUT_TAIL: usize = 8_000;
60
61const EVENT_OUTPUT_TAIL: usize = 2_000;
64
65const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
68
69const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
86
87pub(crate) const STAGNANT_LIMIT: usize = 2;
101
102const BASE_SYNC_ROUNDS: usize = 4;
115
116const MAX_FIX_CONTINUATIONS: usize = 2;
132
133#[derive(Clone)]
139struct SeatJob {
140 spec: AgentSpec,
141 seat: SeatState,
142 cwd: PathBuf,
143 prompt: String,
144 timeout: Duration,
145 allow_write: bool,
146 sessions: bool,
147 artifacts: PathBuf,
148 stem: String,
149 handover: Option<String>,
154}
155
156enum AgentOutcome {
168 Ok(AgentOutput),
170 Quota(AgentOutput),
172 Dropped(AgentOutput),
175 Failed(String),
177}
178
179#[derive(Debug, Clone, Default)]
206pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
207
208impl Pause {
209 #[must_use]
211 pub fn new() -> Self {
212 Self::default()
213 }
214
215 pub fn park(&self) {
217 self.0.store(true, Ordering::SeqCst);
218 }
219
220 pub fn park_because(&self, reason: impl Into<String>) {
226 let mut reason_guard = self
227 .1
228 .lock()
229 .unwrap_or_else(std::sync::PoisonError::into_inner);
230 if reason_guard.is_none() {
231 *reason_guard = Some(reason.into());
232 }
233 drop(reason_guard);
234 self.park();
235 }
236
237 #[must_use]
239 pub fn parked(&self) -> bool {
240 self.0.load(Ordering::SeqCst)
241 }
242
243 #[must_use]
245 pub fn reason(&self) -> Option<String> {
246 self.1
247 .lock()
248 .unwrap_or_else(std::sync::PoisonError::into_inner)
249 .clone()
250 }
251}
252
253pub struct Runner {
255 pub state: RunState,
257 roles: ResolvedRoles,
258 sem: Arc<Semaphore>,
259 pause: Pause,
263 interrupt: Pause,
269}
270
271async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
300 let tracking = format!("{remote}/{branch}");
301 let fetched = git::fetch(repo, remote, branch).await;
302 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
303 let local_exists = git::branch_exists(repo, branch).await?;
304 if !fresh {
305 if !local_exists {
306 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
307 }
308 tracing::warn!(
309 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
310 );
311 return Ok(());
312 }
313 let remote_sha = git::rev_parse(repo, &tracking).await?;
314 if !local_exists {
315 git::git(repo, &["branch", branch, &tracking]).await?;
316 return Ok(());
317 }
318 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
319 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
320 return Ok(());
321 }
322 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
323 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
329 .await?
330 {
331 crate::reconcile::Reconciliation::Pushed => {
332 tracing::warn!(
333 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
334 short(&local_sha),
335 short(&remote_sha)
336 );
337 return Ok(());
338 }
339 crate::reconcile::Reconciliation::Placeholder => {}
340 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
341 }
342 }
343 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
344 if !out.ok() {
345 bail!(
346 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
347 short(&local_sha),
348 short(&remote_sha),
349 out.stderr
350 );
351 }
352 tracing::warn!(
353 "local `{branch}` was stale: fast-forwarded {} -> {}",
354 short(&local_sha),
355 short(&remote_sha)
356 );
357 Ok(())
358}
359
360async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
361 let tracking = format!("{remote}/{base_branch}");
362 let fetched = git::fetch(repo, remote, base_branch).await;
363 if let Ok(out) = &fetched
364 && out.ok()
365 && git::rev_exists(repo, &tracking).await
366 {
367 return git::rev_parse(repo, &tracking).await;
368 }
369 let why = match &fetched {
370 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
371 Ok(_) => format!("{remote} has no {base_branch}"),
372 Err(e) => e.to_string(),
373 };
374 tracing::warn!(
375 "could not read {tracking} ({why}); branching off the local \
376 {base_branch} instead, which may be behind"
377 );
378 git::rev_parse(repo, base_branch).await.with_context(|| {
379 format!(
380 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
381 branch that exists"
382 )
383 })
384}
385
386struct FixClaim {
403 path: PathBuf,
404}
405
406impl FixClaim {
407 fn acquire(dir: &Path) -> Result<Self> {
408 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
409 let path = dir.join("fix.lock");
410 match Self::create(&path) {
411 Ok(claim) => Ok(claim),
412 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
413 if Self::reclaim_if_dead(&path) {
414 Self::create(&path).with_context(|| format!("lock {}", path.display()))
415 } else {
416 bail!(
417 "another `magi fix` is already running for this run ({} exists)",
418 path.display()
419 )
420 }
421 }
422 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
423 }
424 }
425
426 fn create(path: &Path) -> std::io::Result<Self> {
427 let mut f = std::fs::OpenOptions::new()
428 .write(true)
429 .create_new(true)
430 .open(path)?;
431 use std::io::Write as _;
432 writeln!(f, "{}", std::process::id())?;
434 Ok(Self {
435 path: path.to_owned(),
436 })
437 }
438
439 fn reclaim_if_dead(path: &Path) -> bool {
443 let dead = std::fs::read_to_string(path)
444 .ok()
445 .and_then(|body| body.trim().parse::<u32>().ok())
446 .is_some_and(|pid| !crate::proc::pid_alive(pid));
447 dead && std::fs::remove_file(path).is_ok()
448 }
449}
450
451impl Drop for FixClaim {
452 fn drop(&mut self) {
453 let _ = std::fs::remove_file(&self.path);
454 }
455}
456
457impl Runner {
458 pub async fn start(
460 repo: &Path,
461 instruction: String,
462 config: Config,
463 origin: Origin,
464 ) -> Result<Self> {
465 Self::start_naming(repo, instruction, "", config, origin).await
466 }
467
468 pub async fn start_naming(
472 repo: &Path,
473 instruction: String,
474 also_scan: &str,
475 config: Config,
476 origin: Origin,
477 ) -> Result<Self> {
478 let repo = git::toplevel(repo).await?;
479 let missing = agent::missing_programs(&config.agents);
480 if !missing.is_empty() {
481 bail!(
482 "these agent programs are not on PATH: {}. Fix the roster in \
483 magi.toml or install them.",
484 missing.join(", ")
485 );
486 }
487 let base_branch = match config.merge.base.clone() {
488 Some(b) => b,
489 None => git::current_branch(&repo)
490 .await?
491 .context("HEAD is detached; set [merge] base in magi.toml")?,
492 };
493 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
494 if !git::is_clean(&repo).await? {
498 tracing::warn!(
499 "{} has uncommitted changes; they are not part of this run, \
500 which branches off {base_branch} ({})",
501 repo.display(),
502 &base_commit[..base_commit.len().min(8)]
503 );
504 }
505 let roles = config.resolve_roles()?;
506 let max_parallel = config.graph.max_parallel.max(1);
507 let seeds = refs::resolve(
510 &repo,
511 &base_commit,
512 &config.merge.remote,
513 &format!("{also_scan}\n{instruction}"),
514 )
515 .await;
516 refs::plan(&repo, &seeds).await?;
517 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
518 state.origin = Some(origin);
522 for seed in &seeds {
523 state.event(
524 "seed",
525 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
526 );
527 }
528 state.seeds = seeds;
529 state.event("start", format!("run {} created", state.id));
530 state.save()?;
531 Ok(Self {
532 state,
533 roles,
534 sem: Arc::new(Semaphore::new(max_parallel)),
535 pause: Pause::new(),
536 interrupt: Pause::new(),
537 })
538 }
539
540 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
554 Self::review_taking_over(repo, branch, config, None, origin).await
555 }
556
557 pub async fn review_taking_over(
564 repo: &Path,
565 branch: &str,
566 config: Config,
567 takeover: Option<crate::handover::Takeover>,
568 origin: Origin,
569 ) -> Result<Self> {
570 let repo = git::toplevel(repo).await?;
571 let missing = agent::missing_programs(&config.agents);
572 if !missing.is_empty() {
573 bail!(
574 "these agent programs are not on PATH: {}. Fix the roster in \
575 magi.toml or install them.",
576 missing.join(", ")
577 );
578 }
579 let base_branch = match config.merge.base.clone() {
580 Some(b) => b,
581 None => git::current_branch(&repo)
582 .await?
583 .context("HEAD is detached; set [merge] base in magi.toml")?,
584 };
585 if base_branch == branch {
586 bail!("`{branch}` is the base branch; there is nothing to review against");
587 }
588 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
589
590 let roles = config.resolve_roles()?;
591 let max_parallel = config.graph.max_parallel.max(1);
592 let mut state = RunState::new(
593 repo.clone(),
594 base_branch,
595 base_commit.clone(),
596 String::new(),
597 config,
598 );
599 state.origin = Some(origin);
600
601 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
608 earlier: Vec::new(),
609 home: crate::run::home(),
610 choice: None,
611 });
612 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
613 if let Some(released) = &released {
614 state.event(
615 "release",
616 format!(
617 "took `{branch}` over from run {}: its worktree was released: {}",
618 crate::run::short_of(&released.old_id),
619 released.audit
620 ),
621 );
622 }
623 if let Some(choice) = takeover.choice.as_ref()
627 && let Err(e) =
628 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
629 .await
630 {
631 if let Some(released) = &released {
632 released.restore(&repo, branch).await;
633 }
634 return Err(e.context("applying the owner's answer about the diverged branch"));
635 }
636 let opened =
637 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
638 if opened.is_err()
639 && let Some(released) = &released
640 {
641 released.restore(&repo, branch).await;
642 }
643 opened
644 }
645
646 async fn open_review(
649 repo: &Path,
650 branch: &str,
651 mut state: RunState,
652 roles: ResolvedRoles,
653 max_parallel: usize,
654 base_commit: String,
655 ) -> Result<Self> {
656 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
657 let log = git::log_oneline(repo, &base_commit, branch)
660 .await
661 .unwrap_or_default();
662 let instruction = format!(
663 "Review the work already on branch `{branch}`. There is no task \
664 statement: what the change claims to do is whatever its commits \
665 say.\n\n{}",
666 if log.trim().is_empty() {
667 "(no commit messages)"
668 } else {
669 log.trim()
670 }
671 );
672 state.instruction = instruction;
673 state.reviewed_commits = Some(
674 git::subjects(repo, &base_commit, branch)
675 .await
676 .unwrap_or_default(),
677 );
678
679 let worktree = state.worktree_root().join("under-review");
682 if let Some(parent) = worktree.parent() {
683 tokio::fs::create_dir_all(parent).await.ok();
684 }
685 let path = worktree.to_string_lossy().to_string();
686 git::git(repo, &["worktree", "add", &path, branch])
687 .await
688 .with_context(|| {
689 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
690 })?;
691
692 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
693 .await
694 .unwrap_or(0);
695 if commits == 0 {
696 git::worktree_remove(repo, &worktree).await.ok();
697 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
698 }
699 let files = git::changed_files(&worktree, &base_commit, "HEAD")
700 .await
701 .map(|f| f.len())
702 .unwrap_or(0);
703 if files == 0
704 && let (Ok(head_tree), Ok(base_tree)) = (
705 git::tree_of(&worktree, "HEAD").await,
706 git::tree_of(&worktree, &base_commit).await,
707 )
708 && head_tree == base_tree
709 {
710 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
711 git::worktree_remove(repo, &worktree).await.ok();
712 bail!(
713 "`{branch}` at {} has a tree identical to base {}; this usually means \
714 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
715 against `{}/{branch}`) rather than an empty change",
716 short(&head),
717 short(&base_commit),
718 state.config.merge.remote
719 );
720 }
721 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
722 .await
723 .unwrap_or_default();
724
725 state.candidates.push(Candidate {
726 index: 0,
727 label: 'A',
728 agent: EXISTING_BRANCH.to_owned(),
731 branch: branch.to_owned(),
732 worktree,
733 summary: String::new(),
734 stat,
735 files,
736 commits,
737 empty: false,
738 failed: None,
739 verified_noop: None,
740 duration_ms: 0,
741 folded: false,
742 });
743 state.tally = Some(Tally {
744 first_choice: BTreeMap::from([('A', 0)]),
745 borda: BTreeMap::new(),
746 winner: 'A',
747 rankings: 0,
748 unanimous_initial: false,
749 deliberated: false,
750 changed_votes: 0,
751 unanimous_final: false,
752 tie_break: None,
753 judges: 0,
757 present: 0,
758 quorum: 0,
759 met_quorum: true,
760 uncontested: Some("review-only run: nothing competed".to_owned()),
761 });
762 state.status = RunStatus::Reviewing;
763 state.event(
764 "start",
765 format!(
766 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
767 state.id
768 ),
769 );
770 state.save()?;
771 Ok(Self {
772 state,
773 roles,
774 sem: Arc::new(Semaphore::new(max_parallel)),
775 pause: Pause::new(),
776 interrupt: Pause::new(),
777 })
778 }
779
780 pub fn resume(id: &str) -> Result<Self> {
782 let state = RunState::load(id)?;
783 if let Some(to) = &state.released_to {
784 bail!(
785 "run {} cannot be resumed: its worktree was released to run {}",
786 state.short(),
787 crate::run::short_of(to)
788 );
789 }
790 let roles = state.config.resolve_roles()?;
791 let max_parallel = state.config.graph.max_parallel.max(1);
792 Ok(Self {
793 state,
794 roles,
795 sem: Arc::new(Semaphore::new(max_parallel)),
796 pause: Pause::new(),
797 interrupt: Pause::new(),
798 })
799 }
800
801 pub async fn execute(&mut self) -> Result<()> {
808 let result = self.execute_graph().await;
809 self.mark_driver_exited();
810 let ended = if result.is_err() {
811 Some(crate::notices::run_stopped(&self.state.id, &self.state))
812 } else {
813 crate::notices::run_ended(&self.state)
814 };
815 if let Some(notice) = ended {
816 crate::notices::raise(notice);
817 }
818 result
819 }
820
821 fn mark_driver_exited(&mut self) {
830 self.state.driver_exited = true;
831 let pid = std::process::id();
832 let Ok(mut disk) = RunState::load(&self.state.id) else {
833 return;
834 };
835 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
836 return;
837 }
838 disk.driver_exited = true;
839 if let Err(e) = disk.save() {
840 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
841 }
842 }
843
844 async fn execute_graph(&mut self) -> Result<()> {
845 self.state.parked = false;
850 self.state.clear_active();
857 if let Ok(disk) = RunState::load(&self.state.id)
875 && let Some(to) = &disk.released_to
876 {
877 bail!(
878 "run {} cannot continue: its worktree was released to run {}",
879 self.state.short(),
880 crate::run::short_of(to)
881 );
882 }
883 let pid = std::process::id();
884 self.state.driver_pid = Some(pid);
885 self.state.driver_started_at = crate::proc::process_started_at(pid);
886 self.state.driver_exited = false;
887 self.state.save()?;
888 if self.state.status == RunStatus::Stalled {
901 if self.recover_stall().await? {
902 self.finish_after_tally().await?;
903 } else {
904 self.state.save()?;
906 }
907 return Ok(());
908 }
909 if self.state.status == RunStatus::Landing {
919 self.run_land().await?;
920 self.settle_questions();
925 return Ok(());
926 }
927 self.prep().await?;
928 if self.park_here()? {
929 return Ok(());
930 }
931 self.advise().await?;
932 if self.park_here()? {
933 return Ok(());
934 }
935 self.implement().await?;
936 if self.park_here()? {
937 return Ok(());
938 }
939 if self.state.status == RunStatus::VerifiedNoop {
943 return Ok(());
944 }
945 self.judge().await?;
946 if self.park_here()? {
947 return Ok(());
948 }
949 self.deliberate().await?;
950 if self.park_here()? {
951 return Ok(());
952 }
953 self.vote().await?;
954 if self.park_here()? {
955 return Ok(());
956 }
957 self.tally()?;
958 if self.state.status == RunStatus::Stalled {
963 self.state.save()?;
967 return Ok(());
968 }
969 self.finish_after_tally().await?;
970 Ok(())
971 }
972
973 fn park_here(&mut self) -> Result<bool> {
980 if !self.pause.parked() && !self.interrupt.parked() {
985 return Ok(false);
986 }
987 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
988 Some(reason) => format!(
989 "parked after `{}` ({reason}) — resume to carry on from here",
990 self.state.status.as_str()
991 ),
992 None => format!(
993 "parked after `{}` — resume to carry on from here",
994 self.state.status.as_str()
995 ),
996 };
997 self.state.event("park", why);
998 self.state.parked = true;
999 self.state.save()?;
1000 Ok(true)
1001 }
1002
1003 pub fn on_pause(&mut self, pause: Pause) {
1005 self.pause = pause;
1006 }
1007
1008 pub fn watch_interrupt(&mut self, pause: Pause) {
1014 self.interrupt = pause;
1015 }
1016
1017 fn settle_questions(&mut self) {
1037 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1038 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1039 }
1040 }
1041
1042 async fn finish_after_tally(&mut self) -> Result<()> {
1045 self.fold_losers().await?;
1046 self.sync_to_base().await?;
1051 if self.state.status == RunStatus::AlreadyInBase {
1052 return Ok(());
1053 }
1054 self.review_loop().await?;
1055 self.sync_to_base().await?;
1056 if self.state.status == RunStatus::AlreadyInBase {
1057 return Ok(());
1058 }
1059 self.gate().await?;
1060 self.merge().await?;
1061 self.state.save()?;
1062 Ok(())
1063 }
1064
1065 async fn prep(&mut self) -> Result<()> {
1068 if !self.state.candidates.is_empty() {
1069 return Ok(());
1070 }
1071 self.state.status = RunStatus::Prep;
1072 let repo = self.state.repo.clone();
1073 let base = self.state.base_commit.clone();
1074 let plan = refs::plan(&repo, &self.state.seeds).await?;
1075 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1076 let root = self.state.worktree_root();
1077 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1078
1079 let hooks_dir = self.state.dir().join("hooks");
1082 if self.state.config.blind.commit_msg_hook {
1083 std::fs::create_dir_all(&hooks_dir)
1084 .with_context(|| format!("create {}", hooks_dir.display()))?;
1085 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1086 let path = hooks_dir.join("commit-msg");
1087 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1088 make_executable(&path)?;
1089 git::acquire_worktree_config(&repo).await?;
1097 self.state.enabled_worktree_config = true;
1098 }
1099
1100 for (index, (spec, label)) in self
1101 .roles
1102 .implementers
1103 .clone()
1104 .into_iter()
1105 .zip(labels)
1106 .enumerate()
1107 {
1108 let branch = self.state.branch_for(label);
1109 let worktree = root.join(format!("cand-{label}"));
1110 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1111 if self.state.config.blind.commit_msg_hook {
1112 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1113 }
1114 git::local_exclude(&worktree, "/.magi/").await?;
1115 for pick in &plan.picks {
1116 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1117 self.state.status = RunStatus::Blocked;
1118 self.state
1119 .event("prep", format!("cannot apply referenced commit: {e}"));
1120 self.state.save()?;
1121 return Err(e);
1122 }
1123 }
1124 self.state.candidates.push(Candidate {
1125 index,
1126 label,
1127 agent: spec.id.clone(),
1128 branch,
1129 worktree,
1130 summary: String::new(),
1131 stat: String::new(),
1132 files: 0,
1133 commits: 0,
1134 empty: false,
1135 failed: None,
1136 verified_noop: None,
1137 duration_ms: 0,
1138 folded: false,
1139 });
1140 }
1141
1142 for j in 1..=self.roles.judges.len() {
1143 let wt = root.join(format!("judge-{j}"));
1144 if !wt.exists() {
1145 git::worktree_add_detached(&repo, &wt, &base).await?;
1146 }
1147 }
1148
1149 if self.state.config.graph.advise {
1157 for k in 1..=self.state.config.graph.advisors {
1158 let wt = root.join(format!("advisor-{k}"));
1159 if !wt.exists() {
1160 git::worktree_add_detached(&repo, &wt, &base).await?;
1161 }
1162 }
1163 }
1164
1165 let authors: Vec<&str> = self
1170 .roles
1171 .implementers
1172 .iter()
1173 .map(|a| a.id.as_str())
1174 .collect();
1175 let overlap: Vec<String> = self
1176 .roles
1177 .judges
1178 .iter()
1179 .enumerate()
1180 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1181 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1182 .collect();
1183 if !overlap.is_empty() {
1184 let note = format!(
1185 "{} also authored a candidate; blind, but the panel is less \
1186 independent than {} distinct agents would be",
1187 overlap.join(", "),
1188 self.roles.judges.len()
1189 );
1190 self.state.event("prep", note);
1191 }
1192
1193 self.state.event(
1194 "prep",
1195 format!(
1196 "{} candidates, {} judges, base {} ({})",
1197 self.state.candidates.len(),
1198 self.roles.judges.len(),
1199 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1200 self.state.base_branch
1201 ),
1202 );
1203 self.state.status = RunStatus::Implementing;
1204 self.state.save()?;
1205 Ok(())
1206 }
1207
1208 async fn advise(&mut self) -> Result<()> {
1241 let implement_untouched = self
1242 .state
1243 .candidates
1244 .iter()
1245 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1246 if !self.state.config.graph.advise || self.state.advise_attempted {
1247 return Ok(());
1248 }
1249 if !implement_untouched {
1250 self.state.event(
1251 "advise",
1252 "skipping the design-deliberation stage: at least one \
1253 candidate already shows implementation progress, so this \
1254 run is past the point the stage exists to run before"
1255 .to_owned(),
1256 );
1257 self.state.advise_attempted = true;
1258 self.state.save()?;
1259 return Ok(());
1260 }
1261 let run_id = self.state.id.clone();
1262 let prompts = self.state.config.prompts.clone();
1263 let instruction = self.state.instruction.clone();
1264 let language = self.state.config.graph.language.clone();
1265 let root = self.state.worktree_root();
1266 let n = self.state.config.graph.advisors;
1267 let where_recorded = self.state.dir().join("run.json");
1268
1269 let seats = match self.state.config.advisors() {
1270 Ok(seats) if !seats.is_empty() => seats,
1271 Ok(_) => {
1272 self.state.event(
1273 "advise",
1274 format!(
1275 "[graph] advisors is 0; skipping the design-deliberation \
1276 stage and continuing without a synthesis brief (see {})",
1277 where_recorded.display()
1278 ),
1279 );
1280 self.state.advise_attempted = true;
1281 self.state.save()?;
1282 return Ok(());
1283 }
1284 Err(e) => {
1285 self.state.event(
1286 "advise",
1287 format!(
1288 "could not resolve advisor seats ({e:#}); continuing \
1289 without a design-deliberation brief (see {})",
1290 where_recorded.display()
1291 ),
1292 );
1293 self.state.advise_attempted = true;
1294 self.state.save()?;
1295 return Ok(());
1296 }
1297 };
1298
1299 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1300 let artifacts = agent::artifacts_dir(&self.state.dir());
1301 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1302
1303 let mut jobs = Vec::new();
1304 for (i, spec) in seats.iter().cloned().enumerate() {
1305 let seat_key = format!("advisor-{}", i + 1);
1306 let seat = self.seat(&seat_key, &spec.id);
1307 jobs.push(SeatJob {
1308 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1309 spec,
1310 seat,
1311 cwd: worktrees[i % worktrees.len()].clone(),
1312 timeout,
1313 allow_write: false,
1314 sessions: false,
1315 artifacts: artifacts.clone(),
1316 stem: seat_key,
1317 handover: None,
1318 });
1319 }
1320
1321 self.state.event(
1322 "advise",
1323 format!(
1324 "{} advisor seat(s) sketching a design in parallel",
1325 jobs.len()
1326 ),
1327 );
1328 let mut quota_losses = Vec::new();
1329 let cache = self.state.config.cache_dir();
1330 let ctx = WaveCtx {
1331 carry_seats: false,
1332 run: &run_id,
1333 node: "advise",
1334 prompts: &prompts,
1335 cache: cache.as_deref(),
1336 round: None,
1337 };
1338 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1339 let results = ask_json_wave::<Proposal>(
1340 jobs,
1341 Arc::clone(&self.sem),
1342 self.state.config.graph.retries,
1343 &advisor_roster,
1344 &ctx,
1345 &mut quota_losses,
1346 &mut self.state,
1347 &|p: &Proposal| p.validate(),
1348 )
1349 .await;
1350 self.state.quota.extend(quota_losses);
1351
1352 let mut records = Vec::with_capacity(results.len());
1353 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1354 let agent_id = seat.agent.clone();
1355 self.state.seats.insert(seat.key.clone(), seat);
1356 match res {
1357 Ok((proposal, out)) => {
1358 self.state
1359 .event("advise", format!("advisor-{} proposed a design", i + 1));
1360 records.push(advise::AdvisorRecord::proposed(
1361 i + 1,
1362 agent_id,
1363 proposal,
1364 out.duration_ms,
1365 ));
1366 }
1367 Err(e) => {
1368 self.state.event(
1369 "advise",
1370 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1371 );
1372 records.push(advise::AdvisorRecord::failed(
1373 i + 1,
1374 agent_id,
1375 e.to_string(),
1376 ));
1377 }
1378 }
1379 }
1380
1381 let mut advice = advise::Advice {
1382 records,
1383 synthesis: None,
1384 };
1385 if advice.proposals().is_empty() {
1386 self.state.event(
1387 "advise",
1388 "no advisor produced a usable proposal; continuing without a \
1389 synthesis brief"
1390 .to_owned(),
1391 );
1392 } else {
1393 match self
1394 .synthesize_brief(
1395 &advice,
1396 &instruction,
1397 &language,
1398 &worktrees[0],
1399 &artifacts,
1400 &run_id,
1401 &prompts,
1402 cache.as_deref(),
1403 )
1404 .await
1405 {
1406 Ok(Some(text)) => {
1407 self.state.event(
1408 "advise",
1409 "synthesized a design brief for the implementer".to_owned(),
1410 );
1411 advice.synthesis = Some(text);
1412 }
1413 Ok(None) => {
1414 self.state.event(
1415 "advise",
1416 "the synthesis seat produced nothing usable; continuing \
1417 without a design brief"
1418 .to_owned(),
1419 );
1420 }
1421 Err(e) => {
1422 self.state.event(
1423 "advise",
1424 format!("could not synthesize a design brief: {e:#}"),
1425 );
1426 }
1427 }
1428 }
1429 advise::apply_reflection(&mut advice);
1430
1431 self.state.advice = Some(advice);
1432 self.state.advise_attempted = true;
1433 self.state.save()?;
1434 Ok(())
1435 }
1436
1437 #[allow(clippy::too_many_arguments)]
1449 async fn synthesize_brief(
1450 &mut self,
1451 advice: &advise::Advice,
1452 instruction: &str,
1453 language: &str,
1454 cwd: &Path,
1455 artifacts: &Path,
1456 run_id: &str,
1457 prompts: &Prompts,
1458 cache: Option<&Path>,
1459 ) -> Result<Option<String>> {
1460 let chain = agent::pick_chain(
1461 &self.state.config.agents,
1462 self.state.config.roles.synthesizer.as_ref(),
1463 &agent::installed,
1464 "synthesizer",
1465 )?;
1466 let proposals = advice.proposals();
1467 let mut prompt = prompt::with_overlay(
1468 prompt::synthesize_brief(instruction, &proposals, language),
1469 prompts.overlay("advise"),
1470 );
1471 if cache.is_some() {
1472 prompt.push('\n');
1477 prompt.push_str(&prompt::build_cache_note("advise", false));
1478 }
1479 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1480 let mut last = None;
1485 for (n, spec) in chain.iter().enumerate() {
1486 if n > 0 {
1487 self.state
1488 .event("advise", format!("synthesis falling back to {}", spec.id));
1489 }
1490 let mut seat = self.seat("advise-synthesis", &spec.id);
1491 let outcome = agent::invoke(
1492 spec,
1493 &mut seat,
1494 &Invocation {
1495 cwd,
1496 prompt: &prompt,
1497 timeout,
1498 allow_write: false,
1499 sessions: false,
1500 artifacts,
1501 stem: &if n == 0 {
1502 "advise-synthesis".to_owned()
1503 } else {
1504 format!("advise-synthesis-{}", spec.id)
1505 },
1506 run: run_id,
1507 node: "advise",
1508 cache_dir: None,
1509 attachments: &[],
1510 writable: &[],
1511 },
1512 )
1513 .await;
1514 if outcome.is_ok() {
1515 self.state.seats.insert(seat.key.clone(), seat);
1516 }
1517 let advance = agent::chain_advances(&outcome);
1518 last = Some(outcome);
1519 if !advance {
1520 break;
1521 }
1522 }
1523 let out = last.expect("a chain holds at least one agent")?;
1526 if !out.usable() {
1527 return Ok(None);
1528 }
1529 let text =
1530 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1531 Ok((!text.trim().is_empty()).then_some(text))
1532 }
1533
1534 async fn implement(&mut self) -> Result<()> {
1537 let run_id = self.state.id.clone();
1542 let prompts = self.state.config.prompts.clone();
1543 let todo: Vec<usize> = self
1544 .state
1545 .candidates
1546 .iter()
1547 .enumerate()
1548 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1549 .map(|(i, _)| i)
1550 .collect();
1551 if todo.is_empty() {
1552 return self.after_implement();
1553 }
1554 self.state.status = RunStatus::Implementing;
1555
1556 let language = self.state.config.graph.language.clone();
1557 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1558 let sessions = self.state.config.graph.sessions;
1559 let artifacts = agent::artifacts_dir(&self.state.dir());
1560 let brief = self
1564 .state
1565 .advice
1566 .as_ref()
1567 .and_then(|a| a.synthesis.as_deref())
1568 .map(str::to_owned);
1569 let attachments = self.state.attachments.clone();
1570
1571 let mut jobs = Vec::new();
1572 for &i in &todo {
1573 let (index, label, worktree) = {
1574 let c = &self.state.candidates[i];
1575 (c.index, c.label, c.worktree.clone())
1576 };
1577 let spec = self.roles.implementers[index].clone();
1578 let seat_key = format!("impl-{label}");
1579 let seat = self.seat(&seat_key, &spec.id);
1580 let instruction = seeded_instruction(&self.state);
1581 jobs.push(SeatJob {
1582 spec,
1583 seat,
1584 prompt: prompt::implement(
1585 &instruction,
1586 &worktree.to_string_lossy(),
1587 &language,
1588 brief.as_deref(),
1589 &attachments,
1590 ),
1591 cwd: worktree,
1592 timeout,
1593 allow_write: true,
1594 sessions,
1595 artifacts: artifacts.clone(),
1596 stem: format!("impl-{label}"),
1597 handover: None,
1598 });
1599 }
1600
1601 self.state.event(
1602 "implement",
1603 format!("{} candidates in parallel", jobs.len()),
1604 );
1605 let mut sent = jobs.clone();
1611 let cache = self.state.config.cache_dir();
1612 let ctx = WaveCtx {
1613 carry_seats: false,
1614 run: &run_id,
1615 node: "implement",
1616 prompts: &prompts,
1617 cache: cache.as_deref(),
1618 round: None,
1619 };
1620 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1621 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1622 .await;
1623 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1624 .await;
1625 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1626 .await;
1627
1628 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1629 let seat_key = seat.key.clone();
1630 let agent = seat.agent.clone();
1639 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1640 self.state.seats.insert(seat.key.clone(), seat);
1641 let label = self.state.candidates[i].label;
1642 let worktree = self.state.candidates[i].worktree.clone();
1643 let base = self.state.base_commit.clone();
1644
1645 let (summary, duration, failed, verified_claim) = match out {
1646 AgentOutcome::Ok(o) => {
1647 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1648 let failed = (!o.usable()).then(|| {
1649 if o.timed_out {
1650 "agent timed out".to_owned()
1651 } else {
1652 format!("agent exited with {:?}", o.exit_code)
1653 }
1654 });
1655 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1656 (text, o.duration_ms, failed, verified_claim)
1657 }
1658 AgentOutcome::Dropped(o) => {
1664 let why = o
1665 .dropped
1666 .as_ref()
1667 .map(|d| d.why.as_str())
1668 .unwrap_or("the CLI ended the stream without delivering its answer");
1669 (
1670 String::new(),
1671 o.duration_ms,
1672 Some(format!("the CLI dropped the stream ({why})")),
1673 None,
1674 )
1675 }
1676 AgentOutcome::Quota(o) => {
1677 self.state.quota.push(QuotaLoss {
1678 seat: seat_key,
1679 node: "implement".to_owned(),
1680 at: Timestamp::now(),
1681 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1682 });
1683 (
1684 String::new(),
1685 o.duration_ms,
1686 Some("rate limited (quota); produced no change".to_owned()),
1687 None,
1688 )
1689 }
1690 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1691 };
1692
1693 let rescued = match git::rescue_commit(
1696 &worktree,
1697 &format!("magi: candidate {label} (uncommitted work)"),
1698 )
1699 .await
1700 {
1701 Ok(r) => {
1702 self.state.note_withheld("implement", &r.withheld);
1703 r.committed
1704 }
1705 Err(_) => false,
1706 };
1707 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1708 .await
1709 .unwrap_or(0);
1710 let patch = git::diff(&worktree, &base, "HEAD")
1711 .await
1712 .unwrap_or_default();
1713 let stat = git::diff_stat(&worktree, &base, "HEAD")
1714 .await
1715 .unwrap_or_default();
1716 let files = git::changed_files(&worktree, &base, "HEAD")
1717 .await
1718 .map(|f| f.len())
1719 .unwrap_or(0);
1720 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1721
1722 let c = &mut self.state.candidates[i];
1723 if !exhausted_the_fallback_chain {
1724 c.agent = agent;
1725 }
1726 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1727 c.stat = stat;
1728 c.files = files;
1729 c.commits = commits;
1730 c.duration_ms = duration;
1731 c.empty = commits == 0 || patch.trim().is_empty();
1732 c.failed = match failed {
1735 Some(_) if c.empty => failed,
1736 _ => None,
1737 };
1738 c.verified_noop = if c.empty { verified_claim } else { None };
1743 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1744 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1745 (None, true, Some(_), _) => {
1746 format!("candidate {label}: no change produced (agent-verified no-op)")
1747 }
1748 (None, true, None, _) => format!("candidate {label}: no change produced"),
1749 (None, false, _, true) => {
1750 format!(
1751 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1752 )
1753 }
1754 (None, false, _, false) => {
1755 format!("candidate {label}: {files} files, {commits} commits")
1756 }
1757 };
1758 self.state.event("implement", note);
1759 self.state.save()?;
1760 }
1761
1762 self.after_implement()
1763 }
1764
1765 async fn resume_undelivered(
1793 &mut self,
1794 results: &mut [(usize, SeatState, AgentOutcome)],
1795 sent: &[SeatJob],
1796 prompts: &Prompts,
1797 run_id: &str,
1798 ) {
1799 for (wi, seat, out) in results.iter_mut() {
1800 let Some(dropped) = (match &*out {
1801 AgentOutcome::Dropped(o) => o.dropped.clone(),
1802 _ => None,
1803 }) else {
1804 continue;
1805 };
1806 let Some(job) = sent.get(*wi) else { continue };
1807 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1809 self.state.event(
1810 "implement",
1811 format!(
1812 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1813 work is in the tree",
1814 seat.key, dropped.output_tokens, dropped.why
1815 ),
1816 );
1817 continue;
1818 }
1819 if !has_context(&job.spec, seat, job.sessions) {
1827 self.state.event(
1828 "implement",
1829 format!(
1830 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1831 is no session left to resume",
1832 seat.key, dropped.output_tokens, dropped.why
1833 ),
1834 );
1835 continue;
1836 }
1837 self.state.event(
1838 "implement",
1839 format!(
1840 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1841 conversation",
1842 seat.key, dropped.output_tokens, dropped.why
1843 ),
1844 );
1845 let mut retry = job.clone();
1846 retry.seat = seat.clone();
1847 retry.prompt = prompt::resume_after_drop(&dropped.why);
1848 retry.timeout = retry_budget(job.timeout, true);
1849 retry.stem = format!("{}-resume", job.stem);
1850 let cache = self.state.config.cache_dir();
1851 let ctx = WaveCtx {
1852 carry_seats: false,
1853 run: run_id,
1854 node: "implement",
1855 prompts,
1856 cache: cache.as_deref(),
1857 round: None,
1858 };
1859 let (resumed_seat, resumed) =
1860 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1861 *seat = resumed_seat;
1862 *out = resumed;
1863 }
1864 }
1865
1866 async fn resume_seat_handovers(
1931 &mut self,
1932 results: &mut [(usize, SeatState, AgentOutcome)],
1933 sent: &mut [SeatJob],
1934 prompts: &Prompts,
1935 run_id: &str,
1936 ) {
1937 let instruction = seeded_instruction(&self.state);
1938 let language = self.state.config.graph.language.clone();
1939 let brief = self
1940 .state
1941 .advice
1942 .as_ref()
1943 .and_then(|a| a.synthesis.as_deref())
1944 .map(str::to_owned);
1945 let attachments = self.state.attachments.clone();
1946 for (wi, seat, out) in results.iter_mut() {
1947 let Some(job) = sent.get_mut(*wi) else {
1948 continue;
1949 };
1950 let start = self
1955 .roles
1956 .implementer_roster
1957 .iter()
1958 .position(|s| s.id == job.spec.id)
1959 .unwrap_or(0);
1960 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1961 let mut fallback_attempt = 0usize;
1962 let mut prev: Option<FailClass> = None;
1963 while let Some(cur) = FailClass::of(&*out) {
1964 if !should_hand_over(prev.as_ref(), &cur) {
1965 break;
1966 }
1967 let Some(next) =
1968 next_untried_in_roster(&self.roles.implementer_roster, start, &tried).cloned()
1969 else {
1970 break;
1971 };
1972 tried.insert(next.id.clone());
1973 fallback_attempt += 1;
1974
1975 if let Ok(r) = git::rescue_commit(
1976 &job.cwd,
1977 &format!(
1978 "magi: candidate {} (uncommitted work before {} fallback)",
1979 seat.key,
1980 if cur == FailClass::Quota {
1981 "quota"
1982 } else {
1983 "handover"
1984 }
1985 ),
1986 )
1987 .await
1988 {
1989 self.state.note_withheld("implement", &r.withheld);
1990 }
1991
1992 record_handover(
1993 &mut self.state,
1994 "implement",
1995 &seat.key,
1996 &seat.agent,
1997 &next.id,
1998 &cur,
1999 &fail_reason(&*out),
2000 );
2001 prev = Some(cur.clone());
2002
2003 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2004 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2005 job.spec = next.clone();
2013 let mut retry = job.clone();
2014 retry.seat = new_seat;
2015 retry.prompt = prompt::implement(
2016 &instruction,
2017 &job.cwd.to_string_lossy(),
2018 &language,
2019 brief.as_deref(),
2020 &attachments,
2021 );
2022 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2023 let cache = self.state.config.cache_dir();
2024 let ctx = WaveCtx {
2025 carry_seats: false,
2026 run: run_id,
2027 node: "implement",
2028 prompts,
2029 cache: cache.as_deref(),
2030 round: None,
2031 };
2032 let (fallback_seat, fallback_out) = run_one(
2033 retry,
2034 Arc::clone(&self.sem),
2035 &ctx,
2036 &mut self.state,
2037 fallback_attempt,
2038 )
2039 .await;
2040 *seat = fallback_seat;
2041 *out = fallback_out;
2042 }
2043 }
2044 }
2045
2046 async fn resume_unconfirmed_commands(
2070 &mut self,
2071 results: &mut [(usize, SeatState, AgentOutcome)],
2072 sent: &[SeatJob],
2073 prompts: &Prompts,
2074 run_id: &str,
2075 ) {
2076 for (wi, seat, out) in results.iter_mut() {
2077 let AgentOutcome::Ok(o) = &*out else {
2078 continue;
2079 };
2080 if !has_unconfirmed_command(&o.commands) {
2081 continue;
2082 }
2083 let Some(job) = sent.get(*wi) else { continue };
2084 if !has_context(&job.spec, seat, job.sessions) {
2085 self.state.event(
2086 "implement",
2087 format!(
2088 "{}: the reply named a command whose own CLI never confirmed the exit \
2089 status of, but there is no session left to resume",
2090 seat.key
2091 ),
2092 );
2093 continue;
2094 }
2095 self.state.event(
2096 "implement",
2097 format!(
2098 "{}: the reply named a command whose own CLI never confirmed the exit \
2099 status of; resuming the conversation",
2100 seat.key
2101 ),
2102 );
2103 let mut retry = job.clone();
2104 retry.seat = seat.clone();
2105 retry.prompt = prompt::resume_incomplete(
2106 "a command in your last reply had no confirmed exit status",
2107 );
2108 retry.timeout = retry_budget(job.timeout, true);
2109 retry.stem = format!("{}-confirm", job.stem);
2110 let cache = self.state.config.cache_dir();
2111 let ctx = WaveCtx {
2112 carry_seats: false,
2113 run: run_id,
2114 node: "implement",
2115 prompts,
2116 cache: cache.as_deref(),
2117 round: None,
2118 };
2119 let (resumed_seat, resumed) =
2120 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2121 *seat = resumed_seat;
2122 *out = resumed;
2123 }
2124 }
2125
2126 async fn continue_fix_report(
2147 &mut self,
2148 mut seat: SeatState,
2149 parse_err: String,
2150 job: &SeatJob,
2151 prompts: &Prompts,
2152 run_id: &str,
2153 round: usize,
2154 ) -> (
2155 SeatState,
2156 Option<FixReport>,
2157 Option<String>,
2158 ContinuationRecord,
2159 ) {
2160 let mut last_err = parse_err;
2161 let mut cumulative_wait_ms = 0u64;
2162 let mut attempts = 0usize;
2163 loop {
2164 if !has_context(&job.spec, &seat, job.sessions) {
2165 self.state.event(
2166 "fix",
2167 format!(
2168 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2169 session left to resume into"
2170 ),
2171 );
2172 let outcome = if attempts == 0 {
2173 ContinuationOutcome::NoSession
2174 } else {
2175 ContinuationOutcome::Exhausted
2176 };
2177 return (
2178 seat,
2179 None,
2180 Some(format!("unparsable fix report: {last_err}")),
2181 ContinuationRecord {
2182 attempts,
2183 cumulative_wait_ms,
2184 outcome,
2185 },
2186 );
2187 }
2188 if attempts >= MAX_FIX_CONTINUATIONS {
2189 self.state.event(
2190 "fix",
2191 format!(
2192 "round {round}: fixer's reply still had no adoption report after \
2193 {attempts} continuation(s) ({last_err}); giving up"
2194 ),
2195 );
2196 return (
2197 seat,
2198 None,
2199 Some(format!(
2200 "unparsable fix report after {attempts} continuation(s): {last_err}"
2201 )),
2202 ContinuationRecord {
2203 attempts,
2204 cumulative_wait_ms,
2205 outcome: ContinuationOutcome::Exhausted,
2206 },
2207 );
2208 }
2209 attempts += 1;
2210 self.state.event(
2211 "fix",
2212 format!(
2213 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2214 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2215 ),
2216 );
2217 let mut retry = job.clone();
2218 retry.seat = seat.clone();
2219 retry.prompt = prompt::resume_incomplete(&last_err);
2220 retry.timeout = retry_budget(job.timeout, true);
2221 retry.stem = format!("{}-continue{attempts}", job.stem);
2222 let cache = self.state.config.cache_dir();
2223 let ctx = WaveCtx {
2224 carry_seats: false,
2225 run: run_id,
2226 node: "fix",
2227 prompts,
2228 cache: cache.as_deref(),
2229 round: Some(round),
2230 };
2231 let (resumed_seat, resumed_out) = run_one(
2232 retry,
2233 Arc::clone(&self.sem),
2234 &ctx,
2235 &mut self.state,
2236 attempts,
2237 )
2238 .await;
2239 seat = resumed_seat;
2240 match resumed_out {
2241 AgentOutcome::Ok(o) => {
2242 cumulative_wait_ms += o.duration_ms;
2243 match verdict::extract_json::<FixReport>(&o.text) {
2244 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2245 self.state.event(
2246 "fix",
2247 format!(
2248 "round {round}: fixer's adoption report recovered after \
2249 {attempts} continuation(s)"
2250 ),
2251 );
2252 return (
2253 seat,
2254 Some(report),
2255 None,
2256 ContinuationRecord {
2257 attempts,
2258 cumulative_wait_ms,
2259 outcome: ContinuationOutcome::Resumed,
2260 },
2261 );
2262 }
2263 Ok(_) => {
2271 last_err = "the reply parsed, but it reported a command whose own CLI \
2272 never confirmed an exit status"
2273 .to_owned();
2274 }
2275 Err(e) => last_err = e.to_string(),
2276 }
2277 }
2278 AgentOutcome::Quota(o) => {
2279 cumulative_wait_ms += o.duration_ms;
2280 self.state.quota.push(QuotaLoss {
2281 seat: seat.key.clone(),
2282 node: "fix".to_owned(),
2283 at: Timestamp::now(),
2284 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2285 });
2286 self.state.event(
2287 "fix",
2288 format!(
2289 "round {round}: continuation rate limited (quota); not retrying now"
2290 ),
2291 );
2292 return (
2293 seat,
2294 None,
2295 Some("rate limited (quota) while recovering the fix report".to_owned()),
2296 ContinuationRecord {
2297 attempts,
2298 cumulative_wait_ms,
2299 outcome: ContinuationOutcome::QuotaLost,
2300 },
2301 );
2302 }
2303 AgentOutcome::Dropped(o) => {
2304 cumulative_wait_ms += o.duration_ms;
2305 let why = o
2306 .dropped
2307 .as_ref()
2308 .map(|d| d.why.as_str())
2309 .unwrap_or("the CLI ended the stream without delivering its answer");
2310 last_err = format!("the CLI dropped the stream ({why})");
2311 }
2312 AgentOutcome::Failed(e) => last_err = e,
2313 }
2314 }
2315 }
2316
2317 fn after_implement(&mut self) -> Result<()> {
2318 if self.state.leaks.is_empty() {
2320 let cfg = self.state.config.blind.clone();
2321 let mut leaks = Vec::new();
2322 for c in &self.state.candidates {
2323 let Some(patch) =
2324 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2325 else {
2326 continue;
2327 };
2328 leaks.extend(blind::scan(
2329 &format!("candidate {} patch", c.label),
2330 &patch,
2331 &cfg.vendor_tokens,
2332 ));
2333 }
2334 if !leaks.is_empty() {
2335 let summary = leaks
2336 .iter()
2337 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2338 .collect::<Vec<_>>()
2339 .join(", ");
2340 match cfg.on_leak {
2341 LeakPolicy::Fail => {
2342 self.state.status = RunStatus::Failed;
2343 self.state
2344 .event("blind", format!("vendor text in a patch: {summary}"));
2345 self.state.leaks = leaks;
2346 self.state.save()?;
2347 self.settle_questions();
2348 bail!(
2349 "blind.on_leak = \"fail\" and vendor text reached a \
2350 judged patch: {summary}"
2351 );
2352 }
2353 LeakPolicy::Redact => self.state.event(
2354 "blind",
2355 format!("redacting vendor text for judging: {summary}"),
2356 ),
2357 LeakPolicy::Warn => self.state.event(
2358 "blind",
2359 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2360 ),
2361 }
2362 self.state.leaks = leaks;
2363 }
2364 }
2365
2366 if self.state.viable().is_empty() {
2367 if self.state.all_candidates_verified_noop() {
2368 self.state.status = RunStatus::VerifiedNoop;
2379 self.state.save()?;
2380 self.settle_questions();
2381 return Ok(());
2382 }
2383 self.state.status = RunStatus::Failed;
2384 self.state.save()?;
2385 self.settle_questions();
2386 bail!("no candidate produced a change; nothing to judge");
2387 }
2388 self.state.status = RunStatus::Judging;
2389 self.state.save()?;
2390 Ok(())
2391 }
2392
2393 async fn judge(&mut self) -> Result<()> {
2396 let run_id = self.state.id.clone();
2401 let prompts = self.state.config.prompts.clone();
2402 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2403 return Ok(());
2404 }
2405 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2406 if viable.len() == 1 {
2407 self.state.judge_skipped = true;
2414 self.state.event(
2415 "judge",
2416 format!(
2417 "only candidate {} produced a change; judging skipped",
2418 viable[0].label
2419 ),
2420 );
2421 self.state.save()?;
2422 return Ok(());
2423 }
2424 self.state.status = RunStatus::Judging;
2425
2426 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2427 let language = self.state.config.graph.language.clone();
2428 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2429 let sessions = self.state.config.graph.sessions;
2430 let artifacts = agent::artifacts_dir(&self.state.dir());
2431 let root = self.state.worktree_root();
2432 let base_short = short(&self.state.base_commit);
2433
2434 let mut jobs = Vec::new();
2435 let mut orders = Vec::new();
2436 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2437 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2438 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2439 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2440 let seat_key = format!("judge-{}", j + 1);
2441 let seat = self.seat(&seat_key, &spec.id);
2442 jobs.push(SeatJob {
2443 prompt: prompt::judge(
2444 &self.state.instruction,
2445 &views,
2446 self.roles.judges.len(),
2447 &base_short,
2448 &language,
2449 ),
2450 spec,
2451 seat,
2452 cwd: root.join(format!("judge-{}", j + 1)),
2453 timeout,
2454 allow_write: false,
2455 sessions,
2456 artifacts: artifacts.clone(),
2457 stem: format!("judge-{}", j + 1),
2458 handover: None,
2459 });
2460 }
2461
2462 self.state.event(
2463 "judge",
2464 format!(
2465 "{} judges ranking {} candidates blind",
2466 jobs.len(),
2467 viable.len()
2468 ),
2469 );
2470 let labels_for_check = labels.clone();
2471 let mut quota_losses = Vec::new();
2472 let cache = self.state.config.cache_dir();
2473 let ctx = WaveCtx {
2474 carry_seats: false,
2475 run: &run_id,
2476 node: "judge",
2477 prompts: &prompts,
2478 cache: cache.as_deref(),
2479 round: None,
2480 };
2481 let results = ask_json_wave::<Ranking>(
2482 jobs,
2483 Arc::clone(&self.sem),
2484 self.state.config.graph.retries,
2485 &self.roles.judge_roster,
2486 &ctx,
2487 &mut quota_losses,
2488 &mut self.state,
2489 &move |r: &Ranking| r.validate(&labels_for_check),
2490 )
2491 .await;
2492 self.state.quota.extend(quota_losses);
2493
2494 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2495 let agent_id = seat.agent.clone();
2496 self.state.seats.insert(seat.key.clone(), seat);
2497 let mut record = Judgement {
2498 judge: j + 1,
2499 seat: format!("judge-{}", j + 1),
2500 agent: agent_id,
2501 ranking: Vec::new(),
2502 reasons: BTreeMap::new(),
2503 confidence: None,
2504 order: orders[j].clone(),
2505 failed: None,
2506 duration_ms: 0,
2507 };
2508 match res {
2509 Ok((ranking, out)) => {
2510 record.ranking = ranking.normalized();
2511 record.reasons = ranking.reasons;
2512 record.confidence = ranking.confidence;
2513 record.duration_ms = out.duration_ms;
2514 self.state.event(
2515 "judge",
2516 format!(
2517 "judge {} ranked {}",
2518 j + 1,
2519 record.ranking.iter().collect::<String>()
2520 ),
2521 );
2522 }
2523 Err(e) => {
2524 record.failed = Some(e.to_string());
2525 self.state
2526 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2527 }
2528 }
2529 self.state.judgements.push(record);
2530 self.state.save()?;
2531 }
2532 Ok(())
2533 }
2534
2535 async fn deliberate(&mut self) -> Result<()> {
2538 let run_id = self.state.id.clone();
2543 let prompts = self.state.config.prompts.clone();
2544 if !self.state.deliberation.is_empty() {
2545 return Ok(());
2546 }
2547 let tops: Vec<char> = self
2548 .state
2549 .judgements
2550 .iter()
2551 .filter_map(|j| j.ranking.first().copied())
2552 .collect();
2553 let rounds = self.state.config.graph.deliberate_rounds;
2554 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2555 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2556 self.state.event(
2557 "deliberate",
2558 format!("judges agreed on {} outright; no deliberation", tops[0]),
2559 );
2560 }
2561 self.state.status = RunStatus::Voting;
2562 self.state.save()?;
2563 return Ok(());
2564 }
2565
2566 self.state.status = RunStatus::Deliberating;
2567 self.state.event(
2568 "deliberate",
2569 format!(
2570 "split: first choices were {} — opening {rounds} round(s)",
2571 tops.iter().collect::<String>()
2572 ),
2573 );
2574
2575 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2576 let language = self.state.config.graph.language.clone();
2577 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2578 let sessions = self.state.config.graph.sessions;
2579 let artifacts = agent::artifacts_dir(&self.state.dir());
2580 let root = self.state.worktree_root();
2581 let base_short = short(&self.state.base_commit);
2582
2583 for round in 1..=rounds {
2587 let mut turns: Vec<DeliberationTurn> = Vec::new();
2588 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2589 if self.state.judgements[j].failed.is_some() {
2590 continue;
2591 }
2592 let seat_key = format!("judge-{}", j + 1);
2593 let spec = self.occupant(&seat_key, spec);
2594 let mut seat = self.seat(&seat_key, &spec.id);
2595 let transcript = self.transcript(&turns, j);
2596 let build = |context: Option<&str>| {
2597 prompt::deliberate(
2598 &self.state.instruction,
2599 context,
2600 &transcript,
2601 round,
2602 rounds,
2603 &language,
2604 )
2605 };
2606 let block = self.candidate_block(&viable, &base_short);
2607 let full = build(Some(&block));
2608 let text = if has_context(&spec, &seat, sessions) {
2609 build(None)
2610 } else {
2611 full.clone()
2612 };
2613 let job = SeatJob {
2614 spec,
2615 seat: seat.clone(),
2616 prompt: text,
2617 cwd: root.join(format!("judge-{}", j + 1)),
2618 timeout,
2619 allow_write: false,
2620 sessions,
2621 artifacts: artifacts.clone(),
2622 stem: format!("delib-{round}-judge-{}", j + 1),
2623 handover: Some(full),
2624 };
2625 let cache = self.state.config.cache_dir();
2626 let ctx = WaveCtx {
2627 carry_seats: false,
2628 run: &run_id,
2629 node: "deliberate",
2630 prompts: &prompts,
2631 cache: cache.as_deref(),
2632 round: None,
2633 };
2634 let mut losses = Vec::new();
2638 let mut results = ask_wave_with::<String>(
2639 vec![job],
2640 Arc::clone(&self.sem),
2641 0,
2642 &self.roles.judge_roster,
2643 &ctx,
2644 &mut losses,
2645 &mut self.state,
2646 &|text: &str| {
2647 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
2648 },
2649 )
2650 .await;
2651 self.state.quota.extend(losses);
2652 let (updated, res, _) = results.pop().expect("one job in, one result out");
2653 seat = updated;
2654 let agent_id = seat.agent.clone();
2655 self.state.seats.insert(seat.key.clone(), seat);
2656 let body = match res {
2657 Ok((body, _)) => body,
2658 Err(e) => {
2661 self.state
2662 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2663 continue;
2664 }
2665 };
2666 let tentative = verdict::extract_json::<Position>(&body)
2667 .ok()
2668 .and_then(|p| p.tentative)
2669 .and_then(|s| s.trim().chars().next())
2670 .map(|c| c.to_ascii_uppercase());
2671 self.state.event(
2672 "deliberate",
2673 format!(
2674 "round {round}: judge {} now favours {}",
2675 j + 1,
2676 tentative.map_or("—".to_owned(), |c| c.to_string())
2677 ),
2678 );
2679 turns.push(DeliberationTurn {
2680 judge: j + 1,
2681 agent: agent_id,
2682 body: blind::sanitize_prose(&body, &self.state.config.blind),
2683 tentative,
2684 });
2685 }
2686 self.state
2687 .deliberation
2688 .push(DeliberationRound { round, turns });
2689 self.state.save()?;
2690 }
2691
2692 self.state.status = RunStatus::Voting;
2693 self.state.save()?;
2694 Ok(())
2695 }
2696
2697 async fn vote(&mut self) -> Result<()> {
2700 let run_id = self.state.id.clone();
2705 let prompts = self.state.config.prompts.clone();
2706 if !self.state.votes.is_empty() {
2707 return Ok(());
2708 }
2709 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2710 if viable.len() == 1 {
2711 return Ok(());
2712 }
2713 self.state.status = RunStatus::Voting;
2714
2715 let language = self.state.config.graph.language.clone();
2716 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2717 let sessions = self.state.config.graph.sessions;
2718 let artifacts = agent::artifacts_dir(&self.state.dir());
2719 let root = self.state.worktree_root();
2720 let base_short = short(&self.state.base_commit);
2721 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2722
2723 let mut jobs = Vec::new();
2724 let mut seats_at = Vec::new();
2725 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2726 if self
2727 .state
2728 .judgements
2729 .get(j)
2730 .is_some_and(|r| r.failed.is_some())
2731 {
2732 continue;
2733 }
2734 let seat_key = format!("judge-{}", j + 1);
2735 let spec = self.occupant(&seat_key, spec);
2736 let seat = self.seat(&seat_key, &spec.id);
2737 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
2738 let text = if has_context(&spec, &seat, sessions) {
2739 prompt::final_vote(&viable, &language)
2740 } else {
2741 full.clone()
2742 };
2743 jobs.push(SeatJob {
2744 spec,
2745 seat,
2746 prompt: text,
2747 cwd: root.join(format!("judge-{}", j + 1)),
2748 timeout,
2749 allow_write: false,
2750 sessions,
2751 artifacts: artifacts.clone(),
2752 stem: format!("vote-judge-{}", j + 1),
2753 handover: Some(full),
2754 });
2755 seats_at.push(j);
2756 }
2757
2758 self.state.event(
2759 "vote",
2760 format!(
2761 "collecting {} final votes one by one, privately",
2762 jobs.len()
2763 ),
2764 );
2765 let allowed = viable.clone();
2766 let mut quota_losses = Vec::new();
2767 let cache = self.state.config.cache_dir();
2768 let ctx = WaveCtx {
2769 carry_seats: false,
2770 run: &run_id,
2771 node: "vote",
2772 prompts: &prompts,
2773 cache: cache.as_deref(),
2774 round: None,
2775 };
2776 let results = ask_json_wave::<FinalVote>(
2777 jobs,
2778 Arc::clone(&self.sem),
2779 self.state.config.graph.retries,
2780 &self.roles.judge_roster,
2781 &ctx,
2782 &mut quota_losses,
2783 &mut self.state,
2784 &move |v: &FinalVote| match v.label() {
2785 Some(c) if allowed.contains(&c) => Ok(()),
2786 other => bail!("vote {other:?} is not one of {allowed:?}"),
2787 },
2788 )
2789 .await;
2790 self.state.quota.extend(quota_losses);
2791
2792 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2793 let agent_id = seat.agent.clone();
2794 self.state.seats.insert(seat.key.clone(), seat);
2795 let initial = self
2796 .state
2797 .judgements
2798 .get(j)
2799 .and_then(|r| r.ranking.first().copied());
2800 let mut record = VoteRecord {
2801 judge: j + 1,
2802 agent: agent_id,
2803 vote: None,
2804 reason: String::new(),
2805 changed: false,
2806 };
2807 match res {
2808 Ok((v, _)) => {
2809 record.vote = v.label();
2810 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2811 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2812 self.state.event(
2813 "vote",
2814 format!(
2815 "judge {} voted {}{}",
2816 j + 1,
2817 record.vote.unwrap_or('?'),
2818 if record.changed { " (changed)" } else { "" }
2819 ),
2820 );
2821 }
2822 Err(e) => {
2823 self.state
2824 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2825 }
2826 }
2827 self.state.votes.push(record);
2828 self.state.save()?;
2829 }
2830 Ok(())
2831 }
2832
2833 fn tally(&mut self) -> Result<()> {
2836 if self.state.tally.is_some() {
2837 return Ok(());
2838 }
2839 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2840 let tops: Vec<char> = self
2841 .state
2842 .judgements
2843 .iter()
2844 .filter_map(|j| j.ranking.first().copied())
2845 .collect();
2846 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2847
2848 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2851 let mut cast: Vec<char> = Vec::new();
2852 for (i, j) in self.state.judgements.iter().enumerate() {
2853 let vote = self
2854 .state
2855 .votes
2856 .iter()
2857 .find(|v| v.judge == i + 1)
2858 .and_then(|v| v.vote)
2859 .or_else(|| j.ranking.first().copied());
2860 if let Some(v) = vote {
2861 *first_choice.entry(v).or_insert(0) += 1;
2862 cast.push(v);
2863 }
2864 }
2865
2866 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2867 for j in &self.state.judgements {
2868 let n = j.ranking.len();
2869 for (pos, label) in j.ranking.iter().enumerate() {
2870 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2871 }
2872 }
2873
2874 let best = first_choice.values().copied().max().unwrap_or(0);
2875 let mut leaders: Vec<char> = first_choice
2876 .iter()
2877 .filter(|(_, v)| **v == best)
2878 .map(|(k, _)| *k)
2879 .collect();
2880 let mut tie_break = None;
2881 if leaders.len() > 1 {
2882 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2883 let borda_leaders: Vec<char> = leaders
2884 .iter()
2885 .copied()
2886 .filter(|l| borda[l] == top_borda)
2887 .collect();
2888 tie_break = Some(if borda_leaders.len() == 1 {
2889 format!(
2890 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2891 leaders.len()
2892 )
2893 } else {
2894 format!(
2895 "{} way tie on both first-choice votes and Borda points, broken by label order",
2896 leaders.len()
2897 )
2898 });
2899 leaders = borda_leaders;
2900 leaders.sort_unstable();
2901 }
2902 let winner = *leaders
2903 .first()
2904 .or(viable.first())
2905 .context("no candidate to declare a winner from")?;
2906
2907 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2908 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2909 let deliberated = !self.state.deliberation.is_empty();
2910
2911 let quota_seats: std::collections::BTreeSet<&str> =
2915 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2916 let mut present = 0usize;
2917 for (i, j) in self.state.judgements.iter().enumerate() {
2918 if quota_seats.contains(j.seat.as_str()) {
2919 continue;
2920 }
2921 let ranked = !j.ranking.is_empty() && j.failed.is_none();
2922 let voted = self
2923 .state
2924 .votes
2925 .iter()
2926 .any(|v| v.judge == i + 1 && v.vote.is_some());
2927 if ranked || voted {
2928 present += 1;
2929 }
2930 }
2931 let needs_quorum = viable.len() > 1;
2937 let judges_total = if needs_quorum {
2938 self.roles.judges.len()
2939 } else {
2940 0
2941 };
2942 let quorum = if needs_quorum {
2943 judges_total / 2 + 1
2944 } else {
2945 0
2946 };
2947 let met_quorum = !needs_quorum || present >= quorum;
2948 let uncontested = (!needs_quorum).then(|| {
2949 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2950 });
2951
2952 self.state.event(
2953 "tally",
2954 match &uncontested {
2955 Some(reason) => format!("winner {winner} — {reason}"),
2956 None => format!(
2957 "winner {winner} — votes {} | initial {} | {} changed | \
2958 {present}/{judges_total} judges{}",
2959 first_choice
2960 .iter()
2961 .map(|(k, v)| format!("{k}:{v}"))
2962 .collect::<Vec<_>>()
2963 .join(" "),
2964 if unanimous_initial {
2965 "unanimous"
2966 } else {
2967 "split"
2968 },
2969 changed_votes,
2970 if met_quorum {
2971 String::new()
2972 } else {
2973 format!(" — below quorum ({quorum} required)")
2974 },
2975 ),
2976 },
2977 );
2978 if !met_quorum {
2979 self.state.event(
2980 "stall",
2981 format!(
2982 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2983 the run stops here, resumable"
2984 ),
2985 );
2986 }
2987 self.state.tally = Some(Tally {
2988 first_choice,
2989 borda,
2990 winner,
2991 rankings: tops.len(),
2992 unanimous_initial,
2993 deliberated,
2994 changed_votes,
2995 unanimous_final,
2996 tie_break,
2997 judges: judges_total,
2998 present,
2999 quorum,
3000 met_quorum,
3001 uncontested,
3002 });
3003 self.state.status = if met_quorum {
3004 RunStatus::Reviewing
3005 } else {
3006 RunStatus::Stalled
3007 };
3008 self.state.save()?;
3009 Ok(())
3010 }
3011
3012 #[allow(clippy::too_many_lines)]
3033 async fn recover_stall(&mut self) -> Result<bool> {
3034 let run_id = self.state.id.clone();
3039 let prompts = self.state.config.prompts.clone();
3040 let quota_seats: BTreeSet<&str> =
3045 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3046 let absent: Vec<String> = self
3047 .state
3048 .judgements
3049 .iter()
3050 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3051 .map(|j| j.seat.clone())
3052 .collect();
3053 if absent.is_empty() {
3054 return Ok(false);
3055 }
3056 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3057 if viable.len() <= 1 {
3058 return Ok(false);
3059 }
3060 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3061 let language = self.state.config.graph.language.clone();
3062 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3063 let sessions = self.state.config.graph.sessions;
3064 let artifacts = agent::artifacts_dir(&self.state.dir());
3065 let root = self.state.worktree_root();
3066 let base_short = short(&self.state.base_commit);
3067 let candidates: Vec<Candidate> = viable.clone();
3068
3069 let mut positions: Vec<usize> = absent
3071 .iter()
3072 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3073 .collect();
3074 if positions.is_empty() {
3075 return Ok(false);
3076 }
3077 positions.sort_unstable();
3078 positions.dedup();
3079
3080 let mut judge_jobs = Vec::new();
3082 for &j in &positions {
3083 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3084 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3085 let seat_key = format!("judge-{}", j + 1);
3086 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3087 let seat = self.seat(&seat_key, &spec.id);
3088 judge_jobs.push(SeatJob {
3089 spec,
3090 seat,
3091 prompt: prompt::judge(
3092 &self.state.instruction,
3093 &views,
3094 self.roles.judges.len(),
3095 &base_short,
3096 &language,
3097 ),
3098 cwd: root.join(seat_key),
3099 timeout,
3100 allow_write: false,
3101 sessions,
3102 artifacts: artifacts.clone(),
3103 stem: format!("judge-{}-recover", j + 1),
3104 handover: None,
3105 });
3106 }
3107
3108 let labels_for_check = labels.clone();
3109 let mut judge_losses = Vec::new();
3110 let retries = self.state.config.graph.retries;
3111 let cache = self.state.config.cache_dir();
3112 let ctx = WaveCtx {
3113 carry_seats: false,
3114 run: &run_id,
3115 node: "judge",
3116 prompts: &prompts,
3117 cache: cache.as_deref(),
3118 round: None,
3119 };
3120 let results = ask_json_wave::<Ranking>(
3121 judge_jobs,
3122 Arc::clone(&self.sem),
3123 retries,
3124 &self.roles.judge_roster,
3125 &ctx,
3126 &mut judge_losses,
3127 &mut self.state,
3128 &move |r: &Ranking| r.validate(&labels_for_check),
3129 )
3130 .await;
3131
3132 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3134 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3135 let agent_id = seat.agent.clone();
3136 self.state.seats.insert(seat.key.clone(), seat);
3137 let record = &mut self.state.judgements[j];
3138 match res {
3139 Ok((ranking, out)) => {
3140 record.agent = agent_id;
3141 record.ranking = ranking.normalized();
3142 record.reasons = ranking.reasons;
3143 record.confidence = ranking.confidence;
3144 record.failed = None;
3145 record.duration_ms = out.duration_ms;
3146 recovered.insert(j);
3147 self.state.event(
3148 "recover",
3149 format!("judge {} ranked again after the limit", j + 1),
3150 );
3151 }
3152 Err(e) => {
3153 self.state
3154 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3155 }
3156 }
3157 }
3158
3159 let mut vote_jobs = Vec::new();
3161 let mut vote_pos: Vec<usize> = Vec::new();
3162 for &j in &recovered {
3163 let seat_key = format!("judge-{}", j + 1);
3164 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3165 let seat = self.seat(&seat_key, &spec.id);
3166 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3167 let text = if has_context(&spec, &seat, sessions) {
3168 prompt::final_vote(&labels, &language)
3169 } else {
3170 full.clone()
3171 };
3172 vote_jobs.push(SeatJob {
3173 spec,
3174 seat,
3175 prompt: text,
3176 cwd: root.join(seat_key),
3177 timeout,
3178 allow_write: false,
3179 sessions,
3180 artifacts: artifacts.clone(),
3181 stem: format!("vote-judge-{}-recover", j + 1),
3182 handover: Some(full),
3183 });
3184 vote_pos.push(j);
3185 }
3186 let allowed = labels.clone();
3187 let mut vote_losses = Vec::new();
3188 let vote_retries = self.state.config.graph.retries;
3189 let vote_cache = self.state.config.cache_dir();
3190 let ctx = WaveCtx {
3191 carry_seats: false,
3192 run: &run_id,
3193 node: "vote",
3194 prompts: &prompts,
3195 cache: vote_cache.as_deref(),
3196 round: None,
3197 };
3198 let votes = ask_json_wave::<FinalVote>(
3199 vote_jobs,
3200 Arc::clone(&self.sem),
3201 vote_retries,
3202 &self.roles.judge_roster,
3203 &ctx,
3204 &mut vote_losses,
3205 &mut self.state,
3206 &move |v: &FinalVote| match v.label() {
3207 Some(c) if allowed.contains(&c) => Ok(()),
3208 other => bail!("vote {other:?} is not one of {allowed:?}"),
3209 },
3210 )
3211 .await;
3212 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3213 let agent_id = seat.agent.clone();
3214 self.state.seats.insert(seat.key.clone(), seat);
3215 match res {
3216 Ok((v, _)) => {
3217 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3218 rec.vote = v.label();
3219 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3220 } else {
3221 self.state.votes.push(VoteRecord {
3222 judge: j + 1,
3223 agent: agent_id,
3224 vote: v.label(),
3225 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3226 changed: false,
3227 });
3228 }
3229 self.state.event(
3230 "recover",
3231 format!("judge {} voted again after the limit", j + 1),
3232 );
3233 }
3234 Err(e) => {
3235 self.state
3236 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3237 }
3238 }
3239 }
3240
3241 let recovered_keys: BTreeSet<String> = recovered
3245 .iter()
3246 .map(|&j| format!("judge-{}", j + 1))
3247 .collect();
3248 self.state
3249 .quota
3250 .retain(|q| !recovered_keys.contains(&q.seat));
3251 for loss in judge_losses.into_iter().chain(vote_losses) {
3255 if recovered_keys.contains(&loss.seat) {
3256 continue;
3257 }
3258 self.state.quota.retain(|q| q.seat != loss.seat);
3259 self.state.quota.push(loss);
3260 }
3261
3262 self.state.tally = None;
3264 self.tally()?;
3265 Ok(self
3266 .state
3267 .tally
3268 .as_ref()
3269 .map(|t| t.met_quorum)
3270 .unwrap_or(false))
3271 }
3272
3273 async fn fold_losers(&mut self) -> Result<()> {
3276 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3277 return Ok(());
3278 };
3279 let repo = self.state.repo.clone();
3280 let mut folded = Vec::new();
3281 for i in 0..self.state.candidates.len() {
3282 let c = &self.state.candidates[i];
3283 if c.label == winner || c.folded {
3284 continue;
3285 }
3286 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3287 git::worktree_remove(&repo, &wt).await.ok();
3288 git::branch_delete(&repo, &branch).await.ok();
3289 self.state.candidates[i].folded = true;
3290 folded.push(label.to_string());
3291 }
3292 let root = self.state.worktree_root();
3294 for j in 1..=self.roles.judges.len() {
3295 let wt = root.join(format!("judge-{j}"));
3296 if wt.exists() {
3297 git::worktree_remove(&repo, &wt).await.ok();
3298 }
3299 }
3300 if self.state.config.graph.advise {
3303 for k in 1..=self.state.config.graph.advisors {
3304 let wt = root.join(format!("advisor-{k}"));
3305 if wt.exists() {
3306 git::worktree_remove(&repo, &wt).await.ok();
3307 }
3308 }
3309 }
3310 if !folded.is_empty() {
3311 self.state
3312 .event("fold", format!("folded candidates {}", folded.join(", ")));
3313 self.state.save()?;
3314 }
3315 Ok(())
3316 }
3317
3318 async fn sync_to_base(&mut self) -> Result<()> {
3358 if self.state.status == RunStatus::AlreadyInBase {
3359 return Ok(());
3360 }
3361 let conflicted = self
3362 .state
3363 .base_sync
3364 .as_ref()
3365 .is_some_and(|s| s.conflict.is_some());
3366 let Some(winner) = self.state.winner().cloned() else {
3367 return Ok(());
3368 };
3369
3370 let repo = self.state.repo.clone();
3371 let remote = self.state.config.merge.remote.clone();
3372 let base_branch = self.state.base_branch.clone();
3373 let tracking = format!("{remote}/{base_branch}");
3374
3375 git::fetch(&repo, &remote, &base_branch).await.ok();
3376 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3380 return Ok(());
3381 };
3382
3383 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3384 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3385 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3386
3387 if (behind > 0 || conflicted || head == tip)
3396 && self
3397 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3398 .await?
3399 {
3400 return Ok(());
3401 }
3402 if conflicted {
3403 return Ok(());
3404 }
3405
3406 if behind == 0 {
3407 if let Some(from) = self
3416 .state
3417 .rebase_fixes
3418 .iter()
3419 .rev()
3420 .find_map(|r| r.from.clone())
3421 && from != head
3422 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3423 .await
3424 .is_ok_and(|o| o.ok())
3425 {
3426 git::sync_to_head(&winner.worktree).await?;
3427 }
3428 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3434 if let Some(why) = &conflict {
3435 self.state.status = RunStatus::Blocked;
3436 self.state.event("land", why.clone());
3437 }
3438 self.state.base_sync = Some(BaseSync {
3439 tip,
3440 behind: 0,
3441 attempts,
3442 conflict,
3443 already_in: None,
3444 });
3445 self.state.save()?;
3446 return Ok(());
3447 }
3448
3449 if attempts >= BASE_SYNC_ROUNDS {
3450 let why = format!(
3451 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3452 rebase(s); rebasing again would only race it",
3453 winner.branch
3454 );
3455 self.state.status = RunStatus::Blocked;
3456 self.state.base_sync = Some(BaseSync {
3457 tip,
3458 behind,
3459 attempts,
3460 conflict: Some(why.clone()),
3461 already_in: None,
3462 });
3463 self.state.event("land", why);
3464 self.state.save()?;
3465 return Ok(());
3466 }
3467
3468 self.state.event(
3469 "land",
3470 format!(
3471 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3472 winner.branch
3473 ),
3474 );
3475 self.state.save()?;
3476
3477 let branch_tracking = format!("{remote}/{}", winner.branch);
3483 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3484 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3485 git::rev_parse(&repo, &branch_tracking).await.ok()
3486 } else {
3487 None
3488 };
3489 if let Some(theirs) = &remote_tip
3493 && !git::is_ancestor(&repo, theirs, &head).await
3494 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3495 .await
3496 .is_ok_and(|missing| missing.is_empty())
3497 {
3498 let why = format!(
3499 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3500 them",
3501 short(theirs),
3502 winner.branch
3503 );
3504 self.state.status = RunStatus::Blocked;
3505 self.state.base_sync = Some(BaseSync {
3506 tip,
3507 behind,
3508 attempts,
3509 conflict: Some(why.clone()),
3510 already_in: None,
3511 });
3512 self.state.event("land", why);
3513 self.state.save()?;
3514 return Ok(());
3515 }
3516
3517 let scratch = self.state.dir().join("base-sync");
3518 let rebased = match crate::rebase::rebase_with_fixer(
3519 &mut self.state,
3520 &scratch,
3521 &winner.branch,
3522 &tracking,
3523 )
3524 .await
3525 {
3526 Ok(crate::rebase::Rebased::Applied) => Ok(None),
3527 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3528 Err(e) => Err(e),
3529 };
3530 let attempts = attempts + 1;
3531 match rebased {
3532 Ok(None) => {
3533 git::sync_to_head(&winner.worktree).await?;
3537 let mut conflict = None;
3538 if let Some(pinned) = &remote_tip {
3539 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3540 match pushed {
3541 Ok(o) if o.ok() => self.state.event(
3542 "land",
3543 format!("pushed rebased {} to {remote}", winner.branch),
3544 ),
3545 Ok(o) => {
3546 conflict = Some(format!(
3547 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
3548 winner.branch,
3549 short(pinned),
3550 o.stderr.chars().take(600).collect::<String>()
3551 ));
3552 }
3553 Err(e) => {
3554 conflict = Some(format!(
3555 "rebased {} locally but could not push it: {e:#}",
3556 winner.branch
3557 ));
3558 }
3559 }
3560 }
3561 if let Some(why) = &conflict {
3562 self.state.status = RunStatus::Blocked;
3563 self.state.event("land", why.clone());
3564 }
3565 self.state.base_sync = Some(BaseSync {
3566 tip: tip.clone(),
3567 behind: 0,
3568 attempts,
3569 conflict,
3570 already_in: None,
3571 });
3572 self.state
3573 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3574 }
3575 Ok(Some(conflict)) => {
3576 let why = format!(
3577 "{} conflicts with {tracking} and did not rebase: {}",
3578 winner.branch,
3579 conflict.chars().take(600).collect::<String>()
3580 );
3581 self.state.status = RunStatus::Blocked;
3582 self.state.base_sync = Some(BaseSync {
3583 tip,
3584 behind,
3585 attempts,
3586 conflict: Some(why.clone()),
3587 already_in: None,
3588 });
3589 self.state.event("land", why);
3590 }
3591 Err(e) => {
3592 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3593 self.state.status = RunStatus::Blocked;
3594 self.state.base_sync = Some(BaseSync {
3595 tip,
3596 behind,
3597 attempts,
3598 conflict: Some(why.clone()),
3599 already_in: None,
3600 });
3601 self.state.event("land", why);
3602 }
3603 }
3604 self.state.save()?;
3605 Ok(())
3606 }
3607
3608 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
3614 let repo = self.state.repo.clone();
3615 let remote = self.state.config.merge.remote.clone();
3616 let fetched = git::fetch(&repo, &remote, branch).await;
3617 if !matches!(&fetched, Ok(o) if o.ok()) {
3618 return None;
3619 }
3620 let branch_tracking = format!("{remote}/{branch}");
3621 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
3622 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
3623 return None;
3624 }
3625 if !crate::reconcile::origin_missing(&repo, head, &theirs)
3626 .await
3627 .is_ok_and(|missing| missing.is_empty())
3628 {
3629 return Some(format!(
3630 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
3631 them",
3632 short(&theirs)
3633 ));
3634 }
3635 match git::push_pinned(&repo, &remote, branch, &theirs).await {
3636 Ok(o) if o.ok() => {
3637 self.state
3638 .event("land", format!("pushed rebased {branch} to {remote}"));
3639 None
3640 }
3641 Ok(o) => Some(format!(
3642 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
3643 someone may have pushed): {}",
3644 short(&theirs),
3645 o.stderr.chars().take(600).collect::<String>()
3646 )),
3647 Err(e) => Some(format!(
3648 "{branch} is rebased locally but could not be pushed: {e:#}"
3649 )),
3650 }
3651 }
3652
3653 async fn settle_already_in(
3670 &mut self,
3671 branch: &str,
3672 tip: &str,
3673 head: &str,
3674 attempts: usize,
3675 behind: usize,
3676 ) -> Result<bool> {
3677 let repo = self.state.repo.clone();
3678 let remote = self.state.config.merge.remote.clone();
3679 let start = self.state.base_commit.clone();
3680 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3681 Ok(Some(e)) => e,
3682 Ok(None) => return Ok(false),
3683 Err(e) => {
3684 tracing::warn!("already-in-base check for {branch}: {e:#}");
3685 return Ok(false);
3686 }
3687 };
3688 let mut verified = vec![head.to_owned()];
3689 let fetched = git::fetch(&repo, &remote, branch).await;
3690 if matches!(&fetched, Ok(o) if o.ok())
3691 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3692 && theirs != head
3693 {
3694 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3695 Ok(Some(_)) => verified.push(theirs),
3696 _ => return Ok(false),
3697 }
3698 }
3699 let base_branch = self.state.base_branch.clone();
3700 let message = format!(
3701 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3702 land",
3703 evidence.names(),
3704 evidence.proof.as_str()
3705 );
3706 let closed =
3707 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3708 match closed {
3709 Ok(Ok(url)) => self
3710 .state
3711 .event("land", format!("closed {url}: superseded on {base_branch}")),
3712 Ok(Err(why)) => self
3713 .state
3714 .event("land", format!("did not close a pull request: {why}")),
3715 Err(e) => {
3716 let why = format!(
3717 "{branch} is already in {remote}/{base_branch}, but its pull request could \
3718 not be closed ({e:#}); resume to retry"
3719 );
3720 self.state.status = RunStatus::Blocked;
3721 self.state.base_sync = Some(BaseSync {
3722 tip: tip.to_owned(),
3723 behind,
3724 attempts,
3725 conflict: Some(why.clone()),
3726 already_in: None,
3727 });
3728 self.state.event("land", why);
3729 self.state.save()?;
3730 return Ok(true);
3731 }
3732 }
3733 self.state.status = RunStatus::AlreadyInBase;
3734 self.state.base_sync = Some(BaseSync {
3735 tip: tip.to_owned(),
3736 behind,
3737 attempts,
3738 conflict: None,
3739 already_in: Some(evidence),
3740 });
3741 self.state.event("land", message);
3742 self.state.save()?;
3743 self.settle_questions();
3744 Ok(true)
3745 }
3746
3747 fn landing_base(&self) -> String {
3757 self.state
3758 .base_sync
3759 .as_ref()
3760 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3761 }
3762
3763 pub async fn fix_selected(
3796 &mut self,
3797 ids: &[String],
3798 reason: &str,
3799 allow_stale: bool,
3800 ) -> Result<()> {
3801 let reason = reason.trim();
3802 if reason.is_empty() {
3803 bail!("a fix request needs a reason — that is the operator's own record of why");
3804 }
3805 if ids.is_empty() {
3806 bail!("no finding id given");
3807 }
3808 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3809 bail!(
3810 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3811 has already concluded — can be given a targeted fix. A run still \
3812 in progress should simply be resumed; a `merged` run's branch has \
3813 already landed, so its answer is a fresh `magi review <branch>`, \
3814 not reopening this run's own record",
3815 self.state.id,
3816 self.state.status.as_str()
3817 );
3818 }
3819 let Some(winner) = self.state.winner().cloned() else {
3820 bail!("run {} has no winning candidate to fix", self.state.id);
3821 };
3822 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3823 bail!(
3824 "branch `{}` no longer exists; this run cannot be extended",
3825 winner.branch
3826 );
3827 }
3828 let home = crate::run::home();
3829 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3830 bail!(
3831 "run {} is currently being worked on by another magi process",
3832 self.state.id
3833 );
3834 }
3835 let _claim = FixClaim::acquire(&self.state.dir())?;
3841
3842 let mut seen = BTreeSet::new();
3846 let mut findings = Vec::new();
3847 let mut missing = Vec::new();
3848 for id in ids {
3849 if !seen.insert(id.clone()) {
3850 continue;
3851 }
3852 match self.state.finding(id) {
3853 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3854 id: f.id.clone(),
3855 severity: f.severity,
3856 reviewer_vote: rec.vote,
3857 round: round.round,
3858 round_head: round.head.clone(),
3859 reviewer: rec.reviewer,
3860 agent: rec.agent.clone(),
3861 file: f.file.clone(),
3862 line: f.line,
3863 title: f.title.clone(),
3864 detail: f.detail.clone(),
3865 outcome: OperatorFixOutcome::Pending,
3866 }),
3867 None => missing.push(id.clone()),
3868 }
3869 }
3870 if !missing.is_empty() {
3871 bail!(
3872 "unknown finding id(s): {}; nothing was changed",
3873 missing.join(", ")
3874 );
3875 }
3876
3877 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3878 let stale_details: Vec<(String, String)> = findings
3879 .iter()
3880 .filter(|f| f.round_head != head_at_request)
3881 .map(|f| (f.id.clone(), f.round_head.clone()))
3882 .collect();
3883 let stale = !stale_details.is_empty();
3884 if stale && !allow_stale {
3885 bail!(
3886 "the branch has moved since some finding(s) were raised — {} — now \
3887 at {}; pass --allow-stale to fix anyway, or re-run review first",
3888 stale_details
3889 .iter()
3890 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3891 .collect::<Vec<_>>()
3892 .join(", "),
3893 short(&head_at_request)
3894 );
3895 }
3896
3897 let request = OperatorFixRequest {
3898 requested_at: Timestamp::now(),
3899 reason: reason.to_owned(),
3900 findings,
3901 head_at_request: head_at_request.clone(),
3902 allow_stale,
3903 stale,
3904 fix: None,
3905 result_head: None,
3906 follow_up_review_run: None,
3907 };
3908 self.state.event(
3909 "fix",
3910 format!(
3911 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3912 request.findings.len(),
3913 request
3914 .findings
3915 .iter()
3916 .map(|f| f.id.as_str())
3917 .collect::<Vec<_>>()
3918 .join(", "),
3919 ),
3920 );
3921 self.state.operator_fixes.push(request);
3928 self.state.save()?;
3929 let request_index = self.state.operator_fixes.len() - 1;
3930
3931 if winner.worktree.exists() {
3940 let dirty = git::git(
3943 &winner.worktree,
3944 &["status", "--porcelain", "--untracked-files=all"],
3945 )
3946 .await?;
3947 let only_withheld = dirty.lines().all(|l| {
3948 l.strip_prefix("?? ")
3949 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3950 });
3951 if !only_withheld {
3952 bail!(
3953 "`{}` has uncommitted changes; refusing to touch it — commit or \
3954 discard them first",
3955 winner.worktree.display()
3956 );
3957 }
3958 git::worktree_remove(&self.state.repo, &winner.worktree)
3959 .await
3960 .ok();
3961 }
3962 let fix_worktree = self.state.worktree_root().join("operator-fix");
3963 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3964 git::git(
3965 &self.state.repo,
3966 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3967 )
3968 .await
3969 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3970 if !git::is_clean(&fix_worktree).await? {
3971 git::worktree_remove(&self.state.repo, &fix_worktree)
3972 .await
3973 .ok();
3974 bail!(
3975 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3976 winner.branch
3977 );
3978 }
3979
3980 let run_id = self.state.id.clone();
3981 let prompts = self.state.config.prompts.clone();
3982 let language = self.state.config.graph.language.clone();
3983 let sessions = self.state.config.graph.sessions;
3984 let artifacts = agent::artifacts_dir(&self.state.dir());
3985 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3986 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3987 _ => (
3988 self.state
3989 .config
3990 .agent(&winner.agent)
3991 .cloned()
3992 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3993 format!("impl-{}", winner.label),
3994 ),
3995 };
3996 let seat = self.seat(&fix_seat_key, &fix_spec.id);
3997 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3998 .findings
3999 .iter()
4000 .map(|f| Finding {
4001 id: f.id.clone(),
4002 severity: f.severity,
4003 file: f.file.clone(),
4004 line: f.line,
4005 title: f.title.clone(),
4006 detail: f.detail.clone(),
4007 })
4008 .collect();
4009 let job = SeatJob {
4010 prompt: prompt::operator_fix(
4011 &self.state.instruction,
4012 &finding_list,
4013 reason,
4014 &stale_details,
4015 &head_at_request,
4016 &language,
4017 ),
4018 spec: fix_spec.clone(),
4019 seat,
4020 cwd: fix_worktree.clone(),
4021 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4022 allow_write: true,
4023 sessions,
4024 artifacts: artifacts.clone(),
4025 stem: "operator-fix".to_owned(),
4026 handover: None,
4027 };
4028 let cache = self.state.config.cache_dir();
4029 let ctx = WaveCtx {
4030 carry_seats: false,
4031 run: &run_id,
4032 node: "fix",
4033 prompts: &prompts,
4034 cache: cache.as_deref(),
4035 round: None,
4036 };
4037 let (seat, out) =
4038 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4039 let agent_id = seat.agent.clone();
4040
4041 let mut fix = FixRecord {
4042 agent: agent_id,
4043 addressed: Vec::new(),
4044 rejected: Vec::new(),
4045 notes: String::new(),
4046 committed: false,
4047 failed: None,
4048 duration_ms: 0,
4049 continuation: None,
4050 };
4051 let mut final_seat = seat.clone();
4052 match out {
4053 AgentOutcome::Ok(o) => {
4054 fix.duration_ms = o.duration_ms;
4055 let parsed = verdict::extract_json::<FixReport>(&o.text);
4056 let incomplete_reason = match &parsed {
4057 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4058 "the reply parsed, but it reported a command whose own CLI \
4059 never confirmed an exit status"
4060 .to_owned(),
4061 ),
4062 Ok(_) => None,
4063 Err(e) => Some(e.to_string()),
4064 };
4065 match incomplete_reason {
4066 None => {
4067 let report = parsed.expect("checked Ok above");
4068 fix.addressed = report.addressed;
4069 fix.rejected = report.rejected;
4070 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4071 }
4072 Some(reason) => {
4073 let (resumed_seat, resolved, failure, cont) = self
4074 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4075 .await;
4076 fix.duration_ms += cont.cumulative_wait_ms;
4077 fix.continuation = Some(cont);
4078 final_seat = resumed_seat;
4079 match resolved {
4080 Some(report) => {
4081 fix.addressed = report.addressed;
4082 fix.rejected = report.rejected;
4083 fix.notes =
4084 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4085 }
4086 None => fix.failed = failure,
4087 }
4088 }
4089 }
4090 }
4091 AgentOutcome::Dropped(o) => {
4092 fix.duration_ms = o.duration_ms;
4093 let why = o
4094 .dropped
4095 .as_ref()
4096 .map(|d| d.why.as_str())
4097 .unwrap_or("the CLI ended the stream without delivering its answer");
4098 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4099 }
4100 AgentOutcome::Quota(o) => {
4101 self.state.quota.push(QuotaLoss {
4102 seat: final_seat.key.clone(),
4103 node: "fix".to_owned(),
4104 at: Timestamp::now(),
4105 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4106 });
4107 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4108 }
4109 AgentOutcome::Failed(e) => fix.failed = Some(e),
4110 }
4111 if fix.continuation.is_none() {
4112 fix.continuation = Some(ContinuationRecord::not_needed());
4113 }
4114 self.state.seats.insert(final_seat.key.clone(), final_seat);
4115
4116 let rescue_message = format!(
4117 "magi: operator-selected fix ({}) (uncommitted work)",
4118 self.state.operator_fixes[request_index]
4119 .findings
4120 .iter()
4121 .map(|f| f.id.as_str())
4122 .collect::<Vec<_>>()
4123 .join(", ")
4124 );
4125 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4126 self.state.note_withheld("fix", &r.withheld);
4127 }
4128 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4129 fix.committed = after != head_at_request;
4130 git::worktree_remove(&self.state.repo, &fix_worktree)
4131 .await
4132 .ok();
4133
4134 self.state.event(
4135 "fix",
4136 match &fix.failed {
4137 Some(reason) => format!(
4138 "operator fix: adoption report was lost ({reason}); {}",
4139 if fix.committed {
4140 "committed"
4141 } else {
4142 "NO new commit"
4143 }
4144 ),
4145 None => format!(
4146 "operator fix: {} addressed, {} rejected, {}",
4147 fix.addressed.len(),
4148 fix.rejected.len(),
4149 if fix.committed {
4150 "committed"
4151 } else {
4152 "NO new commit"
4153 }
4154 ),
4155 },
4156 );
4157
4158 for f in &mut self.state.operator_fixes[request_index].findings {
4165 f.outcome = if fix.failed.is_some() {
4166 OperatorFixOutcome::Unreported
4167 } else if fix.addressed.contains(&f.id) {
4168 OperatorFixOutcome::Addressed
4169 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4170 OperatorFixOutcome::Rejected { why: r.why.clone() }
4171 } else {
4172 OperatorFixOutcome::Unreported
4173 };
4174 }
4175
4176 let committed = fix.committed;
4177 if committed {
4178 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4179 }
4180 self.state.operator_fixes[request_index].fix = Some(fix);
4181 self.state.save()?;
4184
4185 if committed {
4186 self.state.event(
4187 "fix",
4188 format!(
4189 "operator fix committed {}; opening a follow-up review-only run",
4190 short(&after)
4191 ),
4192 );
4193 let origin =
4196 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4197 match Self::review(
4198 &self.state.repo,
4199 &winner.branch,
4200 self.state.config.clone(),
4201 origin,
4202 )
4203 .await
4204 {
4205 Ok(mut follow_up) => {
4206 follow_up.state.event(
4207 "start",
4208 format!(
4209 "requested by an operator fix on run {} for finding(s) {}",
4210 self.state.id,
4211 self.state.operator_fixes[request_index]
4212 .findings
4213 .iter()
4214 .map(|f| f.id.as_str())
4215 .collect::<Vec<_>>()
4216 .join(", "),
4217 ),
4218 );
4219 follow_up.state.save()?;
4220 let follow_up_id = follow_up.state.id.clone();
4221 if let Err(e) = follow_up.execute().await {
4222 self.state.event(
4223 "fix",
4224 format!(
4225 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4226 ),
4227 );
4228 }
4229 self.state.operator_fixes[request_index].follow_up_review_run =
4230 Some(follow_up_id);
4231 }
4232 Err(e) => {
4233 self.state.event(
4234 "fix",
4235 format!("committed the fix but could not open a follow-up review: {e:#}"),
4236 );
4237 }
4238 }
4239 self.state.save()?;
4240 }
4241
4242 Ok(())
4243 }
4244
4245 fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
4252 match &self.roles.fixer {
4253 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4254 _ => (
4255 self.state
4256 .config
4257 .agent(&winner.agent)
4258 .cloned()
4259 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4260 format!("impl-{}", winner.label),
4261 ),
4262 }
4263 }
4264
4265 async fn review_loop(&mut self) -> Result<()> {
4266 if self
4271 .state
4272 .base_sync
4273 .as_ref()
4274 .is_some_and(|s| s.conflict.is_some())
4275 {
4276 return Ok(());
4277 }
4278 let run_id = self.state.id.clone();
4283 let prompts = self.state.config.prompts.clone();
4284 let Some(winner) = self.state.winner().cloned() else {
4285 return Ok(());
4286 };
4287 let max_rounds = self.state.config.graph.review_rounds;
4288 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4298 if status == RunStatus::Gating {
4301 self.record_contested_handoff();
4302 }
4303 self.state.status = status;
4304 self.state.save()?;
4305 return Ok(());
4306 }
4307 self.state.status = RunStatus::Reviewing;
4308 if self
4318 .state
4319 .reviews
4320 .last()
4321 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4322 {
4323 let shell = self.state.config.shell();
4324 return self
4325 .stop_reviewing(
4326 "the last round's own verification never resolved",
4327 &shell,
4328 &winner.worktree,
4329 )
4330 .await;
4331 }
4332
4333 let repo = self.state.repo.clone();
4334 let root = self.state.worktree_root();
4335 let language = self.state.config.graph.language.clone();
4336 let sessions = self.state.config.graph.sessions;
4337 let artifacts = agent::artifacts_dir(&self.state.dir());
4338 let base = self.landing_base();
4339 let base_short = short(&base);
4340 let reviewers = self.roles.reviewers.clone();
4341 let shell = self.state.config.shell();
4342
4343 for round in (self.state.reviews.len() + 1)..=max_rounds {
4344 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4345 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4346 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4347 let prev_verification = self
4356 .state
4357 .reviews
4358 .last()
4359 .and_then(|r| r.verification_summary(&head));
4360
4361 let mut jobs = Vec::new();
4365 for (r, spec) in reviewers.iter().cloned().enumerate() {
4366 let wt = root.join(format!("review-{}", r + 1));
4367 if wt.exists() {
4368 git::reset_detached(&wt, &head).await?;
4369 } else {
4370 git::worktree_add_detached(&repo, &wt, &head).await?;
4371 }
4372 let seat_key = format!("review-{}", r + 1);
4373 let spec = pick_start_spec(
4376 &self.roles.reviewer_roster,
4377 spec,
4378 self.state.seat_history.get(&seat_key),
4379 );
4380 let seat = self.seat(&seat_key, &spec.id);
4381 jobs.push(SeatJob {
4382 prompt: prompt::review(&prompt::ReviewCtx {
4383 instruction: &self.state.instruction,
4384 branch: &winner.branch,
4385 base_short: &base_short,
4386 stat: &stat,
4387 patch: &patch,
4388 verification: prev_verification.as_ref(),
4389 reviewers: reviewers.len(),
4390 round,
4391 rounds: max_rounds,
4392 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4395 lens: Lens::for_seat(r),
4396 language: &language,
4397 }),
4398 spec,
4399 seat,
4400 cwd: wt,
4401 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4402 allow_write: false,
4403 sessions,
4404 artifacts: artifacts.clone(),
4405 stem: format!("review-{round}-{}", r + 1),
4406 handover: None,
4407 });
4408 }
4409
4410 self.state.event(
4411 "review",
4412 format!(
4413 "round {round}: {} reviewers on {}",
4414 jobs.len(),
4415 short(&head)
4416 ),
4417 );
4418 let mut quota_losses = Vec::new();
4419 let review_retries = self.state.config.graph.retries;
4420 let review_cache = self.state.config.cache_dir();
4421 let ctx = WaveCtx {
4422 carry_seats: true,
4423 run: &run_id,
4424 node: "review",
4425 prompts: &prompts,
4426 cache: review_cache.as_deref(),
4427 round: Some(round),
4428 };
4429 let results = ask_json_wave::<Review>(
4430 jobs,
4431 Arc::clone(&self.sem),
4432 review_retries,
4433 &self.roles.reviewer_roster,
4434 &ctx,
4435 &mut quota_losses,
4436 &mut self.state,
4437 &|_: &Review| Ok(()),
4438 )
4439 .await;
4440 let round_quota_missing = quota_losses.len();
4444 self.state.quota.extend(quota_losses);
4445
4446 let mut records = Vec::new();
4447 let mut all_findings = Vec::new();
4448 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4449 let agent_id = seat.agent.clone();
4450 self.state.seats.insert(seat.key.clone(), seat);
4451 let mut record = ReviewRecord {
4452 reviewer: r + 1,
4453 agent: agent_id,
4454 summary: String::new(),
4455 findings: Vec::new(),
4456 vote: None,
4457 failed: None,
4458 duration_ms: 0,
4459 attempts,
4465 };
4466 match res {
4467 Ok((review, out)) => {
4468 record.summary =
4476 blind::sanitize_prose(&review.summary, &self.state.config.blind);
4477 record.vote = Some(review.vote);
4478 record.duration_ms = out.duration_ms;
4479 for (n, mut f) in review.findings.into_iter().enumerate() {
4480 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4483 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4484 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4485 f.file = f
4491 .file
4492 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4493 all_findings.push(f.clone());
4494 record.findings.push(f);
4495 }
4496 self.state.event(
4497 "review",
4498 format!(
4499 "round {round}: reviewer {} voted {} with {} finding(s)",
4500 r + 1,
4501 review.vote.label(),
4502 record.findings.len()
4503 ),
4504 );
4505 }
4506 Err(e) => {
4507 record.failed = Some(e.to_string());
4508 self.state.event(
4509 "review",
4510 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4511 );
4512 }
4513 }
4514 records.push(record);
4515 }
4516
4517 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4524 let vote_split =
4525 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4526 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4527 if vote_split {
4528 self.state.event(
4529 "review",
4530 format!(
4531 "round {round}: votes split ({}) — one round of reconsideration",
4532 initial_votes
4533 .iter()
4534 .map(|v| v.label())
4535 .collect::<Vec<_>>()
4536 .join(", ")
4537 ),
4538 );
4539 let panel: Vec<ReviewSeatReport<'_>> = records
4542 .iter()
4543 .filter_map(|r| {
4544 r.vote.map(|vote| ReviewSeatReport {
4545 reviewer: r.reviewer,
4546 vote,
4547 summary: &r.summary,
4548 findings: &r.findings,
4549 })
4550 })
4551 .collect();
4552
4553 let mut jobs = Vec::new();
4554 let mut seats_at = Vec::new();
4555 for (r, spec) in reviewers.iter().cloned().enumerate() {
4556 if records[r].vote.is_none() {
4560 continue;
4561 }
4562 let wt = root.join(format!("review-{}", r + 1));
4563 let seat_key = format!("review-{}", r + 1);
4564 let spec = self.occupant(&seat_key, spec);
4565 let seat = self.seat(&seat_key, &spec.id);
4566 let build = |with_patch: bool| {
4574 prompt::review_reconsider(&ReviewReconsiderCtx {
4575 instruction: &self.state.instruction,
4576 reviewer: r + 1,
4577 lens: Lens::for_seat(r),
4578 panel: &panel,
4579 patch: with_patch.then_some(ReviewPatch {
4580 branch: &winner.branch,
4581 base_short: &base_short,
4582 stat: &stat,
4583 patch: &patch,
4584 }),
4585 round,
4586 rounds: max_rounds,
4587 language: &language,
4588 })
4589 };
4590 let full = build(true);
4591 let prompt = if has_context(&spec, &seat, sessions) {
4592 build(false)
4593 } else {
4594 full.clone()
4595 };
4596 jobs.push(SeatJob {
4597 prompt,
4598 spec,
4599 seat,
4600 cwd: wt,
4601 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4602 allow_write: false,
4603 sessions,
4604 artifacts: artifacts.clone(),
4605 stem: format!("review-{round}-reconsider-{}", r + 1),
4606 handover: Some(full),
4607 });
4608 seats_at.push(r);
4609 }
4610
4611 let mut recon_quota_losses = Vec::new();
4612 let recon_cache = self.state.config.cache_dir();
4613 let recon_ctx = WaveCtx {
4614 carry_seats: true,
4615 run: &run_id,
4616 node: "review",
4617 prompts: &prompts,
4618 cache: recon_cache.as_deref(),
4619 round: Some(round),
4620 };
4621 let recon_results = ask_json_wave::<ReviewRevote>(
4622 jobs,
4623 Arc::clone(&self.sem),
4624 review_retries,
4625 &self.roles.reviewer_roster,
4626 &recon_ctx,
4627 &mut recon_quota_losses,
4628 &mut self.state,
4629 &|_: &ReviewRevote| Ok(()),
4630 )
4631 .await;
4632 self.state.quota.extend(recon_quota_losses);
4633
4634 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4635 let agent_id = seat.agent.clone();
4636 self.state.seats.insert(seat.key.clone(), seat);
4637 let mut rec = ReviewRevoteRecord {
4638 reviewer: r + 1,
4639 agent: agent_id,
4640 vote: None,
4641 reason: String::new(),
4642 failed: None,
4643 };
4644 match res {
4645 Ok((rv, _)) => {
4646 rec.vote = Some(rv.vote);
4647 rec.reason =
4648 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4649 self.state.event(
4650 "review",
4651 format!(
4652 "round {round}: reviewer {} revoted {}",
4653 r + 1,
4654 rv.vote.label()
4655 ),
4656 );
4657 }
4658 Err(e) => {
4659 rec.failed = Some(e.to_string());
4660 self.state.event(
4661 "review",
4662 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4663 );
4664 }
4665 }
4666 reconsideration.push(rec);
4667 }
4668 } else if initial_votes.len() > 1 {
4669 self.state.event(
4670 "review",
4671 format!(
4672 "round {round}: votes agreed ({}) — no reconsideration",
4673 initial_votes[0].label()
4674 ),
4675 );
4676 }
4677
4678 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4679 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4680 let defer_e2e =
4691 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4692 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4693 let reason =
4694 format!("{blocking} blocking finding(s) already required a fix this round");
4695 self.state.event(
4696 "verify",
4697 format!(
4698 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4699 {}); it will run once a round has none left",
4700 short(&head)
4701 ),
4702 );
4703 (Vec::new(), false, true, Some(reason))
4704 } else {
4705 let e2e_commands = self.state.config.verify.e2e.clone();
4706 let cache_dir = self.state.config.cache_dir();
4707 let context = format!("round {round}");
4708 let (e2e, verify_retried) = with_cache_lease(
4709 &mut self.state,
4710 cache_dir.as_deref(),
4711 "e2e",
4712 "e2e",
4713 &winner.worktree,
4714 &head,
4715 verify_timeout,
4716 &context,
4717 |state, budget| {
4718 let shell = shell.clone();
4719 let e2e_commands = e2e_commands.clone();
4720 let worktree = winner.worktree.clone();
4721 let context = context.clone();
4722 async move {
4723 run_e2e_with_retry(
4724 state,
4725 &shell,
4726 &e2e_commands,
4727 &worktree,
4728 budget,
4729 &context,
4730 )
4731 .await
4732 }
4733 },
4734 )
4735 .await;
4736 (e2e, verify_retried, false, None)
4737 };
4738
4739 let expected = records.len();
4740 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4741 let incomplete = answered < expected;
4742 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4743 let policy = self.state.config.graph.incomplete_review;
4744 let clean = round_is_clean(
4745 blocking,
4746 e2e_ok,
4747 answered,
4748 expected,
4749 round_quota_missing,
4750 policy,
4751 );
4752
4753 let mut round_record = ReviewRound {
4754 round,
4755 head: head.clone(),
4756 verified_head: None,
4757 verified_at: None,
4758 reviews: records,
4759 e2e,
4760 verify_retried,
4761 e2e_deferred,
4762 e2e_defer_reason,
4763 fix: None,
4764 blocking,
4765 answered,
4766 expected,
4767 clean,
4768 progressed: false,
4769 vote_split,
4770 reconsideration,
4771 verdict: None,
4772 };
4773 round_record.verdict = ReviewVote::worst(
4777 round_record
4778 .final_votes()
4779 .into_iter()
4780 .map(|(_, _, vote)| vote),
4781 );
4782 if !matches!(
4793 round_record.e2e_status(),
4794 E2eStatus::Deferred | E2eStatus::NotConfigured
4795 ) {
4796 round_record.verified_head = Some(head.clone());
4797 round_record.verified_at = Some(Timestamp::now());
4798 }
4799 let this_round_verification = round_record.verification_summary(&head);
4800
4801 if incomplete {
4802 let missing: Vec<String> = round_record
4803 .reviews
4804 .iter()
4805 .filter(|r| r.failed.is_some())
4806 .map(|r| format!("review-{}", r.reviewer))
4807 .collect();
4808 self.state.event(
4809 "review",
4810 format!(
4811 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4812 missing.join(", ")
4813 ),
4814 );
4815 }
4816
4817 if clean {
4818 self.state.event(
4819 "review",
4820 if incomplete && policy == IncompleteReviewPolicy::Warn {
4821 format!(
4822 "round {round}: clean (warn policy, incomplete panel) — no \
4823 blocking findings from the seats that answered, verification green"
4824 )
4825 } else if incomplete {
4826 format!(
4827 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4828 quorum) — no blocking findings from the seats that answered, \
4829 verification green",
4830 expected - answered
4831 )
4832 } else {
4833 format!("round {round}: clean — no blocking findings, verification green")
4834 },
4835 );
4836 self.state.reviews.push(round_record);
4837 self.state.status = RunStatus::Gating;
4838 self.state.save()?;
4839 return Ok(());
4840 }
4841
4842 if incomplete && blocking == 0 && e2e_ok {
4850 self.state.reviews.push(round_record);
4851 self.state.save()?;
4852 if round == max_rounds {
4853 self.state.status = RunStatus::Blocked;
4854 self.state.event(
4855 "review",
4856 format!(
4857 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4858 refusing to call it clean",
4859 expected - answered
4860 ),
4861 );
4862 return Ok(());
4863 }
4864 continue;
4865 }
4866
4867 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4879 self.state.reviews.push(round_record);
4880 return self
4881 .stop_reviewing(
4882 "the round's own verification could not run",
4883 &shell,
4884 &winner.worktree,
4885 )
4886 .await;
4887 }
4888
4889 if round == max_rounds {
4890 self.state.reviews.push(round_record);
4891 return self
4892 .stop_reviewing(
4893 &format!(
4894 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4895 ),
4896 &shell,
4897 &winner.worktree,
4898 )
4899 .await;
4900 }
4901
4902 let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4905 let seat = self.seat(&fix_seat_key, &fix_spec.id);
4906 let blocking_findings: Vec<_> = all_findings
4907 .iter()
4908 .filter(|f| f.severity.blocks())
4909 .cloned()
4910 .collect();
4911 let job = SeatJob {
4912 prompt: prompt::fix(
4913 &self.state.instruction,
4914 &blocking_findings,
4915 this_round_verification.as_ref(),
4916 round,
4917 max_rounds,
4918 &language,
4919 ),
4920 spec: fix_spec.clone(),
4921 seat,
4922 cwd: winner.worktree.clone(),
4923 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4924 allow_write: true,
4925 sessions,
4926 artifacts: artifacts.clone(),
4927 stem: format!("fix-{round}"),
4928 handover: None,
4929 };
4930 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4931 let cache = self.state.config.cache_dir();
4932 let ctx = WaveCtx {
4933 carry_seats: false,
4934 run: &run_id,
4935 node: "fix",
4936 prompts: &prompts,
4937 cache: cache.as_deref(),
4938 round: Some(round),
4939 };
4940 let (seat, out) =
4941 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4942 let agent_id = seat.agent.clone();
4943
4944 let mut fix = FixRecord {
4945 agent: agent_id,
4946 addressed: Vec::new(),
4947 rejected: Vec::new(),
4948 notes: String::new(),
4949 committed: false,
4950 failed: None,
4951 duration_ms: 0,
4952 continuation: None,
4953 };
4954 let mut continuation = ContinuationRecord::not_needed();
4955 let mut final_seat = seat.clone();
4956 match out {
4957 AgentOutcome::Ok(o) => {
4958 fix.duration_ms = o.duration_ms;
4959 let parsed = verdict::extract_json::<FixReport>(&o.text);
4960 let incomplete_reason = match &parsed {
4967 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4968 "the reply parsed, but it reported a command whose own CLI never \
4969 confirmed an exit status"
4970 .to_owned(),
4971 ),
4972 Ok(_) => None,
4973 Err(e) => Some(e.to_string()),
4974 };
4975 match incomplete_reason {
4976 None => {
4977 let report = parsed.expect("checked Ok above");
4978 fix.addressed = report.addressed;
4979 fix.rejected = report.rejected;
4980 fix.notes =
4981 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4982 }
4983 Some(reason) => {
4984 let (resumed_seat, resolved, failure, cont) = self
4985 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4986 .await;
4987 fix.duration_ms += cont.cumulative_wait_ms;
4988 continuation = cont;
4989 final_seat = resumed_seat;
4990 match resolved {
4991 Some(report) => {
4992 fix.addressed = report.addressed;
4993 fix.rejected = report.rejected;
4994 fix.notes = blind::sanitize_prose(
4995 &report.notes,
4996 &self.state.config.blind,
4997 );
4998 }
4999 None => fix.failed = failure,
5000 }
5001 }
5002 }
5003 }
5004 AgentOutcome::Dropped(o) => {
5006 fix.duration_ms = o.duration_ms;
5007 let why = o
5008 .dropped
5009 .as_ref()
5010 .map(|d| d.why.as_str())
5011 .unwrap_or("the CLI ended the stream without delivering its answer");
5012 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5013 }
5014 AgentOutcome::Quota(o) => {
5015 self.state.quota.push(QuotaLoss {
5016 seat: final_seat.key.clone(),
5017 node: "fix".to_owned(),
5018 at: Timestamp::now(),
5019 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5020 });
5021 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5022 }
5023 AgentOutcome::Failed(e) => fix.failed = Some(e),
5024 }
5025 fix.continuation = Some(continuation);
5026 self.state.seats.insert(final_seat.key.clone(), final_seat);
5027 if let Ok(r) = git::rescue_commit(
5028 &winner.worktree,
5029 &format!("magi: review round {round} fixes (uncommitted work)"),
5030 )
5031 .await
5032 {
5033 self.state.note_withheld("fix", &r.withheld);
5034 }
5035 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5036 fix.committed = after != before;
5037 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5045 let progressed = diff_after != patch;
5046 let commit_note = if fix.committed {
5047 "committed"
5048 } else {
5049 "NO new commit"
5050 };
5051 let tree_note = if progressed {
5052 "changed vs base"
5053 } else {
5054 "unchanged vs base"
5055 };
5056 self.state.event(
5057 "fix",
5058 match &fix.failed {
5059 Some(reason) => {
5065 format!(
5066 "round {round}: fixer's adoption report was lost ({reason}); \
5067 {commit_note}, tree {tree_note}"
5068 )
5069 }
5070 None => format!(
5071 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5072 {tree_note}{}",
5073 fix.addressed.len(),
5074 fix.rejected.len(),
5075 if continuation.outcome == ContinuationOutcome::Resumed {
5076 format!(
5077 " (adoption report recovered after {} continuation(s))",
5078 continuation.attempts
5079 )
5080 } else {
5081 String::new()
5082 },
5083 ),
5084 },
5085 );
5086 round_record.fix = Some(fix);
5087 round_record.progressed = progressed;
5088 self.state.reviews.push(round_record);
5089 self.state.save()?;
5090
5091 if matches!(
5104 continuation.outcome,
5105 ContinuationOutcome::Exhausted
5106 | ContinuationOutcome::QuotaLost
5107 | ContinuationOutcome::NoSession
5108 ) {
5109 return self
5110 .stop_reviewing(
5111 "the fixer's adoption report never came back, even after resuming its \
5112 own seat; refusing to start another round against the same worktree \
5113 while that is unresolved",
5114 &shell,
5115 &winner.worktree,
5116 )
5117 .await;
5118 }
5119
5120 let streak = self
5121 .state
5122 .reviews
5123 .iter()
5124 .rev()
5125 .take_while(|r| !r.progressed)
5126 .count();
5127 if streak >= STAGNANT_LIMIT {
5128 return self
5129 .stop_reviewing(
5130 &format!(
5131 "the tree has not moved against base for {streak} round(s) in a row"
5132 ),
5133 &shell,
5134 &winner.worktree,
5135 )
5136 .await;
5137 }
5138 }
5139 Ok(())
5140 }
5141
5142 fn record_contested_handoff(&mut self) {
5176 if self.state.contested_handoff.is_some() {
5177 return;
5178 }
5179 let Some(contested) = self
5180 .state
5181 .reviews
5182 .last()
5183 .and_then(ReviewRound::contested_handoff)
5184 else {
5185 return;
5186 };
5187 self.state.event(
5188 "review",
5189 format!(
5190 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5191 wait for the owner's approval",
5192 contested.findings.len(),
5193 contested.rejecters.len()
5194 ),
5195 );
5196 self.state.contested_handoff = Some(contested);
5197 }
5198
5199 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5200 let round_idx = self.state.reviews.len() - 1;
5201 let needs_catchup_run = matches!(
5209 self.state.reviews[round_idx].e2e_status(),
5210 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5211 );
5212 if needs_catchup_run {
5213 let round = self.state.reviews[round_idx].round;
5214 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5215 let commands = self.state.config.verify.e2e.clone();
5216 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5217 let cache_dir = self.state.config.cache_dir();
5218 let context = format!(
5219 "round {round}: verification unresolved, catching up before the final decision"
5220 );
5221 let (outcomes, verify_retried) = with_cache_lease(
5222 &mut self.state,
5223 cache_dir.as_deref(),
5224 "e2e",
5225 "e2e",
5226 worktree,
5227 &attempted_head,
5228 timeout,
5229 &context,
5230 |state, budget| {
5231 let shell = shell.to_vec();
5232 let commands = commands.clone();
5233 let context = context.clone();
5234 async move {
5235 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5236 .await
5237 }
5238 },
5239 )
5240 .await;
5241 let last = &mut self.state.reviews[round_idx];
5242 last.e2e = outcomes;
5243 last.verify_retried = verify_retried;
5244 last.verified_head = Some(attempted_head);
5251 last.verified_at = Some(Timestamp::now());
5252 if verify_inconclusive(&last.e2e) {
5253 self.state.save()?;
5260 return Ok(());
5261 }
5262 last.e2e_deferred = false;
5263 }
5264 let last = &self.state.reviews[round_idx];
5265 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5266
5267 match last.e2e_status() {
5268 E2eStatus::Failed => {
5269 let red: Vec<String> = last
5270 .e2e
5271 .iter()
5272 .filter(|o| !o.ok())
5273 .map(|o| {
5274 format!(
5275 "`{}` -> {:?}\n{}",
5276 o.command,
5277 o.code,
5278 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5279 )
5280 })
5281 .collect();
5282 self.state
5283 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5284 self.state.status = RunStatus::Blocked;
5285 }
5286 E2eStatus::ResourceBlocked => {
5291 self.state.event(
5292 "review",
5293 format!(
5294 "{why}; e2e could not run (shared build cache unavailable); not \
5295 deciding yet"
5296 ),
5297 );
5298 }
5299 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5300 self.state.event(
5301 "review",
5302 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5303 );
5304 self.record_contested_handoff();
5305 self.state.status = RunStatus::Gating;
5306 }
5307 }
5308 self.state.save()?;
5309 Ok(())
5310 }
5311
5312 async fn gate(&mut self) -> Result<()> {
5315 if self.state.status == RunStatus::Failed
5327 || self
5328 .state
5329 .base_sync
5330 .as_ref()
5331 .is_some_and(|s| s.conflict.is_some())
5332 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5333 != Some(RunStatus::Gating)
5334 {
5335 return Ok(());
5336 }
5337 if self.state.gate_ran {
5338 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5349 self.state.status = RunStatus::Blocked;
5350 self.state.save()?;
5351 }
5352 return Ok(());
5353 }
5354 let Some(winner) = self.state.winner().cloned() else {
5355 return Ok(());
5356 };
5357 self.state.status = RunStatus::Gating;
5358 let mut outcomes = self.run_gate(&winner).await?;
5359 loop {
5360 if verify_inconclusive(&outcomes) {
5371 self.state.save()?;
5372 return Ok(());
5373 }
5374 if outcomes.iter().all(CommandOutcome::ok) {
5375 break;
5376 }
5377 match self.gate_fix_round(&winner, &outcomes).await? {
5378 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5379 GateFix::Stop => break,
5380 GateFix::Defer => {
5381 self.state.save()?;
5382 return Ok(());
5383 }
5384 }
5385 }
5386 let passed = outcomes.iter().all(CommandOutcome::ok);
5387 self.state.gate = outcomes;
5388 self.state.gate_ran = true;
5389 if !passed {
5390 self.state.status = RunStatus::Blocked;
5391 let spent = self.state.gate_fixes.len();
5392 self.state.event(
5393 "gate",
5394 if spent == 0 {
5395 "gate failed; not merging".to_owned()
5396 } else {
5397 format!("gate failed after {spent} gate-fix round(s); not merging")
5398 },
5399 );
5400 }
5401 self.state.save()?;
5402 Ok(())
5403 }
5404
5405 async fn run_pre_gate(&mut self, winner: &Candidate) {
5415 let commands = self.state.config.verify.pre_gate.clone();
5416 if commands.is_empty() {
5417 return;
5418 }
5419 let shell = self.state.config.shell();
5420 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5421 let (outcomes, _) = run_commands(
5422 &mut self.state,
5423 "pre_gate",
5424 "pre_gate",
5425 0,
5426 &shell,
5427 &commands,
5428 &winner.worktree,
5429 timeout,
5430 )
5431 .await;
5432 for o in &outcomes {
5433 if !o.ok() {
5434 tracing::warn!(
5435 "pre_gate `{}` failed ({:?}); the gate decides",
5436 o.command,
5437 o.code
5438 );
5439 }
5440 self.state.event(
5441 "pre_gate",
5442 format!(
5443 "`{}` -> {}",
5444 o.command,
5445 if o.ok() {
5446 "pass".to_owned()
5447 } else {
5448 format!(
5449 "FAIL ({:?})\n{}",
5450 o.code,
5451 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5452 )
5453 }
5454 ),
5455 );
5456 }
5457 self.state.pre_gate = outcomes;
5458 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5459 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5460 Ok(head) => {
5461 self.state
5462 .event("pre_gate", format!("committed mechanical fixes ({head})"));
5463 self.state.pre_gate_commit = Some(head);
5464 }
5465 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5466 },
5467 Ok(false) => {}
5468 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5469 }
5470 if let Err(e) = self.state.save() {
5471 tracing::warn!("could not persist the pre_gate record: {e:#}");
5472 }
5473 }
5474
5475 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5478 self.run_pre_gate(winner).await;
5479 let shell = self.state.config.shell();
5480 let gate_commands = self.state.config.verify.gate.clone();
5481 let outcomes = if gate_commands.is_empty() {
5490 Vec::new()
5491 } else {
5492 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5493 let cache_dir = self.state.config.cache_dir();
5494 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5495 let (outcomes, _) = with_cache_lease(
5496 &mut self.state,
5497 cache_dir.as_deref(),
5498 "gate",
5499 "gate",
5500 &winner.worktree,
5501 &head,
5502 timeout,
5503 "final gate",
5504 |state, budget| {
5505 let shell = shell.clone();
5506 let gate_commands = gate_commands.clone();
5507 let worktree = winner.worktree.clone();
5508 async move {
5509 let (outcomes, timed_out_pids) = run_commands(
5510 state,
5511 "gate",
5512 "gate",
5513 0,
5514 &shell,
5515 &gate_commands,
5516 &worktree,
5517 budget,
5518 )
5519 .await;
5520 (outcomes, false, timed_out_pids)
5521 }
5522 },
5523 )
5524 .await;
5525 outcomes
5526 };
5527 if outcomes.is_empty() {
5528 self.state.event(
5533 "gate",
5534 "no gate commands configured; nothing to check, passing",
5535 );
5536 }
5537 for o in &outcomes {
5538 self.state.event(
5539 "gate",
5540 format!(
5541 "`{}` -> {}",
5542 o.command,
5543 if o.ok() {
5544 "pass".to_owned()
5545 } else {
5546 format!(
5547 "FAIL ({:?})\n{}",
5548 o.code,
5549 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5550 )
5551 }
5552 ),
5553 );
5554 }
5555 Ok(outcomes)
5556 }
5557
5558 async fn gate_fix_round(
5571 &mut self,
5572 winner: &Candidate,
5573 outcomes: &[CommandOutcome],
5574 ) -> Result<GateFix> {
5575 let cap = self.state.config.graph.gate_fix_rounds;
5576 let spent = self.state.gate_fixes.len();
5577 if spent >= cap {
5578 if cap > 0 {
5579 self.state.event(
5580 "gate",
5581 format!("{spent} gate-fix round(s) spent and the gate still fails"),
5582 );
5583 }
5584 return Ok(GateFix::Stop);
5585 }
5586 if !gate_fixable(outcomes) {
5587 self.state.event(
5588 "gate",
5589 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5590 command or similar); not spending a fix round on it",
5591 );
5592 return Ok(GateFix::Stop);
5593 }
5594 let min_free = self.state.config.disk.min_free_bytes;
5595 if min_free > 0 {
5596 match crate::disk::free_bytes(&winner.worktree) {
5597 Ok(free) if crate::disk::enough_space(free, min_free) => {}
5598 Ok(free) => {
5599 self.state.event(
5600 "gate",
5601 format!(
5602 "only {free} bytes free ({min_free} required by `[disk] \
5603 min_free_bytes`); not spending a fix round on a failure the disk \
5604 may explain"
5605 ),
5606 );
5607 return Ok(GateFix::Stop);
5608 }
5609 Err(e) => {
5610 self.state.event(
5611 "gate",
5612 format!("free disk space could not be measured ({e:#}); no fix round"),
5613 );
5614 return Ok(GateFix::Stop);
5615 }
5616 }
5617 }
5618
5619 let attempt = spent + 1;
5620 let run_id = self.state.id.clone();
5621 let prompts = self.state.config.prompts.clone();
5622 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5623 let base = self.landing_base();
5624 let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5625 let seat = self.seat(&fix_seat_key, &fix_spec.id);
5626 let job = SeatJob {
5627 prompt: prompt::gate_fix(
5628 &self.state.instruction,
5629 &failed,
5630 attempt,
5631 cap,
5632 &self.state.config.graph.language,
5633 ),
5634 spec: fix_spec,
5635 seat,
5636 cwd: winner.worktree.clone(),
5637 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5638 allow_write: true,
5639 sessions: self.state.config.graph.sessions,
5640 artifacts: agent::artifacts_dir(&self.state.dir()),
5641 stem: format!("gate-fix-{attempt}"),
5642 handover: None,
5643 };
5644 self.state.event(
5645 "gate",
5646 format!("gate failed; gate-fix round {attempt} of {cap}"),
5647 );
5648 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5649 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5650 let cache = self.state.config.cache_dir();
5651 let ctx = WaveCtx {
5652 carry_seats: false,
5653 run: &run_id,
5654 node: "gate-fix",
5655 prompts: &prompts,
5656 cache: cache.as_deref(),
5657 round: None,
5658 };
5659 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5660 let mut record = GateFixRecord {
5661 agent: seat.agent.clone(),
5662 failed,
5663 notes: String::new(),
5664 committed: false,
5665 error: None,
5666 };
5667 match out {
5668 AgentOutcome::Ok(o) => {
5669 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5672 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5673 }
5674 }
5675 AgentOutcome::Dropped(_) => {
5676 record.error = Some("the CLI dropped the stream".to_owned());
5677 }
5678 AgentOutcome::Quota(o) => {
5679 self.state.quota.push(QuotaLoss {
5680 seat: seat.key.clone(),
5681 node: "gate-fix".to_owned(),
5682 at: Timestamp::now(),
5683 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5684 });
5685 record.error = Some("rate limited (quota); fixer could not run".to_owned());
5686 }
5687 AgentOutcome::Failed(e) => record.error = Some(e),
5688 }
5689 self.state.seats.insert(seat.key.clone(), seat);
5690 if let Ok(r) = git::rescue_commit(
5691 &winner.worktree,
5692 &format!("magi: gate fix {attempt} (uncommitted work)"),
5693 )
5694 .await
5695 {
5696 self.state.note_withheld("gate-fix", &r.withheld);
5697 }
5698 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5699 record.committed = after != before;
5700 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5701 let note = record.error.clone();
5702 self.state.gate_fixes.push(record);
5703 self.state.save()?;
5704 if !changed {
5705 self.state.event(
5706 "gate",
5707 match note {
5708 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5709 None => format!("gate-fix round {attempt}: the tree did not change"),
5710 },
5711 );
5712 return Ok(GateFix::Stop);
5713 }
5714 self.state.event(
5715 "gate",
5716 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5717 );
5718
5719 let commands = self.state.config.verify.e2e.clone();
5720 if !commands.is_empty() {
5721 let shell = self.state.config.shell();
5722 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5723 let cache_dir = self.state.config.cache_dir();
5724 let context = format!("gate-fix round {attempt}");
5725 let (e2e, _) = with_cache_lease(
5726 &mut self.state,
5727 cache_dir.as_deref(),
5728 "e2e",
5729 "e2e",
5730 &winner.worktree,
5731 &after,
5732 timeout,
5733 &context,
5734 |state, budget| {
5735 let shell = shell.clone();
5736 let commands = commands.clone();
5737 let context = context.clone();
5738 let worktree = winner.worktree.clone();
5739 async move {
5740 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5741 .await
5742 }
5743 },
5744 )
5745 .await;
5746 if verify_inconclusive(&e2e) {
5747 return Ok(GateFix::Defer);
5748 }
5749 if e2e.iter().any(|o| !o.ok()) {
5750 self.state.event(
5751 "gate",
5752 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5753 );
5754 return Ok(GateFix::Stop);
5755 }
5756 }
5757 Ok(GateFix::Retry)
5758 }
5759
5760 async fn merge(&mut self) -> Result<()> {
5763 if self
5778 .state
5779 .base_sync
5780 .as_ref()
5781 .is_some_and(|s| s.conflict.is_some())
5782 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5783 != Some(RunStatus::Gating)
5784 || !self.state.gate_status().ok()
5793 {
5794 return Ok(());
5795 }
5796 if self.state.merge.is_some() {
5805 return Ok(());
5806 }
5807 let Some(winner) = self.state.winner().cloned() else {
5808 return Ok(());
5809 };
5810 let repo = self.state.repo.clone();
5811 let base = self.state.base_branch.clone();
5812 let mode = self.state.config.merge.mode;
5813 let style = self.state.config.merge.style;
5814 let facts = if is_review_run(&self.state) {
5815 branch_facts(&repo, &self.state.base_commit, &winner.branch).await
5816 } else {
5817 None
5818 };
5819 let pr = pr_message_with(&self.state, winner.label, facts.as_ref());
5820 let message = pr.commit_message();
5821
5822 let outcome = match mode {
5823 MergeMode::None => MergeOutcome {
5824 mode,
5825 ok: true,
5826 detail: manual_merge_command(style, &repo, &winner.branch, &message),
5827 empty: false,
5828 },
5829 MergeMode::Pr | MergeMode::Local
5830 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
5831 {
5832 MergeOutcome {
5833 mode,
5834 ok: false,
5835 detail: empty_candidate_detail(&self.state, &base),
5836 empty: true,
5837 }
5838 }
5839 MergeMode::Local => {
5840 let on = git::current_branch(&repo).await?;
5841 if on.as_deref() != Some(base.as_str()) {
5842 MergeOutcome {
5843 mode,
5844 ok: false,
5845 detail: format!(
5846 "{} has {} checked out, not the base branch {base}",
5847 repo.display(),
5848 on.unwrap_or_else(|| "a detached HEAD".to_owned())
5849 ),
5850 empty: false,
5851 }
5852 } else if !git::is_clean(&repo).await? {
5853 MergeOutcome {
5854 mode,
5855 ok: false,
5856 detail: format!("{} is dirty; refusing to merge", repo.display()),
5857 empty: false,
5858 }
5859 } else {
5860 let out = match style {
5861 MergeStyle::Merge => {
5862 git::merge_no_ff(&repo, &winner.branch, &message).await?
5863 }
5864 MergeStyle::Squash => {
5865 git::merge_squash(&repo, &winner.branch, &message).await?
5866 }
5867 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5868 };
5869 MergeOutcome {
5870 mode,
5871 ok: out.ok(),
5872 detail: if out.ok() { out.stdout } else { out.stderr },
5873 empty: false,
5874 }
5875 }
5876 }
5877 MergeMode::Pr => {
5878 let remote = self.state.config.merge.remote.clone();
5879 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5880 if !pushed.ok() {
5881 MergeOutcome {
5882 mode,
5883 ok: false,
5884 detail: pushed.stderr,
5885 empty: false,
5886 }
5887 } else {
5888 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
5894 let out = match pr_merge_plan(found) {
5895 PrPlan::Create => {
5896 gh_pr_create(
5897 &winner.worktree,
5898 &base,
5899 &winner.branch,
5900 &pr.title,
5901 &pr.body,
5902 )
5903 .await
5904 }
5905 PrPlan::Adopt { url, title } => {
5906 self.state
5907 .event("merge", format!("Pr: adopted open pull request {url}"));
5908 if title != pr.title
5909 && let Err(e) =
5910 land::set_pr_title(&winner.worktree, &url, &pr.title).await
5911 {
5912 tracing::warn!("could not refresh title of {url}: {e:#}");
5913 self.state
5914 .event("merge", format!("Pr: title refresh failed: {e:#}"));
5915 }
5916 Ok(url)
5917 }
5918 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
5919 };
5920 match out {
5921 Ok(url) => MergeOutcome {
5922 mode,
5923 ok: true,
5924 detail: url,
5925 empty: false,
5926 },
5927 Err(e) => MergeOutcome {
5928 mode,
5929 ok: false,
5930 detail: e.to_string(),
5931 empty: false,
5932 },
5933 }
5934 }
5935 }
5936 };
5937
5938 self.state.status = match (mode, outcome.ok) {
5939 (MergeMode::None, _) => RunStatus::Ready,
5940 (_, true) => RunStatus::Merged,
5941 (_, false) => RunStatus::Blocked,
5942 };
5943 self.state.event(
5944 "merge",
5945 format!(
5946 "{:?}: {}",
5947 mode,
5948 outcome.detail.lines().next().unwrap_or("")
5949 ),
5950 );
5951 self.state.merge = Some(outcome);
5952 self.state.save()?;
5953
5954 if self.state.config.graph.land
5960 && mode == MergeMode::Pr
5961 && self.state.status == RunStatus::Merged
5962 {
5963 self.run_land().await?;
5964 }
5965 self.settle_questions();
5970 Ok(())
5971 }
5972
5973 async fn run_land(&mut self) -> Result<()> {
5984 let url = self
5985 .state
5986 .merge
5987 .as_ref()
5988 .map(|m| m.detail.clone())
5989 .unwrap_or_default();
5990 let url = url.lines().next().unwrap_or("").trim().to_owned();
5991 if !url.starts_with("http") {
5992 return Ok(());
5993 }
5994 match land::land(&mut self.state, &url).await {
5997 Ok(pr) if self.state.parked => {
5998 let _ = pr;
6002 }
6003 Ok(pr) => {
6004 self.state.status = match pr.state {
6005 land::PrLifecycle::Merged => RunStatus::Merged,
6006 _ => RunStatus::Blocked,
6007 };
6008 if bump::should_release_bump(self.state.status)
6015 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
6016 {
6017 self.state
6023 .event("bump", format!("release bump skipped: {e:#}"));
6024 }
6025 if self.state.status == RunStatus::Merged {
6028 crate::followup::after_merge(&mut self.state, &pr.url).await;
6029 }
6030 self.state.save()?;
6031 }
6032 Err(e) => {
6033 self.state.status = RunStatus::Blocked;
6034 self.state.event("land", format!("gave up: {e}"));
6035 self.state.save()?;
6036 }
6037 }
6038 Ok(())
6039 }
6040
6041 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
6045 if let Some(existing) = self.state.seats.get(key)
6046 && existing.agent == agent
6047 {
6048 return existing.clone();
6049 }
6050 let fresh = if self.state.seats.contains_key(key) {
6053 handover_seat(key, agent, self.state.next_seat_seed())
6054 } else {
6055 SeatState::new(key, agent, self.state.seed)
6056 };
6057 self.state.seats.insert(key.to_owned(), fresh.clone());
6058 fresh
6059 }
6060
6061 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
6067 match self.state.seats.get(key) {
6068 Some(s) if s.agent != spec.id => {
6069 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
6070 }
6071 _ => spec,
6072 }
6073 }
6074
6075 fn view(&self, c: &Candidate) -> CandidateView {
6077 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
6078 .unwrap_or_default();
6079 let (patch, _) = blind::sanitize_patch(
6080 &format!("candidate {} patch", c.label),
6081 &raw,
6082 &self.state.config.blind,
6083 );
6084 CandidateView {
6085 label: c.label,
6086 branch: c.branch.clone(),
6087 summary: c.summary.clone(),
6088 stat: c.stat.clone(),
6089 patch,
6090 }
6091 }
6092
6093 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
6095 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
6096 prompt::judge(
6097 "(see above)",
6098 &views,
6099 self.roles.judges.len(),
6100 base_short,
6101 "en",
6102 )
6103 }
6104
6105 fn vote_prompt_full(
6111 &self,
6112 j: usize,
6113 labels: &[char],
6114 language: &str,
6115 candidates: &[Candidate],
6116 base_short: &str,
6117 ) -> String {
6118 let mut text = format!(
6119 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
6120 prompt::final_vote(labels, language),
6121 self.state.instruction,
6122 self.candidate_block(candidates, base_short)
6123 );
6124 if let Some(own) = self
6125 .state
6126 .judgements
6127 .get(j)
6128 .filter(|r| !r.ranking.is_empty())
6129 {
6130 let reasons = own
6131 .reasons
6132 .iter()
6133 .map(|(k, v)| format!("- {k}: {v}"))
6134 .collect::<Vec<_>>()
6135 .join("\n");
6136 text.push_str(&format!(
6137 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
6138 own.ranking.iter().collect::<String>(),
6139 if reasons.is_empty() {
6140 ""
6141 } else {
6142 ", because:\n"
6143 }
6144 ));
6145 }
6146 if !self.state.deliberation.is_empty() {
6147 text.push_str("\n# What was argued before this vote\n");
6148 for t in self.transcript(&[], j) {
6149 text.push_str(&format!(
6150 "\n## {}{}\n\n{}\n",
6151 t.who,
6152 if t.is_self { " (you)" } else { "" },
6153 t.body.trim()
6154 ));
6155 }
6156 }
6157 text
6158 }
6159
6160 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6167 let mut turns = Vec::new();
6168 for j in &self.state.judgements {
6169 if j.ranking.is_empty() {
6170 continue;
6171 }
6172 let reasons = j
6173 .reasons
6174 .iter()
6175 .map(|(k, v)| format!("- {k}: {v}"))
6176 .collect::<Vec<_>>()
6177 .join("\n");
6178 turns.push(Turn {
6179 who: format!("Judge {} (opening ranking)", j.judge),
6180 is_self: j.judge == self_idx + 1,
6181 body: format!(
6182 "Ranked {}{}{reasons}",
6183 j.ranking.iter().collect::<String>(),
6184 if reasons.is_empty() {
6185 ""
6186 } else {
6187 ", because:\n"
6188 }
6189 ),
6190 });
6191 }
6192 for t in self
6193 .state
6194 .deliberation
6195 .iter()
6196 .flat_map(|r| r.turns.iter())
6197 .chain(current)
6198 {
6199 turns.push(Turn {
6200 who: format!("Judge {}", t.judge),
6201 is_self: t.judge == self_idx + 1,
6202 body: t.body.clone(),
6203 });
6204 }
6205 turns
6206 }
6207}
6208
6209fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6211 agent::has_session(spec.kind, seat, sessions)
6212}
6213
6214fn next_untried_in_roster<'a>(
6235 roster: &'a [AgentSpec],
6236 start: usize,
6237 tried: &BTreeSet<String>,
6238) -> Option<&'a AgentSpec> {
6239 roster
6240 .get(start + 1..)?
6241 .iter()
6242 .find(|s| !tried.contains(&s.id))
6243}
6244
6245fn next_for_seat<'a>(
6255 roster: &'a [AgentSpec],
6256 start: usize,
6257 round_tried: &BTreeSet<String>,
6258 carried_failed: &BTreeSet<String>,
6259) -> Option<&'a AgentSpec> {
6260 roster
6261 .get(start + 1..)?
6262 .iter()
6263 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
6264 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
6265}
6266
6267fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
6273 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
6274 return spec;
6275 };
6276 let ok = |id: &str| !h.failed.contains(id);
6277 if let Some(last) = h.last_ok.as_deref()
6278 && ok(last)
6279 && let Some(s) = roster.iter().find(|s| s.id == last)
6280 {
6281 return s.clone();
6282 }
6283 if ok(&spec.id) {
6284 return spec;
6285 }
6286 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
6287 roster
6288 .get(from + 1..)
6289 .into_iter()
6290 .flatten()
6291 .chain(roster.iter())
6292 .find(|s| ok(&s.id))
6293 .cloned()
6294 .unwrap_or(spec)
6295}
6296
6297fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6301 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6302}
6303
6304const TIMED_OUT: &str = "timed out";
6307
6308impl FailClass {
6309 fn of(out: &AgentOutcome) -> Option<Self> {
6311 match out {
6312 AgentOutcome::Ok(_) => None,
6313 AgentOutcome::Quota(_) => Some(Self::Quota),
6314 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6315 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6316 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6317 }
6318 }
6319
6320 fn stem_word(&self) -> &'static str {
6322 match self {
6323 Self::Quota => "quota",
6324 _ => "handover",
6325 }
6326 }
6327}
6328
6329fn failure_signature(msg: &str) -> String {
6333 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6334 let mut out = Vec::new();
6335 for word in line.split_whitespace() {
6336 if word.contains('/') || word.contains('\\') {
6337 out.push("<path>".to_owned());
6338 continue;
6339 }
6340 let mut w = String::new();
6341 let mut in_digits = false;
6342 for c in word.chars() {
6343 if c.is_ascii_digit() {
6344 if !in_digits {
6345 w.push('#');
6346 }
6347 in_digits = true;
6348 } else {
6349 in_digits = false;
6350 w.push(c);
6351 }
6352 }
6353 out.push(w);
6354 }
6355 out.join(" ").chars().take(120).collect()
6356}
6357
6358fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6365 match cur {
6366 FailClass::Quota | FailClass::Timeout => true,
6367 FailClass::Other(_) => prev != Some(cur),
6368 }
6369}
6370
6371fn fail_reason(out: &AgentOutcome) -> String {
6373 match out {
6374 AgentOutcome::Ok(_) => String::new(),
6375 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6376 AgentOutcome::Dropped(o) => format!(
6377 "the CLI dropped the stream ({})",
6378 o.dropped
6379 .as_ref()
6380 .map(|d| d.why.as_str())
6381 .unwrap_or("it ended without delivering its answer")
6382 ),
6383 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6384 }
6385}
6386
6387fn record_handover(
6390 state: &mut RunState,
6391 node: &str,
6392 seat: &str,
6393 from: &str,
6394 to: &str,
6395 class: &FailClass,
6396 reason: &str,
6397) {
6398 let message = if *class == FailClass::Quota {
6399 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6400 } else {
6401 format!("{seat}: handed over {from} -> {to} ({reason})")
6402 };
6403 state.event(node, message);
6404 state.handovers.push(Handover {
6405 at: Timestamp::now(),
6406 node: node.to_owned(),
6407 seat: seat.to_owned(),
6408 from: from.to_owned(),
6409 to: to.to_owned(),
6410 reason: reason.to_owned(),
6411 });
6412}
6413
6414fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6428 commands.iter().any(|c| c.exit_code.is_none())
6429}
6430
6431fn verified_noop_claim(
6444 usable: bool,
6445 commands: &[agent::CommandEvidence],
6446 text: &str,
6447) -> Option<String> {
6448 (usable && !has_unconfirmed_command(commands))
6449 .then(|| verdict::verified_noop(text))
6450 .flatten()
6451}
6452
6453fn short(commit: &str) -> String {
6454 commit.chars().take(7).collect()
6455}
6456
6457fn make_executable(path: &Path) -> Result<()> {
6458 #[cfg(unix)]
6459 {
6460 use std::os::unix::fs::PermissionsExt as _;
6461 let mut perms = std::fs::metadata(path)?.permissions();
6462 perms.set_mode(0o755);
6463 std::fs::set_permissions(path, perms)?;
6464 }
6465 #[cfg(not(unix))]
6466 {
6467 let _ = path;
6468 }
6469 Ok(())
6470}
6471
6472struct WaveCtx<'a> {
6479 run: &'a str,
6482 node: &'a str,
6484 prompts: &'a Prompts,
6485 cache: Option<&'a Path>,
6487 round: Option<usize>,
6490 carry_seats: bool,
6494}
6495
6496async fn run_one(
6498 job: SeatJob,
6499 sem: Arc<Semaphore>,
6500 ctx: &WaveCtx<'_>,
6501 state: &mut RunState,
6502 attempt: usize,
6503) -> (SeatState, AgentOutcome) {
6504 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6505 .await
6506 .pop()
6507 .expect("one job in, one result out");
6508 (seat, out)
6509}
6510
6511async fn wave(
6517 jobs: Vec<SeatJob>,
6518 sem: Arc<Semaphore>,
6519 ctx: &WaveCtx<'_>,
6520 state: &mut RunState,
6521 attempt: usize,
6522) -> Vec<(usize, SeatState, AgentOutcome)> {
6523 let WaveCtx {
6524 run,
6525 node,
6526 prompts,
6527 cache,
6528 round,
6529 carry_seats: _,
6530 } = *ctx;
6531 for job in &jobs {
6532 state.seat_started(node, &job.seat.key, job.timeout, attempt);
6533 }
6534 if let Err(e) = state.save() {
6535 tracing::warn!("could not persist in-progress seats: {e:#}");
6540 }
6541 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6557 let wait_started = Instant::now();
6558 let cache_guard = if let Some(cache_dir) = cache {
6559 if jobs_had_a_writer {
6560 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6561 let budget = jobs
6562 .iter()
6563 .map(|j| j.timeout)
6564 .max()
6565 .unwrap_or(Duration::from_secs(60));
6566 acquire_cache_lease(state, cache_dir, &owner, budget, node)
6567 .await
6568 .ok()
6569 } else {
6570 None
6571 }
6572 } else {
6573 None
6574 };
6575 let waited_for_lease = wait_started.elapsed();
6582 let mut set = tokio::task::JoinSet::new();
6583 let overlay = prompts.overlay(node);
6584 for (i, mut job) in jobs.into_iter().enumerate() {
6585 job.timeout = job.timeout.saturating_sub(waited_for_lease);
6586 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6587 if cache.is_some() {
6588 job.prompt.push('\n');
6589 job.prompt
6590 .push_str(&prompt::build_cache_note(node, job.allow_write));
6591 }
6592 let sem = Arc::clone(&sem);
6593 let run = run.to_owned();
6594 let node = node.to_owned();
6595 let attachments = if node == "implement" {
6598 state.attachments.clone()
6599 } else {
6600 Vec::new()
6601 };
6602 let cache = cache
6613 .filter(|_| job.allow_write && cache_guard.is_some())
6614 .map(Path::to_path_buf);
6615 set.spawn(async move {
6616 let _permit = sem.acquire().await;
6617 let mut seat = job.seat;
6618 let out = agent::invoke(
6619 &job.spec,
6620 &mut seat,
6621 &Invocation {
6622 cwd: &job.cwd,
6623 prompt: &job.prompt,
6624 timeout: job.timeout,
6625 allow_write: job.allow_write,
6626 sessions: job.sessions,
6627 artifacts: &job.artifacts,
6628 stem: &job.stem,
6629 run: &run,
6630 node: &node,
6631 cache_dir: cache.as_deref(),
6632 attachments: &attachments,
6633 writable: &[],
6634 },
6635 )
6636 .await;
6637 let out = match out {
6638 Ok(o) if o.usable() => AgentOutcome::Ok(o),
6639 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
6640 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
6648 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
6649 Ok(o) => AgentOutcome::Failed(format!(
6650 "exited with {:?} and no usable output",
6651 o.exit_code
6652 )),
6653 Err(e) => AgentOutcome::Failed(e.to_string()),
6654 };
6655 (i, seat, out)
6656 });
6657 }
6658 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
6659 while let Some(joined) = set.join_next().await {
6660 let (i, seat, out) = match joined {
6661 Ok(v) => v,
6662 Err(e) => {
6666 tracing::error!("agent task panicked: {e}");
6667 continue;
6668 }
6669 };
6670 state.seat_finished(&seat.key);
6671 record_jobs(state, node, round, &seat.key, &out);
6672 if let Err(e) = state.save() {
6673 tracing::warn!("could not persist a seat's completion: {e:#}");
6674 }
6675 if collected.len() <= i {
6676 collected.resize_with(i + 1, || None);
6677 }
6678 collected[i] = Some((i, seat, out));
6679 }
6680 if state
6686 .active
6687 .values()
6688 .any(|a| a.node == node && a.attempt == attempt)
6689 {
6690 state
6691 .active
6692 .retain(|_, a| !(a.node == node && a.attempt == attempt));
6693 if let Err(e) = state.save() {
6694 tracing::warn!("could not persist the end of a wave: {e:#}");
6695 }
6696 }
6697 if let Some(cache_dir) = cache
6704 && jobs_had_a_writer
6705 {
6706 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
6707 }
6708 if let Some(guard) = cache_guard {
6709 guard.release();
6710 }
6711 collected.into_iter().flatten().collect()
6712}
6713
6714fn record_jobs(
6725 state: &mut RunState,
6726 node: &str,
6727 round: Option<usize>,
6728 seat: &str,
6729 out: &AgentOutcome,
6730) {
6731 let commands: &[agent::CommandEvidence] = match out {
6732 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
6733 AgentOutcome::Failed(_) => &[],
6734 };
6735 let checked_at = Timestamp::now();
6736 for c in commands {
6737 state.jobs.push(JobRecord {
6738 node: node.to_owned(),
6739 round,
6740 seat: seat.to_owned(),
6741 id: c.id.clone(),
6742 description: c.description.clone(),
6743 checked_at,
6744 status: match c.exit_code {
6745 Some(0) => JobStatus::Completed,
6746 Some(_) => JobStatus::Failed,
6747 None => JobStatus::Unknown,
6748 },
6749 exit_code: c.exit_code,
6750 result_summary: c.result_summary.clone(),
6751 source: c.source.clone(),
6752 });
6753 }
6754}
6755
6756fn round_is_clean(
6777 blocking: usize,
6778 e2e_ok: bool,
6779 answered: usize,
6780 expected: usize,
6781 quota_missing: usize,
6782 policy: IncompleteReviewPolicy,
6783) -> bool {
6784 if blocking != 0 || !e2e_ok {
6785 return false;
6786 }
6787 if answered == expected || policy == IncompleteReviewPolicy::Warn {
6788 return true;
6789 }
6790 answered > 0 && expected - answered <= quota_missing
6791}
6792
6793fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6817 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6818 return Some(RunStatus::Gating);
6819 }
6820 let last = reviews.last()?;
6821 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6822 if reviews.len() < max_rounds && !stagnant {
6823 return None;
6824 }
6825 if last.incomplete() && last.blocking == 0 {
6826 return Some(RunStatus::Blocked);
6827 }
6828 if last.e2e_status() == E2eStatus::ResourceBlocked {
6829 return None;
6830 }
6831 Some(if last.e2e.iter().all(CommandOutcome::ok) {
6832 RunStatus::Gating
6833 } else {
6834 RunStatus::Blocked
6835 })
6836}
6837
6838fn retry_budget(full: Duration, nudged: bool) -> Duration {
6853 if nudged {
6854 (full / 4).max(Duration::from_secs(120)).min(full)
6855 } else {
6856 full
6857 }
6858}
6859
6860#[allow(clippy::too_many_arguments)]
6889async fn ask_json_wave<T>(
6890 jobs: Vec<SeatJob>,
6891 sem: Arc<Semaphore>,
6892 retries: usize,
6893 roster: &[AgentSpec],
6894 ctx: &WaveCtx<'_>,
6895 losses: &mut Vec<QuotaLoss>,
6896 state: &mut RunState,
6897 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
6898) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6899where
6900 T: serde::de::DeserializeOwned + Send + 'static,
6901{
6902 ask_wave_with(
6903 jobs,
6904 sem,
6905 retries,
6906 roster,
6907 ctx,
6908 losses,
6909 state,
6910 &|text: &str| {
6911 let v = verdict::extract_json::<T>(text)?;
6912 validate(&v)?;
6913 Ok(v)
6914 },
6915 )
6916 .await
6917}
6918
6919#[allow(clippy::too_many_arguments)]
6928async fn ask_wave_with<T>(
6929 jobs: Vec<SeatJob>,
6930 sem: Arc<Semaphore>,
6931 retries: usize,
6932 roster: &[AgentSpec],
6933 ctx: &WaveCtx<'_>,
6934 losses: &mut Vec<QuotaLoss>,
6935 state: &mut RunState,
6936 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
6937) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6938where
6939 T: Send + 'static,
6940{
6941 let n = jobs.len();
6942 let originals: Vec<SeatJob> = jobs;
6943 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
6944 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
6945 let mut nudges: Vec<usize> = vec![0; n];
6952 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
6956 let mut tried: Vec<BTreeSet<String>> = specs
6957 .iter()
6958 .map(|s| BTreeSet::from([s.id.clone()]))
6959 .collect();
6960 let starts: Vec<usize> = specs
6961 .iter()
6962 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
6963 .collect();
6964 let carry = ctx.carry_seats && !roster.is_empty();
6965 let carried: Vec<BTreeSet<String>> = originals
6968 .iter()
6969 .map(|j| {
6970 state
6971 .seat_history
6972 .get(&j.seat.key)
6973 .filter(|_| carry)
6974 .map(|h| h.failed.clone())
6975 .unwrap_or_default()
6976 })
6977 .collect();
6978 let mut prev: Vec<Option<FailClass>> = originals
6979 .iter()
6980 .map(|j| {
6981 state
6982 .seat_history
6983 .get(&j.seat.key)
6984 .filter(|_| carry)
6985 .and_then(|h| h.last_fail.clone())
6986 })
6987 .collect();
6988 let next_agent = |i: usize, tried: &BTreeSet<String>| -> Option<AgentSpec> {
6989 if carry {
6990 next_for_seat(roster, starts[i], tried, &carried[i]).cloned()
6991 } else {
6992 next_untried_in_roster(roster, starts[i], tried).cloned()
6993 }
6994 };
6995 let mut fresh: Vec<Option<String>> = vec![None; n];
6996 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
6997 let mut pending: Vec<usize> = (0..n).collect();
6998
6999 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
7004 for round in 0..max_rounds {
7005 if pending.is_empty() {
7006 break;
7007 }
7008 let mut batch = Vec::with_capacity(pending.len());
7009 let mut renudged: Vec<&str> = Vec::new();
7010 for &i in &pending {
7011 let src = &originals[i];
7012 let full: &str = match &src.handover {
7017 Some(h) if specs[i].id != src.spec.id => h,
7018 _ => &src.prompt,
7019 };
7020 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
7023 (
7024 full.to_owned(),
7025 src.timeout,
7026 format!("{}-{word}-{}", src.stem, specs[i].id),
7027 )
7028 } else if nudges[i] == 0 {
7029 (full.to_owned(), src.timeout, src.stem.clone())
7030 } else {
7031 renudged.push(src.seat.key.as_str());
7032 let why = done[i]
7033 .as_ref()
7034 .and_then(|r| r.as_ref().err().map(ToString::to_string))
7035 .unwrap_or_else(|| "no parsable answer".to_owned());
7036 let nudge = prompt::nudge(&why);
7037 let nudged = has_context(&specs[i], &seats[i], src.sessions);
7038 let prompt = if nudged {
7039 nudge
7040 } else {
7041 format!("{full}\n\n---\n\n{nudge}")
7042 };
7043 (
7044 prompt,
7045 retry_budget(src.timeout, nudged),
7046 format!("{}-retry{}", src.stem, nudges[i]),
7047 )
7048 };
7049 batch.push(SeatJob {
7050 spec: specs[i].clone(),
7051 seat: seats[i].clone(),
7052 cwd: src.cwd.clone(),
7053 prompt,
7054 timeout,
7055 allow_write: src.allow_write,
7056 sessions: src.sessions,
7057 artifacts: src.artifacts.clone(),
7058 stem,
7059 handover: None,
7060 });
7061 }
7062
7063 if !renudged.is_empty() {
7064 state.event(
7065 ctx.node,
7066 format!("retry {round}: re-asking {}", renudged.join(", ")),
7067 );
7068 }
7069 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
7070 let mut still = Vec::new();
7071 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
7072 seats[i] = seat;
7073 let class = FailClass::of(&out);
7074 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
7077 && nudges[i] < retries
7078 && !roster.is_empty();
7079 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
7080 let next = should_hand_over(prev[i].as_ref(), &cur)
7081 .then(|| next_agent(i, &tried[i]))
7082 .flatten();
7083 if carry {
7084 let h = state
7085 .seat_history
7086 .entry(originals[i].seat.key.clone())
7087 .or_default();
7088 h.failed.insert(specs[i].id.clone());
7089 h.last_fail = Some(cur.clone());
7090 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
7091 h.last_ok = None;
7092 }
7093 if let Err(e) = state.save() {
7096 tracing::warn!("could not persist a seat's failure history: {e:#}");
7097 }
7098 }
7099 if let Some(next) = next {
7100 record_handover(
7101 state,
7102 ctx.node,
7103 &originals[i].seat.key,
7104 &specs[i].id,
7105 &next.id,
7106 &cur,
7107 &fail_reason(&out),
7108 );
7109 tried[i].insert(next.id.clone());
7110 prev[i] = Some(cur.clone());
7111 seats[i] =
7112 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
7113 specs[i] = next;
7114 fresh[i] = Some(cur.stem_word().to_owned());
7115 nudges[i] = 0;
7116 done[i] = Some(Err(anyhow::anyhow!(
7117 "handed over after: {}",
7118 fail_reason(&out)
7119 )));
7120 still.push(i);
7121 continue;
7122 }
7123 }
7124 if carry
7125 && !nudge_first
7126 && let Some(cur) = class.clone()
7127 {
7128 let h = state
7130 .seat_history
7131 .entry(originals[i].seat.key.clone())
7132 .or_default();
7133 h.failed.insert(specs[i].id.clone());
7134 h.last_fail = Some(cur);
7135 }
7136 let parsed = match out {
7137 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
7138 AgentOutcome::Quota(o) => {
7139 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
7140 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
7141 }
7142 AgentOutcome::Dropped(o) => {
7146 let why = o
7147 .dropped
7148 .as_ref()
7149 .map(|d| d.why.as_str())
7150 .unwrap_or("the CLI ended the stream without delivering its answer");
7151 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
7152 }
7153 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
7154 };
7155 let quota = class == Some(FailClass::Quota);
7156 let failed = parsed.is_err();
7157 done[i] = Some(parsed);
7158 if carry && !failed {
7159 let h = state
7160 .seat_history
7161 .entry(originals[i].seat.key.clone())
7162 .or_default();
7163 h.failed.remove(&specs[i].id);
7164 h.last_ok = Some(specs[i].id.clone());
7165 h.last_fail = None;
7166 }
7167 let agent_failure = class.is_some()
7175 && !nudge_first
7176 && !roster.is_empty()
7177 && next_agent(i, &tried[i]).is_some();
7178 if failed && !quota && !agent_failure && nudges[i] < retries {
7179 nudges[i] += 1;
7180 still.push(i);
7181 }
7182 }
7183 pending = still;
7184 }
7185
7186 for (i, q) in last_quota.into_iter().enumerate() {
7189 if let Some(reset) = q {
7190 losses.push(QuotaLoss {
7191 seat: originals[i].seat.key.clone(),
7192 node: ctx.node.to_owned(),
7193 at: Timestamp::now(),
7194 reset,
7195 });
7196 }
7197 }
7198
7199 seats
7200 .into_iter()
7201 .zip(done)
7202 .zip(nudges)
7203 .map(|((seat, res), attempts)| {
7204 (
7205 seat,
7206 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
7207 attempts,
7208 )
7209 })
7210 .collect()
7211}
7212
7213async fn acquire_cache_lease(
7226 state: &mut RunState,
7227 cache_dir: &Path,
7228 owner: &crate::cache::Owner,
7229 budget: Duration,
7230 context: &str,
7231) -> Result<crate::cache::Guard> {
7232 let home = crate::run::home();
7233 let started = Instant::now();
7234 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
7235 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
7236 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
7237 Err(e) => {
7238 state.event(
7239 "verify",
7240 format!("{context}: could not check the shared build cache: {e:#}"),
7241 );
7242 if let Err(e2) = state.save() {
7243 tracing::warn!("could not persist a cache-check failure: {e2:#}");
7244 }
7245 return Err(e);
7246 }
7247 };
7248 state.event(
7249 "verify",
7250 format!(
7251 "{context}: waiting for the shared build cache at {} ({})",
7252 cache_dir.display(),
7253 busy.describe()
7254 ),
7255 );
7256 if let Err(e) = state.save() {
7257 tracing::warn!("could not persist a cache wait: {e:#}");
7258 }
7259 let remaining = budget.saturating_sub(started.elapsed());
7260 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
7261 Ok(g) => Ok(g),
7262 Err(e) => {
7263 state.event("verify", format!("{context}: {e:#}"));
7264 if let Err(e2) = state.save() {
7265 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
7266 }
7267 Err(e)
7268 }
7269 }
7270}
7271
7272#[allow(clippy::too_many_arguments)]
7293async fn with_cache_lease<'s, F, Fut>(
7294 state: &'s mut RunState,
7295 cache_dir: Option<&Path>,
7296 node: &str,
7297 seat: &str,
7298 worktree: &Path,
7299 head: &str,
7300 budget: Duration,
7301 context: &str,
7302 body: F,
7303) -> (Vec<CommandOutcome>, bool)
7304where
7305 F: FnOnce(&'s mut RunState, Duration) -> Fut,
7306 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
7307{
7308 let Some(cache_dir) = cache_dir else {
7309 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
7310 return (outcomes, retried);
7311 };
7312 let home = crate::run::home();
7313 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
7314 let started = Instant::now();
7315 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
7316 Ok(g) => g,
7317 Err(e) => {
7318 return (
7319 vec![CommandOutcome {
7320 command: "(waiting for the shared build cache)".to_owned(),
7321 code: None,
7322 output_tail: e.to_string(),
7323 duration_ms: started.elapsed().as_millis() as u64,
7324 resource_blocked: true,
7325 }],
7326 false,
7327 );
7328 }
7329 };
7330 let identity = crate::cache::Identity::new(worktree, head);
7331 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7332 state.event(
7340 "verify",
7341 format!(
7342 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7343 worktree.display(),
7344 short(head)
7345 ),
7346 );
7347 guard.release();
7348 return (
7349 vec![CommandOutcome {
7350 command: "(confirming the shared build cache is fresh)".to_owned(),
7351 code: None,
7352 output_tail: e.to_string(),
7353 duration_ms: started.elapsed().as_millis() as u64,
7354 resource_blocked: true,
7355 }],
7356 false,
7357 );
7358 }
7359 let remaining = budget.saturating_sub(started.elapsed());
7360 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7361 if !timed_out_pids.is_empty() {
7366 wait_for_timed_out_children_to_die(&timed_out_pids).await;
7367 }
7368 guard.release();
7369 (outcomes, retried)
7370}
7371
7372async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7384 wait_for_pids_with(
7385 pids,
7386 crate::proc::pid_alive,
7387 LEASE_RELEASE_POLL,
7388 LEASE_RELEASE_MAX_WAIT,
7389 )
7390 .await;
7391}
7392
7393async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7399 pids: &[u32],
7400 alive: F,
7401 poll: Duration,
7402 max_wait: Duration,
7403) {
7404 let deadline = Instant::now() + max_wait;
7405 loop {
7406 if pids.iter().all(|&pid| !alive(pid)) {
7407 return;
7408 }
7409 if Instant::now() >= deadline {
7410 return;
7411 }
7412 tokio::time::sleep(poll).await;
7413 }
7414}
7415
7416fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7423 outcomes.iter().any(|o| o.resource_blocked)
7424}
7425
7426enum GateFix {
7428 Retry,
7430 Stop,
7433 Defer,
7436}
7437
7438fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7446 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7447 red.peek().is_some()
7448 && red.all(|o| {
7449 !o.resource_blocked
7450 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7451 && !o.output_tail.trim().is_empty()
7452 })
7453}
7454
7455fn e2e_outcome_label(o: &CommandOutcome) -> String {
7459 if o.ok() {
7460 return "pass".to_owned();
7461 }
7462 let reason = if o.build_failed() {
7463 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7464 } else {
7465 format!("FAIL ({:?})", o.code)
7466 };
7467 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7468}
7469
7470async fn run_e2e_with_retry(
7478 state: &mut RunState,
7479 shell: &[String],
7480 commands: &[String],
7481 worktree: &Path,
7482 timeout: Duration,
7483 context: &str,
7484) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7485 let (mut e2e, mut timed_out_pids) = run_commands(
7486 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7487 )
7488 .await;
7489 for o in &e2e {
7490 state.event(
7491 "verify",
7492 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7493 );
7494 }
7495 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7499 if verify_retried {
7500 state.event(
7501 "verify",
7502 format!(
7503 "{context}: verify could not build/link, not a test result — retrying once \
7504 before concluding"
7505 ),
7506 );
7507 let retried = run_commands(
7508 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7509 )
7510 .await;
7511 e2e = retried.0;
7512 timed_out_pids.extend(retried.1);
7515 for o in &e2e {
7516 state.event(
7517 "verify",
7518 format!(
7519 "{context}: retry `{}` -> {}",
7520 o.command,
7521 e2e_outcome_label(o)
7522 ),
7523 );
7524 }
7525 }
7526 (e2e, verify_retried, timed_out_pids)
7527}
7528
7529#[allow(clippy::too_many_arguments)]
7545async fn run_commands(
7546 state: &mut RunState,
7547 node: &str,
7548 task: &str,
7549 attempt: usize,
7550 shell: &[String],
7551 commands: &[String],
7552 cwd: &Path,
7553 timeout: Duration,
7554) -> (Vec<CommandOutcome>, Vec<u32>) {
7555 if commands.is_empty() {
7556 return (Vec::new(), Vec::new());
7561 }
7562 let mut out = Vec::new();
7563 let mut timed_out_pids = Vec::new();
7564 let total = commands.len();
7565 for (idx, command) in commands.iter().enumerate() {
7566 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7567 if let Err(e) = state.save() {
7568 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7569 }
7570 let started = Instant::now();
7571 let mut cmd = tokio::process::Command::new(&shell[0]);
7572 cmd.quiet();
7573 cmd.args(&shell[1..])
7574 .arg(command)
7575 .current_dir(cwd)
7576 .stdin(std::process::Stdio::null())
7577 .stdout(std::process::Stdio::piped())
7578 .stderr(std::process::Stdio::piped())
7579 .kill_on_drop(true);
7580 let spawned = cmd.spawn();
7581 let (code, body) = match spawned {
7582 Ok(child) => {
7583 let pid = child.id();
7588 match tokio::time::timeout(timeout, child.wait_with_output()).await {
7589 Ok(Ok(o)) => {
7590 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7591 body.push_str(&String::from_utf8_lossy(&o.stderr));
7592 (o.status.code(), body)
7593 }
7594 Ok(Err(e)) => (None, format!("failed to run: {e}")),
7595 Err(_) => {
7596 if let Some(pid) = pid {
7597 timed_out_pids.push(pid);
7598 }
7599 (None, format!("timed out after {}s", timeout.as_secs()))
7600 }
7601 }
7602 }
7603 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7604 };
7605 out.push(CommandOutcome {
7606 command: command.clone(),
7607 code,
7608 output_tail: tail(&body, OUTPUT_TAIL),
7609 duration_ms: started.elapsed().as_millis() as u64,
7610 resource_blocked: false,
7611 });
7612 }
7613 state.task_finished(task);
7614 if let Err(e) = state.save() {
7615 tracing::warn!("could not persist the end of {task}: {e:#}");
7616 }
7617 (out, timed_out_pids)
7618}
7619
7620fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
7632 let repo = repo.display();
7633 match style {
7634 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
7635 MergeStyle::Squash => {
7636 let subject = message
7639 .lines()
7640 .next()
7641 .unwrap_or(branch)
7642 .replace(['\\', '"', '$', '`'], "");
7643 format!(
7644 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
7645 )
7646 }
7647 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
7648 }
7649}
7650
7651const PR_TITLE_MAX: usize = 240;
7662
7663fn english_title(text: &str, max: usize) -> Option<String> {
7674 let line = queue::first_line(text)?;
7675 let chars: Vec<char> = line.chars().collect();
7676 let (kept, mark) = if chars.len() <= max {
7677 (line.to_owned(), "")
7678 } else if let Some(end) = sentence_end(&chars, max) {
7679 (chars[..end].iter().collect::<String>(), "")
7680 } else {
7681 let room = max.saturating_sub(3);
7682 let cut = if chars[room].is_whitespace() {
7685 room
7686 } else {
7687 chars[..room]
7688 .iter()
7689 .rposition(|c| c.is_whitespace())
7690 .unwrap_or(room)
7691 };
7692 let head: String = chars[..cut].iter().collect();
7693 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
7694 (head.to_owned(), "...")
7695 };
7696 if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
7697 return None;
7698 }
7699 Some(format!("{kept}{mark}"))
7700}
7701
7702fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
7706 const MIN: usize = 20;
7707 for i in MIN..max.min(chars.len()) {
7708 if !matches!(chars[i], '.' | '!' | '?') {
7709 continue;
7710 }
7711 let Some(&next) = chars.get(i + 1) else {
7712 continue;
7713 };
7714 if !next.is_whitespace() {
7715 continue;
7716 }
7717 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
7718 if after.is_some_and(|c| c.is_ascii_lowercase()) {
7719 continue;
7720 }
7721 let word: String = chars[..i]
7722 .iter()
7723 .rev()
7724 .take_while(|c| !c.is_whitespace())
7725 .collect::<Vec<_>>()
7726 .into_iter()
7727 .rev()
7728 .collect();
7729 let word = word.to_ascii_lowercase();
7730 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
7731 continue;
7732 }
7733 let end = chars[..i]
7734 .iter()
7735 .rposition(|c| !c.is_whitespace())
7736 .map_or(i, |p| p + 1);
7737 return Some(end);
7738 }
7739 None
7740}
7741
7742struct PrMessage {
7747 title: String,
7748 body: String,
7749}
7750
7751impl PrMessage {
7752 fn commit_message(&self) -> String {
7756 format!("{}\n\n{}", self.title, self.body)
7757 }
7758}
7759
7760fn title_marker(line: &str) -> Option<&str> {
7762 let line = line.trim();
7763 let head = line.get(..6)?;
7764 head.eq_ignore_ascii_case("title:")
7765 .then(|| line[6..].trim())
7766}
7767
7768fn summary_title(summary: &str) -> Option<String> {
7773 let first = summary.lines().find(|l| !l.trim().is_empty())?;
7774 let raw = title_marker(first)?;
7775 if raw.is_empty() {
7776 return None;
7777 }
7778 let title = queue::title_from(raw, PR_TITLE_MAX);
7779 let lower = title.to_ascii_lowercase();
7780 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7781 return None;
7782 }
7783 Some(title)
7784}
7785
7786const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
7788
7789const EXISTING_BRANCH: &str = "(existing branch)";
7791
7792fn is_review_run(state: &RunState) -> bool {
7796 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
7797}
7798
7799fn review_title(state: &RunState) -> Option<String> {
7804 english_subject(state.reviewed_commits.as_ref()?.first()?)
7805}
7806
7807fn english_subject(raw: &str) -> Option<String> {
7810 let raw = raw.trim();
7811 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
7812 return None;
7813 }
7814 let title = queue::title_from(raw, PR_TITLE_MAX);
7815 let lower = title.to_ascii_lowercase();
7816 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7817 return None;
7818 }
7819 Some(title)
7820}
7821
7822#[derive(Debug, Clone, PartialEq, Eq)]
7825struct BranchFacts {
7826 commits: Vec<(String, String)>,
7828 stat: String,
7830}
7831
7832const STAT_FILE_LINES: usize = 25;
7834const COMMITS_MAX_CHARS: usize = 20_000;
7836
7837async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
7840 let commits = git::commit_log(repo, base, branch).await.ok()?;
7841 if commits.is_empty() {
7842 return None;
7843 }
7844 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
7845 let lines: Vec<&str> = stat.lines().collect();
7846 let stat = if lines.len() > STAT_FILE_LINES + 1 {
7847 let omitted = lines.len() - 1 - STAT_FILE_LINES;
7848 let more = format!(" ... {omitted} more file(s)");
7849 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
7850 kept.push(&more);
7851 kept.push(lines[lines.len() - 1]);
7852 kept.join("\n")
7853 } else {
7854 lines.join("\n")
7855 };
7856 Some(BranchFacts { commits, stat })
7857}
7858
7859fn markdown_safe(text: &str) -> String {
7862 text.replace("</details>", "</details>")
7863 .replace("\x60\x60\x60", "~~~")
7864}
7865
7866fn neutral_title(state: &RunState, winner: char) -> String {
7867 format!(
7868 "chore: land candidate {} of run {}",
7869 winner.to_ascii_uppercase(),
7870 state.id
7871 )
7872}
7873
7874pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
7880 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
7881 ""
7882 } else {
7883 pr_title
7884 }
7885}
7886
7887pub fn landing_subject_source(state: &RunState) -> String {
7891 if is_review_run(state) {
7892 let winner = state.candidates.first().map_or('A', |c| c.label);
7893 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
7894 }
7895 state.instruction.clone()
7896}
7897
7898fn summary_without_title(summary: &str) -> String {
7901 let mut lines = summary.trim().lines().peekable();
7902 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
7903 lines.next();
7904 }
7905 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
7906}
7907
7908#[cfg(test)]
7922fn pr_message(state: &RunState, winner: char) -> PrMessage {
7923 pr_message_with(state, winner, None)
7924}
7925
7926fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
7929 let summary = state
7930 .candidates
7931 .iter()
7932 .find(|c| c.label == winner)
7933 .map(|c| c.summary.as_str())
7934 .unwrap_or_default();
7935 let review = is_review_run(state);
7938 let title = if review {
7939 facts
7940 .and_then(|f| english_subject(&f.commits.first()?.0))
7941 .or_else(|| review_title(state))
7942 .or_else(|| {
7943 state
7944 .candidates
7945 .iter()
7946 .find(|c| c.label == winner)
7947 .filter(|c| !c.branch.starts_with("magi/"))
7948 .and_then(|c| english_subject(&c.branch))
7949 })
7950 .unwrap_or_else(|| neutral_title(state, winner))
7951 } else {
7952 summary_title(summary).unwrap_or_else(|| {
7953 english_title(&state.instruction, PR_TITLE_MAX)
7954 .unwrap_or_else(|| neutral_title(state, winner))
7955 })
7956 };
7957
7958 let mut body = String::new();
7959 let what = summary_without_title(summary);
7960 if !what.is_empty() {
7961 body.push_str("## Summary\n\n");
7962 body.push_str(&what);
7963 body.push_str("\n\n");
7964 }
7965
7966 let notes: Vec<(usize, &str)> = state
7969 .reviews
7970 .iter()
7971 .filter_map(|r| {
7972 let n = r.fix.as_ref()?.notes.trim();
7973 (!n.is_empty()).then_some((r.round, n))
7974 })
7975 .collect();
7976 if !notes.is_empty() {
7977 body.push_str("## Review fixes\n\n");
7978 if let [(_, only)] = notes.as_slice() {
7979 body.push_str(only);
7980 body.push_str("\n\n");
7981 } else {
7982 for (round, n) in ¬es {
7983 body.push_str(&format!("### Round {round}\n\n{n}\n\n"));
7984 }
7985 }
7986 }
7987 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
7988
7989 let open = state.open_findings();
7990 if !open.is_empty() {
7991 body.push_str("## Open review findings\n\n");
7992 for f in &open {
7993 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
7994 }
7995 body.push('\n');
7996 }
7997
7998 if let Some(fix) = fix
7999 && !fix.rejected.is_empty()
8000 {
8001 body.push_str("## Declined by the fixer\n\n");
8002 for r in &fix.rejected {
8003 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
8004 }
8005 body.push('\n');
8006 }
8007
8008 if review {
8009 body.push_str("## Commits under review\n\n");
8011 if let Some(facts) = facts {
8012 let mut left = COMMITS_MAX_CHARS;
8013 for (i, (subject, text)) in facts.commits.iter().enumerate() {
8014 let mut entry = format!("- {}\n", markdown_safe(subject));
8015 for l in markdown_safe(text).lines() {
8016 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
8017 }
8018 if left == 0 {
8019 body.push_str(&format!(
8020 "- ... {} more commit(s)\n",
8021 facts.commits.len() - i
8022 ));
8023 break;
8024 }
8025 if entry.len() > left {
8026 let mut end = left;
8029 while !entry.is_char_boundary(end) {
8030 end -= 1;
8031 }
8032 entry.truncate(end);
8033 entry.push_str("\n ... (truncated)\n");
8034 left = 0;
8035 } else {
8036 left -= entry.len();
8037 }
8038 body.push_str(&entry);
8039 }
8040 if !facts.stat.trim().is_empty() {
8041 body.push_str(&format!(
8042 "\n## Diff stat\n\n```\n{}\n```\n",
8043 markdown_safe(facts.stat.trim())
8044 ));
8045 }
8046 } else {
8047 match &state.reviewed_commits {
8048 Some(subjects) => {
8049 for s in subjects {
8050 body.push_str(&format!("- {}\n", s.trim()));
8051 }
8052 }
8053 None => {
8054 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
8057 body.push_str(rest.trim());
8058 body.push('\n');
8059 }
8060 }
8061 }
8062 } else {
8063 let task = state.instruction.trim();
8064 let task = if task.is_empty() {
8065 "(empty task)"
8066 } else {
8067 task
8068 };
8069 body.push_str(&format!(
8070 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
8071 task.replace("</details>", "</details>")
8072 ));
8073 }
8074
8075 body.push_str(&format!(
8076 "\n---\nmagi:run/{} magi:candidate-{}\n",
8077 state.id,
8078 winner.to_ascii_lowercase()
8079 ));
8080
8081 let id = crate::scrub::Identity::current();
8084 PrMessage {
8085 title: crate::scrub::scrub(&title, &id),
8086 body: crate::scrub::scrub(&body, &id),
8087 }
8088}
8089
8090fn seeded_instruction(state: &RunState) -> String {
8094 match refs::describe(&state.seeds) {
8095 Some(facts) => format!(
8096 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
8097 Candidates start from the unmerged branch named above, when there \
8098 is one, and carry any unmerged commit named by sha as a \
8099 cherry-pick. Check that this is what the task meant before \
8100 building on it.",
8101 state.instruction
8102 ),
8103 None => state.instruction.clone(),
8104 }
8105}
8106
8107async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
8111 let base = &state.base_branch;
8112 let mut against = base.clone();
8113 if mode == MergeMode::Pr {
8114 let remote = &state.config.merge.remote;
8115 let tracking = format!("{remote}/{base}");
8116 let fetched = git::fetch(repo, remote, base).await;
8117 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
8118 against = tracking;
8119 }
8120 }
8121 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
8122}
8123
8124fn empty_candidate_detail(state: &RunState, base: &str) -> String {
8127 let mut detail = format!(
8128 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
8129 nothing to open a pull request for"
8130 );
8131 match refs::describe(&state.seeds) {
8132 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
8133 None => detail.push_str(
8134 "\nThe task names no existing branch or commit; if it means to land work \
8135 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
8136 ),
8137 }
8138 detail
8139}
8140
8141#[derive(Debug, PartialEq, Eq)]
8144enum PrPlan {
8145 Create,
8146 Adopt { url: String, title: String },
8147 Stop(String),
8148}
8149
8150fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
8153 match found {
8154 Ok(land::OpenPr::None) => PrPlan::Create,
8155 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
8156 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
8157 "several open pull requests exist for this branch, not picking one: {}",
8158 urls.join(" ")
8159 )),
8160 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
8161 }
8162}
8163
8164async fn gh_pr_create(
8166 cwd: &Path,
8167 base: &str,
8168 head: &str,
8169 title: &str,
8170 body: &str,
8171) -> Result<String> {
8172 let out = tokio::process::Command::new("gh")
8173 .args([
8174 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
8175 ])
8176 .current_dir(cwd)
8177 .quiet()
8178 .stdin(std::process::Stdio::null())
8179 .output()
8180 .await
8181 .context("spawn gh")?;
8182 if out.status.success() {
8183 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
8184 } else {
8185 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
8186 }
8187}
8188
8189pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
8198 let repo = state.repo.clone();
8199 let root = state.worktree_root();
8200 let winner = state.tally.as_ref().map(|t| t.winner);
8201 let mut removed = Vec::new();
8202
8203 for i in 0..state.candidates.len() {
8204 let c = state.candidates[i].clone();
8205 let is_winner = Some(c.label) == winner;
8206 if is_winner && !drop_winner {
8207 continue;
8208 }
8209 if c.worktree.exists() {
8210 git::worktree_remove(&repo, &c.worktree).await.ok();
8211 removed.push(c.worktree.to_string_lossy().into_owned());
8212 }
8213 let handed_over = state.released_branches.contains(&c.branch);
8216 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
8217 git::branch_delete(&repo, &c.branch).await.ok();
8218 removed.push(c.branch.clone());
8219 }
8220 state.candidates[i].folded = true;
8221 }
8222
8223 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
8224 let path = name.path();
8225 let keep = !drop_winner
8226 && winner.is_some_and(|w| {
8227 path.file_name()
8228 .is_some_and(|n| n == format!("cand-{w}").as_str())
8229 });
8230 if keep {
8231 continue;
8232 }
8233 git::worktree_remove(&repo, &path).await.ok();
8234 removed.push(path.to_string_lossy().into_owned());
8235 }
8236
8237 remove_if_empty(&root);
8246
8247 if state.enabled_worktree_config && drop_winner {
8248 git::release_worktree_config(&repo).await.ok();
8252 state.enabled_worktree_config = false;
8253 }
8254 state.save_under(home)?;
8255 Ok(removed)
8256}
8257
8258fn remove_if_empty(dir: &Path) {
8269 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
8270 std::fs::remove_dir(dir).ok();
8271 }
8272}
8273
8274pub fn worst_open(state: &RunState) -> Option<Severity> {
8276 state
8277 .reviews
8278 .last()?
8279 .reviews
8280 .iter()
8281 .flat_map(|r| r.findings.iter())
8282 .map(|f| f.severity)
8283 .max()
8284}
8285
8286#[cfg(test)]
8287mod tests {
8288 #[test]
8289 fn pr_merge_plan_creates_adopts_or_stops() {
8290 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
8291 assert_eq!(
8292 pr_merge_plan(Ok(land::OpenPr::One {
8293 url: "u".into(),
8294 title: "t".into()
8295 })),
8296 PrPlan::Adopt {
8297 url: "u".into(),
8298 title: "t".into()
8299 }
8300 );
8301 let PrPlan::Stop(many) =
8302 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
8303 else {
8304 panic!("many must stop");
8305 };
8306 assert!(many.contains('a') && many.contains('b'));
8307 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
8308 panic!("a failed lookup must stop");
8309 };
8310 assert!(err.contains("bad token"));
8311 }
8312
8313 use super::*;
8314 use crate::run::GateStatus;
8315 use std::collections::BTreeMap;
8316 use std::time::Duration;
8317
8318 fn conductor() -> AgentSpec {
8319 AgentSpec {
8320 id: "conductor".to_owned(),
8321 kind: crate::config::AgentKind::Command,
8322 model: None,
8323 command: vec!["true".to_owned()],
8324 extra_args: Vec::new(),
8325 env: BTreeMap::new(),
8326 prompt_delivery: None,
8327 }
8328 }
8329
8330 fn spec(id: &str) -> AgentSpec {
8331 AgentSpec {
8332 id: id.to_owned(),
8333 kind: crate::config::AgentKind::Command,
8334 model: None,
8335 command: vec!["true".to_owned()],
8336 extra_args: Vec::new(),
8337 env: BTreeMap::new(),
8338 prompt_delivery: None,
8339 }
8340 }
8341
8342 fn ids(xs: &[&str]) -> BTreeSet<String> {
8343 xs.iter().map(|s| (*s).to_owned()).collect()
8344 }
8345
8346 #[test]
8347 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
8348 let roster = [spec("a"), spec("b"), spec("c")];
8349 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
8350 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8351 }
8352
8353 #[test]
8354 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
8355 let roster = [spec("a"), spec("b"), spec("c")];
8356 let failed = ids(&["a", "b", "c"]);
8357 let mut tried = ids(&["b"]);
8359 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8360 assert_eq!(first.id, "a");
8361 tried.insert(first.id.clone());
8362 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8363 assert_eq!(second.id, "c");
8364 tried.insert(second.id.clone());
8365 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
8366 }
8367
8368 #[test]
8369 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
8370 let roster = [spec("a"), spec("b")];
8371 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
8372 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8373 }
8374
8375 #[test]
8376 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
8377 let roster = [spec("a"), spec("b"), spec("c")];
8378 let h = SeatHistory {
8379 failed: ids(&["a"]),
8380 last_ok: Some("b".to_owned()),
8381 last_fail: None,
8382 };
8383 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
8384 let gone = SeatHistory {
8386 last_ok: Some("zzz".to_owned()),
8387 ..h.clone()
8388 };
8389 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
8390 let failed = SeatHistory {
8391 failed: ids(&["a", "b"]),
8392 last_ok: Some("b".to_owned()),
8393 last_fail: None,
8394 };
8395 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
8396 }
8397
8398 #[test]
8399 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
8400 let first = SeatState::new("review-1", "alpha", 7);
8401 let next = handover_seat("review-1", "gamma", 7);
8402 assert_ne!(first.claude_session, next.claude_session);
8403 }
8404
8405 #[test]
8406 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
8407 let mut state = state_with_summary("x", "y");
8408 let a = handover_seat("review-1", "beta", state.next_seat_seed());
8409 let b = handover_seat("review-1", "beta", state.next_seat_seed());
8410 assert_ne!(a.claude_session, b.claude_session);
8411 }
8412
8413 #[test]
8414 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
8415 let roster = [spec("a"), spec("b")];
8416 let h = SeatHistory {
8417 failed: ids(&["a", "b"]),
8418 ..SeatHistory::default()
8419 };
8420 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
8421 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
8422 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
8423 }
8424
8425 #[test]
8432 fn failure_signature_ignores_numbers_and_paths() {
8433 assert_eq!(
8434 failure_signature("exited with Some(2) and no usable output"),
8435 failure_signature("exited with Some(137) and no usable output")
8436 );
8437 assert_eq!(
8438 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
8439 failure_signature("cannot open /var/x.txt: denied")
8440 );
8441 assert_ne!(failure_signature("boom"), failure_signature("bang"));
8442 }
8443
8444 #[test]
8445 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
8446 let other = FailClass::Other("x".into());
8447 assert!(should_hand_over(None, &FailClass::Quota));
8448 assert!(should_hand_over(Some(&other), &FailClass::Quota));
8449 assert!(should_hand_over(
8450 Some(&FailClass::Timeout),
8451 &FailClass::Timeout
8452 ));
8453 assert!(should_hand_over(None, &other));
8454 assert!(!should_hand_over(Some(&other), &other));
8455 assert!(should_hand_over(
8456 Some(&other),
8457 &FailClass::Other("y".into())
8458 ));
8459 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
8461 assert!(should_hand_over(Some(&FailClass::Quota), &other));
8462 }
8463
8464 #[test]
8465 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
8466 let a = SeatState::new("judge-1", "alpha", 7);
8467 let b = handover_seat("judge-1", "beta", 7);
8468 assert_ne!(a.claude_session, b.claude_session);
8469 assert_eq!(b.turns, 0);
8470 }
8471
8472 #[test]
8473 fn a_timeout_is_classified_apart_from_other_failures() {
8474 assert_eq!(
8475 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
8476 Some(FailClass::Timeout)
8477 );
8478 assert!(matches!(
8479 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
8480 Some(FailClass::Other(_))
8481 ));
8482 }
8483
8484 #[test]
8485 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
8486 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8487 let tried = BTreeSet::from(["beta".to_owned()]);
8488 let next = next_untried_in_roster(&roster, 1, &tried);
8491 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
8492 }
8493
8494 #[test]
8495 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
8496 let roster = vec![spec("alpha"), spec("beta")];
8497 let tried = BTreeSet::from(["beta".to_owned()]);
8498 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8502 }
8503
8504 #[test]
8505 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
8506 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8507 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
8508 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8512 }
8513
8514 #[test]
8515 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
8516 let roster = vec![spec("a"), spec("a"), spec("b")];
8517 let tried = BTreeSet::from(["a".to_owned()]);
8518 let next = next_untried_in_roster(&roster, 0, &tried);
8519 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8520 }
8521
8522 #[test]
8523 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
8524 let roster = vec![spec("a"), spec("b")];
8525 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
8526 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
8527 }
8528
8529 #[test]
8530 fn remove_if_empty_only_ever_takes_a_bare_directory() {
8531 let dir = tempfile::tempdir().unwrap();
8532 let bay = dir.path().join("ffff");
8533
8534 remove_if_empty(&bay);
8536 assert!(!bay.exists());
8537
8538 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
8541 remove_if_empty(&bay);
8542 assert!(bay.exists(), "non-empty directory must survive");
8543
8544 std::fs::remove_dir(bay.join("cand-A")).unwrap();
8546 remove_if_empty(&bay);
8547 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
8548 }
8549
8550 #[test]
8559 fn a_full_panel_that_found_nothing_is_clean() {
8560 assert!(round_is_clean(
8561 0,
8562 true,
8563 2,
8564 2,
8565 0,
8566 IncompleteReviewPolicy::Block
8567 ));
8568 }
8569
8570 #[test]
8571 fn a_missing_seat_is_never_clean_under_the_default_policy() {
8572 assert!(!round_is_clean(
8573 0,
8574 true,
8575 1,
8576 2,
8577 0,
8578 IncompleteReviewPolicy::Block
8579 ));
8580 }
8581
8582 #[test]
8583 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
8584 assert!(!round_is_clean(
8585 1,
8586 true,
8587 1,
8588 2,
8589 0,
8590 IncompleteReviewPolicy::Warn
8591 ));
8592 }
8593
8594 #[test]
8595 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
8596 assert!(round_is_clean(
8597 0,
8598 true,
8599 1,
8600 2,
8601 0,
8602 IncompleteReviewPolicy::Warn
8603 ));
8604 }
8605
8606 #[test]
8607 fn a_full_panel_with_an_open_finding_is_not_clean() {
8608 assert!(!round_is_clean(
8609 1,
8610 true,
8611 2,
8612 2,
8613 0,
8614 IncompleteReviewPolicy::Block
8615 ));
8616 }
8617
8618 #[test]
8619 fn a_full_panel_with_a_red_e2e_is_not_clean() {
8620 assert!(!round_is_clean(
8621 0,
8622 false,
8623 2,
8624 2,
8625 0,
8626 IncompleteReviewPolicy::Block
8627 ));
8628 }
8629
8630 #[test]
8637 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
8638 assert!(round_is_clean(
8641 0,
8642 true,
8643 1,
8644 2,
8645 1,
8646 IncompleteReviewPolicy::Block
8647 ));
8648 }
8649
8650 #[test]
8651 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
8652 assert!(!round_is_clean(
8655 0,
8656 true,
8657 1,
8658 2,
8659 0,
8660 IncompleteReviewPolicy::Block
8661 ));
8662 }
8663
8664 #[test]
8665 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
8666 assert!(!round_is_clean(
8667 1,
8668 true,
8669 1,
8670 2,
8671 1,
8672 IncompleteReviewPolicy::Block
8673 ));
8674 assert!(!round_is_clean(
8675 0,
8676 false,
8677 1,
8678 2,
8679 1,
8680 IncompleteReviewPolicy::Block
8681 ));
8682 }
8683
8684 #[test]
8685 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
8686 assert!(!round_is_clean(
8690 0,
8691 true,
8692 0,
8693 2,
8694 2,
8695 IncompleteReviewPolicy::Block
8696 ));
8697 }
8698
8699 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
8700 CommandOutcome {
8701 command: "test".to_owned(),
8702 code,
8703 output_tail: String::new(),
8704 duration_ms: 0,
8705 resource_blocked,
8706 }
8707 }
8708
8709 #[test]
8710 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
8711 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
8712 assert!(
8713 !verify_inconclusive(&[outcome(Some(1), false)]),
8714 "an ordinary failure is still evidence about the patch"
8715 );
8716 assert!(verify_inconclusive(&[outcome(None, true)]));
8717 assert!(
8718 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
8719 "one inconclusive outcome taints the whole batch"
8720 );
8721 assert!(!verify_inconclusive(&[]));
8722 }
8723
8724 #[tokio::test]
8725 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
8726 let calls = std::sync::atomic::AtomicUsize::new(0);
8730 let started = Instant::now();
8731 wait_for_pids_with(
8732 &[123],
8733 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
8734 Duration::from_millis(5),
8735 Duration::from_secs(5),
8736 )
8737 .await;
8738 assert!(
8739 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
8740 "must keep checking rather than deciding on the first answer"
8741 );
8742 assert!(
8743 started.elapsed() < Duration::from_secs(1),
8744 "must return the moment it is confirmed dead, not wait out the ceiling"
8745 );
8746 }
8747
8748 #[tokio::test]
8749 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
8750 let started = Instant::now();
8751 wait_for_pids_with(
8752 &[123],
8753 |_| true, Duration::from_millis(5),
8755 Duration::from_millis(30),
8756 )
8757 .await;
8758 let elapsed = started.elapsed();
8759 assert!(
8760 elapsed >= Duration::from_millis(30),
8761 "must not give up before its own ceiling: {elapsed:?}"
8762 );
8763 assert!(
8764 elapsed < Duration::from_secs(1),
8765 "must not wait past its own ceiling either: {elapsed:?}"
8766 );
8767 }
8768
8769 #[tokio::test]
8770 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
8771 let started = Instant::now();
8772 wait_for_pids_with(
8773 &[],
8774 |_| true,
8775 Duration::from_secs(5),
8776 Duration::from_secs(5),
8777 )
8778 .await;
8779 assert!(
8780 started.elapsed() < Duration::from_millis(200),
8781 "an empty pid list has nothing to confirm"
8782 );
8783 }
8784
8785 fn review_round(
8791 clean: bool,
8792 blocking: usize,
8793 answered: usize,
8794 expected: usize,
8795 progressed: bool,
8796 e2e_ok: bool,
8797 ) -> ReviewRound {
8798 ReviewRound {
8799 round: 1,
8800 head: "h".to_owned(),
8801 verified_head: None,
8802 verified_at: None,
8803 reviews: Vec::new(),
8804 e2e: vec![CommandOutcome {
8805 command: "test".to_owned(),
8806 code: Some(if e2e_ok { 0 } else { 1 }),
8807 output_tail: String::new(),
8808 duration_ms: 0,
8809 resource_blocked: false,
8810 }],
8811 verify_retried: false,
8812 e2e_deferred: false,
8813 e2e_defer_reason: None,
8814 fix: None,
8815 blocking,
8816 answered,
8817 expected,
8818 clean,
8819 progressed,
8820 vote_split: false,
8821 reconsideration: Vec::new(),
8822 verdict: None,
8823 }
8824 }
8825
8826 #[test]
8827 fn review_conclusion_is_none_when_nothing_has_run() {
8828 assert_eq!(review_conclusion(&[], 3), None);
8829 }
8830
8831 #[test]
8832 fn review_conclusion_is_none_while_rounds_remain() {
8833 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
8834 assert_eq!(review_conclusion(&rounds, 3), None);
8835 }
8836
8837 #[test]
8838 fn review_conclusion_is_gating_once_a_round_is_clean() {
8839 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
8840 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
8841 }
8842
8843 #[test]
8844 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
8845 let rounds = vec![
8846 review_round(false, 1, 2, 2, true, true),
8847 review_round(false, 1, 2, 2, true, true),
8848 ];
8849 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
8850 }
8851
8852 #[test]
8853 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
8854 let rounds = vec![
8855 review_round(false, 1, 2, 2, true, true),
8856 review_round(false, 1, 2, 2, true, false),
8857 ];
8858 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
8859 }
8860
8861 #[test]
8862 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
8863 let mut blocked = review_round(false, 1, 2, 2, true, false);
8870 blocked.e2e[0].resource_blocked = true;
8871 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
8872 assert_eq!(review_conclusion(&rounds, 2), None);
8873 }
8874
8875 #[test]
8876 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
8877 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
8879 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
8880 }
8881
8882 #[test]
8883 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
8884 let rounds = vec![
8885 review_round(false, 1, 2, 2, false, true),
8886 review_round(false, 1, 2, 2, false, true),
8887 ];
8888 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
8889 }
8890
8891 fn secs(n: u64) -> Duration {
8892 Duration::from_secs(n)
8893 }
8894
8895 fn init_repo(dir: &Path) {
8898 let run = |args: &[&str]| {
8899 let out = std::process::Command::new("git")
8900 .args(args)
8901 .current_dir(dir)
8902 .quiet()
8903 .output()
8904 .expect("spawn git");
8905 assert!(
8906 out.status.success(),
8907 "git {args:?} failed: {}",
8908 String::from_utf8_lossy(&out.stderr)
8909 );
8910 };
8911 run(&["init", "-b", "main"]);
8912 run(&["config", "user.name", "magi test"]);
8913 run(&["config", "user.email", "magi@example.com"]);
8914 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
8915 run(&["add", "-A"]);
8916 run(&["commit", "-m", "init"]);
8917 }
8918
8919 fn ask_test_home() {
8927 crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
8928 }
8929
8930 fn runner_at(status: RunStatus) -> Runner {
8933 let mut state = RunState::new(
8934 PathBuf::from("/nonexistent/repo"),
8935 "main".to_owned(),
8936 "deadbeef".to_owned(),
8937 "task".to_owned(),
8938 Config::default(),
8939 );
8940 state.status = status;
8941 Runner {
8942 state,
8943 roles: ResolvedRoles {
8944 implementers: Vec::new(),
8945 judges: Vec::new(),
8946 reviewers: Vec::new(),
8947 fixer: None,
8948 conductor: conductor(),
8949 implementer_roster: Vec::new(),
8950 judge_roster: Vec::new(),
8951 reviewer_roster: Vec::new(),
8952 },
8953 sem: Arc::new(Semaphore::new(1)),
8954 pause: Pause::new(),
8955 interrupt: Pause::new(),
8956 }
8957 }
8958
8959 #[test]
8963 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
8964 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8965 let mut runner = runner_at(RunStatus::Implementing);
8966 let interrupt = Pause::new();
8967 runner.watch_interrupt(interrupt.clone());
8968
8969 interrupt.park_because("task a1b2 asked to run first");
8970
8971 assert!(runner.park_here().expect("park_here"));
8972 assert!(runner.state.parked);
8973 let last = runner.state.events.last().expect("a park event");
8974 assert_eq!(last.node, "park");
8975 assert!(
8976 last.message.contains("task a1b2 asked to run first"),
8977 "expected the interrupt reason in {:?}",
8978 last.message
8979 );
8980 }
8981
8982 #[test]
8990 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
8991 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8992 let mut runner = runner_at(RunStatus::Implementing);
8993 let shutdown = Pause::new();
8994 runner.on_pause(shutdown.clone());
8995 let interrupt = Pause::new();
8996 runner.watch_interrupt(interrupt.clone());
8997
8998 assert!(!runner.park_here().expect("park_here"));
9000 assert!(!runner.state.parked);
9001
9002 interrupt.park_because("test");
9004 assert!(!shutdown.parked());
9005 assert!(runner.park_here().expect("park_here"));
9006 }
9007
9008 #[tokio::test]
9022 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
9023 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
9024 let mut runner = runner_at(RunStatus::Implementing);
9025 let interrupt = Pause::new();
9026 runner.watch_interrupt(interrupt.clone());
9027
9028 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
9029 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
9030
9031 let node = async move {
9035 started_tx.send(()).expect("send started");
9036 finish_rx.await.expect("recv finish");
9037 "node finished"
9038 };
9039
9040 let interrupter = async move {
9041 started_rx.await.expect("recv started");
9042 interrupt.park_because("higher-priority task waiting");
9044 tokio::task::yield_now().await;
9048 finish_tx.send(()).expect("send finish");
9049 };
9050
9051 let (node_result, ()) = tokio::join!(node, interrupter);
9052 assert_eq!(
9053 node_result, "node finished",
9054 "the in-flight call ran to completion"
9055 );
9056
9057 assert!(runner.park_here().expect("park_here"));
9060 assert!(runner.state.parked);
9061 }
9062
9063 #[test]
9069 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
9070 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
9071 let mut runner = runner_at(RunStatus::Judging);
9072 runner.state.config.agents = vec![conductor()];
9076 runner.state.candidates = vec![Candidate {
9077 index: 0,
9078 label: 'A',
9079 agent: "alpha".to_owned(),
9080 branch: "magi/x/A".to_owned(),
9081 worktree: PathBuf::from("/nonexistent/worktree"),
9082 summary: "did the thing".to_owned(),
9083 stat: "1 file changed".to_owned(),
9084 files: 1,
9085 commits: 1,
9086 empty: false,
9087 failed: None,
9088 verified_noop: None,
9089 duration_ms: 1234,
9090 folded: false,
9091 }];
9092 let run_id = runner.state.id.clone();
9093
9094 let interrupt = Pause::new();
9095 runner.watch_interrupt(interrupt.clone());
9096 interrupt.park_because("task c3d4 asked to run first");
9097 assert!(runner.park_here().expect("park_here"));
9098
9099 let resumed = Runner::resume(&run_id).expect("resume");
9100 assert_eq!(resumed.state.candidates.len(), 1);
9101 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
9102 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
9103 assert_eq!(resumed.state.status, runner.state.status);
9104 assert!(
9105 resumed.state.parked,
9106 "still parked until `execute` actually walks the graph again"
9107 );
9108 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
9109 }
9110
9111 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
9113 let mut q = ask::Question::new(
9114 run.to_owned(),
9115 "implement".to_owned(),
9116 "impl-A".to_owned(),
9117 "Which storage backend should the cache use?".to_owned(),
9118 String::new(),
9119 vec!["SQLite".to_owned(), "Redis".to_owned()],
9120 );
9121 store.put(&mut q).unwrap();
9122 q
9123 }
9124
9125 #[test]
9126 fn a_failed_runs_open_question_is_abandoned() {
9127 ask_test_home();
9128 let store = ask::Questions::open();
9129 let mut runner = runner_at(RunStatus::Failed);
9130 let run = runner.state.id.clone();
9131 let q = ask_open_question(&store, &run);
9132
9133 runner.settle_questions();
9134
9135 let back = store.get(&q.id).unwrap();
9136 assert!(
9137 !back.status.open(),
9138 "the seat that asked died with the run; nobody is left to read an answer"
9139 );
9140 assert!(
9141 back.detail.contains(&run) && back.detail.contains("failed"),
9142 "the reason names what the run became, not just that it is gone: {}",
9143 back.detail
9144 );
9145 }
9146
9147 #[test]
9148 fn a_merged_runs_open_question_is_abandoned_too() {
9149 ask_test_home();
9150 let store = ask::Questions::open();
9151 for status in [RunStatus::Merged, RunStatus::Ready] {
9154 let mut runner = runner_at(status);
9155 let run = runner.state.id.clone();
9156 let q = ask_open_question(&store, &run);
9157
9158 runner.settle_questions();
9159
9160 let back = store.get(&q.id).unwrap();
9161 assert!(
9162 !back.status.open(),
9163 "{status:?} run's question must not outlive the run"
9164 );
9165 }
9166 }
9167
9168 #[test]
9169 fn a_still_resumable_runs_open_question_is_left_alone() {
9170 ask_test_home();
9171 let store = ask::Questions::open();
9172 for status in [RunStatus::Blocked, RunStatus::Stalled] {
9178 let mut runner = runner_at(status);
9179 let run = runner.state.id.clone();
9180 let q = ask_open_question(&store, &run);
9181
9182 runner.settle_questions();
9183
9184 let back = store.get(&q.id).unwrap();
9185 assert!(
9186 back.status.open(),
9187 "{status:?} is still alive; the question must still be waiting"
9188 );
9189 }
9190 }
9191
9192 #[test]
9193 fn settle_questions_never_touches_an_already_answered_question() {
9194 ask_test_home();
9195 let store = ask::Questions::open();
9196 let mut runner = runner_at(RunStatus::Failed);
9197 let run = runner.state.id.clone();
9198 let mut q = ask_open_question(&store, &run);
9199 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
9200 .unwrap();
9201 store.put(&mut q).unwrap();
9202
9203 runner.settle_questions();
9208 runner.settle_questions();
9209
9210 let back = store.get(&q.id).unwrap();
9211 assert_eq!(
9212 back.status,
9213 ask::QuestionStatus::Answered,
9214 "a real answer is a decision on record, never overwritten by a sweep"
9215 );
9216 }
9217
9218 #[tokio::test]
9229 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
9230 crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
9231 let tmp = tempfile::tempdir().expect("tempdir");
9232 let repo = tmp.path().join("repo");
9233 std::fs::create_dir_all(&repo).unwrap();
9234 init_repo(&repo);
9235
9236 let mut config = Config::default();
9237 config.graph.worktree_root = Some(tmp.path().join("wt"));
9238
9239 let mut state = RunState::new(
9240 repo.clone(),
9241 "main".to_owned(),
9242 "deadbeef".to_owned(),
9243 "task".to_owned(),
9244 config,
9245 );
9246 let root = state.worktree_root();
9247 let wt_a = root.join("cand-A");
9248 let wt_b = root.join("cand-B");
9249 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
9250 .await
9251 .expect("worktree A");
9252 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
9253 .await
9254 .expect("worktree B");
9255
9256 state.candidates = vec![
9257 Candidate {
9258 index: 0,
9259 label: 'A',
9260 agent: "alpha".to_owned(),
9261 branch: "magi/x/A".to_owned(),
9262 worktree: wt_a.clone(),
9263 summary: String::new(),
9264 stat: String::new(),
9265 files: 0,
9266 commits: 0,
9267 empty: false,
9268 failed: None,
9269 verified_noop: None,
9270 duration_ms: 0,
9271 folded: false,
9272 },
9273 Candidate {
9274 index: 1,
9275 label: 'B',
9276 agent: "beta".to_owned(),
9277 branch: "magi/x/B".to_owned(),
9278 worktree: wt_b.clone(),
9279 summary: String::new(),
9280 stat: String::new(),
9281 files: 0,
9282 commits: 0,
9283 empty: false,
9284 failed: None,
9285 verified_noop: None,
9286 duration_ms: 0,
9287 folded: false,
9288 },
9289 ];
9290 state.tally = Some(Tally {
9291 first_choice: BTreeMap::from([('A', 1)]),
9292 borda: BTreeMap::new(),
9293 winner: 'A',
9294 rankings: 1,
9295 unanimous_initial: true,
9296 deliberated: false,
9297 changed_votes: 0,
9298 unanimous_final: true,
9299 tie_break: None,
9300 judges: 1,
9301 present: 1,
9302 quorum: 1,
9303 met_quorum: true,
9304 uncontested: None,
9305 });
9306 state.status = RunStatus::Ready;
9307
9308 fold_run(&mut state, false, &crate::run::home())
9309 .await
9310 .expect("fold_run");
9311
9312 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
9313 assert!(
9314 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9315 "the unmerged winner's branch survives"
9316 );
9317 assert!(
9318 !state.candidates[0].folded,
9319 "the winner is not marked folded"
9320 );
9321
9322 assert!(!wt_b.exists(), "the loser's worktree is removed");
9323 assert!(
9324 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
9325 "the loser's branch is removed"
9326 );
9327 assert!(state.candidates[1].folded, "the loser is marked folded");
9328 }
9329
9330 #[tokio::test]
9333 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
9334 let tmp = tempfile::tempdir().expect("tempdir");
9335 let repo = tmp.path().join("repo");
9336 std::fs::create_dir_all(&repo).unwrap();
9337 init_repo(&repo);
9338 let home = tmp.path().join("home");
9339
9340 let mut config = Config::default();
9341 config.graph.worktree_root = Some(tmp.path().join("wt"));
9342 let mut state = RunState::new(
9343 repo.clone(),
9344 "main".to_owned(),
9345 "deadbeef".to_owned(),
9346 "task".to_owned(),
9347 config,
9348 );
9349 git::git(&repo, &["branch", "magi/x/A", "main"])
9351 .await
9352 .expect("branch");
9353 state.candidates = vec![Candidate {
9354 index: 0,
9355 label: 'A',
9356 agent: "alpha".to_owned(),
9357 branch: "magi/x/A".to_owned(),
9358 worktree: state.worktree_root().join("cand-A"),
9359 summary: String::new(),
9360 stat: String::new(),
9361 files: 0,
9362 commits: 0,
9363 empty: false,
9364 failed: None,
9365 verified_noop: None,
9366 duration_ms: 0,
9367 folded: true,
9368 }];
9369 state.released_to = Some("20260901-000000-new1".to_owned());
9370 state.released_branches = vec!["magi/x/A".to_owned()];
9371
9372 fold_run(&mut state, true, &home).await.expect("fold_run");
9373
9374 assert!(
9375 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9376 "the handed-over branch survives a fold"
9377 );
9378 }
9379
9380 #[tokio::test]
9384 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
9385 let tmp = tempfile::tempdir().expect("tempdir");
9386 let repo = tmp.path().join("repo");
9387 std::fs::create_dir_all(&repo).unwrap();
9388 init_repo(&repo);
9389 let run = |args: &[&str]| {
9390 let out = std::process::Command::new("git")
9391 .quiet()
9392 .args(args)
9393 .current_dir(&repo)
9394 .output()
9395 .expect("spawn git");
9396 assert!(out.status.success(), "git {args:?}");
9397 };
9398 run(&["branch", "magi/x/A"]);
9399 run(&["checkout", "-q", "-b", "magi/x/B"]);
9400 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
9401 run(&["add", "-A"]);
9402 run(&["commit", "-q", "-m", "work"]);
9403 run(&["checkout", "-q", "main"]);
9404
9405 let mut state = RunState::new(
9406 repo.clone(),
9407 "main".to_owned(),
9408 "deadbeef".to_owned(),
9409 "task".to_owned(),
9410 Config::default(),
9411 );
9412 state.seeds = vec![refs::Seed {
9413 token: "magi/27b2/A".to_owned(),
9414 kind: refs::SeedKind::Unresolved,
9415 sha: String::new(),
9416 branch: true,
9417 detail: "no branch or commit named magi/27b2/A".to_owned(),
9418 }];
9419
9420 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
9421 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
9422 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
9423 let detail = empty_candidate_detail(&state, "main");
9424 assert!(detail.starts_with("empty candidate"), "{detail}");
9425 assert!(detail.contains("magi/27b2/A"), "{detail}");
9426 }
9427
9428 #[tokio::test]
9437 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
9438 let tmp = tempfile::tempdir().expect("tempdir");
9439 let repo = tmp.path().join("repo");
9440 std::fs::create_dir_all(&repo).unwrap();
9441 init_repo(&repo);
9442
9443 let mut config = Config::default();
9444 config.merge.mode = MergeMode::Local;
9445
9446 let mut state = RunState::new(
9447 repo.clone(),
9448 "main".to_owned(),
9449 "deadbeef".to_owned(),
9450 "task".to_owned(),
9451 config,
9452 );
9453 state.candidates = vec![Candidate {
9454 index: 0,
9455 label: 'A',
9456 agent: "alpha".to_owned(),
9457 branch: "does-not-exist".to_owned(),
9458 worktree: repo.clone(),
9459 summary: String::new(),
9460 stat: String::new(),
9461 files: 0,
9462 commits: 0,
9463 empty: false,
9464 failed: None,
9465 verified_noop: None,
9466 duration_ms: 0,
9467 folded: false,
9468 }];
9469 state.tally = Some(Tally {
9470 first_choice: BTreeMap::from([('A', 1)]),
9471 borda: BTreeMap::new(),
9472 winner: 'A',
9473 rankings: 1,
9474 unanimous_initial: true,
9475 deliberated: false,
9476 changed_votes: 0,
9477 unanimous_final: true,
9478 tie_break: None,
9479 judges: 0,
9480 present: 0,
9481 quorum: 0,
9482 met_quorum: true,
9483 uncontested: Some("only candidate A produced a change".to_owned()),
9484 });
9485 state.reviews = vec![ReviewRound {
9486 round: 1,
9487 head: "deadbeef".to_owned(),
9488 verified_head: None,
9489 verified_at: None,
9490 reviews: Vec::new(),
9491 e2e: Vec::new(),
9492 fix: None,
9493 blocking: 0,
9494 answered: 0,
9495 expected: 0,
9496 clean: true,
9497 verify_retried: false,
9498 e2e_deferred: false,
9499 e2e_defer_reason: None,
9500 progressed: false,
9501 vote_split: false,
9502 reconsideration: Vec::new(),
9503 verdict: None,
9504 }];
9505 state.gate = vec![CommandOutcome {
9506 command: "test".to_owned(),
9507 code: Some(0),
9508 output_tail: String::new(),
9509 duration_ms: 0,
9510 resource_blocked: false,
9511 }];
9512 state.gate_ran = true;
9513 state.status = RunStatus::Ready;
9518 state.merge = Some(MergeOutcome {
9519 mode: MergeMode::Local,
9520 ok: false,
9521 detail: "already concluded".to_owned(),
9522 empty: false,
9523 });
9524
9525 let mut runner = Runner {
9526 state,
9527 roles: ResolvedRoles {
9528 implementers: Vec::new(),
9529 judges: Vec::new(),
9530 reviewers: Vec::new(),
9531 fixer: None,
9532 conductor: conductor(),
9533 implementer_roster: Vec::new(),
9534 judge_roster: Vec::new(),
9535 reviewer_roster: Vec::new(),
9536 },
9537 sem: Arc::new(Semaphore::new(1)),
9538 pause: Pause::new(),
9539 interrupt: Pause::new(),
9540 };
9541
9542 runner.merge().await.expect("merge");
9543
9544 assert_eq!(
9545 runner.state.status,
9546 RunStatus::Ready,
9547 "a concluded run's status must not change on reentry"
9548 );
9549 assert_eq!(
9550 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
9551 Some("already concluded"),
9552 "merge must not run again once the node already recorded an outcome"
9553 );
9554 }
9555
9556 #[tokio::test]
9565 async fn merge_refuses_a_gate_that_has_not_actually_run() {
9566 let tmp = tempfile::tempdir().expect("tempdir");
9567 let repo = tmp.path().join("repo");
9568 std::fs::create_dir_all(&repo).unwrap();
9569 init_repo(&repo);
9570
9571 let mut config = Config::default();
9572 config.merge.mode = MergeMode::Local;
9573
9574 let mut state = RunState::new(
9575 repo.clone(),
9576 "main".to_owned(),
9577 "deadbeef".to_owned(),
9578 "task".to_owned(),
9579 config,
9580 );
9581 state.candidates = vec![Candidate {
9582 index: 0,
9583 label: 'A',
9584 agent: "alpha".to_owned(),
9585 branch: "does-not-exist".to_owned(),
9586 worktree: repo.clone(),
9587 summary: String::new(),
9588 stat: String::new(),
9589 files: 0,
9590 commits: 0,
9591 empty: false,
9592 failed: None,
9593 verified_noop: None,
9594 duration_ms: 0,
9595 folded: false,
9596 }];
9597 state.tally = Some(Tally {
9598 first_choice: BTreeMap::from([('A', 1)]),
9599 borda: BTreeMap::new(),
9600 winner: 'A',
9601 rankings: 1,
9602 unanimous_initial: true,
9603 deliberated: false,
9604 changed_votes: 0,
9605 unanimous_final: true,
9606 tie_break: None,
9607 judges: 0,
9608 present: 0,
9609 quorum: 0,
9610 met_quorum: true,
9611 uncontested: Some("only candidate A produced a change".to_owned()),
9612 });
9613 state.reviews = vec![ReviewRound {
9614 round: 1,
9615 head: "deadbeef".to_owned(),
9616 verified_head: None,
9617 verified_at: None,
9618 reviews: Vec::new(),
9619 e2e: Vec::new(),
9620 fix: None,
9621 blocking: 0,
9622 answered: 0,
9623 expected: 0,
9624 clean: true,
9625 verify_retried: false,
9626 e2e_deferred: false,
9627 e2e_defer_reason: None,
9628 progressed: false,
9629 vote_split: false,
9630 reconsideration: Vec::new(),
9631 verdict: None,
9632 }];
9633 state.gate = Vec::new();
9635 state.gate_ran = false;
9636 state.status = RunStatus::Gating;
9637
9638 let mut runner = Runner {
9639 state,
9640 roles: ResolvedRoles {
9641 implementers: Vec::new(),
9642 judges: Vec::new(),
9643 reviewers: Vec::new(),
9644 fixer: None,
9645 conductor: conductor(),
9646 implementer_roster: Vec::new(),
9647 judge_roster: Vec::new(),
9648 reviewer_roster: Vec::new(),
9649 },
9650 sem: Arc::new(Semaphore::new(1)),
9651 pause: Pause::new(),
9652 interrupt: Pause::new(),
9653 };
9654
9655 runner.merge().await.expect("merge");
9656
9657 assert!(
9658 runner.state.merge.is_none(),
9659 "an empty gate must never be read as a passing one: {:?}",
9660 runner.state.merge
9661 );
9662 }
9663
9664 #[tokio::test]
9671 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
9672 let tmp = tempfile::tempdir().expect("tempdir");
9673 let repo = tmp.path().join("repo");
9674 std::fs::create_dir_all(&repo).unwrap();
9675 init_repo(&repo);
9676
9677 let config = Config::default();
9679
9680 let mut state = RunState::new(
9681 repo.clone(),
9682 "main".to_owned(),
9683 "deadbeef".to_owned(),
9684 "task".to_owned(),
9685 config,
9686 );
9687 state.candidates = vec![Candidate {
9688 index: 0,
9689 label: 'A',
9690 agent: "alpha".to_owned(),
9691 branch: "does-not-exist".to_owned(),
9692 worktree: repo.clone(),
9693 summary: String::new(),
9694 stat: String::new(),
9695 files: 0,
9696 commits: 0,
9697 empty: false,
9698 failed: None,
9699 verified_noop: None,
9700 duration_ms: 0,
9701 folded: false,
9702 }];
9703 state.tally = Some(Tally {
9704 first_choice: BTreeMap::from([('A', 1)]),
9705 borda: BTreeMap::new(),
9706 winner: 'A',
9707 rankings: 1,
9708 unanimous_initial: true,
9709 deliberated: false,
9710 changed_votes: 0,
9711 unanimous_final: true,
9712 tie_break: None,
9713 judges: 0,
9714 present: 0,
9715 quorum: 0,
9716 met_quorum: true,
9717 uncontested: Some("only candidate A produced a change".to_owned()),
9718 });
9719 state.reviews = vec![ReviewRound {
9720 round: 1,
9721 head: "deadbeef".to_owned(),
9722 verified_head: None,
9723 verified_at: None,
9724 reviews: Vec::new(),
9725 e2e: Vec::new(),
9726 fix: None,
9727 blocking: 0,
9728 answered: 0,
9729 expected: 0,
9730 clean: true,
9731 verify_retried: false,
9732 e2e_deferred: false,
9733 e2e_defer_reason: None,
9734 progressed: false,
9735 vote_split: false,
9736 reconsideration: Vec::new(),
9737 verdict: None,
9738 }];
9739
9740 let mut runner = Runner {
9741 state,
9742 roles: ResolvedRoles {
9743 implementers: Vec::new(),
9744 judges: Vec::new(),
9745 reviewers: Vec::new(),
9746 fixer: None,
9747 conductor: conductor(),
9748 implementer_roster: Vec::new(),
9749 judge_roster: Vec::new(),
9750 reviewer_roster: Vec::new(),
9751 },
9752 sem: Arc::new(Semaphore::new(1)),
9753 pause: Pause::new(),
9754 interrupt: Pause::new(),
9755 };
9756
9757 runner.gate().await.expect("gate");
9758 assert!(
9759 runner.state.gate_ran,
9760 "zero configured commands is still a real attempt, not an unrun gate"
9761 );
9762 assert!(runner.state.gate.is_empty());
9763 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
9764 assert_ne!(
9765 runner.state.status,
9766 RunStatus::Blocked,
9767 "a gate with nothing to check must not read as failed"
9768 );
9769
9770 runner.merge().await.expect("merge");
9771 assert_eq!(
9772 runner.state.status,
9773 RunStatus::Ready,
9774 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
9775 );
9776 }
9777
9778 #[tokio::test]
9789 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
9790 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9791 let home = crate::run::home();
9792
9793 let tmp = tempfile::tempdir().expect("tempdir");
9794 let repo = tmp.path().join("repo");
9795 std::fs::create_dir_all(&repo).unwrap();
9796 init_repo(&repo);
9797 let cache_dir = tmp.path().join("target");
9800
9801 let mut config = Config::default();
9802 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
9803 config.graph.timeout_verify = Some(2);
9806
9807 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9808 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9809 .expect("no io error acquiring directly")
9810 {
9811 crate::cache::AcquireOutcome::Acquired(g) => g,
9812 crate::cache::AcquireOutcome::Busy(b) => {
9813 panic!("expected the direct acquire to win the lease first: {b:?}")
9814 }
9815 };
9816
9817 let mut state = RunState::new(
9818 repo.clone(),
9819 "main".to_owned(),
9820 "deadbeef".to_owned(),
9821 "task".to_owned(),
9822 config,
9823 );
9824 state.candidates = vec![Candidate {
9825 index: 0,
9826 label: 'A',
9827 agent: "alpha".to_owned(),
9828 branch: "does-not-exist".to_owned(),
9829 worktree: repo.clone(),
9830 summary: String::new(),
9831 stat: String::new(),
9832 files: 0,
9833 commits: 0,
9834 empty: false,
9835 failed: None,
9836 verified_noop: None,
9837 duration_ms: 0,
9838 folded: false,
9839 }];
9840 state.tally = Some(Tally {
9841 first_choice: BTreeMap::from([('A', 1)]),
9842 borda: BTreeMap::new(),
9843 winner: 'A',
9844 rankings: 1,
9845 unanimous_initial: true,
9846 deliberated: false,
9847 changed_votes: 0,
9848 unanimous_final: true,
9849 tie_break: None,
9850 judges: 0,
9851 present: 0,
9852 quorum: 0,
9853 met_quorum: true,
9854 uncontested: Some("only candidate A produced a change".to_owned()),
9855 });
9856 state.reviews = vec![ReviewRound {
9857 round: 1,
9858 head: "deadbeef".to_owned(),
9859 verified_head: None,
9860 verified_at: None,
9861 reviews: Vec::new(),
9862 e2e: Vec::new(),
9863 fix: None,
9864 blocking: 0,
9865 answered: 0,
9866 expected: 0,
9867 clean: true,
9868 verify_retried: false,
9869 e2e_deferred: false,
9870 e2e_defer_reason: None,
9871 progressed: false,
9872 vote_split: false,
9873 reconsideration: Vec::new(),
9874 verdict: None,
9875 }];
9876
9877 let mut runner = Runner {
9878 state,
9879 roles: ResolvedRoles {
9880 implementers: Vec::new(),
9881 judges: Vec::new(),
9882 reviewers: Vec::new(),
9883 fixer: None,
9884 conductor: conductor(),
9885 implementer_roster: Vec::new(),
9886 judge_roster: Vec::new(),
9887 reviewer_roster: Vec::new(),
9888 },
9889 sem: Arc::new(Semaphore::new(1)),
9890 pause: Pause::new(),
9891 interrupt: Pause::new(),
9892 };
9893
9894 let started = std::time::Instant::now();
9895 runner.gate().await.expect("gate");
9896 assert!(
9897 started.elapsed() < Duration::from_secs(1),
9898 "a gate with nothing to run must never wait on a lease it never needed"
9899 );
9900 assert!(
9901 runner.state.gate_ran,
9902 "zero commands is still a real, immediate attempt"
9903 );
9904 assert!(runner.state.gate.is_empty());
9905 assert_ne!(
9906 runner.state.status,
9907 RunStatus::Blocked,
9908 "must not read as resource-blocked on a lease it never asked for"
9909 );
9910 }
9911
9912 #[tokio::test]
9922 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
9923 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9924
9925 let tmp = tempfile::tempdir().expect("tempdir");
9926 let repo = tmp.path().join("repo");
9927 std::fs::create_dir_all(&repo).unwrap();
9928 init_repo(&repo);
9929
9930 let mut config = Config::default();
9931 config.verify.gate = vec![
9932 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
9933 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
9934 .to_owned(),
9935 ];
9936
9937 let mut state = RunState::new(
9938 repo.clone(),
9939 "main".to_owned(),
9940 "deadbeef".to_owned(),
9941 "task".to_owned(),
9942 config,
9943 );
9944 let run_id = state.id.clone();
9945 state.candidates = vec![Candidate {
9946 index: 0,
9947 label: 'A',
9948 agent: "alpha".to_owned(),
9949 branch: "does-not-exist".to_owned(),
9950 worktree: repo.clone(),
9951 summary: String::new(),
9952 stat: String::new(),
9953 files: 0,
9954 commits: 0,
9955 empty: false,
9956 failed: None,
9957 verified_noop: None,
9958 duration_ms: 0,
9959 folded: false,
9960 }];
9961 state.tally = Some(Tally {
9962 first_choice: BTreeMap::from([('A', 1)]),
9963 borda: BTreeMap::new(),
9964 winner: 'A',
9965 rankings: 1,
9966 unanimous_initial: true,
9967 deliberated: false,
9968 changed_votes: 0,
9969 unanimous_final: true,
9970 tie_break: None,
9971 judges: 0,
9972 present: 0,
9973 quorum: 0,
9974 met_quorum: true,
9975 uncontested: Some("only candidate A produced a change".to_owned()),
9976 });
9977 state.reviews = vec![ReviewRound {
9978 round: 1,
9979 head: "deadbeef".to_owned(),
9980 verified_head: None,
9981 verified_at: None,
9982 reviews: Vec::new(),
9983 e2e: Vec::new(),
9984 fix: None,
9985 blocking: 0,
9986 answered: 0,
9987 expected: 0,
9988 clean: true,
9989 verify_retried: false,
9990 e2e_deferred: false,
9991 e2e_defer_reason: None,
9992 progressed: false,
9993 vote_split: false,
9994 reconsideration: Vec::new(),
9995 verdict: None,
9996 }];
9997
9998 let mut runner = Runner {
9999 state,
10000 roles: ResolvedRoles {
10001 implementers: Vec::new(),
10002 judges: Vec::new(),
10003 reviewers: Vec::new(),
10004 fixer: None,
10005 conductor: conductor(),
10006 implementer_roster: Vec::new(),
10007 judge_roster: Vec::new(),
10008 reviewer_roster: Vec::new(),
10009 },
10010 sem: Arc::new(Semaphore::new(1)),
10011 pause: Pause::new(),
10012 interrupt: Pause::new(),
10013 };
10014
10015 let started_marker = repo.join("started.marker");
10016 let release_marker = repo.join("release.marker");
10017 let poller = tokio::spawn(async move {
10018 for _ in 0..100 {
10023 if started_marker.exists()
10024 && let Ok(s) = crate::run::RunState::load(&run_id)
10025 && let Some(a) = s.active.get("gate")
10026 {
10027 std::fs::write(&release_marker, b"go").expect("release marker");
10028 return Some(a.clone());
10029 }
10030 tokio::time::sleep(Duration::from_millis(50)).await;
10031 }
10032 None
10033 });
10034
10035 runner.gate().await.expect("gate");
10036 let captured = poller.await.expect("poller task");
10037 let captured = captured.expect(
10038 "the poller never saw a `gate` task entry in run.json while the command was \
10039 still blocked on its own release marker",
10040 );
10041
10042 assert_eq!(captured.task.as_deref(), Some("gate"));
10043 assert_eq!(captured.node, "gate");
10044 assert_eq!(captured.index, Some(1));
10045 assert_eq!(captured.total, Some(1));
10046 assert!(
10047 captured
10048 .command
10049 .as_deref()
10050 .is_some_and(|c| c.contains("started.marker")),
10051 "{captured:?}"
10052 );
10053
10054 assert!(
10055 runner.state.active.is_empty(),
10056 "the entry must be cleared once the command actually finished: {:?}",
10057 runner.state.active
10058 );
10059 assert!(runner.state.gate_ran);
10060 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
10061 }
10062
10063 #[tokio::test]
10066 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
10067 use crate::verdict::{Finding, ReviewVote, Severity};
10068 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10069 let tmp = tempfile::tempdir().expect("tempdir");
10070 let repo = tmp.path().join("repo");
10071 std::fs::create_dir_all(&repo).unwrap();
10072 init_repo(&repo);
10073
10074 for (severity, vote, expect) in [
10075 (Severity::Major, ReviewVote::Reject, true),
10076 (Severity::Minor, ReviewVote::Reject, false),
10077 (Severity::Major, ReviewVote::Approve, false),
10078 ] {
10079 let mut round = review_round(false, 1, 1, 1, false, true);
10080 round.reviews = vec![ReviewRecord {
10081 reviewer: 1,
10082 agent: "alpha".to_owned(),
10083 summary: String::new(),
10084 findings: vec![Finding {
10085 id: "R1-1-1".to_owned(),
10086 severity,
10087 file: None,
10088 line: None,
10089 title: "t".to_owned(),
10090 detail: String::new(),
10091 }],
10092 vote: Some(vote),
10093 failed: None,
10094 duration_ms: 0,
10095 attempts: 0,
10096 }];
10097 let mut state = RunState::new(
10098 repo.clone(),
10099 "main".to_owned(),
10100 "deadbeef".to_owned(),
10101 "task".to_owned(),
10102 Config::default(),
10103 );
10104 state.reviews = vec![round];
10105 let mut runner = Runner {
10106 state,
10107 roles: ResolvedRoles {
10108 implementers: Vec::new(),
10109 judges: Vec::new(),
10110 reviewers: Vec::new(),
10111 fixer: None,
10112 conductor: conductor(),
10113 implementer_roster: Vec::new(),
10114 judge_roster: Vec::new(),
10115 reviewer_roster: Vec::new(),
10116 },
10117 sem: Arc::new(Semaphore::new(1)),
10118 pause: Pause::new(),
10119 interrupt: Pause::new(),
10120 };
10121 let shell = runner.state.config.shell();
10122 runner
10123 .stop_reviewing("round budget spent", &shell, &repo)
10124 .await
10125 .expect("stop_reviewing");
10126 assert_eq!(runner.state.status, RunStatus::Gating);
10127 assert_eq!(
10128 runner.state.contested_handoff.is_some(),
10129 expect,
10130 "{severity:?} + {vote:?}"
10131 );
10132 }
10133 }
10134
10135 #[tokio::test]
10148 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
10149 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10150 let home = crate::run::home();
10151
10152 let tmp = tempfile::tempdir().expect("tempdir");
10153 let repo = tmp.path().join("repo");
10154 std::fs::create_dir_all(&repo).unwrap();
10155 init_repo(&repo);
10156 let head = crate::git::rev_parse(&repo, "HEAD")
10157 .await
10158 .expect("rev-parse");
10159 let cache_dir = tmp.path().join("target");
10162
10163 let mut config = Config::default();
10164 config.verify.e2e = vec![format!(
10165 "CARGO_TARGET_DIR='{}' test -f README.md",
10166 cache_dir.display()
10167 )];
10168 config.graph.review_rounds = 1;
10169 config.graph.timeout_verify = Some(2);
10172
10173 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10174 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10175 .expect("no io error acquiring directly")
10176 {
10177 crate::cache::AcquireOutcome::Acquired(g) => g,
10178 crate::cache::AcquireOutcome::Busy(b) => {
10179 panic!("expected the direct acquire to win the lease first: {b:?}")
10180 }
10181 };
10182
10183 let mut state = RunState::new(
10184 repo.clone(),
10185 "main".to_owned(),
10186 head.clone(),
10187 "task".to_owned(),
10188 config,
10189 );
10190 state.candidates = vec![Candidate {
10191 index: 0,
10192 label: 'A',
10193 agent: "alpha".to_owned(),
10194 branch: "does-not-exist".to_owned(),
10195 worktree: repo.clone(),
10196 summary: String::new(),
10197 stat: String::new(),
10198 files: 0,
10199 commits: 0,
10200 empty: false,
10201 failed: None,
10202 verified_noop: None,
10203 duration_ms: 0,
10204 folded: false,
10205 }];
10206 state.tally = Some(Tally {
10207 first_choice: BTreeMap::from([('A', 1)]),
10208 borda: BTreeMap::new(),
10209 winner: 'A',
10210 rankings: 1,
10211 unanimous_initial: true,
10212 deliberated: false,
10213 changed_votes: 0,
10214 unanimous_final: true,
10215 tie_break: None,
10216 judges: 0,
10217 present: 0,
10218 quorum: 0,
10219 met_quorum: true,
10220 uncontested: Some("only candidate A produced a change".to_owned()),
10221 });
10222 state.reviews = vec![ReviewRound {
10226 round: 1,
10227 head: head.clone(),
10228 verified_head: None,
10229 verified_at: None,
10230 reviews: Vec::new(),
10231 e2e: Vec::new(),
10232 fix: None,
10233 blocking: 1,
10234 answered: 1,
10235 expected: 1,
10236 clean: false,
10237 verify_retried: false,
10238 e2e_deferred: true,
10239 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
10240 progressed: false,
10241 vote_split: false,
10242 reconsideration: Vec::new(),
10243 verdict: None,
10244 }];
10245
10246 let mut runner = Runner {
10247 state,
10248 roles: ResolvedRoles {
10249 implementers: Vec::new(),
10250 judges: Vec::new(),
10251 reviewers: Vec::new(),
10252 fixer: None,
10253 conductor: conductor(),
10254 implementer_roster: Vec::new(),
10255 judge_roster: Vec::new(),
10256 reviewer_roster: Vec::new(),
10257 },
10258 sem: Arc::new(Semaphore::new(1)),
10259 pause: Pause::new(),
10260 interrupt: Pause::new(),
10261 };
10262
10263 let shell = runner.state.config.shell();
10264 runner
10265 .stop_reviewing("round budget spent", &shell, &repo)
10266 .await
10267 .expect("stop_reviewing");
10268
10269 let last = runner.state.reviews.last().expect("round record");
10270 assert_eq!(
10271 last.e2e_status(),
10272 E2eStatus::ResourceBlocked,
10273 "the shared cache is still held; the attempt must read as blocked, not deferred or \
10274 failed: {last:?}"
10275 );
10276 assert_eq!(
10277 last.verified_head.as_deref(),
10278 Some(head.as_str()),
10279 "which commit this attempt targeted is known even though nothing finished checking \
10280 it"
10281 );
10282 let first_attempt_at = last
10283 .verified_at
10284 .expect("when this attempt ran is known too");
10285 assert_ne!(
10286 runner.state.status,
10287 RunStatus::Blocked,
10288 "contention is evidence about the machine, not the patch — it must not settle the \
10289 run as blocked: {:?}",
10290 runner.state.status
10291 );
10292 assert!(
10293 !runner
10294 .state
10295 .events
10296 .iter()
10297 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
10298 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
10299 runner.state.events
10300 );
10301
10302 runner
10307 .stop_reviewing("round budget spent", &shell, &repo)
10308 .await
10309 .expect("stop_reviewing retry");
10310 assert_eq!(
10311 runner.state.reviews.len(),
10312 1,
10313 "no new round was started: {:?}",
10314 runner.state.reviews
10315 );
10316 let last = runner.state.reviews.last().expect("round record");
10317 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
10318 assert!(
10319 last.verified_at.expect("still known") > first_attempt_at,
10320 "a second reentry must be a fresh attempt, not a stale copy of the first"
10321 );
10322 assert_ne!(runner.state.status, RunStatus::Blocked);
10323
10324 held.release();
10325 }
10326
10327 #[tokio::test]
10339 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
10340 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10341 let home = crate::run::home();
10342
10343 let tmp = tempfile::tempdir().expect("tempdir");
10344 let repo = tmp.path().join("repo");
10345 std::fs::create_dir_all(&repo).unwrap();
10346 init_repo(&repo);
10347 let head = crate::git::rev_parse(&repo, "HEAD")
10348 .await
10349 .expect("rev-parse");
10350 let cache_dir = tmp.path().join("target");
10351
10352 let mut config = Config::default();
10353 config.verify.e2e = vec![format!(
10354 "CARGO_TARGET_DIR='{}' test -f README.md",
10355 cache_dir.display()
10356 )];
10357 config.graph.review_rounds = 1;
10358 config.graph.timeout_verify = Some(2);
10359
10360 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10361 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10362 .expect("no io error acquiring directly")
10363 {
10364 crate::cache::AcquireOutcome::Acquired(g) => g,
10365 crate::cache::AcquireOutcome::Busy(b) => {
10366 panic!("expected the direct acquire to win the lease first: {b:?}")
10367 }
10368 };
10369
10370 let mut state = RunState::new(
10371 repo.clone(),
10372 "main".to_owned(),
10373 head.clone(),
10374 "task".to_owned(),
10375 config,
10376 );
10377 state.candidates = vec![Candidate {
10378 index: 0,
10379 label: 'A',
10380 agent: "alpha".to_owned(),
10381 branch: "does-not-exist".to_owned(),
10382 worktree: repo.clone(),
10383 summary: String::new(),
10384 stat: String::new(),
10385 files: 0,
10386 commits: 0,
10387 empty: false,
10388 failed: None,
10389 verified_noop: None,
10390 duration_ms: 0,
10391 folded: false,
10392 }];
10393 state.tally = Some(Tally {
10394 first_choice: BTreeMap::from([('A', 1)]),
10395 borda: BTreeMap::new(),
10396 winner: 'A',
10397 rankings: 1,
10398 unanimous_initial: true,
10399 deliberated: false,
10400 changed_votes: 0,
10401 unanimous_final: true,
10402 tie_break: None,
10403 judges: 0,
10404 present: 0,
10405 quorum: 0,
10406 met_quorum: true,
10407 uncontested: Some("only candidate A produced a change".to_owned()),
10408 });
10409 state.reviews = vec![ReviewRound {
10413 round: 1,
10414 head: head.clone(),
10415 verified_head: Some(head.clone()),
10416 verified_at: Some(jiff::Timestamp::now()),
10417 reviews: Vec::new(),
10418 e2e: vec![CommandOutcome {
10419 command: format!(
10420 "CARGO_TARGET_DIR='{}' test -f README.md",
10421 cache_dir.display()
10422 ),
10423 code: None,
10424 output_tail: "waiting for the shared build cache".to_owned(),
10425 duration_ms: 0,
10426 resource_blocked: true,
10427 }],
10428 fix: None,
10429 blocking: 1,
10430 answered: 1,
10431 expected: 1,
10432 clean: false,
10433 verify_retried: false,
10434 e2e_deferred: false,
10435 e2e_defer_reason: None,
10436 progressed: false,
10437 vote_split: false,
10438 reconsideration: Vec::new(),
10439 verdict: None,
10440 }];
10441
10442 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
10443 let mut runner = Runner {
10444 state,
10445 roles: ResolvedRoles {
10446 implementers: Vec::new(),
10447 judges: Vec::new(),
10448 reviewers: Vec::new(),
10449 fixer: None,
10450 conductor: conductor(),
10451 implementer_roster: Vec::new(),
10452 judge_roster: Vec::new(),
10453 reviewer_roster: Vec::new(),
10454 },
10455 sem: Arc::new(Semaphore::new(1)),
10456 pause: Pause::new(),
10457 interrupt: Pause::new(),
10458 };
10459
10460 runner.review_loop().await.expect("review_loop");
10465
10466 assert_eq!(
10467 runner.state.reviews.len(),
10468 1,
10469 "no new round was started on top of the unresolved one: {:?}",
10470 runner.state.reviews
10471 );
10472 let last = &runner.state.reviews[0];
10473 assert_eq!(
10474 last.e2e_status(),
10475 E2eStatus::ResourceBlocked,
10476 "still contended: {last:?}"
10477 );
10478 assert!(
10479 last.verified_at.expect("still known") > first_attempt_at,
10480 "review_loop must have actually retried the check, not left it exactly as found"
10481 );
10482 assert_ne!(
10483 runner.state.status,
10484 RunStatus::Blocked,
10485 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
10486 runner.state.status
10487 );
10488
10489 held.release();
10490 }
10491
10492 #[tokio::test]
10493 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
10494 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10495 let tmp = tempfile::tempdir().expect("tempdir");
10496 let repo = tmp.path().join("repo");
10497 std::fs::create_dir_all(&repo).unwrap();
10498 init_repo(&repo);
10499
10500 let mut config = Config::default();
10501 config.merge.mode = MergeMode::Pr;
10502 config.graph.land = true;
10503 config.graph.land_approval = false;
10504
10505 let mut state = RunState::new(
10506 repo.clone(),
10507 "main".to_owned(),
10508 "deadbeef".to_owned(),
10509 "task".to_owned(),
10510 config,
10511 );
10512 state.candidates = vec![Candidate {
10513 index: 0,
10514 label: 'A',
10515 agent: "alpha".to_owned(),
10516 branch: "does-not-exist".to_owned(),
10517 worktree: repo.clone(),
10518 summary: String::new(),
10519 stat: String::new(),
10520 files: 0,
10521 commits: 0,
10522 empty: false,
10523 failed: None,
10524 verified_noop: None,
10525 duration_ms: 0,
10526 folded: false,
10527 }];
10528 state.tally = Some(Tally {
10529 first_choice: BTreeMap::from([('A', 1)]),
10530 borda: BTreeMap::new(),
10531 winner: 'A',
10532 rankings: 1,
10533 unanimous_initial: true,
10534 deliberated: false,
10535 changed_votes: 0,
10536 unanimous_final: true,
10537 tie_break: None,
10538 judges: 0,
10539 present: 0,
10540 quorum: 0,
10541 met_quorum: true,
10542 uncontested: Some("only candidate A produced a change".to_owned()),
10543 });
10544 state.reviews = vec![ReviewRound {
10545 round: 1,
10546 head: "deadbeef".to_owned(),
10547 verified_head: None,
10548 verified_at: None,
10549 reviews: Vec::new(),
10550 e2e: Vec::new(),
10551 fix: None,
10552 blocking: 0,
10553 answered: 0,
10554 expected: 0,
10555 clean: true,
10556 verify_retried: false,
10557 e2e_deferred: false,
10558 e2e_defer_reason: None,
10559 progressed: false,
10560 vote_split: false,
10561 reconsideration: Vec::new(),
10562 verdict: None,
10563 }];
10564 state.gate = vec![CommandOutcome {
10565 command: "test".to_owned(),
10566 code: Some(0),
10567 output_tail: String::new(),
10568 duration_ms: 0,
10569 resource_blocked: false,
10570 }];
10571 state.gate_ran = true;
10572 state.status = RunStatus::Landing;
10576 state.merge = Some(MergeOutcome {
10577 mode: MergeMode::Pr,
10578 ok: true,
10579 detail: "https://example.invalid/x/y/pull/1".to_owned(),
10580 empty: false,
10581 });
10582
10583 ask_test_home();
10587 let store = ask::Questions::open();
10588 let q = ask_open_question(&store, &state.id);
10589
10590 let mut runner = Runner {
10591 state,
10592 roles: ResolvedRoles {
10593 implementers: Vec::new(),
10594 judges: Vec::new(),
10595 reviewers: Vec::new(),
10596 fixer: None,
10597 conductor: conductor(),
10598 implementer_roster: Vec::new(),
10599 judge_roster: Vec::new(),
10600 reviewer_roster: Vec::new(),
10601 },
10602 sem: Arc::new(Semaphore::new(1)),
10603 pause: Pause::new(),
10604 interrupt: Pause::new(),
10605 };
10606
10607 runner.execute().await.expect("execute");
10612
10613 assert_eq!(
10614 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10615 Some("https://example.invalid/x/y/pull/1"),
10616 "reentry must not push again or open a second pull request over the \
10617 one `land` is already watching"
10618 );
10619 assert_ne!(
10620 runner.state.status,
10621 RunStatus::Landing,
10622 "land could not actually reach the fake pull request, so it must \
10623 have given up rather than left the run silently parked forever"
10624 );
10625 assert_eq!(runner.state.status, RunStatus::Blocked);
10629 assert!(
10630 store.get(&q.id).unwrap().status.open(),
10631 "Blocked is still alive; settle_questions must have been a no-op here"
10632 );
10633 }
10634
10635 fn state_with_round(round: ReviewRound) -> RunState {
10636 let mut s = RunState::new(
10637 PathBuf::from("/repo"),
10638 "main".to_owned(),
10639 "abc1234".to_owned(),
10640 "add retries".to_owned(),
10641 Config::default(),
10642 );
10643 s.reviews = vec![round];
10644 s
10645 }
10646
10647 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
10648 crate::verdict::Finding {
10649 id: id.to_owned(),
10650 severity,
10651 file: None,
10652 line: None,
10653 title: title.to_owned(),
10654 detail: String::new(),
10655 }
10656 }
10657
10658 #[test]
10659 fn pr_body_names_open_findings_and_declined_ones() {
10660 let round = ReviewRound {
10661 round: 2,
10662 head: "deadbee".to_owned(),
10663 verified_head: None,
10664 verified_at: None,
10665 reviews: vec![ReviewRecord {
10666 attempts: 0,
10667 reviewer: 1,
10668 agent: "alpha".to_owned(),
10669 summary: String::new(),
10670 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
10671 vote: None,
10672 failed: None,
10673 duration_ms: 0,
10674 }],
10675 e2e: vec![CommandOutcome {
10676 command: "cargo test".to_owned(),
10677 code: Some(0),
10678 output_tail: String::new(),
10679 duration_ms: 0,
10680 resource_blocked: false,
10681 }],
10682 verify_retried: false,
10683 e2e_deferred: false,
10684 e2e_defer_reason: None,
10685 fix: Some(FixRecord {
10686 agent: "alpha".to_owned(),
10687 addressed: Vec::new(),
10688 rejected: vec![crate::verdict::Rejection {
10689 id: "R1-1-1".to_owned(),
10690 why: "not reachable from any caller".to_owned(),
10691 }],
10692 notes: String::new(),
10693 committed: true,
10694 failed: None,
10695 duration_ms: 0,
10696 continuation: None,
10697 }),
10698 blocking: 0,
10699 answered: 1,
10700 expected: 1,
10701 clean: false,
10702 progressed: true,
10703 vote_split: false,
10704 reconsideration: Vec::new(),
10705 verdict: None,
10706 };
10707 let state = state_with_round(round);
10708 let body = pr_message(&state, 'A').body;
10709
10710 assert!(body.contains("add retries"), "the task must still be there");
10711 assert!(body.contains("R2-1-1"), "{body}");
10712 assert!(body.contains("unused import"), "{body}");
10713 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
10714 assert!(
10715 body.contains("not reachable from any caller"),
10716 "the reason it was declined: {body}"
10717 );
10718 }
10719
10720 #[test]
10721 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
10722 let round = ReviewRound {
10723 round: 1,
10724 head: "deadbee".to_owned(),
10725 verified_head: None,
10726 verified_at: None,
10727 reviews: vec![ReviewRecord {
10728 attempts: 0,
10729 reviewer: 1,
10730 agent: "alpha".to_owned(),
10731 summary: String::new(),
10732 findings: Vec::new(),
10733 vote: None,
10734 failed: None,
10735 duration_ms: 0,
10736 }],
10737 e2e: Vec::new(),
10738 verify_retried: false,
10739 e2e_deferred: false,
10740 e2e_defer_reason: None,
10741 fix: None,
10742 blocking: 0,
10743 answered: 1,
10744 expected: 1,
10745 clean: true,
10746 progressed: false,
10747 vote_split: false,
10748 reconsideration: Vec::new(),
10749 verdict: None,
10750 };
10751 let state = state_with_round(round);
10752 let body = pr_message(&state, 'A').body;
10753 assert!(!body.contains("Open review findings"), "{body}");
10754 assert!(!body.contains("Declined"), "{body}");
10755 }
10756
10757 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
10758 let mut state = RunState::new(
10759 PathBuf::from("/repo"),
10760 "main".to_owned(),
10761 "abc1234".to_owned(),
10762 instruction.to_owned(),
10763 Config::default(),
10764 );
10765 state.candidates.push(Candidate {
10766 index: 0,
10767 label: 'A',
10768 agent: "alpha".to_owned(),
10769 branch: "magi/x/A".to_owned(),
10770 worktree: PathBuf::from("/wt"),
10771 summary: summary.to_owned(),
10772 stat: String::new(),
10773 files: 1,
10774 commits: 1,
10775 empty: false,
10776 failed: None,
10777 verified_noop: None,
10778 folded: false,
10779 duration_ms: 0,
10780 });
10781 state
10782 }
10783
10784 fn review_state(subjects: &[&str]) -> RunState {
10785 let mut state = state_with_summary(
10786 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
10787 "",
10788 );
10789 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
10790 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
10791 state
10792 }
10793
10794 #[test]
10795 fn pr_message_review_single_commit_uses_its_subject() {
10796 let state = review_state(&["feat(nats): per-role user"]);
10797 let m = pr_message(&state, 'A');
10798 assert_eq!(m.title, "feat(nats): per-role user");
10799 assert!(!m.body.contains("Review the work already"), "{}", m.body);
10800 assert!(m.body.contains("## Commits under review"), "{}", m.body);
10801 }
10802
10803 #[test]
10804 fn pr_message_review_multi_commit_takes_the_oldest() {
10805 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
10806 let m = pr_message(&state, 'A');
10807 assert_eq!(m.title, "feat: the change");
10808 for s in ["feat: the change", "fix: typo", "fix: again"] {
10809 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
10810 }
10811 }
10812
10813 #[test]
10814 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
10815 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
10816 let mut state = review_state(&[first, "fix: later fixup"]);
10817 state.candidates[0].branch = "機能/ログイン".to_owned();
10818 let m = pr_message(&state, 'A');
10819 assert!(
10820 m.title.starts_with("chore: land candidate A of run"),
10821 "{}",
10822 m.title
10823 );
10824 }
10825 }
10826
10827 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
10828 BranchFacts {
10829 commits: commits
10830 .iter()
10831 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
10832 .collect(),
10833 stat: stat.to_owned(),
10834 }
10835 }
10836
10837 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
10838 let mut r = review_round(true, 0, 1, 1, true, true);
10839 r.round = round;
10840 r.fix = notes.map(|n| FixRecord {
10841 agent: "fixer".to_owned(),
10842 addressed: Vec::new(),
10843 rejected: Vec::new(),
10844 notes: n.to_owned(),
10845 committed: true,
10846 failed: None,
10847 duration_ms: 0,
10848 continuation: None,
10849 });
10850 r
10851 }
10852
10853 #[test]
10854 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
10855 let state = review_state(&["ignored"]);
10856 let f = facts(
10857 &[
10858 (
10859 "fix(login): resolve PATH on macOS",
10860 "Login shells skip rc files.",
10861 ),
10862 ("fix: address review", ""),
10863 ],
10864 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
10865 );
10866 let m = pr_message_with(&state, 'A', Some(&f));
10867 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
10868 assert!(
10869 m.body
10870 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
10871 "{}",
10872 m.body
10873 );
10874 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
10875 assert!(m.body.contains("## Diff stat"), "{}", m.body);
10876 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
10877 for banned in [
10878 "Review the work already",
10879 "no task statement",
10880 "Original task",
10881 ] {
10882 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
10883 }
10884 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
10885 }
10886
10887 #[test]
10888 fn pr_message_review_truncates_a_huge_first_commit_body() {
10889 let state = review_state(&["ignored"]);
10890 let f = facts(
10891 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
10892 "s",
10893 );
10894 let m = pr_message_with(&state, 'A', Some(&f));
10895 assert!(m.body.len() < 30_000, "{}", m.body.len());
10896 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
10897 assert!(
10898 m.body.contains("- ... 1 more commit(s)"),
10899 "{}",
10900 m.body.len()
10901 );
10902 assert!(m.body.ends_with("magi:candidate-a\n"));
10903 }
10904
10905 #[test]
10906 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
10907 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
10908 assert_eq!(m.title, "feat: x");
10909 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
10910 assert!(!m.body.contains("Diff stat"), "{}", m.body);
10911 assert!(!m.body.contains("no task statement"), "{}", m.body);
10912 }
10913
10914 #[test]
10915 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
10916 let mut state = review_state(&["日本語の件名"]);
10917 state.candidates[0].branch = "fix/macos-login-path".to_owned();
10918 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
10919 state.candidates[0].branch = "機能/ログイン".to_owned();
10920 assert!(
10921 pr_message(&state, 'A')
10922 .title
10923 .starts_with("chore: land candidate A")
10924 );
10925 }
10926
10927 #[test]
10928 fn pr_message_review_fixes_survive_a_clean_final_round() {
10929 let mut state = review_state(&["feat: x"]);
10930 state.reviews = vec![
10931 round_with_notes(1, Some("handled the PATH case")),
10932 round_with_notes(2, None),
10933 ];
10934 let body = pr_message(&state, 'A').body;
10935 assert!(
10936 body.contains("## Review fixes\n\nhandled the PATH case\n"),
10937 "{body}"
10938 );
10939 assert!(!body.contains("### Round"), "{body}");
10940
10941 state.reviews = vec![
10942 round_with_notes(1, Some("first fix")),
10943 round_with_notes(2, Some("")),
10944 round_with_notes(3, Some("second fix")),
10945 round_with_notes(4, None),
10946 ];
10947 let body = pr_message(&state, 'A').body;
10948 assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
10949 assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
10950 assert!(!body.contains("### Round 2"), "{body}");
10951 }
10952
10953 #[test]
10954 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
10955 let state = state_with_summary(
10956 "add retries to the client",
10957 "TITLE: feat: retries\n\nDid it.",
10958 );
10959 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
10960 assert_eq!(m.title, "feat: retries");
10961 assert!(
10962 m.body.contains("<summary>Original task</summary>"),
10963 "{}",
10964 m.body
10965 );
10966 assert!(!m.body.contains("Commits under review"), "{}", m.body);
10967 assert!(
10968 m.body
10969 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
10970 "{}",
10971 m.body
10972 );
10973 }
10974
10975 #[test]
10976 fn pr_message_review_bounds_a_long_english_subject() {
10977 let long = format!("feat: {}", "word ".repeat(100));
10978 let m = pr_message(&review_state(&[&long]), 'A');
10979 assert!(m.title.starts_with("feat: word"), "{}", m.title);
10980 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10981 }
10982
10983 #[test]
10984 fn pr_message_implementation_run_is_unchanged_by_review_support() {
10985 let state = state_with_summary("add retries\n\ndetails", "- did some things");
10986 let m = pr_message(&state, 'A');
10987 assert_eq!(m.title, "add retries");
10988 assert!(m.body.contains("<summary>Original task</summary>"));
10989 assert!(!m.body.contains("Commits under review"));
10990 assert_eq!(landing_subject_source(&state), state.instruction);
10991 }
10992
10993 #[test]
10994 fn review_run_squash_subject_is_the_change_not_the_prompt() {
10995 let state = review_state(&["feat: the change", "fix: typo"]);
10996 let source = landing_subject_source(&state);
10997 assert_eq!(land::merge_subject("", &source), "feat: the change");
10998 assert_eq!(
10999 land::merge_subject("magi: candidate A (uncommitted work)", &source),
11000 "feat: the change"
11001 );
11002 assert_eq!(
11004 land::merge_subject("feat: renamed by hand", &source),
11005 "feat: renamed by hand"
11006 );
11007 let blank = review_state(&["日本語"]);
11008 assert!(
11009 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
11010 );
11011 }
11012
11013 #[test]
11014 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
11015 let state = review_state(&["feat: the change"]);
11016 let source = landing_subject_source(&state);
11017 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
11018 assert_eq!(
11019 land::merge_subject(landing_title(&state, old), &source),
11020 "feat: the change"
11021 );
11022 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
11023 let task = state_with_summary("add retries", "");
11024 assert_eq!(landing_title(&task, old), old);
11025 }
11026
11027 #[test]
11028 fn pr_message_describes_the_change_not_the_task() {
11029 let state = state_with_summary(
11030 "今回やってほしいこと: results projector を直す",
11031 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
11032 );
11033 let m = pr_message(&state, 'A');
11034 assert_eq!(m.title, "fix(web): batch the runs list reads");
11035 assert!(
11036 m.body.starts_with("## Summary\n\n- reads run.json once"),
11037 "{}",
11038 m.body
11039 );
11040 assert!(!m.body.contains("TITLE:"), "{}", m.body);
11041 let task_at = m.body.find("今回やってほしいこと").unwrap();
11042 let details_at = m.body.find("<details>").unwrap();
11043 assert!(
11044 details_at < task_at,
11045 "the task lives inside <details>: {}",
11046 m.body
11047 );
11048 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
11049 assert!(m.body.contains("magi:candidate-a"));
11050 }
11051
11052 #[test]
11053 fn pr_message_falls_back_to_the_task_without_a_title_line() {
11054 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
11055 let m = pr_message(&state, 'A');
11056 assert_eq!(m.title, "add retries");
11057 assert!(
11058 m.body.contains("## Summary\n\n- did some things"),
11059 "{}",
11060 m.body
11061 );
11062
11063 let none = RunState::new(
11064 PathBuf::from("/repo"),
11065 "main".to_owned(),
11066 "abc1234".to_owned(),
11067 "add retries".to_owned(),
11068 Config::default(),
11069 );
11070 let m = pr_message(&none, 'A');
11071 assert_eq!(m.title, "add retries");
11072 assert!(!m.body.contains("## Summary"), "{}", m.body);
11073 }
11074
11075 #[test]
11076 fn pr_message_refuses_the_candidate_commit_subject() {
11077 for bad in [
11078 "TITLE: magi: candidate A (uncommitted work)",
11079 "TITLE: chore: stuff (uncommitted work)",
11080 "TITLE: ",
11081 ] {
11082 let state = state_with_summary("add retries", bad);
11083 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
11084 }
11085 }
11086
11087 #[test]
11088 fn pr_message_bounds_a_very_long_task_and_title() {
11089 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
11090 let state = state_with_summary(&long, "- nothing");
11091 let m = pr_message(&state, 'A');
11092 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11093 assert!(!m.title.contains('\n'));
11094
11095 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
11096 let m = pr_message(&state, 'A');
11097 assert!(m.title.starts_with("feat: "));
11098 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11099 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
11100 }
11101
11102 fn long_title_of(instruction: &str) -> String {
11103 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
11104 }
11105
11106 #[test]
11107 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
11108 let first = "Make the landing path keep a readable title for long tasks";
11109 let line = format!(
11110 "{first}. {}",
11111 "And then keep going with more words ".repeat(20)
11112 );
11113 let t = long_title_of(&line);
11114 assert_eq!(t, first);
11115 assert!(!t.starts_with("chore: land"));
11116 }
11117
11118 #[test]
11119 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
11120 let line = "word ".repeat(200);
11121 let t = long_title_of(&line);
11122 assert!(t.ends_with("word..."), "{t}");
11123 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
11124 }
11125
11126 #[test]
11127 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
11128 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
11129 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
11130 }
11131 }
11132
11133 #[test]
11134 fn pr_message_title_limit_is_exact() {
11135 let at = "a".repeat(PR_TITLE_MAX);
11136 assert_eq!(long_title_of(&at), at);
11137 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
11138 assert!(over.ends_with("..."), "{over}");
11139 assert_eq!(over.chars().count(), PR_TITLE_MAX);
11140 }
11141
11142 #[test]
11143 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
11144 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
11145 let t = long_title_of(&line);
11146 assert!(t.ends_with("..."), "{t}");
11147 assert!(t.is_ascii(), "{t}");
11148 }
11149
11150 #[test]
11151 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
11152 let line = format!(
11153 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
11154 "plus more filler words ".repeat(20)
11155 );
11156 let t = long_title_of(&line);
11157 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
11158 }
11159
11160 #[test]
11161 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
11162 let line = format!("\n\n# {}", "title words ".repeat(40));
11163 let state = state_with_summary(&line, "- nothing");
11164 let m = pr_message(&state, 'A');
11165 assert!(m.title.starts_with("title words"), "{}", m.title);
11166 assert_eq!(
11167 land::merge_subject(&m.title, &landing_subject_source(&state)),
11168 m.title
11169 );
11170 assert_eq!(
11172 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
11173 "feat: renamed by hand"
11174 );
11175 }
11176
11177 #[test]
11178 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
11179 let mut state = state_with_summary(
11183 "add retries",
11184 "TITLE: fix(web): batch reads\n- reads run.json once",
11185 );
11186 state.config.graph.language = "ja".to_owned();
11187 let m = pr_message(&state, 'A');
11188 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
11189
11190 let task = "今回やってほしいこと: results projector を直す";
11193 let mut state = state_with_summary(task, "- no title line");
11194 state.config.graph.language = "ja".to_owned();
11195 let m = pr_message(&state, 'A');
11196 assert_eq!(
11197 m.title,
11198 format!("chore: land candidate A of run {}", state.id)
11199 );
11200 assert!(
11201 m.body.contains(&format!(
11202 "<summary>Original task</summary>\n\n{task}\n\n</details>"
11203 )),
11204 "{}",
11205 m.body
11206 );
11207 }
11208
11209 #[test]
11210 fn pr_message_scrubs_home_paths_and_addresses() {
11211 let state = state_with_summary(
11212 "fix it in /Users/someone/src/x",
11213 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
11214 );
11215 let m = pr_message(&state, 'A');
11216 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
11217 assert!(!m.body.contains(leak), "{}", m.body);
11218 }
11219 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
11220 }
11221
11222 #[test]
11223 fn pr_message_survives_a_task_that_closes_details() {
11224 let state = state_with_summary("a </details> b", "TITLE: fix: x");
11225 let m = pr_message(&state, 'A');
11226 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
11227 }
11228
11229 #[test]
11230 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
11231 let cmd = manual_merge_command(
11232 MergeStyle::Squash,
11233 Path::new("/repo"),
11234 "b",
11235 "fix: \"quoted\" $(x) `y`\n\nbody",
11236 );
11237 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
11238 }
11239
11240 #[test]
11241 fn manual_merge_command_matches_the_configured_style() {
11242 let repo = Path::new("/repo");
11243 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
11244
11245 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
11246 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
11247
11248 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
11249 assert_eq!(
11250 squash,
11251 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
11252 \"Merge magi run 0832 (candidate A)\""
11253 );
11254
11255 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
11256 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
11257 }
11258
11259 #[test]
11260 fn a_nudge_gets_a_quarter_of_the_budget() {
11261 assert_eq!(retry_budget(secs(1200), true), secs(300));
11263 assert_eq!(retry_budget(secs(3600), true), secs(900));
11264 }
11265
11266 #[test]
11267 fn a_resent_prompt_keeps_the_whole_budget() {
11268 assert_eq!(retry_budget(secs(1200), false), secs(1200));
11271 assert_eq!(retry_budget(secs(60), false), secs(60));
11272 }
11273
11274 #[test]
11275 fn the_floor_never_exceeds_the_original_budget() {
11276 assert_eq!(retry_budget(secs(60), true), secs(60));
11280 assert_eq!(retry_budget(secs(480), true), secs(120));
11281 assert_eq!(retry_budget(secs(0), true), secs(0));
11282 }
11283
11284 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
11285 agent::CommandEvidence {
11286 id: "item1".to_owned(),
11287 description: "cargo test".to_owned(),
11288 exit_code,
11289 result_summary: String::new(),
11290 source: "codex".to_owned(),
11291 }
11292 }
11293
11294 #[test]
11295 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
11296 assert!(!has_unconfirmed_command(&[]));
11300 }
11301
11302 #[test]
11303 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
11304 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
11308 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
11309 assert!(!has_unconfirmed_command(&[
11310 evidence(Some(0)),
11311 evidence(Some(101))
11312 ]));
11313 }
11314
11315 #[test]
11316 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
11317 assert!(has_unconfirmed_command(&[
11318 evidence(Some(0)),
11319 evidence(None)
11320 ]));
11321 }
11322
11323 #[test]
11324 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
11325 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11326 assert_eq!(
11327 verified_noop_claim(true, &[], text).as_deref(),
11328 Some("already fixed by b32cfc4, on main.")
11329 );
11330 }
11331
11332 #[test]
11333 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
11334 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11337 assert!(verified_noop_claim(false, &[], text).is_none());
11338 }
11339
11340 #[test]
11341 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
11342 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11343 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
11344 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
11346 }
11347
11348 #[test]
11349 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
11350 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
11351 }
11352
11353 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
11356 runner.state.candidates = shape
11357 .iter()
11358 .enumerate()
11359 .map(|(i, &(empty, verified))| Candidate {
11360 index: i,
11361 label: (b'A' + i as u8) as char,
11362 agent: "sonnet".to_owned(),
11363 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
11364 worktree: PathBuf::from(format!("/wt/{i}")),
11365 summary: String::new(),
11366 stat: String::new(),
11367 files: 0,
11368 commits: 0,
11369 empty,
11370 failed: None,
11371 verified_noop: verified.map(str::to_owned),
11372 duration_ms: 0,
11373 folded: false,
11374 })
11375 .collect();
11376 }
11377
11378 #[test]
11379 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
11380 ask_test_home();
11381 let mut runner = runner_at(RunStatus::Implementing);
11382 set_candidates(
11383 &mut runner,
11384 &[
11385 (true, Some("already on main at b32cfc4")),
11386 (true, Some("same fix, see the existing test")),
11387 ],
11388 );
11389
11390 runner
11391 .after_implement()
11392 .expect("a verified no-op is not an error");
11393
11394 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
11395 }
11396
11397 #[test]
11398 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
11399 ask_test_home();
11400 let mut runner = runner_at(RunStatus::Implementing);
11401 set_candidates(
11405 &mut runner,
11406 &[(true, Some("already on main at b32cfc4")), (true, None)],
11407 );
11408
11409 let err = runner
11410 .after_implement()
11411 .expect_err("an unverified empty candidate must still fail the run");
11412
11413 assert!(
11414 err.to_string().contains("no candidate produced a change"),
11415 "{err}"
11416 );
11417 assert_eq!(runner.state.status, RunStatus::Failed);
11418 }
11419
11420 #[test]
11421 fn after_implement_still_fails_an_ordinary_all_empty_run() {
11422 ask_test_home();
11423 let mut runner = runner_at(RunStatus::Implementing);
11424 set_candidates(&mut runner, &[(true, None), (true, None)]);
11425
11426 let err = runner
11427 .after_implement()
11428 .expect_err("no candidate declared anything; this is an ordinary failure");
11429
11430 assert!(
11431 err.to_string().contains("no candidate produced a change"),
11432 "{err}"
11433 );
11434 assert_eq!(runner.state.status, RunStatus::Failed);
11435 }
11436}