1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::refs;
46use crate::run::{
47 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
48 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord, Handover,
49 JobRecord, JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome,
50 OperatorFixRequest, Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState,
51 RunStatus, SeatHistory, Tally, VoteRecord, tail, write_artifact,
52};
53use crate::verdict::{
54 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
55 ReviewVote, Severity,
56};
57
58const OUTPUT_TAIL: usize = 8_000;
60
61const EVENT_OUTPUT_TAIL: usize = 2_000;
64
65const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
68
69const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
86
87pub(crate) const STAGNANT_LIMIT: usize = 2;
101
102const BASE_SYNC_ROUNDS: usize = 4;
115
116const MAX_FIX_CONTINUATIONS: usize = 2;
132
133#[derive(Clone)]
139struct SeatJob {
140 spec: AgentSpec,
141 seat: SeatState,
142 cwd: PathBuf,
143 prompt: String,
144 timeout: Duration,
145 allow_write: bool,
146 sessions: bool,
147 artifacts: PathBuf,
148 stem: String,
149 handover: Option<String>,
154}
155
156enum AgentOutcome {
168 Ok(AgentOutput),
170 Quota(AgentOutput),
172 Dropped(AgentOutput),
175 Failed(String),
177}
178
179#[derive(Debug, Clone, Default)]
206pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
207
208impl Pause {
209 #[must_use]
211 pub fn new() -> Self {
212 Self::default()
213 }
214
215 pub fn park(&self) {
217 self.0.store(true, Ordering::SeqCst);
218 }
219
220 pub fn park_because(&self, reason: impl Into<String>) {
226 let mut reason_guard = self
227 .1
228 .lock()
229 .unwrap_or_else(std::sync::PoisonError::into_inner);
230 if reason_guard.is_none() {
231 *reason_guard = Some(reason.into());
232 }
233 drop(reason_guard);
234 self.park();
235 }
236
237 #[must_use]
239 pub fn parked(&self) -> bool {
240 self.0.load(Ordering::SeqCst)
241 }
242
243 #[must_use]
245 pub fn reason(&self) -> Option<String> {
246 self.1
247 .lock()
248 .unwrap_or_else(std::sync::PoisonError::into_inner)
249 .clone()
250 }
251}
252
253pub struct Runner {
255 pub state: RunState,
257 roles: ResolvedRoles,
258 sem: Arc<Semaphore>,
259 pause: Pause,
263 interrupt: Pause,
269}
270
271async fn review_base(
276 repo: &Path,
277 remote: &str,
278 base_branch: &str,
279 base_commit: &str,
280 branch: &str,
281) -> String {
282 review_base_checked(repo, remote, base_branch, base_commit, branch)
283 .await
284 .0
285}
286
287async fn review_base_checked(
292 repo: &Path,
293 remote: &str,
294 base_branch: &str,
295 base_commit: &str,
296 branch: &str,
297) -> (String, bool) {
298 let tracking = format!("{remote}/{base_branch}");
299 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
300 if git::rev_exists(repo, &tracking).await
301 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
302 && !mb.is_empty()
303 {
304 return (mb, fresh);
305 }
306 let mb = git::merge_base(repo, base_commit, branch)
307 .await
308 .ok()
309 .filter(|mb| !mb.is_empty())
310 .unwrap_or_else(|| base_commit.to_owned());
311 (mb, false)
312}
313
314pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
318 if !is_review_run(state) {
319 return;
320 }
321 let base = review_base(
322 &state.repo,
323 &state.config.merge.remote,
324 &state.base_branch,
325 &state.base_commit,
326 branch,
327 )
328 .await;
329 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
330 && !subjects.is_empty()
331 && state.reviewed_commits.as_ref() != Some(&subjects)
332 {
333 state.reviewed_commits = Some(subjects);
334 state.save().ok();
335 }
336}
337
338async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
342 let (base, trusted) = review_base_checked(
343 &state.repo,
344 &state.config.merge.remote,
345 &state.base_branch,
346 &state.base_commit,
347 branch,
348 )
349 .await;
350 if !trusted {
351 return None;
352 }
353 git::subjects(&state.repo, &state.base_commit, &base)
354 .await
355 .ok()
356}
357
358fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
362 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
363 if is_leak(computed) {
364 return false;
365 }
366 let cur = current.trim();
367 cur.is_empty()
368 || cur.starts_with(REVIEW_PROMPT_OPENING)
369 || cur.starts_with("chore: land candidate")
370 || cur.starts_with("magi: candidate")
371 || is_leak(cur)
372}
373
374async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
403 let tracking = format!("{remote}/{branch}");
404 let fetched = git::fetch(repo, remote, branch).await;
405 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
406 let local_exists = git::branch_exists(repo, branch).await?;
407 if !fresh {
408 if !local_exists {
409 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
410 }
411 tracing::warn!(
412 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
413 );
414 return Ok(());
415 }
416 let remote_sha = git::rev_parse(repo, &tracking).await?;
417 if !local_exists {
418 git::git(repo, &["branch", branch, &tracking]).await?;
419 return Ok(());
420 }
421 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
422 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
423 return Ok(());
424 }
425 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
426 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
432 .await?
433 {
434 crate::reconcile::Reconciliation::Pushed => {
435 tracing::warn!(
436 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
437 short(&local_sha),
438 short(&remote_sha)
439 );
440 return Ok(());
441 }
442 crate::reconcile::Reconciliation::Placeholder => {}
443 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
444 }
445 }
446 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
447 if !out.ok() {
448 bail!(
449 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
450 short(&local_sha),
451 short(&remote_sha),
452 out.stderr
453 );
454 }
455 tracing::warn!(
456 "local `{branch}` was stale: fast-forwarded {} -> {}",
457 short(&local_sha),
458 short(&remote_sha)
459 );
460 Ok(())
461}
462
463async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
464 let tracking = format!("{remote}/{base_branch}");
465 let fetched = git::fetch(repo, remote, base_branch).await;
466 if let Ok(out) = &fetched
467 && out.ok()
468 && git::rev_exists(repo, &tracking).await
469 {
470 return git::rev_parse(repo, &tracking).await;
471 }
472 let why = match &fetched {
473 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
474 Ok(_) => format!("{remote} has no {base_branch}"),
475 Err(e) => e.to_string(),
476 };
477 bail!(
480 "cannot read {tracking} ({why}); refusing to branch off the local \
481 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
482 base / remote in magi.toml"
483 )
484}
485
486struct FixClaim {
503 path: PathBuf,
504}
505
506impl FixClaim {
507 fn acquire(dir: &Path) -> Result<Self> {
508 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
509 let path = dir.join("fix.lock");
510 match Self::create(&path) {
511 Ok(claim) => Ok(claim),
512 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
513 if Self::reclaim_if_dead(&path) {
514 Self::create(&path).with_context(|| format!("lock {}", path.display()))
515 } else {
516 bail!(
517 "another `magi fix` is already running for this run ({} exists)",
518 path.display()
519 )
520 }
521 }
522 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
523 }
524 }
525
526 fn create(path: &Path) -> std::io::Result<Self> {
527 let mut f = std::fs::OpenOptions::new()
528 .write(true)
529 .create_new(true)
530 .open(path)?;
531 use std::io::Write as _;
532 writeln!(f, "{}", std::process::id())?;
534 Ok(Self {
535 path: path.to_owned(),
536 })
537 }
538
539 fn reclaim_if_dead(path: &Path) -> bool {
543 let dead = std::fs::read_to_string(path)
544 .ok()
545 .and_then(|body| body.trim().parse::<u32>().ok())
546 .is_some_and(|pid| !crate::proc::pid_alive(pid));
547 dead && std::fs::remove_file(path).is_ok()
548 }
549}
550
551impl Drop for FixClaim {
552 fn drop(&mut self) {
553 let _ = std::fs::remove_file(&self.path);
554 }
555}
556
557impl Runner {
558 pub async fn start(
560 repo: &Path,
561 instruction: String,
562 config: Config,
563 origin: Origin,
564 ) -> Result<Self> {
565 Self::start_naming(repo, instruction, "", config, origin).await
566 }
567
568 pub async fn start_naming(
572 repo: &Path,
573 instruction: String,
574 also_scan: &str,
575 config: Config,
576 origin: Origin,
577 ) -> Result<Self> {
578 let repo = git::toplevel(repo).await?;
579 let missing = agent::missing_programs(&config.agents);
580 if !missing.is_empty() {
581 bail!(
582 "these agent programs are not on PATH: {}. Fix the roster in \
583 magi.toml or install them.",
584 missing.join(", ")
585 );
586 }
587 let base_branch = match config.merge.base.clone() {
588 Some(b) => b,
589 None => git::current_branch(&repo)
590 .await?
591 .context("HEAD is detached; set [merge] base in magi.toml")?,
592 };
593 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
594 if !git::is_clean(&repo).await? {
598 tracing::warn!(
599 "{} has uncommitted changes; they are not part of this run, \
600 which branches off {base_branch} ({})",
601 repo.display(),
602 &base_commit[..base_commit.len().min(8)]
603 );
604 }
605 let roles = config.resolve_roles()?;
606 let max_parallel = config.graph.max_parallel.max(1);
607 let seeds = refs::resolve(
610 &repo,
611 &base_commit,
612 &config.merge.remote,
613 &format!("{also_scan}\n{instruction}"),
614 )
615 .await;
616 refs::plan(&repo, &seeds).await?;
617 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
618 state.origin = Some(origin);
622 for seed in &seeds {
623 state.event(
624 "seed",
625 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
626 );
627 }
628 state.seeds = seeds;
629 state.event("start", format!("run {} created", state.id));
630 state.save()?;
631 Ok(Self {
632 state,
633 roles,
634 sem: Arc::new(Semaphore::new(max_parallel)),
635 pause: Pause::new(),
636 interrupt: Pause::new(),
637 })
638 }
639
640 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
654 Self::review_taking_over(repo, branch, config, None, origin).await
655 }
656
657 pub async fn review_taking_over(
664 repo: &Path,
665 branch: &str,
666 config: Config,
667 takeover: Option<crate::handover::Takeover>,
668 origin: Origin,
669 ) -> Result<Self> {
670 let repo = git::toplevel(repo).await?;
671 let missing = agent::missing_programs(&config.agents);
672 if !missing.is_empty() {
673 bail!(
674 "these agent programs are not on PATH: {}. Fix the roster in \
675 magi.toml or install them.",
676 missing.join(", ")
677 );
678 }
679 let base_branch = match config.merge.base.clone() {
680 Some(b) => b,
681 None => git::current_branch(&repo)
682 .await?
683 .context("HEAD is detached; set [merge] base in magi.toml")?,
684 };
685 if base_branch == branch {
686 bail!("`{branch}` is the base branch; there is nothing to review against");
687 }
688 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
689
690 let roles = config.resolve_roles()?;
691 let max_parallel = config.graph.max_parallel.max(1);
692 let mut state = RunState::new(
693 repo.clone(),
694 base_branch,
695 base_commit.clone(),
696 String::new(),
697 config,
698 );
699 state.origin = Some(origin);
700
701 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
708 earlier: Vec::new(),
709 home: crate::run::home(),
710 choice: None,
711 });
712 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
713 if let Some(released) = &released {
714 state.event(
715 "release",
716 format!(
717 "took `{branch}` over from run {}: its worktree was released: {}",
718 crate::run::short_of(&released.old_id),
719 released.audit
720 ),
721 );
722 }
723 if let Some(choice) = takeover.choice.as_ref()
727 && let Err(e) =
728 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
729 .await
730 {
731 if let Some(released) = &released {
732 released.restore(&repo, branch).await;
733 }
734 return Err(e.context("applying the owner's answer about the diverged branch"));
735 }
736 let opened =
737 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
738 if opened.is_err()
739 && let Some(released) = &released
740 {
741 released.restore(&repo, branch).await;
742 }
743 opened
744 }
745
746 async fn open_review(
749 repo: &Path,
750 branch: &str,
751 mut state: RunState,
752 roles: ResolvedRoles,
753 max_parallel: usize,
754 base_commit: String,
755 ) -> Result<Self> {
756 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
757 let start = review_base(
760 repo,
761 &state.config.merge.remote,
762 &state.base_branch,
763 &base_commit,
764 branch,
765 )
766 .await;
767 let log = git::log_oneline(repo, &start, branch)
768 .await
769 .unwrap_or_default();
770 let instruction = format!(
771 "Review the work already on branch `{branch}`. There is no task \
772 statement: what the change claims to do is whatever its commits \
773 say.\n\n{}",
774 if log.trim().is_empty() {
775 "(no commit messages)"
776 } else {
777 log.trim()
778 }
779 );
780 state.instruction = instruction;
781 state.reviewed_commits = Some(
782 git::subjects(repo, &start, branch)
783 .await
784 .unwrap_or_default(),
785 );
786
787 let worktree = state.worktree_root().join("under-review");
790 if let Some(parent) = worktree.parent() {
791 tokio::fs::create_dir_all(parent).await.ok();
792 }
793 let path = worktree.to_string_lossy().to_string();
794 git::git(repo, &["worktree", "add", &path, branch])
795 .await
796 .with_context(|| {
797 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
798 })?;
799
800 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
801 .await
802 .unwrap_or(0);
803 if commits == 0 {
804 git::worktree_remove(repo, &worktree).await.ok();
805 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
806 }
807 let files = git::changed_files(&worktree, &base_commit, "HEAD")
808 .await
809 .map(|f| f.len())
810 .unwrap_or(0);
811 if files == 0
812 && let (Ok(head_tree), Ok(base_tree)) = (
813 git::tree_of(&worktree, "HEAD").await,
814 git::tree_of(&worktree, &base_commit).await,
815 )
816 && head_tree == base_tree
817 {
818 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
819 git::worktree_remove(repo, &worktree).await.ok();
820 bail!(
821 "`{branch}` at {} has a tree identical to base {}; this usually means \
822 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
823 against `{}/{branch}`) rather than an empty change",
824 short(&head),
825 short(&base_commit),
826 state.config.merge.remote
827 );
828 }
829 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
830 .await
831 .unwrap_or_default();
832
833 state.candidates.push(Candidate {
834 index: 0,
835 label: 'A',
836 agent: EXISTING_BRANCH.to_owned(),
839 branch: branch.to_owned(),
840 worktree,
841 summary: String::new(),
842 stat,
843 files,
844 commits,
845 empty: false,
846 failed: None,
847 verified_noop: None,
848 duration_ms: 0,
849 folded: false,
850 });
851 state.tally = Some(Tally {
852 first_choice: BTreeMap::from([('A', 0)]),
853 borda: BTreeMap::new(),
854 winner: 'A',
855 rankings: 0,
856 unanimous_initial: false,
857 deliberated: false,
858 changed_votes: 0,
859 unanimous_final: false,
860 tie_break: None,
861 judges: 0,
865 present: 0,
866 quorum: 0,
867 met_quorum: true,
868 uncontested: Some("review-only run: nothing competed".to_owned()),
869 });
870 state.status = RunStatus::Reviewing;
871 state.event(
872 "start",
873 format!(
874 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
875 state.id
876 ),
877 );
878 state.save()?;
879 Ok(Self {
880 state,
881 roles,
882 sem: Arc::new(Semaphore::new(max_parallel)),
883 pause: Pause::new(),
884 interrupt: Pause::new(),
885 })
886 }
887
888 pub fn resume(id: &str) -> Result<Self> {
890 let state = RunState::load(id)?;
891 if let Some(to) = &state.released_to {
892 bail!(
893 "run {} cannot be resumed: its worktree was released to run {}",
894 state.short(),
895 crate::run::short_of(to)
896 );
897 }
898 let roles = state.config.resolve_roles()?;
899 let max_parallel = state.config.graph.max_parallel.max(1);
900 Ok(Self {
901 state,
902 roles,
903 sem: Arc::new(Semaphore::new(max_parallel)),
904 pause: Pause::new(),
905 interrupt: Pause::new(),
906 })
907 }
908
909 pub async fn execute(&mut self) -> Result<()> {
916 let result = self.execute_graph().await;
917 self.mark_driver_exited();
918 let ended = if result.is_err() {
919 Some(crate::notices::run_stopped(&self.state.id, &self.state))
920 } else {
921 crate::notices::run_ended(&self.state)
922 };
923 if let Some(notice) = ended {
924 crate::notices::raise(notice);
925 }
926 result
927 }
928
929 fn mark_driver_exited(&mut self) {
938 self.state.driver_exited = true;
939 let pid = std::process::id();
940 let Ok(mut disk) = RunState::load(&self.state.id) else {
941 return;
942 };
943 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
944 return;
945 }
946 disk.driver_exited = true;
947 if let Err(e) = disk.save() {
948 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
949 }
950 }
951
952 async fn execute_graph(&mut self) -> Result<()> {
953 self.state.parked = false;
958 self.state.clear_active();
965 if let Ok(disk) = RunState::load(&self.state.id)
983 && let Some(to) = &disk.released_to
984 {
985 bail!(
986 "run {} cannot continue: its worktree was released to run {}",
987 self.state.short(),
988 crate::run::short_of(to)
989 );
990 }
991 let pid = std::process::id();
992 self.state.driver_pid = Some(pid);
993 self.state.driver_started_at = crate::proc::process_started_at(pid);
994 self.state.driver_exited = false;
995 self.state.save()?;
996 if self.state.status == RunStatus::Stalled {
1009 if self.recover_stall().await? {
1010 self.finish_after_tally().await?;
1011 } else {
1012 self.state.save()?;
1014 }
1015 return Ok(());
1016 }
1017 if self.state.status == RunStatus::Landing {
1027 self.run_land().await?;
1028 self.settle_questions();
1033 return Ok(());
1034 }
1035 self.prep().await?;
1036 if self.park_here()? {
1037 return Ok(());
1038 }
1039 self.advise().await?;
1040 if self.park_here()? {
1041 return Ok(());
1042 }
1043 self.implement().await?;
1044 if self.park_here()? {
1045 return Ok(());
1046 }
1047 if self.state.status == RunStatus::VerifiedNoop {
1051 return Ok(());
1052 }
1053 self.judge().await?;
1054 if self.park_here()? {
1055 return Ok(());
1056 }
1057 self.deliberate().await?;
1058 if self.park_here()? {
1059 return Ok(());
1060 }
1061 self.vote().await?;
1062 if self.park_here()? {
1063 return Ok(());
1064 }
1065 self.tally()?;
1066 if self.state.status == RunStatus::Stalled {
1071 self.state.save()?;
1075 return Ok(());
1076 }
1077 self.finish_after_tally().await?;
1078 Ok(())
1079 }
1080
1081 fn park_here(&mut self) -> Result<bool> {
1088 if !self.pause.parked() && !self.interrupt.parked() {
1093 return Ok(false);
1094 }
1095 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
1096 Some(reason) => format!(
1097 "parked after `{}` ({reason}) — resume to carry on from here",
1098 self.state.status.as_str()
1099 ),
1100 None => format!(
1101 "parked after `{}` — resume to carry on from here",
1102 self.state.status.as_str()
1103 ),
1104 };
1105 self.state.event("park", why);
1106 self.state.parked = true;
1107 self.state.save()?;
1108 Ok(true)
1109 }
1110
1111 pub fn on_pause(&mut self, pause: Pause) {
1113 self.pause = pause;
1114 }
1115
1116 pub fn watch_interrupt(&mut self, pause: Pause) {
1122 self.interrupt = pause;
1123 }
1124
1125 fn settle_questions(&mut self) {
1145 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1146 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1147 }
1148 }
1149
1150 async fn finish_after_tally(&mut self) -> Result<()> {
1153 self.fold_losers().await?;
1154 self.sync_to_base().await?;
1159 if self.state.status == RunStatus::AlreadyInBase {
1160 return Ok(());
1161 }
1162 self.review_loop().await?;
1163 self.sync_to_base().await?;
1164 if self.state.status == RunStatus::AlreadyInBase {
1165 return Ok(());
1166 }
1167 self.gate().await?;
1168 self.merge().await?;
1169 self.state.save()?;
1170 Ok(())
1171 }
1172
1173 async fn prep(&mut self) -> Result<()> {
1176 if !self.state.candidates.is_empty() {
1177 return Ok(());
1178 }
1179 self.state.status = RunStatus::Prep;
1180 let repo = self.state.repo.clone();
1181 let base = self.state.base_commit.clone();
1182 let plan = refs::plan(&repo, &self.state.seeds).await?;
1183 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1184 let root = self.state.worktree_root();
1185 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1186
1187 let hooks_dir = self.state.dir().join("hooks");
1190 if self.state.config.blind.commit_msg_hook {
1191 std::fs::create_dir_all(&hooks_dir)
1192 .with_context(|| format!("create {}", hooks_dir.display()))?;
1193 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1194 let path = hooks_dir.join("commit-msg");
1195 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1196 make_executable(&path)?;
1197 git::acquire_worktree_config(&repo).await?;
1205 self.state.enabled_worktree_config = true;
1206 }
1207
1208 for (index, (spec, label)) in self
1209 .roles
1210 .implementers
1211 .clone()
1212 .into_iter()
1213 .zip(labels)
1214 .enumerate()
1215 {
1216 let branch = self.state.branch_for(label);
1217 let worktree = root.join(format!("cand-{label}"));
1218 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1219 if self.state.config.blind.commit_msg_hook {
1220 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1221 }
1222 git::local_exclude(&worktree, "/.magi/").await?;
1223 for pick in &plan.picks {
1224 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1225 self.state.status = RunStatus::Blocked;
1226 self.state
1227 .event("prep", format!("cannot apply referenced commit: {e}"));
1228 self.state.save()?;
1229 return Err(e);
1230 }
1231 }
1232 self.state.candidates.push(Candidate {
1233 index,
1234 label,
1235 agent: spec.id.clone(),
1236 branch,
1237 worktree,
1238 summary: String::new(),
1239 stat: String::new(),
1240 files: 0,
1241 commits: 0,
1242 empty: false,
1243 failed: None,
1244 verified_noop: None,
1245 duration_ms: 0,
1246 folded: false,
1247 });
1248 }
1249
1250 for j in 1..=self.roles.judges.len() {
1251 let wt = root.join(format!("judge-{j}"));
1252 if !wt.exists() {
1253 git::worktree_add_detached(&repo, &wt, &base).await?;
1254 }
1255 }
1256
1257 if self.state.config.graph.advise {
1265 for k in 1..=self.state.config.graph.advisors {
1266 let wt = root.join(format!("advisor-{k}"));
1267 if !wt.exists() {
1268 git::worktree_add_detached(&repo, &wt, &base).await?;
1269 }
1270 }
1271 }
1272
1273 let authors: Vec<&str> = self
1278 .roles
1279 .implementers
1280 .iter()
1281 .map(|a| a.id.as_str())
1282 .collect();
1283 let overlap: Vec<String> = self
1284 .roles
1285 .judges
1286 .iter()
1287 .enumerate()
1288 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1289 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1290 .collect();
1291 if !overlap.is_empty() {
1292 let note = format!(
1293 "{} also authored a candidate; blind, but the panel is less \
1294 independent than {} distinct agents would be",
1295 overlap.join(", "),
1296 self.roles.judges.len()
1297 );
1298 self.state.event("prep", note);
1299 }
1300
1301 self.state.event(
1302 "prep",
1303 format!(
1304 "{} candidates, {} judges, base {} ({})",
1305 self.state.candidates.len(),
1306 self.roles.judges.len(),
1307 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1308 self.state.base_branch
1309 ),
1310 );
1311 self.state.status = RunStatus::Implementing;
1312 self.state.save()?;
1313 Ok(())
1314 }
1315
1316 async fn advise(&mut self) -> Result<()> {
1349 let implement_untouched = self
1350 .state
1351 .candidates
1352 .iter()
1353 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1354 if !self.state.config.graph.advise || self.state.advise_attempted {
1355 return Ok(());
1356 }
1357 if !implement_untouched {
1358 self.state.event(
1359 "advise",
1360 "skipping the design-deliberation stage: at least one \
1361 candidate already shows implementation progress, so this \
1362 run is past the point the stage exists to run before"
1363 .to_owned(),
1364 );
1365 self.state.advise_attempted = true;
1366 self.state.save()?;
1367 return Ok(());
1368 }
1369 let run_id = self.state.id.clone();
1370 let prompts = self.state.config.prompts.clone();
1371 let instruction = self.state.instruction.clone();
1372 let language = self.state.config.graph.language.clone();
1373 let root = self.state.worktree_root();
1374 let n = self.state.config.graph.advisors;
1375 let where_recorded = self.state.dir().join("run.json");
1376
1377 let seats = match self.state.config.advisors() {
1378 Ok(seats) if !seats.is_empty() => seats,
1379 Ok(_) => {
1380 self.state.event(
1381 "advise",
1382 format!(
1383 "[graph] advisors is 0; skipping the design-deliberation \
1384 stage and continuing without a synthesis brief (see {})",
1385 where_recorded.display()
1386 ),
1387 );
1388 self.state.advise_attempted = true;
1389 self.state.save()?;
1390 return Ok(());
1391 }
1392 Err(e) => {
1393 self.state.event(
1394 "advise",
1395 format!(
1396 "could not resolve advisor seats ({e:#}); continuing \
1397 without a design-deliberation brief (see {})",
1398 where_recorded.display()
1399 ),
1400 );
1401 self.state.advise_attempted = true;
1402 self.state.save()?;
1403 return Ok(());
1404 }
1405 };
1406
1407 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1408 let artifacts = agent::artifacts_dir(&self.state.dir());
1409 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1410
1411 let mut jobs = Vec::new();
1412 for (i, spec) in seats.iter().cloned().enumerate() {
1413 let seat_key = format!("advisor-{}", i + 1);
1414 let seat = self.seat(&seat_key, &spec.id);
1415 jobs.push(SeatJob {
1416 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1417 spec,
1418 seat,
1419 cwd: worktrees[i % worktrees.len()].clone(),
1420 timeout,
1421 allow_write: false,
1422 sessions: false,
1423 artifacts: artifacts.clone(),
1424 stem: seat_key,
1425 handover: None,
1426 });
1427 }
1428
1429 self.state.event(
1430 "advise",
1431 format!(
1432 "{} advisor seat(s) sketching a design in parallel",
1433 jobs.len()
1434 ),
1435 );
1436 let mut quota_losses = Vec::new();
1437 let cache = self.state.config.cache_dir();
1438 let ctx = WaveCtx {
1439 carry_seats: false,
1440 run: &run_id,
1441 node: "advise",
1442 prompts: &prompts,
1443 cache: cache.as_deref(),
1444 round: None,
1445 };
1446 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1447 let results = ask_json_wave::<Proposal>(
1448 jobs,
1449 Arc::clone(&self.sem),
1450 self.state.config.graph.retries,
1451 &advisor_roster,
1452 &ctx,
1453 &mut quota_losses,
1454 &mut self.state,
1455 &|p: &Proposal| p.validate(),
1456 )
1457 .await;
1458 self.state.quota.extend(quota_losses);
1459
1460 let mut records = Vec::with_capacity(results.len());
1461 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1462 let agent_id = seat.agent.clone();
1463 self.state.seats.insert(seat.key.clone(), seat);
1464 match res {
1465 Ok((proposal, out)) => {
1466 self.state
1467 .event("advise", format!("advisor-{} proposed a design", i + 1));
1468 records.push(advise::AdvisorRecord::proposed(
1469 i + 1,
1470 agent_id,
1471 proposal,
1472 out.duration_ms,
1473 ));
1474 }
1475 Err(e) => {
1476 self.state.event(
1477 "advise",
1478 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1479 );
1480 records.push(advise::AdvisorRecord::failed(
1481 i + 1,
1482 agent_id,
1483 e.to_string(),
1484 ));
1485 }
1486 }
1487 }
1488
1489 let mut advice = advise::Advice {
1490 records,
1491 synthesis: None,
1492 };
1493 if advice.proposals().is_empty() {
1494 self.state.event(
1495 "advise",
1496 "no advisor produced a usable proposal; continuing without a \
1497 synthesis brief"
1498 .to_owned(),
1499 );
1500 } else {
1501 match self
1502 .synthesize_brief(
1503 &advice,
1504 &instruction,
1505 &language,
1506 &worktrees[0],
1507 &artifacts,
1508 &run_id,
1509 &prompts,
1510 cache.as_deref(),
1511 )
1512 .await
1513 {
1514 Ok(Some(text)) => {
1515 self.state.event(
1516 "advise",
1517 "synthesized a design brief for the implementer".to_owned(),
1518 );
1519 advice.synthesis = Some(text);
1520 }
1521 Ok(None) => {
1522 self.state.event(
1523 "advise",
1524 "the synthesis seat produced nothing usable; continuing \
1525 without a design brief"
1526 .to_owned(),
1527 );
1528 }
1529 Err(e) => {
1530 self.state.event(
1531 "advise",
1532 format!("could not synthesize a design brief: {e:#}"),
1533 );
1534 }
1535 }
1536 }
1537 advise::apply_reflection(&mut advice);
1538
1539 self.state.advice = Some(advice);
1540 self.state.advise_attempted = true;
1541 self.state.save()?;
1542 Ok(())
1543 }
1544
1545 #[allow(clippy::too_many_arguments)]
1557 async fn synthesize_brief(
1558 &mut self,
1559 advice: &advise::Advice,
1560 instruction: &str,
1561 language: &str,
1562 cwd: &Path,
1563 artifacts: &Path,
1564 run_id: &str,
1565 prompts: &Prompts,
1566 cache: Option<&Path>,
1567 ) -> Result<Option<String>> {
1568 let chain = agent::pick_chain(
1569 &self.state.config.agents,
1570 self.state.config.roles.synthesizer.as_ref(),
1571 &agent::installed,
1572 "synthesizer",
1573 )?;
1574 let proposals = advice.proposals();
1575 let mut prompt = prompt::with_overlay(
1576 prompt::synthesize_brief(instruction, &proposals, language),
1577 prompts.overlay("advise"),
1578 );
1579 if cache.is_some() {
1580 prompt.push('\n');
1585 prompt.push_str(&prompt::build_cache_note("advise", false));
1586 }
1587 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1588 let mut last = None;
1593 for (n, spec) in chain.iter().enumerate() {
1594 if n > 0 {
1595 self.state
1596 .event("advise", format!("synthesis falling back to {}", spec.id));
1597 }
1598 let mut seat = self.seat("advise-synthesis", &spec.id);
1599 let outcome = agent::invoke(
1600 spec,
1601 &mut seat,
1602 &Invocation {
1603 cwd,
1604 prompt: &prompt,
1605 timeout,
1606 allow_write: false,
1607 sessions: false,
1608 artifacts,
1609 stem: &if n == 0 {
1610 "advise-synthesis".to_owned()
1611 } else {
1612 format!("advise-synthesis-{}", spec.id)
1613 },
1614 run: run_id,
1615 node: "advise",
1616 cache_dir: None,
1617 attachments: &[],
1618 writable: &[],
1619 },
1620 )
1621 .await;
1622 if outcome.is_ok() {
1623 self.state.seats.insert(seat.key.clone(), seat);
1624 }
1625 let advance = agent::chain_advances(&outcome);
1626 last = Some(outcome);
1627 if !advance {
1628 break;
1629 }
1630 }
1631 let out = last.expect("a chain holds at least one agent")?;
1634 if !out.usable() {
1635 return Ok(None);
1636 }
1637 let text =
1638 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1639 Ok((!text.trim().is_empty()).then_some(text))
1640 }
1641
1642 async fn implement(&mut self) -> Result<()> {
1645 let run_id = self.state.id.clone();
1650 let prompts = self.state.config.prompts.clone();
1651 let todo: Vec<usize> = self
1652 .state
1653 .candidates
1654 .iter()
1655 .enumerate()
1656 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1657 .map(|(i, _)| i)
1658 .collect();
1659 if todo.is_empty() {
1660 return self.after_implement();
1661 }
1662 self.state.status = RunStatus::Implementing;
1663
1664 let language = self.state.config.graph.language.clone();
1665 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1666 let sessions = self.state.config.graph.sessions;
1667 let artifacts = agent::artifacts_dir(&self.state.dir());
1668 let brief = self
1672 .state
1673 .advice
1674 .as_ref()
1675 .and_then(|a| a.synthesis.as_deref())
1676 .map(str::to_owned);
1677 let attachments = self.state.attachments.clone();
1678
1679 let mut jobs = Vec::new();
1680 for &i in &todo {
1681 let (index, label, worktree) = {
1682 let c = &self.state.candidates[i];
1683 (c.index, c.label, c.worktree.clone())
1684 };
1685 let spec = self.roles.implementers[index].clone();
1686 let seat_key = format!("impl-{label}");
1687 let seat = self.seat(&seat_key, &spec.id);
1688 let instruction = seeded_instruction(&self.state);
1689 jobs.push(SeatJob {
1690 spec,
1691 seat,
1692 prompt: prompt::implement(
1693 &instruction,
1694 &worktree.to_string_lossy(),
1695 &language,
1696 brief.as_deref(),
1697 &attachments,
1698 ),
1699 cwd: worktree,
1700 timeout,
1701 allow_write: true,
1702 sessions,
1703 artifacts: artifacts.clone(),
1704 stem: format!("impl-{label}"),
1705 handover: None,
1706 });
1707 }
1708
1709 self.state.event(
1710 "implement",
1711 format!("{} candidates in parallel", jobs.len()),
1712 );
1713 let mut sent = jobs.clone();
1719 let cache = self.state.config.cache_dir();
1720 let ctx = WaveCtx {
1721 carry_seats: false,
1722 run: &run_id,
1723 node: "implement",
1724 prompts: &prompts,
1725 cache: cache.as_deref(),
1726 round: None,
1727 };
1728 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1729 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1730 .await;
1731 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1732 .await;
1733 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1734 .await;
1735
1736 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1737 let seat_key = seat.key.clone();
1738 let agent = seat.agent.clone();
1747 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1748 self.state.seats.insert(seat.key.clone(), seat);
1749 let label = self.state.candidates[i].label;
1750 let worktree = self.state.candidates[i].worktree.clone();
1751 let base = self.state.base_commit.clone();
1752
1753 let (summary, duration, failed, verified_claim) = match out {
1754 AgentOutcome::Ok(o) => {
1755 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1756 let failed = (!o.usable()).then(|| {
1757 if o.timed_out {
1758 "agent timed out".to_owned()
1759 } else {
1760 format!("agent exited with {:?}", o.exit_code)
1761 }
1762 });
1763 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1764 (text, o.duration_ms, failed, verified_claim)
1765 }
1766 AgentOutcome::Dropped(o) => {
1772 let why = o
1773 .dropped
1774 .as_ref()
1775 .map(|d| d.why.as_str())
1776 .unwrap_or("the CLI ended the stream without delivering its answer");
1777 (
1778 String::new(),
1779 o.duration_ms,
1780 Some(format!("the CLI dropped the stream ({why})")),
1781 None,
1782 )
1783 }
1784 AgentOutcome::Quota(o) => {
1785 self.state.quota.push(QuotaLoss {
1786 seat: seat_key,
1787 node: "implement".to_owned(),
1788 at: Timestamp::now(),
1789 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1790 });
1791 (
1792 String::new(),
1793 o.duration_ms,
1794 Some("rate limited (quota); produced no change".to_owned()),
1795 None,
1796 )
1797 }
1798 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1799 };
1800
1801 let rescued = match git::rescue_commit(
1804 &worktree,
1805 &format!("magi: candidate {label} (uncommitted work)"),
1806 )
1807 .await
1808 {
1809 Ok(r) => {
1810 self.state.note_withheld("implement", &r.withheld);
1811 r.committed
1812 }
1813 Err(_) => false,
1814 };
1815 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1816 .await
1817 .unwrap_or(0);
1818 let patch = git::diff(&worktree, &base, "HEAD")
1819 .await
1820 .unwrap_or_default();
1821 let stat = git::diff_stat(&worktree, &base, "HEAD")
1822 .await
1823 .unwrap_or_default();
1824 let files = git::changed_files(&worktree, &base, "HEAD")
1825 .await
1826 .map(|f| f.len())
1827 .unwrap_or(0);
1828 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1829
1830 let c = &mut self.state.candidates[i];
1831 if !exhausted_the_fallback_chain {
1832 c.agent = agent;
1833 }
1834 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1835 c.stat = stat;
1836 c.files = files;
1837 c.commits = commits;
1838 c.duration_ms = duration;
1839 c.empty = commits == 0 || patch.trim().is_empty();
1840 c.failed = match failed {
1843 Some(_) if c.empty => failed,
1844 _ => None,
1845 };
1846 c.verified_noop = if c.empty { verified_claim } else { None };
1851 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1852 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1853 (None, true, Some(_), _) => {
1854 format!("candidate {label}: no change produced (agent-verified no-op)")
1855 }
1856 (None, true, None, _) => format!("candidate {label}: no change produced"),
1857 (None, false, _, true) => {
1858 format!(
1859 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1860 )
1861 }
1862 (None, false, _, false) => {
1863 format!("candidate {label}: {files} files, {commits} commits")
1864 }
1865 };
1866 self.state.event("implement", note);
1867 self.state.save()?;
1868 }
1869
1870 self.after_implement()
1871 }
1872
1873 async fn resume_undelivered(
1901 &mut self,
1902 results: &mut [(usize, SeatState, AgentOutcome)],
1903 sent: &[SeatJob],
1904 prompts: &Prompts,
1905 run_id: &str,
1906 ) {
1907 for (wi, seat, out) in results.iter_mut() {
1908 let Some(dropped) = (match &*out {
1909 AgentOutcome::Dropped(o) => o.dropped.clone(),
1910 _ => None,
1911 }) else {
1912 continue;
1913 };
1914 let Some(job) = sent.get(*wi) else { continue };
1915 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1917 self.state.event(
1918 "implement",
1919 format!(
1920 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1921 work is in the tree",
1922 seat.key, dropped.output_tokens, dropped.why
1923 ),
1924 );
1925 continue;
1926 }
1927 if !has_context(&job.spec, seat, job.sessions) {
1935 self.state.event(
1936 "implement",
1937 format!(
1938 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1939 is no session left to resume",
1940 seat.key, dropped.output_tokens, dropped.why
1941 ),
1942 );
1943 continue;
1944 }
1945 self.state.event(
1946 "implement",
1947 format!(
1948 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1949 conversation",
1950 seat.key, dropped.output_tokens, dropped.why
1951 ),
1952 );
1953 let mut retry = job.clone();
1954 retry.seat = seat.clone();
1955 retry.prompt = prompt::resume_after_drop(&dropped.why);
1956 retry.timeout = retry_budget(job.timeout, true);
1957 retry.stem = format!("{}-resume", job.stem);
1958 let cache = self.state.config.cache_dir();
1959 let ctx = WaveCtx {
1960 carry_seats: false,
1961 run: run_id,
1962 node: "implement",
1963 prompts,
1964 cache: cache.as_deref(),
1965 round: None,
1966 };
1967 let (resumed_seat, resumed) =
1968 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1969 *seat = resumed_seat;
1970 *out = resumed;
1971 }
1972 }
1973
1974 async fn resume_seat_handovers(
2039 &mut self,
2040 results: &mut [(usize, SeatState, AgentOutcome)],
2041 sent: &mut [SeatJob],
2042 prompts: &Prompts,
2043 run_id: &str,
2044 ) {
2045 let instruction = seeded_instruction(&self.state);
2046 let language = self.state.config.graph.language.clone();
2047 let brief = self
2048 .state
2049 .advice
2050 .as_ref()
2051 .and_then(|a| a.synthesis.as_deref())
2052 .map(str::to_owned);
2053 let attachments = self.state.attachments.clone();
2054 for (wi, seat, out) in results.iter_mut() {
2055 let Some(job) = sent.get_mut(*wi) else {
2056 continue;
2057 };
2058 let start = self
2063 .roles
2064 .implementer_roster
2065 .iter()
2066 .position(|s| s.id == job.spec.id)
2067 .unwrap_or(0);
2068 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2069 let mut fallback_attempt = 0usize;
2070 let mut prev: Option<FailClass> = None;
2071 while let Some(cur) = FailClass::of(&*out) {
2072 if !should_hand_over(prev.as_ref(), &cur) {
2073 break;
2074 }
2075 let Some(next) =
2076 next_untried_in_roster(&self.roles.implementer_roster, start, &tried).cloned()
2077 else {
2078 break;
2079 };
2080 tried.insert(next.id.clone());
2081 fallback_attempt += 1;
2082
2083 if let Ok(r) = git::rescue_commit(
2084 &job.cwd,
2085 &format!(
2086 "magi: candidate {} (uncommitted work before {} fallback)",
2087 seat.key,
2088 if cur == FailClass::Quota {
2089 "quota"
2090 } else {
2091 "handover"
2092 }
2093 ),
2094 )
2095 .await
2096 {
2097 self.state.note_withheld("implement", &r.withheld);
2098 }
2099
2100 record_handover(
2101 &mut self.state,
2102 "implement",
2103 &seat.key,
2104 &seat.agent,
2105 &next.id,
2106 &cur,
2107 &fail_reason(&*out),
2108 );
2109 prev = Some(cur.clone());
2110
2111 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2112 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2113 job.spec = next.clone();
2121 let mut retry = job.clone();
2122 retry.seat = new_seat;
2123 retry.prompt = prompt::implement(
2124 &instruction,
2125 &job.cwd.to_string_lossy(),
2126 &language,
2127 brief.as_deref(),
2128 &attachments,
2129 );
2130 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2131 let cache = self.state.config.cache_dir();
2132 let ctx = WaveCtx {
2133 carry_seats: false,
2134 run: run_id,
2135 node: "implement",
2136 prompts,
2137 cache: cache.as_deref(),
2138 round: None,
2139 };
2140 let (fallback_seat, fallback_out) = run_one(
2141 retry,
2142 Arc::clone(&self.sem),
2143 &ctx,
2144 &mut self.state,
2145 fallback_attempt,
2146 )
2147 .await;
2148 *seat = fallback_seat;
2149 *out = fallback_out;
2150 }
2151 }
2152 }
2153
2154 async fn resume_unconfirmed_commands(
2178 &mut self,
2179 results: &mut [(usize, SeatState, AgentOutcome)],
2180 sent: &[SeatJob],
2181 prompts: &Prompts,
2182 run_id: &str,
2183 ) {
2184 for (wi, seat, out) in results.iter_mut() {
2185 let AgentOutcome::Ok(o) = &*out else {
2186 continue;
2187 };
2188 if !has_unconfirmed_command(&o.commands) {
2189 continue;
2190 }
2191 let Some(job) = sent.get(*wi) else { continue };
2192 if !has_context(&job.spec, seat, job.sessions) {
2193 self.state.event(
2194 "implement",
2195 format!(
2196 "{}: the reply named a command whose own CLI never confirmed the exit \
2197 status of, but there is no session left to resume",
2198 seat.key
2199 ),
2200 );
2201 continue;
2202 }
2203 self.state.event(
2204 "implement",
2205 format!(
2206 "{}: the reply named a command whose own CLI never confirmed the exit \
2207 status of; resuming the conversation",
2208 seat.key
2209 ),
2210 );
2211 let mut retry = job.clone();
2212 retry.seat = seat.clone();
2213 retry.prompt = prompt::resume_incomplete(
2214 "a command in your last reply had no confirmed exit status",
2215 );
2216 retry.timeout = retry_budget(job.timeout, true);
2217 retry.stem = format!("{}-confirm", job.stem);
2218 let cache = self.state.config.cache_dir();
2219 let ctx = WaveCtx {
2220 carry_seats: false,
2221 run: run_id,
2222 node: "implement",
2223 prompts,
2224 cache: cache.as_deref(),
2225 round: None,
2226 };
2227 let (resumed_seat, resumed) =
2228 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2229 *seat = resumed_seat;
2230 *out = resumed;
2231 }
2232 }
2233
2234 async fn continue_fix_report(
2255 &mut self,
2256 mut seat: SeatState,
2257 parse_err: String,
2258 job: &SeatJob,
2259 prompts: &Prompts,
2260 run_id: &str,
2261 round: usize,
2262 ) -> (
2263 SeatState,
2264 Option<FixReport>,
2265 Option<String>,
2266 ContinuationRecord,
2267 ) {
2268 let mut last_err = parse_err;
2269 let mut cumulative_wait_ms = 0u64;
2270 let mut attempts = 0usize;
2271 loop {
2272 if !has_context(&job.spec, &seat, job.sessions) {
2273 self.state.event(
2274 "fix",
2275 format!(
2276 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2277 session left to resume into"
2278 ),
2279 );
2280 let outcome = if attempts == 0 {
2281 ContinuationOutcome::NoSession
2282 } else {
2283 ContinuationOutcome::Exhausted
2284 };
2285 return (
2286 seat,
2287 None,
2288 Some(format!("unparsable fix report: {last_err}")),
2289 ContinuationRecord {
2290 attempts,
2291 cumulative_wait_ms,
2292 outcome,
2293 },
2294 );
2295 }
2296 if attempts >= MAX_FIX_CONTINUATIONS {
2297 self.state.event(
2298 "fix",
2299 format!(
2300 "round {round}: fixer's reply still had no adoption report after \
2301 {attempts} continuation(s) ({last_err}); giving up"
2302 ),
2303 );
2304 return (
2305 seat,
2306 None,
2307 Some(format!(
2308 "unparsable fix report after {attempts} continuation(s): {last_err}"
2309 )),
2310 ContinuationRecord {
2311 attempts,
2312 cumulative_wait_ms,
2313 outcome: ContinuationOutcome::Exhausted,
2314 },
2315 );
2316 }
2317 attempts += 1;
2318 self.state.event(
2319 "fix",
2320 format!(
2321 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2322 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2323 ),
2324 );
2325 let mut retry = job.clone();
2326 retry.seat = seat.clone();
2327 retry.prompt = prompt::resume_incomplete(&last_err);
2328 retry.timeout = retry_budget(job.timeout, true);
2329 retry.stem = format!("{}-continue{attempts}", job.stem);
2330 let cache = self.state.config.cache_dir();
2331 let ctx = WaveCtx {
2332 carry_seats: false,
2333 run: run_id,
2334 node: "fix",
2335 prompts,
2336 cache: cache.as_deref(),
2337 round: Some(round),
2338 };
2339 let (resumed_seat, resumed_out) = run_one(
2340 retry,
2341 Arc::clone(&self.sem),
2342 &ctx,
2343 &mut self.state,
2344 attempts,
2345 )
2346 .await;
2347 seat = resumed_seat;
2348 match resumed_out {
2349 AgentOutcome::Ok(o) => {
2350 cumulative_wait_ms += o.duration_ms;
2351 match verdict::extract_json::<FixReport>(&o.text) {
2352 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2353 self.state.event(
2354 "fix",
2355 format!(
2356 "round {round}: fixer's adoption report recovered after \
2357 {attempts} continuation(s)"
2358 ),
2359 );
2360 return (
2361 seat,
2362 Some(report),
2363 None,
2364 ContinuationRecord {
2365 attempts,
2366 cumulative_wait_ms,
2367 outcome: ContinuationOutcome::Resumed,
2368 },
2369 );
2370 }
2371 Ok(_) => {
2379 last_err = "the reply parsed, but it reported a command whose own CLI \
2380 never confirmed an exit status"
2381 .to_owned();
2382 }
2383 Err(e) => last_err = e.to_string(),
2384 }
2385 }
2386 AgentOutcome::Quota(o) => {
2387 cumulative_wait_ms += o.duration_ms;
2388 self.state.quota.push(QuotaLoss {
2389 seat: seat.key.clone(),
2390 node: "fix".to_owned(),
2391 at: Timestamp::now(),
2392 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2393 });
2394 self.state.event(
2395 "fix",
2396 format!(
2397 "round {round}: continuation rate limited (quota); not retrying now"
2398 ),
2399 );
2400 return (
2401 seat,
2402 None,
2403 Some("rate limited (quota) while recovering the fix report".to_owned()),
2404 ContinuationRecord {
2405 attempts,
2406 cumulative_wait_ms,
2407 outcome: ContinuationOutcome::QuotaLost,
2408 },
2409 );
2410 }
2411 AgentOutcome::Dropped(o) => {
2412 cumulative_wait_ms += o.duration_ms;
2413 let why = o
2414 .dropped
2415 .as_ref()
2416 .map(|d| d.why.as_str())
2417 .unwrap_or("the CLI ended the stream without delivering its answer");
2418 last_err = format!("the CLI dropped the stream ({why})");
2419 }
2420 AgentOutcome::Failed(e) => last_err = e,
2421 }
2422 }
2423 }
2424
2425 fn after_implement(&mut self) -> Result<()> {
2426 if self.state.leaks.is_empty() {
2428 let cfg = self.state.config.blind.clone();
2429 let mut leaks = Vec::new();
2430 for c in &self.state.candidates {
2431 let Some(patch) =
2432 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2433 else {
2434 continue;
2435 };
2436 leaks.extend(blind::scan(
2437 &format!("candidate {} patch", c.label),
2438 &patch,
2439 &cfg.vendor_tokens,
2440 ));
2441 }
2442 if !leaks.is_empty() {
2443 let summary = leaks
2444 .iter()
2445 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2446 .collect::<Vec<_>>()
2447 .join(", ");
2448 match cfg.on_leak {
2449 LeakPolicy::Fail => {
2450 self.state.status = RunStatus::Failed;
2451 self.state
2452 .event("blind", format!("vendor text in a patch: {summary}"));
2453 self.state.leaks = leaks;
2454 self.state.save()?;
2455 self.settle_questions();
2456 bail!(
2457 "blind.on_leak = \"fail\" and vendor text reached a \
2458 judged patch: {summary}"
2459 );
2460 }
2461 LeakPolicy::Redact => self.state.event(
2462 "blind",
2463 format!("redacting vendor text for judging: {summary}"),
2464 ),
2465 LeakPolicy::Warn => self.state.event(
2466 "blind",
2467 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2468 ),
2469 }
2470 self.state.leaks = leaks;
2471 }
2472 }
2473
2474 if self.state.viable().is_empty() {
2475 if self.state.all_candidates_verified_noop() {
2476 self.state.status = RunStatus::VerifiedNoop;
2487 self.state.save()?;
2488 self.settle_questions();
2489 return Ok(());
2490 }
2491 self.state.status = RunStatus::Failed;
2492 self.state.save()?;
2493 self.settle_questions();
2494 bail!("no candidate produced a change; nothing to judge");
2495 }
2496 self.state.status = RunStatus::Judging;
2497 self.state.save()?;
2498 Ok(())
2499 }
2500
2501 async fn judge(&mut self) -> Result<()> {
2504 let run_id = self.state.id.clone();
2509 let prompts = self.state.config.prompts.clone();
2510 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2511 return Ok(());
2512 }
2513 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2514 if viable.len() == 1 {
2515 self.state.judge_skipped = true;
2522 self.state.event(
2523 "judge",
2524 format!(
2525 "only candidate {} produced a change; judging skipped",
2526 viable[0].label
2527 ),
2528 );
2529 self.state.save()?;
2530 return Ok(());
2531 }
2532 self.state.status = RunStatus::Judging;
2533
2534 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2535 let language = self.state.config.graph.language.clone();
2536 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2537 let sessions = self.state.config.graph.sessions;
2538 let artifacts = agent::artifacts_dir(&self.state.dir());
2539 let root = self.state.worktree_root();
2540 let base_short = short(&self.state.base_commit);
2541
2542 let mut jobs = Vec::new();
2543 let mut orders = Vec::new();
2544 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2545 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2546 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2547 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2548 let seat_key = format!("judge-{}", j + 1);
2549 let seat = self.seat(&seat_key, &spec.id);
2550 jobs.push(SeatJob {
2551 prompt: prompt::judge(
2552 &self.state.instruction,
2553 &views,
2554 self.roles.judges.len(),
2555 &base_short,
2556 &language,
2557 ),
2558 spec,
2559 seat,
2560 cwd: root.join(format!("judge-{}", j + 1)),
2561 timeout,
2562 allow_write: false,
2563 sessions,
2564 artifacts: artifacts.clone(),
2565 stem: format!("judge-{}", j + 1),
2566 handover: None,
2567 });
2568 }
2569
2570 self.state.event(
2571 "judge",
2572 format!(
2573 "{} judges ranking {} candidates blind",
2574 jobs.len(),
2575 viable.len()
2576 ),
2577 );
2578 let labels_for_check = labels.clone();
2579 let mut quota_losses = Vec::new();
2580 let cache = self.state.config.cache_dir();
2581 let ctx = WaveCtx {
2582 carry_seats: false,
2583 run: &run_id,
2584 node: "judge",
2585 prompts: &prompts,
2586 cache: cache.as_deref(),
2587 round: None,
2588 };
2589 let results = ask_json_wave::<Ranking>(
2590 jobs,
2591 Arc::clone(&self.sem),
2592 self.state.config.graph.retries,
2593 &self.roles.judge_roster,
2594 &ctx,
2595 &mut quota_losses,
2596 &mut self.state,
2597 &move |r: &Ranking| r.validate(&labels_for_check),
2598 )
2599 .await;
2600 self.state.quota.extend(quota_losses);
2601
2602 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2603 let agent_id = seat.agent.clone();
2604 self.state.seats.insert(seat.key.clone(), seat);
2605 let mut record = Judgement {
2606 judge: j + 1,
2607 seat: format!("judge-{}", j + 1),
2608 agent: agent_id,
2609 ranking: Vec::new(),
2610 reasons: BTreeMap::new(),
2611 confidence: None,
2612 order: orders[j].clone(),
2613 failed: None,
2614 duration_ms: 0,
2615 };
2616 match res {
2617 Ok((ranking, out)) => {
2618 record.ranking = ranking.normalized();
2619 record.reasons = ranking.reasons;
2620 record.confidence = ranking.confidence;
2621 record.duration_ms = out.duration_ms;
2622 self.state.event(
2623 "judge",
2624 format!(
2625 "judge {} ranked {}",
2626 j + 1,
2627 record.ranking.iter().collect::<String>()
2628 ),
2629 );
2630 }
2631 Err(e) => {
2632 record.failed = Some(e.to_string());
2633 self.state
2634 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2635 }
2636 }
2637 self.state.judgements.push(record);
2638 self.state.save()?;
2639 }
2640 Ok(())
2641 }
2642
2643 async fn deliberate(&mut self) -> Result<()> {
2646 let run_id = self.state.id.clone();
2651 let prompts = self.state.config.prompts.clone();
2652 if !self.state.deliberation.is_empty() {
2653 return Ok(());
2654 }
2655 let tops: Vec<char> = self
2656 .state
2657 .judgements
2658 .iter()
2659 .filter_map(|j| j.ranking.first().copied())
2660 .collect();
2661 let rounds = self.state.config.graph.deliberate_rounds;
2662 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2663 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2664 self.state.event(
2665 "deliberate",
2666 format!("judges agreed on {} outright; no deliberation", tops[0]),
2667 );
2668 }
2669 self.state.status = RunStatus::Voting;
2670 self.state.save()?;
2671 return Ok(());
2672 }
2673
2674 self.state.status = RunStatus::Deliberating;
2675 self.state.event(
2676 "deliberate",
2677 format!(
2678 "split: first choices were {} — opening {rounds} round(s)",
2679 tops.iter().collect::<String>()
2680 ),
2681 );
2682
2683 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2684 let language = self.state.config.graph.language.clone();
2685 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2686 let sessions = self.state.config.graph.sessions;
2687 let artifacts = agent::artifacts_dir(&self.state.dir());
2688 let root = self.state.worktree_root();
2689 let base_short = short(&self.state.base_commit);
2690
2691 for round in 1..=rounds {
2695 let mut turns: Vec<DeliberationTurn> = Vec::new();
2696 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2697 if self.state.judgements[j].failed.is_some() {
2698 continue;
2699 }
2700 let seat_key = format!("judge-{}", j + 1);
2701 let spec = self.occupant(&seat_key, spec);
2702 let mut seat = self.seat(&seat_key, &spec.id);
2703 let transcript = self.transcript(&turns, j);
2704 let build = |context: Option<&str>| {
2705 prompt::deliberate(
2706 &self.state.instruction,
2707 context,
2708 &transcript,
2709 round,
2710 rounds,
2711 &language,
2712 )
2713 };
2714 let block = self.candidate_block(&viable, &base_short);
2715 let full = build(Some(&block));
2716 let text = if has_context(&spec, &seat, sessions) {
2717 build(None)
2718 } else {
2719 full.clone()
2720 };
2721 let job = SeatJob {
2722 spec,
2723 seat: seat.clone(),
2724 prompt: text,
2725 cwd: root.join(format!("judge-{}", j + 1)),
2726 timeout,
2727 allow_write: false,
2728 sessions,
2729 artifacts: artifacts.clone(),
2730 stem: format!("delib-{round}-judge-{}", j + 1),
2731 handover: Some(full),
2732 };
2733 let cache = self.state.config.cache_dir();
2734 let ctx = WaveCtx {
2735 carry_seats: false,
2736 run: &run_id,
2737 node: "deliberate",
2738 prompts: &prompts,
2739 cache: cache.as_deref(),
2740 round: None,
2741 };
2742 let mut losses = Vec::new();
2746 let mut results = ask_wave_with::<String>(
2747 vec![job],
2748 Arc::clone(&self.sem),
2749 0,
2750 &self.roles.judge_roster,
2751 &ctx,
2752 &mut losses,
2753 &mut self.state,
2754 &|text: &str| {
2755 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
2756 },
2757 )
2758 .await;
2759 self.state.quota.extend(losses);
2760 let (updated, res, _) = results.pop().expect("one job in, one result out");
2761 seat = updated;
2762 let agent_id = seat.agent.clone();
2763 self.state.seats.insert(seat.key.clone(), seat);
2764 let body = match res {
2765 Ok((body, _)) => body,
2766 Err(e) => {
2769 self.state
2770 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2771 continue;
2772 }
2773 };
2774 let tentative = verdict::extract_json::<Position>(&body)
2775 .ok()
2776 .and_then(|p| p.tentative)
2777 .and_then(|s| s.trim().chars().next())
2778 .map(|c| c.to_ascii_uppercase());
2779 self.state.event(
2780 "deliberate",
2781 format!(
2782 "round {round}: judge {} now favours {}",
2783 j + 1,
2784 tentative.map_or("—".to_owned(), |c| c.to_string())
2785 ),
2786 );
2787 turns.push(DeliberationTurn {
2788 judge: j + 1,
2789 agent: agent_id,
2790 body: blind::sanitize_prose(&body, &self.state.config.blind),
2791 tentative,
2792 });
2793 }
2794 self.state
2795 .deliberation
2796 .push(DeliberationRound { round, turns });
2797 self.state.save()?;
2798 }
2799
2800 self.state.status = RunStatus::Voting;
2801 self.state.save()?;
2802 Ok(())
2803 }
2804
2805 async fn vote(&mut self) -> Result<()> {
2808 let run_id = self.state.id.clone();
2813 let prompts = self.state.config.prompts.clone();
2814 if !self.state.votes.is_empty() {
2815 return Ok(());
2816 }
2817 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2818 if viable.len() == 1 {
2819 return Ok(());
2820 }
2821 self.state.status = RunStatus::Voting;
2822
2823 let language = self.state.config.graph.language.clone();
2824 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2825 let sessions = self.state.config.graph.sessions;
2826 let artifacts = agent::artifacts_dir(&self.state.dir());
2827 let root = self.state.worktree_root();
2828 let base_short = short(&self.state.base_commit);
2829 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2830
2831 let mut jobs = Vec::new();
2832 let mut seats_at = Vec::new();
2833 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2834 if self
2835 .state
2836 .judgements
2837 .get(j)
2838 .is_some_and(|r| r.failed.is_some())
2839 {
2840 continue;
2841 }
2842 let seat_key = format!("judge-{}", j + 1);
2843 let spec = self.occupant(&seat_key, spec);
2844 let seat = self.seat(&seat_key, &spec.id);
2845 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
2846 let text = if has_context(&spec, &seat, sessions) {
2847 prompt::final_vote(&viable, &language)
2848 } else {
2849 full.clone()
2850 };
2851 jobs.push(SeatJob {
2852 spec,
2853 seat,
2854 prompt: text,
2855 cwd: root.join(format!("judge-{}", j + 1)),
2856 timeout,
2857 allow_write: false,
2858 sessions,
2859 artifacts: artifacts.clone(),
2860 stem: format!("vote-judge-{}", j + 1),
2861 handover: Some(full),
2862 });
2863 seats_at.push(j);
2864 }
2865
2866 self.state.event(
2867 "vote",
2868 format!(
2869 "collecting {} final votes one by one, privately",
2870 jobs.len()
2871 ),
2872 );
2873 let allowed = viable.clone();
2874 let mut quota_losses = Vec::new();
2875 let cache = self.state.config.cache_dir();
2876 let ctx = WaveCtx {
2877 carry_seats: false,
2878 run: &run_id,
2879 node: "vote",
2880 prompts: &prompts,
2881 cache: cache.as_deref(),
2882 round: None,
2883 };
2884 let results = ask_json_wave::<FinalVote>(
2885 jobs,
2886 Arc::clone(&self.sem),
2887 self.state.config.graph.retries,
2888 &self.roles.judge_roster,
2889 &ctx,
2890 &mut quota_losses,
2891 &mut self.state,
2892 &move |v: &FinalVote| match v.label() {
2893 Some(c) if allowed.contains(&c) => Ok(()),
2894 other => bail!("vote {other:?} is not one of {allowed:?}"),
2895 },
2896 )
2897 .await;
2898 self.state.quota.extend(quota_losses);
2899
2900 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2901 let agent_id = seat.agent.clone();
2902 self.state.seats.insert(seat.key.clone(), seat);
2903 let initial = self
2904 .state
2905 .judgements
2906 .get(j)
2907 .and_then(|r| r.ranking.first().copied());
2908 let mut record = VoteRecord {
2909 judge: j + 1,
2910 agent: agent_id,
2911 vote: None,
2912 reason: String::new(),
2913 changed: false,
2914 };
2915 match res {
2916 Ok((v, _)) => {
2917 record.vote = v.label();
2918 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2919 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2920 self.state.event(
2921 "vote",
2922 format!(
2923 "judge {} voted {}{}",
2924 j + 1,
2925 record.vote.unwrap_or('?'),
2926 if record.changed { " (changed)" } else { "" }
2927 ),
2928 );
2929 }
2930 Err(e) => {
2931 self.state
2932 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2933 }
2934 }
2935 self.state.votes.push(record);
2936 self.state.save()?;
2937 }
2938 Ok(())
2939 }
2940
2941 fn tally(&mut self) -> Result<()> {
2944 if self.state.tally.is_some() {
2945 return Ok(());
2946 }
2947 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2948 let tops: Vec<char> = self
2949 .state
2950 .judgements
2951 .iter()
2952 .filter_map(|j| j.ranking.first().copied())
2953 .collect();
2954 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2955
2956 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2959 let mut cast: Vec<char> = Vec::new();
2960 for (i, j) in self.state.judgements.iter().enumerate() {
2961 let vote = self
2962 .state
2963 .votes
2964 .iter()
2965 .find(|v| v.judge == i + 1)
2966 .and_then(|v| v.vote)
2967 .or_else(|| j.ranking.first().copied());
2968 if let Some(v) = vote {
2969 *first_choice.entry(v).or_insert(0) += 1;
2970 cast.push(v);
2971 }
2972 }
2973
2974 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2975 for j in &self.state.judgements {
2976 let n = j.ranking.len();
2977 for (pos, label) in j.ranking.iter().enumerate() {
2978 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2979 }
2980 }
2981
2982 let best = first_choice.values().copied().max().unwrap_or(0);
2983 let mut leaders: Vec<char> = first_choice
2984 .iter()
2985 .filter(|(_, v)| **v == best)
2986 .map(|(k, _)| *k)
2987 .collect();
2988 let mut tie_break = None;
2989 if leaders.len() > 1 {
2990 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2991 let borda_leaders: Vec<char> = leaders
2992 .iter()
2993 .copied()
2994 .filter(|l| borda[l] == top_borda)
2995 .collect();
2996 tie_break = Some(if borda_leaders.len() == 1 {
2997 format!(
2998 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2999 leaders.len()
3000 )
3001 } else {
3002 format!(
3003 "{} way tie on both first-choice votes and Borda points, broken by label order",
3004 leaders.len()
3005 )
3006 });
3007 leaders = borda_leaders;
3008 leaders.sort_unstable();
3009 }
3010 let winner = *leaders
3011 .first()
3012 .or(viable.first())
3013 .context("no candidate to declare a winner from")?;
3014
3015 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3016 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3017 let deliberated = !self.state.deliberation.is_empty();
3018
3019 let quota_seats: std::collections::BTreeSet<&str> =
3023 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3024 let mut present = 0usize;
3025 for (i, j) in self.state.judgements.iter().enumerate() {
3026 if quota_seats.contains(j.seat.as_str()) {
3027 continue;
3028 }
3029 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3030 let voted = self
3031 .state
3032 .votes
3033 .iter()
3034 .any(|v| v.judge == i + 1 && v.vote.is_some());
3035 if ranked || voted {
3036 present += 1;
3037 }
3038 }
3039 let needs_quorum = viable.len() > 1;
3045 let judges_total = if needs_quorum {
3046 self.roles.judges.len()
3047 } else {
3048 0
3049 };
3050 let quorum = if needs_quorum {
3051 judges_total / 2 + 1
3052 } else {
3053 0
3054 };
3055 let met_quorum = !needs_quorum || present >= quorum;
3056 let uncontested = (!needs_quorum).then(|| {
3057 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3058 });
3059
3060 self.state.event(
3061 "tally",
3062 match &uncontested {
3063 Some(reason) => format!("winner {winner} — {reason}"),
3064 None => format!(
3065 "winner {winner} — votes {} | initial {} | {} changed | \
3066 {present}/{judges_total} judges{}",
3067 first_choice
3068 .iter()
3069 .map(|(k, v)| format!("{k}:{v}"))
3070 .collect::<Vec<_>>()
3071 .join(" "),
3072 if unanimous_initial {
3073 "unanimous"
3074 } else {
3075 "split"
3076 },
3077 changed_votes,
3078 if met_quorum {
3079 String::new()
3080 } else {
3081 format!(" — below quorum ({quorum} required)")
3082 },
3083 ),
3084 },
3085 );
3086 if !met_quorum {
3087 self.state.event(
3088 "stall",
3089 format!(
3090 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3091 the run stops here, resumable"
3092 ),
3093 );
3094 }
3095 self.state.tally = Some(Tally {
3096 first_choice,
3097 borda,
3098 winner,
3099 rankings: tops.len(),
3100 unanimous_initial,
3101 deliberated,
3102 changed_votes,
3103 unanimous_final,
3104 tie_break,
3105 judges: judges_total,
3106 present,
3107 quorum,
3108 met_quorum,
3109 uncontested,
3110 });
3111 self.state.status = if met_quorum {
3112 RunStatus::Reviewing
3113 } else {
3114 RunStatus::Stalled
3115 };
3116 self.state.save()?;
3117 Ok(())
3118 }
3119
3120 #[allow(clippy::too_many_lines)]
3141 async fn recover_stall(&mut self) -> Result<bool> {
3142 let run_id = self.state.id.clone();
3147 let prompts = self.state.config.prompts.clone();
3148 let quota_seats: BTreeSet<&str> =
3153 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3154 let absent: Vec<String> = self
3155 .state
3156 .judgements
3157 .iter()
3158 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3159 .map(|j| j.seat.clone())
3160 .collect();
3161 if absent.is_empty() {
3162 return Ok(false);
3163 }
3164 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3165 if viable.len() <= 1 {
3166 return Ok(false);
3167 }
3168 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3169 let language = self.state.config.graph.language.clone();
3170 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3171 let sessions = self.state.config.graph.sessions;
3172 let artifacts = agent::artifacts_dir(&self.state.dir());
3173 let root = self.state.worktree_root();
3174 let base_short = short(&self.state.base_commit);
3175 let candidates: Vec<Candidate> = viable.clone();
3176
3177 let mut positions: Vec<usize> = absent
3179 .iter()
3180 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3181 .collect();
3182 if positions.is_empty() {
3183 return Ok(false);
3184 }
3185 positions.sort_unstable();
3186 positions.dedup();
3187
3188 let mut judge_jobs = Vec::new();
3190 for &j in &positions {
3191 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3192 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3193 let seat_key = format!("judge-{}", j + 1);
3194 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3195 let seat = self.seat(&seat_key, &spec.id);
3196 judge_jobs.push(SeatJob {
3197 spec,
3198 seat,
3199 prompt: prompt::judge(
3200 &self.state.instruction,
3201 &views,
3202 self.roles.judges.len(),
3203 &base_short,
3204 &language,
3205 ),
3206 cwd: root.join(seat_key),
3207 timeout,
3208 allow_write: false,
3209 sessions,
3210 artifacts: artifacts.clone(),
3211 stem: format!("judge-{}-recover", j + 1),
3212 handover: None,
3213 });
3214 }
3215
3216 let labels_for_check = labels.clone();
3217 let mut judge_losses = Vec::new();
3218 let retries = self.state.config.graph.retries;
3219 let cache = self.state.config.cache_dir();
3220 let ctx = WaveCtx {
3221 carry_seats: false,
3222 run: &run_id,
3223 node: "judge",
3224 prompts: &prompts,
3225 cache: cache.as_deref(),
3226 round: None,
3227 };
3228 let results = ask_json_wave::<Ranking>(
3229 judge_jobs,
3230 Arc::clone(&self.sem),
3231 retries,
3232 &self.roles.judge_roster,
3233 &ctx,
3234 &mut judge_losses,
3235 &mut self.state,
3236 &move |r: &Ranking| r.validate(&labels_for_check),
3237 )
3238 .await;
3239
3240 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3242 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3243 let agent_id = seat.agent.clone();
3244 self.state.seats.insert(seat.key.clone(), seat);
3245 let record = &mut self.state.judgements[j];
3246 match res {
3247 Ok((ranking, out)) => {
3248 record.agent = agent_id;
3249 record.ranking = ranking.normalized();
3250 record.reasons = ranking.reasons;
3251 record.confidence = ranking.confidence;
3252 record.failed = None;
3253 record.duration_ms = out.duration_ms;
3254 recovered.insert(j);
3255 self.state.event(
3256 "recover",
3257 format!("judge {} ranked again after the limit", j + 1),
3258 );
3259 }
3260 Err(e) => {
3261 self.state
3262 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3263 }
3264 }
3265 }
3266
3267 let mut vote_jobs = Vec::new();
3269 let mut vote_pos: Vec<usize> = Vec::new();
3270 for &j in &recovered {
3271 let seat_key = format!("judge-{}", j + 1);
3272 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3273 let seat = self.seat(&seat_key, &spec.id);
3274 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3275 let text = if has_context(&spec, &seat, sessions) {
3276 prompt::final_vote(&labels, &language)
3277 } else {
3278 full.clone()
3279 };
3280 vote_jobs.push(SeatJob {
3281 spec,
3282 seat,
3283 prompt: text,
3284 cwd: root.join(seat_key),
3285 timeout,
3286 allow_write: false,
3287 sessions,
3288 artifacts: artifacts.clone(),
3289 stem: format!("vote-judge-{}-recover", j + 1),
3290 handover: Some(full),
3291 });
3292 vote_pos.push(j);
3293 }
3294 let allowed = labels.clone();
3295 let mut vote_losses = Vec::new();
3296 let vote_retries = self.state.config.graph.retries;
3297 let vote_cache = self.state.config.cache_dir();
3298 let ctx = WaveCtx {
3299 carry_seats: false,
3300 run: &run_id,
3301 node: "vote",
3302 prompts: &prompts,
3303 cache: vote_cache.as_deref(),
3304 round: None,
3305 };
3306 let votes = ask_json_wave::<FinalVote>(
3307 vote_jobs,
3308 Arc::clone(&self.sem),
3309 vote_retries,
3310 &self.roles.judge_roster,
3311 &ctx,
3312 &mut vote_losses,
3313 &mut self.state,
3314 &move |v: &FinalVote| match v.label() {
3315 Some(c) if allowed.contains(&c) => Ok(()),
3316 other => bail!("vote {other:?} is not one of {allowed:?}"),
3317 },
3318 )
3319 .await;
3320 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3321 let agent_id = seat.agent.clone();
3322 self.state.seats.insert(seat.key.clone(), seat);
3323 match res {
3324 Ok((v, _)) => {
3325 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3326 rec.vote = v.label();
3327 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3328 } else {
3329 self.state.votes.push(VoteRecord {
3330 judge: j + 1,
3331 agent: agent_id,
3332 vote: v.label(),
3333 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3334 changed: false,
3335 });
3336 }
3337 self.state.event(
3338 "recover",
3339 format!("judge {} voted again after the limit", j + 1),
3340 );
3341 }
3342 Err(e) => {
3343 self.state
3344 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3345 }
3346 }
3347 }
3348
3349 let recovered_keys: BTreeSet<String> = recovered
3353 .iter()
3354 .map(|&j| format!("judge-{}", j + 1))
3355 .collect();
3356 self.state
3357 .quota
3358 .retain(|q| !recovered_keys.contains(&q.seat));
3359 for loss in judge_losses.into_iter().chain(vote_losses) {
3363 if recovered_keys.contains(&loss.seat) {
3364 continue;
3365 }
3366 self.state.quota.retain(|q| q.seat != loss.seat);
3367 self.state.quota.push(loss);
3368 }
3369
3370 self.state.tally = None;
3372 self.tally()?;
3373 Ok(self
3374 .state
3375 .tally
3376 .as_ref()
3377 .map(|t| t.met_quorum)
3378 .unwrap_or(false))
3379 }
3380
3381 async fn fold_losers(&mut self) -> Result<()> {
3384 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3385 return Ok(());
3386 };
3387 let repo = self.state.repo.clone();
3388 let mut folded = Vec::new();
3389 for i in 0..self.state.candidates.len() {
3390 let c = &self.state.candidates[i];
3391 if c.label == winner || c.folded {
3392 continue;
3393 }
3394 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3395 git::worktree_remove(&repo, &wt).await.ok();
3396 git::branch_delete(&repo, &branch).await.ok();
3397 self.state.candidates[i].folded = true;
3398 folded.push(label.to_string());
3399 }
3400 let root = self.state.worktree_root();
3402 for j in 1..=self.roles.judges.len() {
3403 let wt = root.join(format!("judge-{j}"));
3404 if wt.exists() {
3405 git::worktree_remove(&repo, &wt).await.ok();
3406 }
3407 }
3408 if self.state.config.graph.advise {
3411 for k in 1..=self.state.config.graph.advisors {
3412 let wt = root.join(format!("advisor-{k}"));
3413 if wt.exists() {
3414 git::worktree_remove(&repo, &wt).await.ok();
3415 }
3416 }
3417 }
3418 if !folded.is_empty() {
3419 self.state
3420 .event("fold", format!("folded candidates {}", folded.join(", ")));
3421 self.state.save()?;
3422 }
3423 Ok(())
3424 }
3425
3426 async fn sync_to_base(&mut self) -> Result<()> {
3466 if self.state.status == RunStatus::AlreadyInBase {
3467 return Ok(());
3468 }
3469 let conflicted = self
3470 .state
3471 .base_sync
3472 .as_ref()
3473 .is_some_and(|s| s.conflict.is_some());
3474 let Some(winner) = self.state.winner().cloned() else {
3475 return Ok(());
3476 };
3477
3478 let repo = self.state.repo.clone();
3479 let remote = self.state.config.merge.remote.clone();
3480 let base_branch = self.state.base_branch.clone();
3481 let tracking = format!("{remote}/{base_branch}");
3482
3483 git::fetch(&repo, &remote, &base_branch).await.ok();
3484 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3488 return Ok(());
3489 };
3490
3491 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3492 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3493 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3494
3495 if (behind > 0 || conflicted || head == tip)
3504 && self
3505 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3506 .await?
3507 {
3508 return Ok(());
3509 }
3510 if conflicted {
3511 return Ok(());
3512 }
3513
3514 if behind == 0 {
3515 if let Some(from) = self
3524 .state
3525 .rebase_fixes
3526 .iter()
3527 .rev()
3528 .find_map(|r| r.from.clone())
3529 && from != head
3530 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3531 .await
3532 .is_ok_and(|o| o.ok())
3533 {
3534 git::sync_to_head(&winner.worktree).await?;
3535 }
3536 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3542 if let Some(why) = &conflict {
3543 self.state.status = RunStatus::Blocked;
3544 self.state.event("land", why.clone());
3545 }
3546 self.state.base_sync = Some(BaseSync {
3547 tip,
3548 behind: 0,
3549 attempts,
3550 conflict,
3551 already_in: None,
3552 });
3553 self.state.save()?;
3554 return Ok(());
3555 }
3556
3557 if attempts >= BASE_SYNC_ROUNDS {
3558 let why = format!(
3559 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3560 rebase(s); rebasing again would only race it",
3561 winner.branch
3562 );
3563 self.state.status = RunStatus::Blocked;
3564 self.state.base_sync = Some(BaseSync {
3565 tip,
3566 behind,
3567 attempts,
3568 conflict: Some(why.clone()),
3569 already_in: None,
3570 });
3571 self.state.event("land", why);
3572 self.state.save()?;
3573 return Ok(());
3574 }
3575
3576 self.state.event(
3577 "land",
3578 format!(
3579 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3580 winner.branch
3581 ),
3582 );
3583 self.state.save()?;
3584
3585 let branch_tracking = format!("{remote}/{}", winner.branch);
3591 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3592 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3593 git::rev_parse(&repo, &branch_tracking).await.ok()
3594 } else {
3595 None
3596 };
3597 if let Some(theirs) = &remote_tip
3601 && !git::is_ancestor(&repo, theirs, &head).await
3602 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3603 .await
3604 .is_ok_and(|missing| missing.is_empty())
3605 {
3606 let why = format!(
3607 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3608 them",
3609 short(theirs),
3610 winner.branch
3611 );
3612 self.state.status = RunStatus::Blocked;
3613 self.state.base_sync = Some(BaseSync {
3614 tip,
3615 behind,
3616 attempts,
3617 conflict: Some(why.clone()),
3618 already_in: None,
3619 });
3620 self.state.event("land", why);
3621 self.state.save()?;
3622 return Ok(());
3623 }
3624
3625 let scratch = self.state.dir().join("base-sync");
3626 let rebased = match crate::rebase::rebase_with_fixer(
3627 &mut self.state,
3628 &scratch,
3629 &winner.branch,
3630 &tracking,
3631 )
3632 .await
3633 {
3634 Ok(crate::rebase::Rebased::Applied) => Ok(None),
3635 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3636 Err(e) => Err(e),
3637 };
3638 let attempts = attempts + 1;
3639 match rebased {
3640 Ok(None) => {
3641 git::sync_to_head(&winner.worktree).await?;
3645 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
3646 let mut conflict = None;
3647 if let Some(pinned) = &remote_tip {
3648 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3649 match pushed {
3650 Ok(o) if o.ok() => self.state.event(
3651 "land",
3652 format!("pushed rebased {} to {remote}", winner.branch),
3653 ),
3654 Ok(o) => {
3655 conflict = Some(format!(
3656 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
3657 winner.branch,
3658 short(pinned),
3659 o.stderr.chars().take(600).collect::<String>()
3660 ));
3661 }
3662 Err(e) => {
3663 conflict = Some(format!(
3664 "rebased {} locally but could not push it: {e:#}",
3665 winner.branch
3666 ));
3667 }
3668 }
3669 }
3670 if let Some(why) = &conflict {
3671 self.state.status = RunStatus::Blocked;
3672 self.state.event("land", why.clone());
3673 }
3674 self.state.base_sync = Some(BaseSync {
3675 tip: tip.clone(),
3676 behind: 0,
3677 attempts,
3678 conflict,
3679 already_in: None,
3680 });
3681 self.state
3682 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3683 }
3684 Ok(Some(conflict)) => {
3685 let why = format!(
3686 "{} conflicts with {tracking} and did not rebase: {}",
3687 winner.branch,
3688 conflict.chars().take(600).collect::<String>()
3689 );
3690 self.state.status = RunStatus::Blocked;
3691 self.state.base_sync = Some(BaseSync {
3692 tip,
3693 behind,
3694 attempts,
3695 conflict: Some(why.clone()),
3696 already_in: None,
3697 });
3698 self.state.event("land", why);
3699 }
3700 Err(e) => {
3701 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3702 self.state.status = RunStatus::Blocked;
3703 self.state.base_sync = Some(BaseSync {
3704 tip,
3705 behind,
3706 attempts,
3707 conflict: Some(why.clone()),
3708 already_in: None,
3709 });
3710 self.state.event("land", why);
3711 }
3712 }
3713 self.state.save()?;
3714 Ok(())
3715 }
3716
3717 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
3723 let repo = self.state.repo.clone();
3724 let remote = self.state.config.merge.remote.clone();
3725 let fetched = git::fetch(&repo, &remote, branch).await;
3726 if !matches!(&fetched, Ok(o) if o.ok()) {
3727 return None;
3728 }
3729 let branch_tracking = format!("{remote}/{branch}");
3730 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
3731 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
3732 return None;
3733 }
3734 if !crate::reconcile::origin_missing(&repo, head, &theirs)
3735 .await
3736 .is_ok_and(|missing| missing.is_empty())
3737 {
3738 return Some(format!(
3739 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
3740 them",
3741 short(&theirs)
3742 ));
3743 }
3744 match git::push_pinned(&repo, &remote, branch, &theirs).await {
3745 Ok(o) if o.ok() => {
3746 self.state
3747 .event("land", format!("pushed rebased {branch} to {remote}"));
3748 None
3749 }
3750 Ok(o) => Some(format!(
3751 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
3752 someone may have pushed): {}",
3753 short(&theirs),
3754 o.stderr.chars().take(600).collect::<String>()
3755 )),
3756 Err(e) => Some(format!(
3757 "{branch} is rebased locally but could not be pushed: {e:#}"
3758 )),
3759 }
3760 }
3761
3762 async fn settle_already_in(
3779 &mut self,
3780 branch: &str,
3781 tip: &str,
3782 head: &str,
3783 attempts: usize,
3784 behind: usize,
3785 ) -> Result<bool> {
3786 let repo = self.state.repo.clone();
3787 let remote = self.state.config.merge.remote.clone();
3788 let start = self.state.base_commit.clone();
3789 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3790 Ok(Some(e)) => e,
3791 Ok(None) => return Ok(false),
3792 Err(e) => {
3793 tracing::warn!("already-in-base check for {branch}: {e:#}");
3794 return Ok(false);
3795 }
3796 };
3797 let mut verified = vec![head.to_owned()];
3798 let fetched = git::fetch(&repo, &remote, branch).await;
3799 if matches!(&fetched, Ok(o) if o.ok())
3800 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3801 && theirs != head
3802 {
3803 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3804 Ok(Some(_)) => verified.push(theirs),
3805 _ => return Ok(false),
3806 }
3807 }
3808 let base_branch = self.state.base_branch.clone();
3809 let message = format!(
3810 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3811 land",
3812 evidence.names(),
3813 evidence.proof.as_str()
3814 );
3815 let closed =
3816 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3817 match closed {
3818 Ok(Ok(url)) => self
3819 .state
3820 .event("land", format!("closed {url}: superseded on {base_branch}")),
3821 Ok(Err(why)) => self
3822 .state
3823 .event("land", format!("did not close a pull request: {why}")),
3824 Err(e) => {
3825 let why = format!(
3826 "{branch} is already in {remote}/{base_branch}, but its pull request could \
3827 not be closed ({e:#}); resume to retry"
3828 );
3829 self.state.status = RunStatus::Blocked;
3830 self.state.base_sync = Some(BaseSync {
3831 tip: tip.to_owned(),
3832 behind,
3833 attempts,
3834 conflict: Some(why.clone()),
3835 already_in: None,
3836 });
3837 self.state.event("land", why);
3838 self.state.save()?;
3839 return Ok(true);
3840 }
3841 }
3842 self.state.status = RunStatus::AlreadyInBase;
3843 self.state.base_sync = Some(BaseSync {
3844 tip: tip.to_owned(),
3845 behind,
3846 attempts,
3847 conflict: None,
3848 already_in: Some(evidence),
3849 });
3850 self.state.event("land", message);
3851 self.state.save()?;
3852 self.settle_questions();
3853 Ok(true)
3854 }
3855
3856 fn landing_base(&self) -> String {
3866 self.state
3867 .base_sync
3868 .as_ref()
3869 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3870 }
3871
3872 pub async fn fix_selected(
3905 &mut self,
3906 ids: &[String],
3907 reason: &str,
3908 allow_stale: bool,
3909 ) -> Result<()> {
3910 let reason = reason.trim();
3911 if reason.is_empty() {
3912 bail!("a fix request needs a reason — that is the operator's own record of why");
3913 }
3914 if ids.is_empty() {
3915 bail!("no finding id given");
3916 }
3917 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3918 bail!(
3919 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3920 has already concluded — can be given a targeted fix. A run still \
3921 in progress should simply be resumed; a `merged` run's branch has \
3922 already landed, so its answer is a fresh `magi review <branch>`, \
3923 not reopening this run's own record",
3924 self.state.id,
3925 self.state.status.as_str()
3926 );
3927 }
3928 let Some(winner) = self.state.winner().cloned() else {
3929 bail!("run {} has no winning candidate to fix", self.state.id);
3930 };
3931 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3932 bail!(
3933 "branch `{}` no longer exists; this run cannot be extended",
3934 winner.branch
3935 );
3936 }
3937 let home = crate::run::home();
3938 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3939 bail!(
3940 "run {} is currently being worked on by another magi process",
3941 self.state.id
3942 );
3943 }
3944 let _claim = FixClaim::acquire(&self.state.dir())?;
3950
3951 let mut seen = BTreeSet::new();
3955 let mut findings = Vec::new();
3956 let mut missing = Vec::new();
3957 for id in ids {
3958 if !seen.insert(id.clone()) {
3959 continue;
3960 }
3961 match self.state.finding(id) {
3962 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3963 id: f.id.clone(),
3964 severity: f.severity,
3965 reviewer_vote: rec.vote,
3966 round: round.round,
3967 round_head: round.head.clone(),
3968 reviewer: rec.reviewer,
3969 agent: rec.agent.clone(),
3970 file: f.file.clone(),
3971 line: f.line,
3972 title: f.title.clone(),
3973 detail: f.detail.clone(),
3974 outcome: OperatorFixOutcome::Pending,
3975 }),
3976 None => missing.push(id.clone()),
3977 }
3978 }
3979 if !missing.is_empty() {
3980 bail!(
3981 "unknown finding id(s): {}; nothing was changed",
3982 missing.join(", ")
3983 );
3984 }
3985
3986 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3987 let stale_details: Vec<(String, String)> = findings
3988 .iter()
3989 .filter(|f| f.round_head != head_at_request)
3990 .map(|f| (f.id.clone(), f.round_head.clone()))
3991 .collect();
3992 let stale = !stale_details.is_empty();
3993 if stale && !allow_stale {
3994 bail!(
3995 "the branch has moved since some finding(s) were raised — {} — now \
3996 at {}; pass --allow-stale to fix anyway, or re-run review first",
3997 stale_details
3998 .iter()
3999 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
4000 .collect::<Vec<_>>()
4001 .join(", "),
4002 short(&head_at_request)
4003 );
4004 }
4005
4006 let request = OperatorFixRequest {
4007 requested_at: Timestamp::now(),
4008 reason: reason.to_owned(),
4009 findings,
4010 head_at_request: head_at_request.clone(),
4011 allow_stale,
4012 stale,
4013 fix: None,
4014 result_head: None,
4015 follow_up_review_run: None,
4016 };
4017 self.state.event(
4018 "fix",
4019 format!(
4020 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4021 request.findings.len(),
4022 request
4023 .findings
4024 .iter()
4025 .map(|f| f.id.as_str())
4026 .collect::<Vec<_>>()
4027 .join(", "),
4028 ),
4029 );
4030 self.state.operator_fixes.push(request);
4037 self.state.save()?;
4038 let request_index = self.state.operator_fixes.len() - 1;
4039
4040 if winner.worktree.exists() {
4049 let dirty = git::git(
4052 &winner.worktree,
4053 &["status", "--porcelain", "--untracked-files=all"],
4054 )
4055 .await?;
4056 let only_withheld = dirty.lines().all(|l| {
4057 l.strip_prefix("?? ")
4058 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4059 });
4060 if !only_withheld {
4061 bail!(
4062 "`{}` has uncommitted changes; refusing to touch it — commit or \
4063 discard them first",
4064 winner.worktree.display()
4065 );
4066 }
4067 git::worktree_remove(&self.state.repo, &winner.worktree)
4068 .await
4069 .ok();
4070 }
4071 let fix_worktree = self.state.worktree_root().join("operator-fix");
4072 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
4073 git::git(
4074 &self.state.repo,
4075 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
4076 )
4077 .await
4078 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
4079 if !git::is_clean(&fix_worktree).await? {
4080 git::worktree_remove(&self.state.repo, &fix_worktree)
4081 .await
4082 .ok();
4083 bail!(
4084 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
4085 winner.branch
4086 );
4087 }
4088
4089 let run_id = self.state.id.clone();
4090 let prompts = self.state.config.prompts.clone();
4091 let language = self.state.config.graph.language.clone();
4092 let sessions = self.state.config.graph.sessions;
4093 let artifacts = agent::artifacts_dir(&self.state.dir());
4094 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
4095 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4096 _ => (
4097 self.state
4098 .config
4099 .agent(&winner.agent)
4100 .cloned()
4101 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4102 format!("impl-{}", winner.label),
4103 ),
4104 };
4105 let seat = self.seat(&fix_seat_key, &fix_spec.id);
4106 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
4107 .findings
4108 .iter()
4109 .map(|f| Finding {
4110 id: f.id.clone(),
4111 severity: f.severity,
4112 file: f.file.clone(),
4113 line: f.line,
4114 title: f.title.clone(),
4115 detail: f.detail.clone(),
4116 })
4117 .collect();
4118 let job = SeatJob {
4119 prompt: prompt::operator_fix(
4120 &self.state.instruction,
4121 &finding_list,
4122 reason,
4123 &stale_details,
4124 &head_at_request,
4125 &language,
4126 ),
4127 spec: fix_spec.clone(),
4128 seat,
4129 cwd: fix_worktree.clone(),
4130 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4131 allow_write: true,
4132 sessions,
4133 artifacts: artifacts.clone(),
4134 stem: "operator-fix".to_owned(),
4135 handover: None,
4136 };
4137 let cache = self.state.config.cache_dir();
4138 let ctx = WaveCtx {
4139 carry_seats: false,
4140 run: &run_id,
4141 node: "fix",
4142 prompts: &prompts,
4143 cache: cache.as_deref(),
4144 round: None,
4145 };
4146 let (seat, out) =
4147 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4148 let agent_id = seat.agent.clone();
4149
4150 let mut fix = FixRecord {
4151 agent: agent_id,
4152 addressed: Vec::new(),
4153 rejected: Vec::new(),
4154 notes: String::new(),
4155 committed: false,
4156 failed: None,
4157 duration_ms: 0,
4158 continuation: None,
4159 };
4160 let mut final_seat = seat.clone();
4161 match out {
4162 AgentOutcome::Ok(o) => {
4163 fix.duration_ms = o.duration_ms;
4164 let parsed = verdict::extract_json::<FixReport>(&o.text);
4165 let incomplete_reason = match &parsed {
4166 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4167 "the reply parsed, but it reported a command whose own CLI \
4168 never confirmed an exit status"
4169 .to_owned(),
4170 ),
4171 Ok(_) => None,
4172 Err(e) => Some(e.to_string()),
4173 };
4174 match incomplete_reason {
4175 None => {
4176 let report = parsed.expect("checked Ok above");
4177 fix.addressed = report.addressed;
4178 fix.rejected = report.rejected;
4179 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4180 }
4181 Some(reason) => {
4182 let (resumed_seat, resolved, failure, cont) = self
4183 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4184 .await;
4185 fix.duration_ms += cont.cumulative_wait_ms;
4186 fix.continuation = Some(cont);
4187 final_seat = resumed_seat;
4188 match resolved {
4189 Some(report) => {
4190 fix.addressed = report.addressed;
4191 fix.rejected = report.rejected;
4192 fix.notes =
4193 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4194 }
4195 None => fix.failed = failure,
4196 }
4197 }
4198 }
4199 }
4200 AgentOutcome::Dropped(o) => {
4201 fix.duration_ms = o.duration_ms;
4202 let why = o
4203 .dropped
4204 .as_ref()
4205 .map(|d| d.why.as_str())
4206 .unwrap_or("the CLI ended the stream without delivering its answer");
4207 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4208 }
4209 AgentOutcome::Quota(o) => {
4210 self.state.quota.push(QuotaLoss {
4211 seat: final_seat.key.clone(),
4212 node: "fix".to_owned(),
4213 at: Timestamp::now(),
4214 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4215 });
4216 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4217 }
4218 AgentOutcome::Failed(e) => fix.failed = Some(e),
4219 }
4220 if fix.continuation.is_none() {
4221 fix.continuation = Some(ContinuationRecord::not_needed());
4222 }
4223 self.state.seats.insert(final_seat.key.clone(), final_seat);
4224
4225 let rescue_message = format!(
4226 "magi: operator-selected fix ({}) (uncommitted work)",
4227 self.state.operator_fixes[request_index]
4228 .findings
4229 .iter()
4230 .map(|f| f.id.as_str())
4231 .collect::<Vec<_>>()
4232 .join(", ")
4233 );
4234 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4235 self.state.note_withheld("fix", &r.withheld);
4236 }
4237 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4238 fix.committed = after != head_at_request;
4239 git::worktree_remove(&self.state.repo, &fix_worktree)
4240 .await
4241 .ok();
4242
4243 self.state.event(
4244 "fix",
4245 match &fix.failed {
4246 Some(reason) => format!(
4247 "operator fix: adoption report was lost ({reason}); {}",
4248 if fix.committed {
4249 "committed"
4250 } else {
4251 "NO new commit"
4252 }
4253 ),
4254 None => format!(
4255 "operator fix: {} addressed, {} rejected, {}",
4256 fix.addressed.len(),
4257 fix.rejected.len(),
4258 if fix.committed {
4259 "committed"
4260 } else {
4261 "NO new commit"
4262 }
4263 ),
4264 },
4265 );
4266
4267 for f in &mut self.state.operator_fixes[request_index].findings {
4274 f.outcome = if fix.failed.is_some() {
4275 OperatorFixOutcome::Unreported
4276 } else if fix.addressed.contains(&f.id) {
4277 OperatorFixOutcome::Addressed
4278 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4279 OperatorFixOutcome::Rejected { why: r.why.clone() }
4280 } else {
4281 OperatorFixOutcome::Unreported
4282 };
4283 }
4284
4285 let committed = fix.committed;
4286 if committed {
4287 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4288 }
4289 self.state.operator_fixes[request_index].fix = Some(fix);
4290 self.state.save()?;
4293
4294 if committed {
4295 self.state.event(
4296 "fix",
4297 format!(
4298 "operator fix committed {}; opening a follow-up review-only run",
4299 short(&after)
4300 ),
4301 );
4302 let origin =
4305 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4306 match Self::review(
4307 &self.state.repo,
4308 &winner.branch,
4309 self.state.config.clone(),
4310 origin,
4311 )
4312 .await
4313 {
4314 Ok(mut follow_up) => {
4315 follow_up.state.event(
4316 "start",
4317 format!(
4318 "requested by an operator fix on run {} for finding(s) {}",
4319 self.state.id,
4320 self.state.operator_fixes[request_index]
4321 .findings
4322 .iter()
4323 .map(|f| f.id.as_str())
4324 .collect::<Vec<_>>()
4325 .join(", "),
4326 ),
4327 );
4328 follow_up.state.save()?;
4329 let follow_up_id = follow_up.state.id.clone();
4330 let adopted = match crate::direct::adopt(
4333 &follow_up.state,
4334 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
4335 ) {
4336 Ok(a) => a,
4337 Err(e) => {
4338 self.state.event(
4341 "fix",
4342 format!(
4343 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
4344 ),
4345 );
4346 self.state.save()?;
4347 return Ok(());
4348 }
4349 };
4350 let executed = follow_up.execute().await;
4351 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
4352 if let Some(a) = adopted {
4353 a.finish(&follow_up.state, executed);
4354 }
4355 if let Some(e) = failure {
4356 self.state.event(
4357 "fix",
4358 format!(
4359 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4360 ),
4361 );
4362 }
4363 self.state.operator_fixes[request_index].follow_up_review_run =
4364 Some(follow_up_id);
4365 }
4366 Err(e) => {
4367 self.state.event(
4368 "fix",
4369 format!("committed the fix but could not open a follow-up review: {e:#}"),
4370 );
4371 }
4372 }
4373 self.state.save()?;
4374 }
4375
4376 Ok(())
4377 }
4378
4379 fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
4386 match &self.roles.fixer {
4387 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4388 _ => (
4389 self.state
4390 .config
4391 .agent(&winner.agent)
4392 .cloned()
4393 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4394 format!("impl-{}", winner.label),
4395 ),
4396 }
4397 }
4398
4399 async fn review_loop(&mut self) -> Result<()> {
4400 if self
4405 .state
4406 .base_sync
4407 .as_ref()
4408 .is_some_and(|s| s.conflict.is_some())
4409 {
4410 return Ok(());
4411 }
4412 let run_id = self.state.id.clone();
4417 let prompts = self.state.config.prompts.clone();
4418 let Some(winner) = self.state.winner().cloned() else {
4419 return Ok(());
4420 };
4421 let max_rounds = self.state.config.graph.review_rounds;
4422 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4432 if status == RunStatus::Gating {
4435 self.record_contested_handoff();
4436 }
4437 self.state.status = status;
4438 self.state.save()?;
4439 return Ok(());
4440 }
4441 self.state.status = RunStatus::Reviewing;
4442 if self
4452 .state
4453 .reviews
4454 .last()
4455 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4456 {
4457 let shell = self.state.config.shell();
4458 return self
4459 .stop_reviewing(
4460 "the last round's own verification never resolved",
4461 &shell,
4462 &winner.worktree,
4463 )
4464 .await;
4465 }
4466
4467 let repo = self.state.repo.clone();
4468 let root = self.state.worktree_root();
4469 let language = self.state.config.graph.language.clone();
4470 let sessions = self.state.config.graph.sessions;
4471 let artifacts = agent::artifacts_dir(&self.state.dir());
4472 let base = self.landing_base();
4473 let base_short = short(&base);
4474 let reviewers = self.roles.reviewers.clone();
4475 let shell = self.state.config.shell();
4476
4477 for round in (self.state.reviews.len() + 1)..=max_rounds {
4478 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4479 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4480 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4481 let prev_verification = self
4490 .state
4491 .reviews
4492 .last()
4493 .and_then(|r| r.verification_summary(&head));
4494
4495 let mut jobs = Vec::new();
4499 for (r, spec) in reviewers.iter().cloned().enumerate() {
4500 let wt = root.join(format!("review-{}", r + 1));
4501 if wt.exists() {
4502 git::reset_detached(&wt, &head).await?;
4503 } else {
4504 git::worktree_add_detached(&repo, &wt, &head).await?;
4505 }
4506 let seat_key = format!("review-{}", r + 1);
4507 let spec = pick_start_spec(
4510 &self.roles.reviewer_roster,
4511 spec,
4512 self.state.seat_history.get(&seat_key),
4513 );
4514 let seat = self.seat(&seat_key, &spec.id);
4515 jobs.push(SeatJob {
4516 prompt: prompt::review(&prompt::ReviewCtx {
4517 instruction: &self.state.instruction,
4518 branch: &winner.branch,
4519 base_short: &base_short,
4520 stat: &stat,
4521 patch: &patch,
4522 verification: prev_verification.as_ref(),
4523 reviewers: reviewers.len(),
4524 round,
4525 rounds: max_rounds,
4526 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4529 lens: Lens::for_seat(r),
4530 language: &language,
4531 }),
4532 spec,
4533 seat,
4534 cwd: wt,
4535 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4536 allow_write: false,
4537 sessions,
4538 artifacts: artifacts.clone(),
4539 stem: format!("review-{round}-{}", r + 1),
4540 handover: None,
4541 });
4542 }
4543
4544 self.state.event(
4545 "review",
4546 format!(
4547 "round {round}: {} reviewers on {}",
4548 jobs.len(),
4549 short(&head)
4550 ),
4551 );
4552 let mut quota_losses = Vec::new();
4553 let review_retries = self.state.config.graph.retries;
4554 let review_cache = self.state.config.cache_dir();
4555 let ctx = WaveCtx {
4556 carry_seats: true,
4557 run: &run_id,
4558 node: "review",
4559 prompts: &prompts,
4560 cache: review_cache.as_deref(),
4561 round: Some(round),
4562 };
4563 let results = ask_json_wave::<Review>(
4564 jobs,
4565 Arc::clone(&self.sem),
4566 review_retries,
4567 &self.roles.reviewer_roster,
4568 &ctx,
4569 &mut quota_losses,
4570 &mut self.state,
4571 &|_: &Review| Ok(()),
4572 )
4573 .await;
4574 let round_quota_missing = quota_losses.len();
4578 self.state.quota.extend(quota_losses);
4579
4580 let mut records = Vec::new();
4581 let mut all_findings = Vec::new();
4582 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4583 let agent_id = seat.agent.clone();
4584 self.state.seats.insert(seat.key.clone(), seat);
4585 let mut record = ReviewRecord {
4586 reviewer: r + 1,
4587 agent: agent_id,
4588 summary: String::new(),
4589 findings: Vec::new(),
4590 vote: None,
4591 failed: None,
4592 duration_ms: 0,
4593 attempts,
4599 };
4600 match res {
4601 Ok((review, out)) => {
4602 record.summary =
4610 blind::sanitize_prose(&review.summary, &self.state.config.blind);
4611 record.vote = Some(review.vote);
4612 record.duration_ms = out.duration_ms;
4613 for (n, mut f) in review.findings.into_iter().enumerate() {
4614 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4617 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4618 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4619 f.file = f
4625 .file
4626 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4627 all_findings.push(f.clone());
4628 record.findings.push(f);
4629 }
4630 self.state.event(
4631 "review",
4632 format!(
4633 "round {round}: reviewer {} voted {} with {} finding(s)",
4634 r + 1,
4635 review.vote.label(),
4636 record.findings.len()
4637 ),
4638 );
4639 }
4640 Err(e) => {
4641 record.failed = Some(e.to_string());
4642 self.state.event(
4643 "review",
4644 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4645 );
4646 }
4647 }
4648 records.push(record);
4649 }
4650
4651 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4658 let vote_split =
4659 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4660 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4661 if vote_split {
4662 self.state.event(
4663 "review",
4664 format!(
4665 "round {round}: votes split ({}) — one round of reconsideration",
4666 initial_votes
4667 .iter()
4668 .map(|v| v.label())
4669 .collect::<Vec<_>>()
4670 .join(", ")
4671 ),
4672 );
4673 let panel: Vec<ReviewSeatReport<'_>> = records
4676 .iter()
4677 .filter_map(|r| {
4678 r.vote.map(|vote| ReviewSeatReport {
4679 reviewer: r.reviewer,
4680 vote,
4681 summary: &r.summary,
4682 findings: &r.findings,
4683 })
4684 })
4685 .collect();
4686
4687 let mut jobs = Vec::new();
4688 let mut seats_at = Vec::new();
4689 for (r, spec) in reviewers.iter().cloned().enumerate() {
4690 if records[r].vote.is_none() {
4694 continue;
4695 }
4696 let wt = root.join(format!("review-{}", r + 1));
4697 let seat_key = format!("review-{}", r + 1);
4698 let spec = self.occupant(&seat_key, spec);
4699 let seat = self.seat(&seat_key, &spec.id);
4700 let build = |with_patch: bool| {
4708 prompt::review_reconsider(&ReviewReconsiderCtx {
4709 instruction: &self.state.instruction,
4710 reviewer: r + 1,
4711 lens: Lens::for_seat(r),
4712 panel: &panel,
4713 patch: with_patch.then_some(ReviewPatch {
4714 branch: &winner.branch,
4715 base_short: &base_short,
4716 stat: &stat,
4717 patch: &patch,
4718 }),
4719 round,
4720 rounds: max_rounds,
4721 language: &language,
4722 })
4723 };
4724 let full = build(true);
4725 let prompt = if has_context(&spec, &seat, sessions) {
4726 build(false)
4727 } else {
4728 full.clone()
4729 };
4730 jobs.push(SeatJob {
4731 prompt,
4732 spec,
4733 seat,
4734 cwd: wt,
4735 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4736 allow_write: false,
4737 sessions,
4738 artifacts: artifacts.clone(),
4739 stem: format!("review-{round}-reconsider-{}", r + 1),
4740 handover: Some(full),
4741 });
4742 seats_at.push(r);
4743 }
4744
4745 let mut recon_quota_losses = Vec::new();
4746 let recon_cache = self.state.config.cache_dir();
4747 let recon_ctx = WaveCtx {
4748 carry_seats: true,
4749 run: &run_id,
4750 node: "review",
4751 prompts: &prompts,
4752 cache: recon_cache.as_deref(),
4753 round: Some(round),
4754 };
4755 let recon_results = ask_json_wave::<ReviewRevote>(
4756 jobs,
4757 Arc::clone(&self.sem),
4758 review_retries,
4759 &self.roles.reviewer_roster,
4760 &recon_ctx,
4761 &mut recon_quota_losses,
4762 &mut self.state,
4763 &|_: &ReviewRevote| Ok(()),
4764 )
4765 .await;
4766 self.state.quota.extend(recon_quota_losses);
4767
4768 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4769 let agent_id = seat.agent.clone();
4770 self.state.seats.insert(seat.key.clone(), seat);
4771 let mut rec = ReviewRevoteRecord {
4772 reviewer: r + 1,
4773 agent: agent_id,
4774 vote: None,
4775 reason: String::new(),
4776 failed: None,
4777 };
4778 match res {
4779 Ok((rv, _)) => {
4780 rec.vote = Some(rv.vote);
4781 rec.reason =
4782 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4783 self.state.event(
4784 "review",
4785 format!(
4786 "round {round}: reviewer {} revoted {}",
4787 r + 1,
4788 rv.vote.label()
4789 ),
4790 );
4791 }
4792 Err(e) => {
4793 rec.failed = Some(e.to_string());
4794 self.state.event(
4795 "review",
4796 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4797 );
4798 }
4799 }
4800 reconsideration.push(rec);
4801 }
4802 } else if initial_votes.len() > 1 {
4803 self.state.event(
4804 "review",
4805 format!(
4806 "round {round}: votes agreed ({}) — no reconsideration",
4807 initial_votes[0].label()
4808 ),
4809 );
4810 }
4811
4812 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4813 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4814 let defer_e2e =
4825 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4826 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4827 let reason =
4828 format!("{blocking} blocking finding(s) already required a fix this round");
4829 self.state.event(
4830 "verify",
4831 format!(
4832 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4833 {}); it will run once a round has none left",
4834 short(&head)
4835 ),
4836 );
4837 (Vec::new(), false, true, Some(reason))
4838 } else {
4839 let e2e_commands = self.state.config.verify.e2e.clone();
4840 let cache_dir = self.state.config.cache_dir();
4841 let context = format!("round {round}");
4842 let (e2e, verify_retried) = with_cache_lease(
4843 &mut self.state,
4844 cache_dir.as_deref(),
4845 "e2e",
4846 "e2e",
4847 &winner.worktree,
4848 &head,
4849 verify_timeout,
4850 &context,
4851 |state, budget| {
4852 let shell = shell.clone();
4853 let e2e_commands = e2e_commands.clone();
4854 let worktree = winner.worktree.clone();
4855 let context = context.clone();
4856 async move {
4857 run_e2e_with_retry(
4858 state,
4859 &shell,
4860 &e2e_commands,
4861 &worktree,
4862 budget,
4863 &context,
4864 )
4865 .await
4866 }
4867 },
4868 )
4869 .await;
4870 (e2e, verify_retried, false, None)
4871 };
4872
4873 let expected = records.len();
4874 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4875 let incomplete = answered < expected;
4876 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4877 let policy = self.state.config.graph.incomplete_review;
4878 let clean = round_is_clean(
4879 blocking,
4880 e2e_ok,
4881 answered,
4882 expected,
4883 round_quota_missing,
4884 policy,
4885 );
4886
4887 let mut round_record = ReviewRound {
4888 round,
4889 head: head.clone(),
4890 verified_head: None,
4891 verified_at: None,
4892 reviews: records,
4893 e2e,
4894 verify_retried,
4895 e2e_deferred,
4896 e2e_defer_reason,
4897 fix: None,
4898 blocking,
4899 answered,
4900 expected,
4901 clean,
4902 progressed: false,
4903 vote_split,
4904 reconsideration,
4905 verdict: None,
4906 };
4907 round_record.verdict = ReviewVote::worst(
4911 round_record
4912 .final_votes()
4913 .into_iter()
4914 .map(|(_, _, vote)| vote),
4915 );
4916 if !matches!(
4927 round_record.e2e_status(),
4928 E2eStatus::Deferred | E2eStatus::NotConfigured
4929 ) {
4930 round_record.verified_head = Some(head.clone());
4931 round_record.verified_at = Some(Timestamp::now());
4932 }
4933 let this_round_verification = round_record.verification_summary(&head);
4934
4935 if incomplete {
4936 let missing: Vec<String> = round_record
4937 .reviews
4938 .iter()
4939 .filter(|r| r.failed.is_some())
4940 .map(|r| format!("review-{}", r.reviewer))
4941 .collect();
4942 self.state.event(
4943 "review",
4944 format!(
4945 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4946 missing.join(", ")
4947 ),
4948 );
4949 }
4950
4951 if clean {
4952 self.state.event(
4953 "review",
4954 if incomplete && policy == IncompleteReviewPolicy::Warn {
4955 format!(
4956 "round {round}: clean (warn policy, incomplete panel) — no \
4957 blocking findings from the seats that answered, verification green"
4958 )
4959 } else if incomplete {
4960 format!(
4961 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4962 quorum) — no blocking findings from the seats that answered, \
4963 verification green",
4964 expected - answered
4965 )
4966 } else {
4967 format!("round {round}: clean — no blocking findings, verification green")
4968 },
4969 );
4970 self.state.reviews.push(round_record);
4971 self.state.status = RunStatus::Gating;
4972 self.state.save()?;
4973 return Ok(());
4974 }
4975
4976 if incomplete && blocking == 0 && e2e_ok {
4984 self.state.reviews.push(round_record);
4985 self.state.save()?;
4986 if round == max_rounds {
4987 self.state.status = RunStatus::Blocked;
4988 self.state.event(
4989 "review",
4990 format!(
4991 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4992 refusing to call it clean",
4993 expected - answered
4994 ),
4995 );
4996 return Ok(());
4997 }
4998 continue;
4999 }
5000
5001 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
5013 self.state.reviews.push(round_record);
5014 return self
5015 .stop_reviewing(
5016 "the round's own verification could not run",
5017 &shell,
5018 &winner.worktree,
5019 )
5020 .await;
5021 }
5022
5023 if round == max_rounds {
5024 self.state.reviews.push(round_record);
5025 return self
5026 .stop_reviewing(
5027 &format!(
5028 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
5029 ),
5030 &shell,
5031 &winner.worktree,
5032 )
5033 .await;
5034 }
5035
5036 let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
5039 let seat = self.seat(&fix_seat_key, &fix_spec.id);
5040 let blocking_findings: Vec<_> = all_findings
5041 .iter()
5042 .filter(|f| f.severity.blocks())
5043 .cloned()
5044 .collect();
5045 let job = SeatJob {
5046 prompt: prompt::fix(
5047 &self.state.instruction,
5048 &blocking_findings,
5049 this_round_verification.as_ref(),
5050 round,
5051 max_rounds,
5052 &language,
5053 ),
5054 spec: fix_spec.clone(),
5055 seat,
5056 cwd: winner.worktree.clone(),
5057 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5058 allow_write: true,
5059 sessions,
5060 artifacts: artifacts.clone(),
5061 stem: format!("fix-{round}"),
5062 handover: None,
5063 };
5064 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5065 let cache = self.state.config.cache_dir();
5066 let ctx = WaveCtx {
5067 carry_seats: false,
5068 run: &run_id,
5069 node: "fix",
5070 prompts: &prompts,
5071 cache: cache.as_deref(),
5072 round: Some(round),
5073 };
5074 let (seat, out) =
5075 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5076 let agent_id = seat.agent.clone();
5077
5078 let mut fix = FixRecord {
5079 agent: agent_id,
5080 addressed: Vec::new(),
5081 rejected: Vec::new(),
5082 notes: String::new(),
5083 committed: false,
5084 failed: None,
5085 duration_ms: 0,
5086 continuation: None,
5087 };
5088 let mut continuation = ContinuationRecord::not_needed();
5089 let mut final_seat = seat.clone();
5090 match out {
5091 AgentOutcome::Ok(o) => {
5092 fix.duration_ms = o.duration_ms;
5093 let parsed = verdict::extract_json::<FixReport>(&o.text);
5094 let incomplete_reason = match &parsed {
5101 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5102 "the reply parsed, but it reported a command whose own CLI never \
5103 confirmed an exit status"
5104 .to_owned(),
5105 ),
5106 Ok(_) => None,
5107 Err(e) => Some(e.to_string()),
5108 };
5109 match incomplete_reason {
5110 None => {
5111 let report = parsed.expect("checked Ok above");
5112 fix.addressed = report.addressed;
5113 fix.rejected = report.rejected;
5114 fix.notes =
5115 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5116 }
5117 Some(reason) => {
5118 let (resumed_seat, resolved, failure, cont) = self
5119 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
5120 .await;
5121 fix.duration_ms += cont.cumulative_wait_ms;
5122 continuation = cont;
5123 final_seat = resumed_seat;
5124 match resolved {
5125 Some(report) => {
5126 fix.addressed = report.addressed;
5127 fix.rejected = report.rejected;
5128 fix.notes = blind::sanitize_prose(
5129 &report.notes,
5130 &self.state.config.blind,
5131 );
5132 }
5133 None => fix.failed = failure,
5134 }
5135 }
5136 }
5137 }
5138 AgentOutcome::Dropped(o) => {
5140 fix.duration_ms = o.duration_ms;
5141 let why = o
5142 .dropped
5143 .as_ref()
5144 .map(|d| d.why.as_str())
5145 .unwrap_or("the CLI ended the stream without delivering its answer");
5146 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5147 }
5148 AgentOutcome::Quota(o) => {
5149 self.state.quota.push(QuotaLoss {
5150 seat: final_seat.key.clone(),
5151 node: "fix".to_owned(),
5152 at: Timestamp::now(),
5153 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5154 });
5155 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5156 }
5157 AgentOutcome::Failed(e) => fix.failed = Some(e),
5158 }
5159 fix.continuation = Some(continuation);
5160 self.state.seats.insert(final_seat.key.clone(), final_seat);
5161 if let Ok(r) = git::rescue_commit(
5162 &winner.worktree,
5163 &format!("magi: review round {round} fixes (uncommitted work)"),
5164 )
5165 .await
5166 {
5167 self.state.note_withheld("fix", &r.withheld);
5168 }
5169 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5170 fix.committed = after != before;
5171 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5179 let progressed = diff_after != patch;
5180 let commit_note = if fix.committed {
5181 "committed"
5182 } else {
5183 "NO new commit"
5184 };
5185 let tree_note = if progressed {
5186 "changed vs base"
5187 } else {
5188 "unchanged vs base"
5189 };
5190 self.state.event(
5191 "fix",
5192 match &fix.failed {
5193 Some(reason) => {
5199 format!(
5200 "round {round}: fixer's adoption report was lost ({reason}); \
5201 {commit_note}, tree {tree_note}"
5202 )
5203 }
5204 None => format!(
5205 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5206 {tree_note}{}",
5207 fix.addressed.len(),
5208 fix.rejected.len(),
5209 if continuation.outcome == ContinuationOutcome::Resumed {
5210 format!(
5211 " (adoption report recovered after {} continuation(s))",
5212 continuation.attempts
5213 )
5214 } else {
5215 String::new()
5216 },
5217 ),
5218 },
5219 );
5220 round_record.fix = Some(fix);
5221 round_record.progressed = progressed;
5222 self.state.reviews.push(round_record);
5223 self.state.save()?;
5224
5225 if matches!(
5238 continuation.outcome,
5239 ContinuationOutcome::Exhausted
5240 | ContinuationOutcome::QuotaLost
5241 | ContinuationOutcome::NoSession
5242 ) {
5243 return self
5244 .stop_reviewing(
5245 "the fixer's adoption report never came back, even after resuming its \
5246 own seat; refusing to start another round against the same worktree \
5247 while that is unresolved",
5248 &shell,
5249 &winner.worktree,
5250 )
5251 .await;
5252 }
5253
5254 let streak = self
5255 .state
5256 .reviews
5257 .iter()
5258 .rev()
5259 .take_while(|r| !r.progressed)
5260 .count();
5261 if streak >= STAGNANT_LIMIT {
5262 return self
5263 .stop_reviewing(
5264 &format!(
5265 "the tree has not moved against base for {streak} round(s) in a row"
5266 ),
5267 &shell,
5268 &winner.worktree,
5269 )
5270 .await;
5271 }
5272 }
5273 Ok(())
5274 }
5275
5276 fn record_contested_handoff(&mut self) {
5310 if self.state.contested_handoff.is_some() {
5311 return;
5312 }
5313 let Some(contested) = self
5314 .state
5315 .reviews
5316 .last()
5317 .and_then(ReviewRound::contested_handoff)
5318 else {
5319 return;
5320 };
5321 self.state.event(
5322 "review",
5323 format!(
5324 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5325 wait for the owner's approval",
5326 contested.findings.len(),
5327 contested.rejecters.len()
5328 ),
5329 );
5330 self.state.contested_handoff = Some(contested);
5331 }
5332
5333 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5334 let round_idx = self.state.reviews.len() - 1;
5335 let needs_catchup_run = matches!(
5343 self.state.reviews[round_idx].e2e_status(),
5344 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5345 );
5346 if needs_catchup_run {
5347 let round = self.state.reviews[round_idx].round;
5348 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5349 let commands = self.state.config.verify.e2e.clone();
5350 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5351 let cache_dir = self.state.config.cache_dir();
5352 let context = format!(
5353 "round {round}: verification unresolved, catching up before the final decision"
5354 );
5355 let (outcomes, verify_retried) = with_cache_lease(
5356 &mut self.state,
5357 cache_dir.as_deref(),
5358 "e2e",
5359 "e2e",
5360 worktree,
5361 &attempted_head,
5362 timeout,
5363 &context,
5364 |state, budget| {
5365 let shell = shell.to_vec();
5366 let commands = commands.clone();
5367 let context = context.clone();
5368 async move {
5369 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5370 .await
5371 }
5372 },
5373 )
5374 .await;
5375 let last = &mut self.state.reviews[round_idx];
5376 last.e2e = outcomes;
5377 last.verify_retried = verify_retried;
5378 last.verified_head = Some(attempted_head);
5385 last.verified_at = Some(Timestamp::now());
5386 if verify_inconclusive(&last.e2e) {
5387 self.state.save()?;
5394 return Ok(());
5395 }
5396 last.e2e_deferred = false;
5397 }
5398 let last = &self.state.reviews[round_idx];
5399 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5400
5401 match last.e2e_status() {
5402 E2eStatus::Failed => {
5403 let red: Vec<String> = last
5404 .e2e
5405 .iter()
5406 .filter(|o| !o.ok())
5407 .map(|o| {
5408 format!(
5409 "`{}` -> {:?}\n{}",
5410 o.command,
5411 o.code,
5412 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5413 )
5414 })
5415 .collect();
5416 self.state
5417 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5418 self.state.status = RunStatus::Blocked;
5419 }
5420 E2eStatus::ResourceBlocked => {
5425 self.state.event(
5426 "review",
5427 format!(
5428 "{why}; e2e could not run (shared build cache unavailable); not \
5429 deciding yet"
5430 ),
5431 );
5432 }
5433 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5434 self.state.event(
5435 "review",
5436 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5437 );
5438 self.record_contested_handoff();
5439 self.state.status = RunStatus::Gating;
5440 }
5441 }
5442 self.state.save()?;
5443 Ok(())
5444 }
5445
5446 async fn gate(&mut self) -> Result<()> {
5449 if self.state.status == RunStatus::Failed
5461 || self
5462 .state
5463 .base_sync
5464 .as_ref()
5465 .is_some_and(|s| s.conflict.is_some())
5466 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5467 != Some(RunStatus::Gating)
5468 {
5469 return Ok(());
5470 }
5471 if self.state.gate_ran {
5472 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5483 self.state.status = RunStatus::Blocked;
5484 self.state.save()?;
5485 }
5486 return Ok(());
5487 }
5488 let Some(winner) = self.state.winner().cloned() else {
5489 return Ok(());
5490 };
5491 self.state.status = RunStatus::Gating;
5492 let mut outcomes = self.run_gate(&winner).await?;
5493 loop {
5494 if verify_inconclusive(&outcomes) {
5505 self.state.save()?;
5506 return Ok(());
5507 }
5508 if outcomes.iter().all(CommandOutcome::ok) {
5509 break;
5510 }
5511 match self.gate_fix_round(&winner, &outcomes).await? {
5512 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5513 GateFix::Stop => break,
5514 GateFix::Defer => {
5515 self.state.save()?;
5516 return Ok(());
5517 }
5518 }
5519 }
5520 let passed = outcomes.iter().all(CommandOutcome::ok);
5521 self.state.gate = outcomes;
5522 self.state.gate_ran = true;
5523 if !passed {
5524 self.state.status = RunStatus::Blocked;
5525 let spent = self.state.gate_fixes.len();
5526 self.state.event(
5527 "gate",
5528 if spent == 0 {
5529 "gate failed; not merging".to_owned()
5530 } else {
5531 format!("gate failed after {spent} gate-fix round(s); not merging")
5532 },
5533 );
5534 }
5535 self.state.save()?;
5536 Ok(())
5537 }
5538
5539 async fn run_pre_gate(&mut self, winner: &Candidate) {
5549 let commands = self.state.config.verify.pre_gate.clone();
5550 if commands.is_empty() {
5551 return;
5552 }
5553 let shell = self.state.config.shell();
5554 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5555 let (outcomes, _) = run_commands(
5556 &mut self.state,
5557 "pre_gate",
5558 "pre_gate",
5559 0,
5560 &shell,
5561 &commands,
5562 &winner.worktree,
5563 timeout,
5564 )
5565 .await;
5566 for o in &outcomes {
5567 if !o.ok() {
5568 tracing::warn!(
5569 "pre_gate `{}` failed ({:?}); the gate decides",
5570 o.command,
5571 o.code
5572 );
5573 }
5574 self.state.event(
5575 "pre_gate",
5576 format!(
5577 "`{}` -> {}",
5578 o.command,
5579 if o.ok() {
5580 "pass".to_owned()
5581 } else {
5582 format!(
5583 "FAIL ({:?})\n{}",
5584 o.code,
5585 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5586 )
5587 }
5588 ),
5589 );
5590 }
5591 self.state.pre_gate = outcomes;
5592 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5593 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5594 Ok(head) => {
5595 self.state
5596 .event("pre_gate", format!("committed mechanical fixes ({head})"));
5597 self.state.pre_gate_commit = Some(head);
5598 }
5599 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5600 },
5601 Ok(false) => {}
5602 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5603 }
5604 if let Err(e) = self.state.save() {
5605 tracing::warn!("could not persist the pre_gate record: {e:#}");
5606 }
5607 }
5608
5609 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5612 self.run_pre_gate(winner).await;
5613 let shell = self.state.config.shell();
5614 let gate_commands = self.state.config.verify.gate.clone();
5615 let outcomes = if gate_commands.is_empty() {
5624 Vec::new()
5625 } else {
5626 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5627 let cache_dir = self.state.config.cache_dir();
5628 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5629 let (outcomes, _) = with_cache_lease(
5630 &mut self.state,
5631 cache_dir.as_deref(),
5632 "gate",
5633 "gate",
5634 &winner.worktree,
5635 &head,
5636 timeout,
5637 "final gate",
5638 |state, budget| {
5639 let shell = shell.clone();
5640 let gate_commands = gate_commands.clone();
5641 let worktree = winner.worktree.clone();
5642 async move {
5643 let (outcomes, timed_out_pids) = run_commands(
5644 state,
5645 "gate",
5646 "gate",
5647 0,
5648 &shell,
5649 &gate_commands,
5650 &worktree,
5651 budget,
5652 )
5653 .await;
5654 (outcomes, false, timed_out_pids)
5655 }
5656 },
5657 )
5658 .await;
5659 outcomes
5660 };
5661 if outcomes.is_empty() {
5662 self.state.event(
5667 "gate",
5668 "no gate commands configured; nothing to check, passing",
5669 );
5670 }
5671 for o in &outcomes {
5672 self.state.event(
5673 "gate",
5674 format!(
5675 "`{}` -> {}",
5676 o.command,
5677 if o.ok() {
5678 "pass".to_owned()
5679 } else {
5680 format!(
5681 "FAIL ({:?})\n{}",
5682 o.code,
5683 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5684 )
5685 }
5686 ),
5687 );
5688 }
5689 Ok(outcomes)
5690 }
5691
5692 async fn gate_fix_round(
5705 &mut self,
5706 winner: &Candidate,
5707 outcomes: &[CommandOutcome],
5708 ) -> Result<GateFix> {
5709 let cap = self.state.config.graph.gate_fix_rounds;
5710 let spent = self.state.gate_fixes.len();
5711 if spent >= cap {
5712 if cap > 0 {
5713 self.state.event(
5714 "gate",
5715 format!("{spent} gate-fix round(s) spent and the gate still fails"),
5716 );
5717 }
5718 return Ok(GateFix::Stop);
5719 }
5720 if !gate_fixable(outcomes) {
5721 self.state.event(
5722 "gate",
5723 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5724 command or similar); not spending a fix round on it",
5725 );
5726 return Ok(GateFix::Stop);
5727 }
5728 let min_free = self.state.config.disk.min_free_bytes;
5729 if min_free > 0 {
5730 match crate::disk::free_bytes(&winner.worktree) {
5731 Ok(free) if crate::disk::enough_space(free, min_free) => {}
5732 Ok(free) => {
5733 self.state.event(
5734 "gate",
5735 format!(
5736 "only {free} bytes free ({min_free} required by `[disk] \
5737 min_free_bytes`); not spending a fix round on a failure the disk \
5738 may explain"
5739 ),
5740 );
5741 return Ok(GateFix::Stop);
5742 }
5743 Err(e) => {
5744 self.state.event(
5745 "gate",
5746 format!("free disk space could not be measured ({e:#}); no fix round"),
5747 );
5748 return Ok(GateFix::Stop);
5749 }
5750 }
5751 }
5752
5753 let attempt = spent + 1;
5754 let run_id = self.state.id.clone();
5755 let prompts = self.state.config.prompts.clone();
5756 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5757 let base = self.landing_base();
5758 let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5759 let seat = self.seat(&fix_seat_key, &fix_spec.id);
5760 let job = SeatJob {
5761 prompt: prompt::gate_fix(
5762 &self.state.instruction,
5763 &failed,
5764 attempt,
5765 cap,
5766 &self.state.config.graph.language,
5767 ),
5768 spec: fix_spec,
5769 seat,
5770 cwd: winner.worktree.clone(),
5771 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5772 allow_write: true,
5773 sessions: self.state.config.graph.sessions,
5774 artifacts: agent::artifacts_dir(&self.state.dir()),
5775 stem: format!("gate-fix-{attempt}"),
5776 handover: None,
5777 };
5778 self.state.event(
5779 "gate",
5780 format!("gate failed; gate-fix round {attempt} of {cap}"),
5781 );
5782 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5783 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5784 let cache = self.state.config.cache_dir();
5785 let ctx = WaveCtx {
5786 carry_seats: false,
5787 run: &run_id,
5788 node: "gate-fix",
5789 prompts: &prompts,
5790 cache: cache.as_deref(),
5791 round: None,
5792 };
5793 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5794 let mut record = GateFixRecord {
5795 agent: seat.agent.clone(),
5796 failed,
5797 notes: String::new(),
5798 committed: false,
5799 error: None,
5800 };
5801 match out {
5802 AgentOutcome::Ok(o) => {
5803 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5806 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5807 }
5808 }
5809 AgentOutcome::Dropped(_) => {
5810 record.error = Some("the CLI dropped the stream".to_owned());
5811 }
5812 AgentOutcome::Quota(o) => {
5813 self.state.quota.push(QuotaLoss {
5814 seat: seat.key.clone(),
5815 node: "gate-fix".to_owned(),
5816 at: Timestamp::now(),
5817 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5818 });
5819 record.error = Some("rate limited (quota); fixer could not run".to_owned());
5820 }
5821 AgentOutcome::Failed(e) => record.error = Some(e),
5822 }
5823 self.state.seats.insert(seat.key.clone(), seat);
5824 if let Ok(r) = git::rescue_commit(
5825 &winner.worktree,
5826 &format!("magi: gate fix {attempt} (uncommitted work)"),
5827 )
5828 .await
5829 {
5830 self.state.note_withheld("gate-fix", &r.withheld);
5831 }
5832 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5833 record.committed = after != before;
5834 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5835 let note = record.error.clone();
5836 self.state.gate_fixes.push(record);
5837 self.state.save()?;
5838 if !changed {
5839 self.state.event(
5840 "gate",
5841 match note {
5842 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5843 None => format!("gate-fix round {attempt}: the tree did not change"),
5844 },
5845 );
5846 return Ok(GateFix::Stop);
5847 }
5848 self.state.event(
5849 "gate",
5850 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5851 );
5852
5853 let commands = self.state.config.verify.e2e.clone();
5854 if !commands.is_empty() {
5855 let shell = self.state.config.shell();
5856 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5857 let cache_dir = self.state.config.cache_dir();
5858 let context = format!("gate-fix round {attempt}");
5859 let (e2e, _) = with_cache_lease(
5860 &mut self.state,
5861 cache_dir.as_deref(),
5862 "e2e",
5863 "e2e",
5864 &winner.worktree,
5865 &after,
5866 timeout,
5867 &context,
5868 |state, budget| {
5869 let shell = shell.clone();
5870 let commands = commands.clone();
5871 let context = context.clone();
5872 let worktree = winner.worktree.clone();
5873 async move {
5874 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5875 .await
5876 }
5877 },
5878 )
5879 .await;
5880 if verify_inconclusive(&e2e) {
5881 return Ok(GateFix::Defer);
5882 }
5883 if e2e.iter().any(|o| !o.ok()) {
5884 self.state.event(
5885 "gate",
5886 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5887 );
5888 return Ok(GateFix::Stop);
5889 }
5890 }
5891 Ok(GateFix::Retry)
5892 }
5893
5894 async fn merge(&mut self) -> Result<()> {
5897 if self
5912 .state
5913 .base_sync
5914 .as_ref()
5915 .is_some_and(|s| s.conflict.is_some())
5916 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5917 != Some(RunStatus::Gating)
5918 || !self.state.gate_status().ok()
5927 {
5928 return Ok(());
5929 }
5930 if self.state.merge.is_some() {
5939 return Ok(());
5940 }
5941 let Some(winner) = self.state.winner().cloned() else {
5942 return Ok(());
5943 };
5944 let repo = self.state.repo.clone();
5945 let base = self.state.base_branch.clone();
5946 let mode = self.state.config.merge.mode;
5947 let style = self.state.config.merge.style;
5948 let facts = if is_review_run(&self.state) {
5949 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5950 let start = review_base(
5951 &repo,
5952 &self.state.config.merge.remote,
5953 &base,
5954 &self.state.base_commit,
5955 &winner.branch,
5956 )
5957 .await;
5958 branch_facts(&repo, &start, &winner.branch).await
5959 } else {
5960 None
5961 };
5962 let leaked = if is_review_run(&self.state) {
5965 leaked_subjects(&self.state, &winner.branch).await
5966 } else {
5967 Some(Vec::new())
5968 };
5969 let pr = pr_message_with(&self.state, winner.label, facts.as_ref());
5970 let message = pr.commit_message();
5971
5972 let outcome = match mode {
5973 MergeMode::None => MergeOutcome {
5974 mode,
5975 ok: true,
5976 detail: manual_merge_command(style, &repo, &winner.branch, &message),
5977 empty: false,
5978 },
5979 MergeMode::Pr | MergeMode::Local
5980 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
5981 {
5982 MergeOutcome {
5983 mode,
5984 ok: false,
5985 detail: empty_candidate_detail(&self.state, &base),
5986 empty: true,
5987 }
5988 }
5989 MergeMode::Local => {
5990 let on = git::current_branch(&repo).await?;
5991 if on.as_deref() != Some(base.as_str()) {
5992 MergeOutcome {
5993 mode,
5994 ok: false,
5995 detail: format!(
5996 "{} has {} checked out, not the base branch {base}",
5997 repo.display(),
5998 on.unwrap_or_else(|| "a detached HEAD".to_owned())
5999 ),
6000 empty: false,
6001 }
6002 } else if !git::is_clean(&repo).await? {
6003 MergeOutcome {
6004 mode,
6005 ok: false,
6006 detail: format!("{} is dirty; refusing to merge", repo.display()),
6007 empty: false,
6008 }
6009 } else {
6010 let out = match style {
6011 MergeStyle::Merge => {
6012 git::merge_no_ff(&repo, &winner.branch, &message).await?
6013 }
6014 MergeStyle::Squash => {
6015 git::merge_squash(&repo, &winner.branch, &message).await?
6016 }
6017 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
6018 };
6019 MergeOutcome {
6020 mode,
6021 ok: out.ok(),
6022 detail: if out.ok() { out.stdout } else { out.stderr },
6023 empty: false,
6024 }
6025 }
6026 }
6027 MergeMode::Pr => {
6028 let remote = self.state.config.merge.remote.clone();
6029 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
6030 if !pushed.ok() {
6031 MergeOutcome {
6032 mode,
6033 ok: false,
6034 detail: pushed.stderr,
6035 empty: false,
6036 }
6037 } else {
6038 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
6044 let out = match pr_merge_plan(found) {
6045 PrPlan::Create => {
6046 gh_pr_create(
6047 &winner.worktree,
6048 &base,
6049 &winner.branch,
6050 &pr.title,
6051 &pr.body,
6052 )
6053 .await
6054 }
6055 PrPlan::Adopt { url, title } => {
6056 self.state
6057 .event("merge", format!("Pr: adopted open pull request {url}"));
6058 if title != pr.title
6059 && (!is_review_run(&self.state)
6060 || leaked
6061 .as_deref()
6062 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
6063 && let Err(e) =
6064 land::set_pr_title(&winner.worktree, &url, &pr.title).await
6065 {
6066 tracing::warn!("could not refresh title of {url}: {e:#}");
6067 self.state
6068 .event("merge", format!("Pr: title refresh failed: {e:#}"));
6069 }
6070 Ok(url)
6071 }
6072 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
6073 };
6074 match out {
6075 Ok(url) => MergeOutcome {
6076 mode,
6077 ok: true,
6078 detail: url,
6079 empty: false,
6080 },
6081 Err(e) => MergeOutcome {
6082 mode,
6083 ok: false,
6084 detail: e.to_string(),
6085 empty: false,
6086 },
6087 }
6088 }
6089 }
6090 };
6091
6092 self.state.status = match (mode, outcome.ok) {
6093 (MergeMode::None, _) => RunStatus::Ready,
6094 (_, true) => RunStatus::Merged,
6095 (_, false) => RunStatus::Blocked,
6096 };
6097 self.state.event(
6098 "merge",
6099 format!(
6100 "{:?}: {}",
6101 mode,
6102 outcome.detail.lines().next().unwrap_or("")
6103 ),
6104 );
6105 self.state.merge = Some(outcome);
6106 self.state.save()?;
6107
6108 if self.state.config.graph.land
6114 && mode == MergeMode::Pr
6115 && self.state.status == RunStatus::Merged
6116 {
6117 self.run_land().await?;
6118 }
6119 self.settle_questions();
6124 Ok(())
6125 }
6126
6127 async fn run_land(&mut self) -> Result<()> {
6138 let url = self
6139 .state
6140 .merge
6141 .as_ref()
6142 .map(|m| m.detail.clone())
6143 .unwrap_or_default();
6144 let url = url.lines().next().unwrap_or("").trim().to_owned();
6145 if !url.starts_with("http") {
6146 return Ok(());
6147 }
6148 match land::land(&mut self.state, &url).await {
6151 Ok(pr) if self.state.parked => {
6152 let _ = pr;
6156 }
6157 Ok(pr) => {
6158 self.state.status = match pr.state {
6159 land::PrLifecycle::Merged => RunStatus::Merged,
6160 _ => RunStatus::Blocked,
6161 };
6162 if bump::should_release_bump(self.state.status)
6169 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
6170 {
6171 self.state
6177 .event("bump", format!("release bump skipped: {e:#}"));
6178 }
6179 if self.state.status == RunStatus::Merged {
6182 crate::followup::after_merge(&mut self.state, &pr.url).await;
6183 }
6184 self.state.save()?;
6185 }
6186 Err(e) => {
6187 self.state.status = RunStatus::Blocked;
6188 self.state.event("land", format!("gave up: {e}"));
6189 self.state.save()?;
6190 }
6191 }
6192 Ok(())
6193 }
6194
6195 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
6199 if let Some(existing) = self.state.seats.get(key)
6200 && existing.agent == agent
6201 {
6202 return existing.clone();
6203 }
6204 let fresh = if self.state.seats.contains_key(key) {
6207 handover_seat(key, agent, self.state.next_seat_seed())
6208 } else {
6209 SeatState::new(key, agent, self.state.seed)
6210 };
6211 self.state.seats.insert(key.to_owned(), fresh.clone());
6212 fresh
6213 }
6214
6215 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
6221 match self.state.seats.get(key) {
6222 Some(s) if s.agent != spec.id => {
6223 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
6224 }
6225 _ => spec,
6226 }
6227 }
6228
6229 fn view(&self, c: &Candidate) -> CandidateView {
6231 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
6232 .unwrap_or_default();
6233 let (patch, _) = blind::sanitize_patch(
6234 &format!("candidate {} patch", c.label),
6235 &raw,
6236 &self.state.config.blind,
6237 );
6238 CandidateView {
6239 label: c.label,
6240 branch: c.branch.clone(),
6241 summary: c.summary.clone(),
6242 stat: c.stat.clone(),
6243 patch,
6244 }
6245 }
6246
6247 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
6249 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
6250 prompt::judge(
6251 "(see above)",
6252 &views,
6253 self.roles.judges.len(),
6254 base_short,
6255 "en",
6256 )
6257 }
6258
6259 fn vote_prompt_full(
6265 &self,
6266 j: usize,
6267 labels: &[char],
6268 language: &str,
6269 candidates: &[Candidate],
6270 base_short: &str,
6271 ) -> String {
6272 let mut text = format!(
6273 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
6274 prompt::final_vote(labels, language),
6275 self.state.instruction,
6276 self.candidate_block(candidates, base_short)
6277 );
6278 if let Some(own) = self
6279 .state
6280 .judgements
6281 .get(j)
6282 .filter(|r| !r.ranking.is_empty())
6283 {
6284 let reasons = own
6285 .reasons
6286 .iter()
6287 .map(|(k, v)| format!("- {k}: {v}"))
6288 .collect::<Vec<_>>()
6289 .join("\n");
6290 text.push_str(&format!(
6291 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
6292 own.ranking.iter().collect::<String>(),
6293 if reasons.is_empty() {
6294 ""
6295 } else {
6296 ", because:\n"
6297 }
6298 ));
6299 }
6300 if !self.state.deliberation.is_empty() {
6301 text.push_str("\n# What was argued before this vote\n");
6302 for t in self.transcript(&[], j) {
6303 text.push_str(&format!(
6304 "\n## {}{}\n\n{}\n",
6305 t.who,
6306 if t.is_self { " (you)" } else { "" },
6307 t.body.trim()
6308 ));
6309 }
6310 }
6311 text
6312 }
6313
6314 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6321 let mut turns = Vec::new();
6322 for j in &self.state.judgements {
6323 if j.ranking.is_empty() {
6324 continue;
6325 }
6326 let reasons = j
6327 .reasons
6328 .iter()
6329 .map(|(k, v)| format!("- {k}: {v}"))
6330 .collect::<Vec<_>>()
6331 .join("\n");
6332 turns.push(Turn {
6333 who: format!("Judge {} (opening ranking)", j.judge),
6334 is_self: j.judge == self_idx + 1,
6335 body: format!(
6336 "Ranked {}{}{reasons}",
6337 j.ranking.iter().collect::<String>(),
6338 if reasons.is_empty() {
6339 ""
6340 } else {
6341 ", because:\n"
6342 }
6343 ),
6344 });
6345 }
6346 for t in self
6347 .state
6348 .deliberation
6349 .iter()
6350 .flat_map(|r| r.turns.iter())
6351 .chain(current)
6352 {
6353 turns.push(Turn {
6354 who: format!("Judge {}", t.judge),
6355 is_self: t.judge == self_idx + 1,
6356 body: t.body.clone(),
6357 });
6358 }
6359 turns
6360 }
6361}
6362
6363fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6365 agent::has_session(spec.kind, seat, sessions)
6366}
6367
6368fn next_untried_in_roster<'a>(
6389 roster: &'a [AgentSpec],
6390 start: usize,
6391 tried: &BTreeSet<String>,
6392) -> Option<&'a AgentSpec> {
6393 roster
6394 .get(start + 1..)?
6395 .iter()
6396 .find(|s| !tried.contains(&s.id))
6397}
6398
6399fn next_for_seat<'a>(
6409 roster: &'a [AgentSpec],
6410 start: usize,
6411 round_tried: &BTreeSet<String>,
6412 carried_failed: &BTreeSet<String>,
6413) -> Option<&'a AgentSpec> {
6414 roster
6415 .get(start + 1..)?
6416 .iter()
6417 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
6418 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
6419}
6420
6421fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
6427 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
6428 return spec;
6429 };
6430 let ok = |id: &str| !h.failed.contains(id);
6431 if let Some(last) = h.last_ok.as_deref()
6432 && ok(last)
6433 && let Some(s) = roster.iter().find(|s| s.id == last)
6434 {
6435 return s.clone();
6436 }
6437 if ok(&spec.id) {
6438 return spec;
6439 }
6440 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
6441 roster
6442 .get(from + 1..)
6443 .into_iter()
6444 .flatten()
6445 .chain(roster.iter())
6446 .find(|s| ok(&s.id))
6447 .cloned()
6448 .unwrap_or(spec)
6449}
6450
6451fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6455 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6456}
6457
6458const TIMED_OUT: &str = "timed out";
6461
6462impl FailClass {
6463 fn of(out: &AgentOutcome) -> Option<Self> {
6465 match out {
6466 AgentOutcome::Ok(_) => None,
6467 AgentOutcome::Quota(_) => Some(Self::Quota),
6468 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6469 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6470 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6471 }
6472 }
6473
6474 fn stem_word(&self) -> &'static str {
6476 match self {
6477 Self::Quota => "quota",
6478 _ => "handover",
6479 }
6480 }
6481}
6482
6483fn failure_signature(msg: &str) -> String {
6487 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6488 let mut out = Vec::new();
6489 for word in line.split_whitespace() {
6490 if word.contains('/') || word.contains('\\') {
6491 out.push("<path>".to_owned());
6492 continue;
6493 }
6494 let mut w = String::new();
6495 let mut in_digits = false;
6496 for c in word.chars() {
6497 if c.is_ascii_digit() {
6498 if !in_digits {
6499 w.push('#');
6500 }
6501 in_digits = true;
6502 } else {
6503 in_digits = false;
6504 w.push(c);
6505 }
6506 }
6507 out.push(w);
6508 }
6509 out.join(" ").chars().take(120).collect()
6510}
6511
6512fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6519 match cur {
6520 FailClass::Quota | FailClass::Timeout => true,
6521 FailClass::Other(_) => prev != Some(cur),
6522 }
6523}
6524
6525fn fail_reason(out: &AgentOutcome) -> String {
6527 match out {
6528 AgentOutcome::Ok(_) => String::new(),
6529 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6530 AgentOutcome::Dropped(o) => format!(
6531 "the CLI dropped the stream ({})",
6532 o.dropped
6533 .as_ref()
6534 .map(|d| d.why.as_str())
6535 .unwrap_or("it ended without delivering its answer")
6536 ),
6537 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6538 }
6539}
6540
6541fn record_handover(
6544 state: &mut RunState,
6545 node: &str,
6546 seat: &str,
6547 from: &str,
6548 to: &str,
6549 class: &FailClass,
6550 reason: &str,
6551) {
6552 let message = if *class == FailClass::Quota {
6553 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6554 } else {
6555 format!("{seat}: handed over {from} -> {to} ({reason})")
6556 };
6557 state.event(node, message);
6558 state.handovers.push(Handover {
6559 at: Timestamp::now(),
6560 node: node.to_owned(),
6561 seat: seat.to_owned(),
6562 from: from.to_owned(),
6563 to: to.to_owned(),
6564 reason: reason.to_owned(),
6565 });
6566}
6567
6568fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6582 commands.iter().any(|c| c.exit_code.is_none())
6583}
6584
6585fn verified_noop_claim(
6598 usable: bool,
6599 commands: &[agent::CommandEvidence],
6600 text: &str,
6601) -> Option<String> {
6602 (usable && !has_unconfirmed_command(commands))
6603 .then(|| verdict::verified_noop(text))
6604 .flatten()
6605}
6606
6607fn short(commit: &str) -> String {
6608 commit.chars().take(7).collect()
6609}
6610
6611fn make_executable(path: &Path) -> Result<()> {
6612 #[cfg(unix)]
6613 {
6614 use std::os::unix::fs::PermissionsExt as _;
6615 let mut perms = std::fs::metadata(path)?.permissions();
6616 perms.set_mode(0o755);
6617 std::fs::set_permissions(path, perms)?;
6618 }
6619 #[cfg(not(unix))]
6620 {
6621 let _ = path;
6622 }
6623 Ok(())
6624}
6625
6626struct WaveCtx<'a> {
6633 run: &'a str,
6636 node: &'a str,
6638 prompts: &'a Prompts,
6639 cache: Option<&'a Path>,
6641 round: Option<usize>,
6644 carry_seats: bool,
6648}
6649
6650async fn run_one(
6652 job: SeatJob,
6653 sem: Arc<Semaphore>,
6654 ctx: &WaveCtx<'_>,
6655 state: &mut RunState,
6656 attempt: usize,
6657) -> (SeatState, AgentOutcome) {
6658 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6659 .await
6660 .pop()
6661 .expect("one job in, one result out");
6662 (seat, out)
6663}
6664
6665async fn wave(
6671 jobs: Vec<SeatJob>,
6672 sem: Arc<Semaphore>,
6673 ctx: &WaveCtx<'_>,
6674 state: &mut RunState,
6675 attempt: usize,
6676) -> Vec<(usize, SeatState, AgentOutcome)> {
6677 let WaveCtx {
6678 run,
6679 node,
6680 prompts,
6681 cache,
6682 round,
6683 carry_seats: _,
6684 } = *ctx;
6685 for job in &jobs {
6686 state.seat_started(node, &job.seat.key, job.timeout, attempt);
6687 }
6688 if let Err(e) = state.save() {
6689 tracing::warn!("could not persist in-progress seats: {e:#}");
6694 }
6695 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6711 let wait_started = Instant::now();
6712 let cache_guard = if let Some(cache_dir) = cache {
6713 if jobs_had_a_writer {
6714 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6715 let budget = jobs
6716 .iter()
6717 .map(|j| j.timeout)
6718 .max()
6719 .unwrap_or(Duration::from_secs(60));
6720 acquire_cache_lease(state, cache_dir, &owner, budget, node)
6721 .await
6722 .ok()
6723 } else {
6724 None
6725 }
6726 } else {
6727 None
6728 };
6729 let waited_for_lease = wait_started.elapsed();
6736 let mut set = tokio::task::JoinSet::new();
6737 let overlay = prompts.overlay(node);
6738 for (i, mut job) in jobs.into_iter().enumerate() {
6739 job.timeout = job.timeout.saturating_sub(waited_for_lease);
6740 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6741 if cache.is_some() {
6742 job.prompt.push('\n');
6743 job.prompt
6744 .push_str(&prompt::build_cache_note(node, job.allow_write));
6745 }
6746 let sem = Arc::clone(&sem);
6747 let run = run.to_owned();
6748 let node = node.to_owned();
6749 let attachments = if node == "implement" {
6752 state.attachments.clone()
6753 } else {
6754 Vec::new()
6755 };
6756 let cache = cache
6767 .filter(|_| job.allow_write && cache_guard.is_some())
6768 .map(Path::to_path_buf);
6769 set.spawn(async move {
6770 let _permit = sem.acquire().await;
6771 let mut seat = job.seat;
6772 let out = agent::invoke(
6773 &job.spec,
6774 &mut seat,
6775 &Invocation {
6776 cwd: &job.cwd,
6777 prompt: &job.prompt,
6778 timeout: job.timeout,
6779 allow_write: job.allow_write,
6780 sessions: job.sessions,
6781 artifacts: &job.artifacts,
6782 stem: &job.stem,
6783 run: &run,
6784 node: &node,
6785 cache_dir: cache.as_deref(),
6786 attachments: &attachments,
6787 writable: &[],
6788 },
6789 )
6790 .await;
6791 let out = match out {
6792 Ok(o) if o.usable() => AgentOutcome::Ok(o),
6793 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
6794 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
6802 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
6803 Ok(o) => AgentOutcome::Failed(format!(
6804 "exited with {:?} and no usable output",
6805 o.exit_code
6806 )),
6807 Err(e) => AgentOutcome::Failed(e.to_string()),
6808 };
6809 (i, seat, out)
6810 });
6811 }
6812 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
6813 while let Some(joined) = set.join_next().await {
6814 let (i, seat, out) = match joined {
6815 Ok(v) => v,
6816 Err(e) => {
6820 tracing::error!("agent task panicked: {e}");
6821 continue;
6822 }
6823 };
6824 state.seat_finished(&seat.key);
6825 record_jobs(state, node, round, &seat.key, &out);
6826 if let Err(e) = state.save() {
6827 tracing::warn!("could not persist a seat's completion: {e:#}");
6828 }
6829 if collected.len() <= i {
6830 collected.resize_with(i + 1, || None);
6831 }
6832 collected[i] = Some((i, seat, out));
6833 }
6834 if state
6840 .active
6841 .values()
6842 .any(|a| a.node == node && a.attempt == attempt)
6843 {
6844 state
6845 .active
6846 .retain(|_, a| !(a.node == node && a.attempt == attempt));
6847 if let Err(e) = state.save() {
6848 tracing::warn!("could not persist the end of a wave: {e:#}");
6849 }
6850 }
6851 if let Some(cache_dir) = cache
6858 && jobs_had_a_writer
6859 {
6860 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
6861 }
6862 if let Some(guard) = cache_guard {
6863 guard.release();
6864 }
6865 collected.into_iter().flatten().collect()
6866}
6867
6868fn record_jobs(
6879 state: &mut RunState,
6880 node: &str,
6881 round: Option<usize>,
6882 seat: &str,
6883 out: &AgentOutcome,
6884) {
6885 let commands: &[agent::CommandEvidence] = match out {
6886 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
6887 AgentOutcome::Failed(_) => &[],
6888 };
6889 let checked_at = Timestamp::now();
6890 for c in commands {
6891 state.jobs.push(JobRecord {
6892 node: node.to_owned(),
6893 round,
6894 seat: seat.to_owned(),
6895 id: c.id.clone(),
6896 description: c.description.clone(),
6897 checked_at,
6898 status: match c.exit_code {
6899 Some(0) => JobStatus::Completed,
6900 Some(_) => JobStatus::Failed,
6901 None => JobStatus::Unknown,
6902 },
6903 exit_code: c.exit_code,
6904 result_summary: c.result_summary.clone(),
6905 source: c.source.clone(),
6906 });
6907 }
6908}
6909
6910fn round_is_clean(
6931 blocking: usize,
6932 e2e_ok: bool,
6933 answered: usize,
6934 expected: usize,
6935 quota_missing: usize,
6936 policy: IncompleteReviewPolicy,
6937) -> bool {
6938 if blocking != 0 || !e2e_ok {
6939 return false;
6940 }
6941 if answered == expected || policy == IncompleteReviewPolicy::Warn {
6942 return true;
6943 }
6944 answered > 0 && expected - answered <= quota_missing
6945}
6946
6947fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6971 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6972 return Some(RunStatus::Gating);
6973 }
6974 let last = reviews.last()?;
6975 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6976 if reviews.len() < max_rounds && !stagnant {
6977 return None;
6978 }
6979 if last.incomplete() && last.blocking == 0 {
6980 return Some(RunStatus::Blocked);
6981 }
6982 if last.e2e_status() == E2eStatus::ResourceBlocked {
6983 return None;
6984 }
6985 Some(if last.e2e.iter().all(CommandOutcome::ok) {
6986 RunStatus::Gating
6987 } else {
6988 RunStatus::Blocked
6989 })
6990}
6991
6992fn retry_budget(full: Duration, nudged: bool) -> Duration {
7007 if nudged {
7008 (full / 4).max(Duration::from_secs(120)).min(full)
7009 } else {
7010 full
7011 }
7012}
7013
7014#[allow(clippy::too_many_arguments)]
7043async fn ask_json_wave<T>(
7044 jobs: Vec<SeatJob>,
7045 sem: Arc<Semaphore>,
7046 retries: usize,
7047 roster: &[AgentSpec],
7048 ctx: &WaveCtx<'_>,
7049 losses: &mut Vec<QuotaLoss>,
7050 state: &mut RunState,
7051 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
7052) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7053where
7054 T: serde::de::DeserializeOwned + Send + 'static,
7055{
7056 ask_wave_with(
7057 jobs,
7058 sem,
7059 retries,
7060 roster,
7061 ctx,
7062 losses,
7063 state,
7064 &|text: &str| {
7065 let v = verdict::extract_json::<T>(text)?;
7066 validate(&v)?;
7067 Ok(v)
7068 },
7069 )
7070 .await
7071}
7072
7073#[allow(clippy::too_many_arguments)]
7082async fn ask_wave_with<T>(
7083 jobs: Vec<SeatJob>,
7084 sem: Arc<Semaphore>,
7085 retries: usize,
7086 roster: &[AgentSpec],
7087 ctx: &WaveCtx<'_>,
7088 losses: &mut Vec<QuotaLoss>,
7089 state: &mut RunState,
7090 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
7091) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7092where
7093 T: Send + 'static,
7094{
7095 let n = jobs.len();
7096 let originals: Vec<SeatJob> = jobs;
7097 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
7098 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
7099 let mut nudges: Vec<usize> = vec![0; n];
7106 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
7110 let mut tried: Vec<BTreeSet<String>> = specs
7111 .iter()
7112 .map(|s| BTreeSet::from([s.id.clone()]))
7113 .collect();
7114 let starts: Vec<usize> = specs
7115 .iter()
7116 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
7117 .collect();
7118 let carry = ctx.carry_seats && !roster.is_empty();
7119 let carried: Vec<BTreeSet<String>> = originals
7122 .iter()
7123 .map(|j| {
7124 state
7125 .seat_history
7126 .get(&j.seat.key)
7127 .filter(|_| carry)
7128 .map(|h| h.failed.clone())
7129 .unwrap_or_default()
7130 })
7131 .collect();
7132 let mut prev: Vec<Option<FailClass>> = originals
7133 .iter()
7134 .map(|j| {
7135 state
7136 .seat_history
7137 .get(&j.seat.key)
7138 .filter(|_| carry)
7139 .and_then(|h| h.last_fail.clone())
7140 })
7141 .collect();
7142 let next_agent = |i: usize, tried: &BTreeSet<String>| -> Option<AgentSpec> {
7143 if carry {
7144 next_for_seat(roster, starts[i], tried, &carried[i]).cloned()
7145 } else {
7146 next_untried_in_roster(roster, starts[i], tried).cloned()
7147 }
7148 };
7149 let mut fresh: Vec<Option<String>> = vec![None; n];
7150 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
7151 let mut pending: Vec<usize> = (0..n).collect();
7152
7153 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
7158 for round in 0..max_rounds {
7159 if pending.is_empty() {
7160 break;
7161 }
7162 let mut batch = Vec::with_capacity(pending.len());
7163 let mut renudged: Vec<&str> = Vec::new();
7164 for &i in &pending {
7165 let src = &originals[i];
7166 let full: &str = match &src.handover {
7171 Some(h) if specs[i].id != src.spec.id => h,
7172 _ => &src.prompt,
7173 };
7174 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
7177 (
7178 full.to_owned(),
7179 src.timeout,
7180 format!("{}-{word}-{}", src.stem, specs[i].id),
7181 )
7182 } else if nudges[i] == 0 {
7183 (full.to_owned(), src.timeout, src.stem.clone())
7184 } else {
7185 renudged.push(src.seat.key.as_str());
7186 let why = done[i]
7187 .as_ref()
7188 .and_then(|r| r.as_ref().err().map(ToString::to_string))
7189 .unwrap_or_else(|| "no parsable answer".to_owned());
7190 let nudge = prompt::nudge(&why);
7191 let nudged = has_context(&specs[i], &seats[i], src.sessions);
7192 let prompt = if nudged {
7193 nudge
7194 } else {
7195 format!("{full}\n\n---\n\n{nudge}")
7196 };
7197 (
7198 prompt,
7199 retry_budget(src.timeout, nudged),
7200 format!("{}-retry{}", src.stem, nudges[i]),
7201 )
7202 };
7203 batch.push(SeatJob {
7204 spec: specs[i].clone(),
7205 seat: seats[i].clone(),
7206 cwd: src.cwd.clone(),
7207 prompt,
7208 timeout,
7209 allow_write: src.allow_write,
7210 sessions: src.sessions,
7211 artifacts: src.artifacts.clone(),
7212 stem,
7213 handover: None,
7214 });
7215 }
7216
7217 if !renudged.is_empty() {
7218 state.event(
7219 ctx.node,
7220 format!("retry {round}: re-asking {}", renudged.join(", ")),
7221 );
7222 }
7223 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
7224 let mut still = Vec::new();
7225 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
7226 seats[i] = seat;
7227 let class = FailClass::of(&out);
7228 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
7231 && nudges[i] < retries
7232 && !roster.is_empty();
7233 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
7234 let next = should_hand_over(prev[i].as_ref(), &cur)
7235 .then(|| next_agent(i, &tried[i]))
7236 .flatten();
7237 if carry {
7238 let h = state
7239 .seat_history
7240 .entry(originals[i].seat.key.clone())
7241 .or_default();
7242 h.failed.insert(specs[i].id.clone());
7243 h.last_fail = Some(cur.clone());
7244 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
7245 h.last_ok = None;
7246 }
7247 if let Err(e) = state.save() {
7250 tracing::warn!("could not persist a seat's failure history: {e:#}");
7251 }
7252 }
7253 if let Some(next) = next {
7254 record_handover(
7255 state,
7256 ctx.node,
7257 &originals[i].seat.key,
7258 &specs[i].id,
7259 &next.id,
7260 &cur,
7261 &fail_reason(&out),
7262 );
7263 tried[i].insert(next.id.clone());
7264 prev[i] = Some(cur.clone());
7265 seats[i] =
7266 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
7267 specs[i] = next;
7268 fresh[i] = Some(cur.stem_word().to_owned());
7269 nudges[i] = 0;
7270 done[i] = Some(Err(anyhow::anyhow!(
7271 "handed over after: {}",
7272 fail_reason(&out)
7273 )));
7274 still.push(i);
7275 continue;
7276 }
7277 }
7278 if carry
7279 && !nudge_first
7280 && let Some(cur) = class.clone()
7281 {
7282 let h = state
7284 .seat_history
7285 .entry(originals[i].seat.key.clone())
7286 .or_default();
7287 h.failed.insert(specs[i].id.clone());
7288 h.last_fail = Some(cur);
7289 }
7290 let parsed = match out {
7291 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
7292 AgentOutcome::Quota(o) => {
7293 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
7294 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
7295 }
7296 AgentOutcome::Dropped(o) => {
7300 let why = o
7301 .dropped
7302 .as_ref()
7303 .map(|d| d.why.as_str())
7304 .unwrap_or("the CLI ended the stream without delivering its answer");
7305 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
7306 }
7307 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
7308 };
7309 let quota = class == Some(FailClass::Quota);
7310 let failed = parsed.is_err();
7311 done[i] = Some(parsed);
7312 if carry && !failed {
7313 let h = state
7314 .seat_history
7315 .entry(originals[i].seat.key.clone())
7316 .or_default();
7317 h.failed.remove(&specs[i].id);
7318 h.last_ok = Some(specs[i].id.clone());
7319 h.last_fail = None;
7320 }
7321 let agent_failure = class.is_some()
7329 && !nudge_first
7330 && !roster.is_empty()
7331 && next_agent(i, &tried[i]).is_some();
7332 if failed && !quota && !agent_failure && nudges[i] < retries {
7333 nudges[i] += 1;
7334 still.push(i);
7335 }
7336 }
7337 pending = still;
7338 }
7339
7340 for (i, q) in last_quota.into_iter().enumerate() {
7343 if let Some(reset) = q {
7344 losses.push(QuotaLoss {
7345 seat: originals[i].seat.key.clone(),
7346 node: ctx.node.to_owned(),
7347 at: Timestamp::now(),
7348 reset,
7349 });
7350 }
7351 }
7352
7353 seats
7354 .into_iter()
7355 .zip(done)
7356 .zip(nudges)
7357 .map(|((seat, res), attempts)| {
7358 (
7359 seat,
7360 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
7361 attempts,
7362 )
7363 })
7364 .collect()
7365}
7366
7367async fn acquire_cache_lease(
7380 state: &mut RunState,
7381 cache_dir: &Path,
7382 owner: &crate::cache::Owner,
7383 budget: Duration,
7384 context: &str,
7385) -> Result<crate::cache::Guard> {
7386 let home = crate::run::home();
7387 let started = Instant::now();
7388 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
7389 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
7390 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
7391 Err(e) => {
7392 state.event(
7393 "verify",
7394 format!("{context}: could not check the shared build cache: {e:#}"),
7395 );
7396 if let Err(e2) = state.save() {
7397 tracing::warn!("could not persist a cache-check failure: {e2:#}");
7398 }
7399 return Err(e);
7400 }
7401 };
7402 state.event(
7403 "verify",
7404 format!(
7405 "{context}: waiting for the shared build cache at {} ({})",
7406 cache_dir.display(),
7407 busy.describe()
7408 ),
7409 );
7410 if let Err(e) = state.save() {
7411 tracing::warn!("could not persist a cache wait: {e:#}");
7412 }
7413 let remaining = budget.saturating_sub(started.elapsed());
7414 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
7415 Ok(g) => Ok(g),
7416 Err(e) => {
7417 state.event("verify", format!("{context}: {e:#}"));
7418 if let Err(e2) = state.save() {
7419 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
7420 }
7421 Err(e)
7422 }
7423 }
7424}
7425
7426#[allow(clippy::too_many_arguments)]
7447async fn with_cache_lease<'s, F, Fut>(
7448 state: &'s mut RunState,
7449 cache_dir: Option<&Path>,
7450 node: &str,
7451 seat: &str,
7452 worktree: &Path,
7453 head: &str,
7454 budget: Duration,
7455 context: &str,
7456 body: F,
7457) -> (Vec<CommandOutcome>, bool)
7458where
7459 F: FnOnce(&'s mut RunState, Duration) -> Fut,
7460 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
7461{
7462 let Some(cache_dir) = cache_dir else {
7463 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
7464 return (outcomes, retried);
7465 };
7466 let home = crate::run::home();
7467 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
7468 let started = Instant::now();
7469 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
7470 Ok(g) => g,
7471 Err(e) => {
7472 return (
7473 vec![CommandOutcome {
7474 command: "(waiting for the shared build cache)".to_owned(),
7475 code: None,
7476 output_tail: e.to_string(),
7477 duration_ms: started.elapsed().as_millis() as u64,
7478 resource_blocked: true,
7479 }],
7480 false,
7481 );
7482 }
7483 };
7484 let identity = crate::cache::Identity::new(worktree, head);
7485 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7486 state.event(
7494 "verify",
7495 format!(
7496 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7497 worktree.display(),
7498 short(head)
7499 ),
7500 );
7501 guard.release();
7502 return (
7503 vec![CommandOutcome {
7504 command: "(confirming the shared build cache is fresh)".to_owned(),
7505 code: None,
7506 output_tail: e.to_string(),
7507 duration_ms: started.elapsed().as_millis() as u64,
7508 resource_blocked: true,
7509 }],
7510 false,
7511 );
7512 }
7513 let remaining = budget.saturating_sub(started.elapsed());
7514 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7515 if !timed_out_pids.is_empty() {
7520 wait_for_timed_out_children_to_die(&timed_out_pids).await;
7521 }
7522 guard.release();
7523 (outcomes, retried)
7524}
7525
7526async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7538 wait_for_pids_with(
7539 pids,
7540 crate::proc::pid_alive,
7541 LEASE_RELEASE_POLL,
7542 LEASE_RELEASE_MAX_WAIT,
7543 )
7544 .await;
7545}
7546
7547async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7553 pids: &[u32],
7554 alive: F,
7555 poll: Duration,
7556 max_wait: Duration,
7557) {
7558 let deadline = Instant::now() + max_wait;
7559 loop {
7560 if pids.iter().all(|&pid| !alive(pid)) {
7561 return;
7562 }
7563 if Instant::now() >= deadline {
7564 return;
7565 }
7566 tokio::time::sleep(poll).await;
7567 }
7568}
7569
7570fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7577 outcomes.iter().any(|o| o.resource_blocked)
7578}
7579
7580enum GateFix {
7582 Retry,
7584 Stop,
7587 Defer,
7590}
7591
7592fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7600 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7601 red.peek().is_some()
7602 && red.all(|o| {
7603 !o.resource_blocked
7604 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7605 && !o.output_tail.trim().is_empty()
7606 })
7607}
7608
7609fn e2e_outcome_label(o: &CommandOutcome) -> String {
7613 if o.ok() {
7614 return "pass".to_owned();
7615 }
7616 let reason = if o.build_failed() {
7617 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7618 } else {
7619 format!("FAIL ({:?})", o.code)
7620 };
7621 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7622}
7623
7624async fn run_e2e_with_retry(
7632 state: &mut RunState,
7633 shell: &[String],
7634 commands: &[String],
7635 worktree: &Path,
7636 timeout: Duration,
7637 context: &str,
7638) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7639 let (mut e2e, mut timed_out_pids) = run_commands(
7640 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7641 )
7642 .await;
7643 for o in &e2e {
7644 state.event(
7645 "verify",
7646 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7647 );
7648 }
7649 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7653 if verify_retried {
7654 state.event(
7655 "verify",
7656 format!(
7657 "{context}: verify could not build/link, not a test result — retrying once \
7658 before concluding"
7659 ),
7660 );
7661 let retried = run_commands(
7662 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7663 )
7664 .await;
7665 e2e = retried.0;
7666 timed_out_pids.extend(retried.1);
7669 for o in &e2e {
7670 state.event(
7671 "verify",
7672 format!(
7673 "{context}: retry `{}` -> {}",
7674 o.command,
7675 e2e_outcome_label(o)
7676 ),
7677 );
7678 }
7679 }
7680 (e2e, verify_retried, timed_out_pids)
7681}
7682
7683#[allow(clippy::too_many_arguments)]
7699async fn run_commands(
7700 state: &mut RunState,
7701 node: &str,
7702 task: &str,
7703 attempt: usize,
7704 shell: &[String],
7705 commands: &[String],
7706 cwd: &Path,
7707 timeout: Duration,
7708) -> (Vec<CommandOutcome>, Vec<u32>) {
7709 if commands.is_empty() {
7710 return (Vec::new(), Vec::new());
7715 }
7716 let mut out = Vec::new();
7717 let mut timed_out_pids = Vec::new();
7718 let total = commands.len();
7719 for (idx, command) in commands.iter().enumerate() {
7720 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7721 if let Err(e) = state.save() {
7722 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7723 }
7724 let started = Instant::now();
7725 let mut cmd = tokio::process::Command::new(&shell[0]);
7726 cmd.quiet();
7727 cmd.args(&shell[1..])
7728 .arg(command)
7729 .current_dir(cwd)
7730 .stdin(std::process::Stdio::null())
7731 .stdout(std::process::Stdio::piped())
7732 .stderr(std::process::Stdio::piped())
7733 .kill_on_drop(true);
7734 let spawned = cmd.spawn();
7735 let (code, body) = match spawned {
7736 Ok(child) => {
7737 let pid = child.id();
7742 match tokio::time::timeout(timeout, child.wait_with_output()).await {
7743 Ok(Ok(o)) => {
7744 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7745 body.push_str(&String::from_utf8_lossy(&o.stderr));
7746 (o.status.code(), body)
7747 }
7748 Ok(Err(e)) => (None, format!("failed to run: {e}")),
7749 Err(_) => {
7750 if let Some(pid) = pid {
7751 timed_out_pids.push(pid);
7752 }
7753 (None, format!("timed out after {}s", timeout.as_secs()))
7754 }
7755 }
7756 }
7757 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7758 };
7759 out.push(CommandOutcome {
7760 command: command.clone(),
7761 code,
7762 output_tail: tail(&body, OUTPUT_TAIL),
7763 duration_ms: started.elapsed().as_millis() as u64,
7764 resource_blocked: false,
7765 });
7766 }
7767 state.task_finished(task);
7768 if let Err(e) = state.save() {
7769 tracing::warn!("could not persist the end of {task}: {e:#}");
7770 }
7771 (out, timed_out_pids)
7772}
7773
7774fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
7786 let repo = repo.display();
7787 match style {
7788 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
7789 MergeStyle::Squash => {
7790 let subject = message
7793 .lines()
7794 .next()
7795 .unwrap_or(branch)
7796 .replace(['\\', '"', '$', '`'], "");
7797 format!(
7798 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
7799 )
7800 }
7801 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
7802 }
7803}
7804
7805const PR_TITLE_MAX: usize = 240;
7816
7817fn english_title(text: &str, max: usize) -> Option<String> {
7828 let line = queue::first_line(text)?;
7829 let chars: Vec<char> = line.chars().collect();
7830 let (kept, mark) = if chars.len() <= max {
7831 (line.to_owned(), "")
7832 } else if let Some(end) = sentence_end(&chars, max) {
7833 (chars[..end].iter().collect::<String>(), "")
7834 } else {
7835 let room = max.saturating_sub(3);
7836 let cut = if chars[room].is_whitespace() {
7839 room
7840 } else {
7841 chars[..room]
7842 .iter()
7843 .rposition(|c| c.is_whitespace())
7844 .unwrap_or(room)
7845 };
7846 let head: String = chars[..cut].iter().collect();
7847 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
7848 (head.to_owned(), "...")
7849 };
7850 if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
7851 return None;
7852 }
7853 Some(format!("{kept}{mark}"))
7854}
7855
7856fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
7860 const MIN: usize = 20;
7861 for i in MIN..max.min(chars.len()) {
7862 if !matches!(chars[i], '.' | '!' | '?') {
7863 continue;
7864 }
7865 let Some(&next) = chars.get(i + 1) else {
7866 continue;
7867 };
7868 if !next.is_whitespace() {
7869 continue;
7870 }
7871 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
7872 if after.is_some_and(|c| c.is_ascii_lowercase()) {
7873 continue;
7874 }
7875 let word: String = chars[..i]
7876 .iter()
7877 .rev()
7878 .take_while(|c| !c.is_whitespace())
7879 .collect::<Vec<_>>()
7880 .into_iter()
7881 .rev()
7882 .collect();
7883 let word = word.to_ascii_lowercase();
7884 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
7885 continue;
7886 }
7887 let end = chars[..i]
7888 .iter()
7889 .rposition(|c| !c.is_whitespace())
7890 .map_or(i, |p| p + 1);
7891 return Some(end);
7892 }
7893 None
7894}
7895
7896struct PrMessage {
7901 title: String,
7902 body: String,
7903}
7904
7905impl PrMessage {
7906 fn commit_message(&self) -> String {
7910 format!("{}\n\n{}", self.title, self.body)
7911 }
7912}
7913
7914fn title_marker(line: &str) -> Option<&str> {
7916 let line = line.trim();
7917 let head = line.get(..6)?;
7918 head.eq_ignore_ascii_case("title:")
7919 .then(|| line[6..].trim())
7920}
7921
7922fn summary_title(summary: &str) -> Option<String> {
7927 let first = summary.lines().find(|l| !l.trim().is_empty())?;
7928 let raw = title_marker(first)?;
7929 if raw.is_empty() {
7930 return None;
7931 }
7932 let title = queue::title_from(raw, PR_TITLE_MAX);
7933 let lower = title.to_ascii_lowercase();
7934 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7935 return None;
7936 }
7937 Some(title)
7938}
7939
7940const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
7942
7943const EXISTING_BRANCH: &str = "(existing branch)";
7945
7946fn is_review_run(state: &RunState) -> bool {
7950 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
7951}
7952
7953fn review_title(state: &RunState) -> Option<String> {
7958 english_subject(state.reviewed_commits.as_ref()?.first()?)
7959}
7960
7961fn english_subject(raw: &str) -> Option<String> {
7964 let raw = raw.trim();
7965 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
7966 return None;
7967 }
7968 let title = queue::title_from(raw, PR_TITLE_MAX);
7969 let lower = title.to_ascii_lowercase();
7970 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7971 return None;
7972 }
7973 Some(title)
7974}
7975
7976#[derive(Debug, Clone, PartialEq, Eq)]
7979struct BranchFacts {
7980 commits: Vec<(String, String)>,
7982 stat: String,
7984}
7985
7986const STAT_FILE_LINES: usize = 25;
7988const COMMITS_MAX_CHARS: usize = 20_000;
7990
7991async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
7994 let commits = git::commit_log(repo, base, branch).await.ok()?;
7995 if commits.is_empty() {
7996 return None;
7997 }
7998 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
7999 let lines: Vec<&str> = stat.lines().collect();
8000 let stat = if lines.len() > STAT_FILE_LINES + 1 {
8001 let omitted = lines.len() - 1 - STAT_FILE_LINES;
8002 let more = format!(" ... {omitted} more file(s)");
8003 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
8004 kept.push(&more);
8005 kept.push(lines[lines.len() - 1]);
8006 kept.join("\n")
8007 } else {
8008 lines.join("\n")
8009 };
8010 Some(BranchFacts { commits, stat })
8011}
8012
8013fn markdown_safe(text: &str) -> String {
8016 text.replace("</details>", "</details>")
8017 .replace("\x60\x60\x60", "~~~")
8018}
8019
8020fn neutral_title(state: &RunState, winner: char) -> String {
8021 format!(
8022 "chore: land candidate {} of run {}",
8023 winner.to_ascii_uppercase(),
8024 state.id
8025 )
8026}
8027
8028pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
8034 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
8035 ""
8036 } else {
8037 pr_title
8038 }
8039}
8040
8041pub fn landing_subject_source(state: &RunState) -> String {
8045 if is_review_run(state) {
8046 let winner = state.candidates.first().map_or('A', |c| c.label);
8047 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
8048 }
8049 state.instruction.clone()
8050}
8051
8052fn summary_without_title(summary: &str) -> String {
8055 let mut lines = summary.trim().lines().peekable();
8056 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
8057 lines.next();
8058 }
8059 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
8060}
8061
8062#[cfg(test)]
8076fn pr_message(state: &RunState, winner: char) -> PrMessage {
8077 pr_message_with(state, winner, None)
8078}
8079
8080fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8083 let summary = state
8084 .candidates
8085 .iter()
8086 .find(|c| c.label == winner)
8087 .map(|c| c.summary.as_str())
8088 .unwrap_or_default();
8089 let review = is_review_run(state);
8092 let title = if review {
8093 facts
8094 .and_then(|f| english_subject(&f.commits.first()?.0))
8095 .or_else(|| review_title(state))
8096 .or_else(|| {
8097 state
8098 .candidates
8099 .iter()
8100 .find(|c| c.label == winner)
8101 .filter(|c| !c.branch.starts_with("magi/"))
8102 .and_then(|c| english_subject(&c.branch))
8103 })
8104 .unwrap_or_else(|| neutral_title(state, winner))
8105 } else {
8106 summary_title(summary).unwrap_or_else(|| {
8107 english_title(&state.instruction, PR_TITLE_MAX)
8108 .unwrap_or_else(|| neutral_title(state, winner))
8109 })
8110 };
8111
8112 let mut body = String::new();
8113 let what = summary_without_title(summary);
8114 if !what.is_empty() {
8115 body.push_str("## Summary\n\n");
8116 body.push_str(&what);
8117 body.push_str("\n\n");
8118 }
8119
8120 let notes: Vec<(usize, &str)> = state
8123 .reviews
8124 .iter()
8125 .filter_map(|r| {
8126 let n = r.fix.as_ref()?.notes.trim();
8127 (!n.is_empty()).then_some((r.round, n))
8128 })
8129 .collect();
8130 if !notes.is_empty() {
8131 body.push_str("## Review fixes\n\n");
8132 if let [(_, only)] = notes.as_slice() {
8133 body.push_str(only);
8134 body.push_str("\n\n");
8135 } else {
8136 for (round, n) in ¬es {
8137 body.push_str(&format!("### Round {round}\n\n{n}\n\n"));
8138 }
8139 }
8140 }
8141 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
8142
8143 let open = state.open_findings();
8144 if !open.is_empty() {
8145 body.push_str("## Open review findings\n\n");
8146 for f in &open {
8147 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
8148 }
8149 body.push('\n');
8150 }
8151
8152 if let Some(fix) = fix
8153 && !fix.rejected.is_empty()
8154 {
8155 body.push_str("## Declined by the fixer\n\n");
8156 for r in &fix.rejected {
8157 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
8158 }
8159 body.push('\n');
8160 }
8161
8162 if review {
8163 body.push_str("## Commits under review\n\n");
8165 if let Some(facts) = facts {
8166 let mut left = COMMITS_MAX_CHARS;
8167 for (i, (subject, text)) in facts.commits.iter().enumerate() {
8168 let mut entry = format!("- {}\n", markdown_safe(subject));
8169 for l in markdown_safe(text).lines() {
8170 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
8171 }
8172 if left == 0 {
8173 body.push_str(&format!(
8174 "- ... {} more commit(s)\n",
8175 facts.commits.len() - i
8176 ));
8177 break;
8178 }
8179 if entry.len() > left {
8180 let mut end = left;
8183 while !entry.is_char_boundary(end) {
8184 end -= 1;
8185 }
8186 entry.truncate(end);
8187 entry.push_str("\n ... (truncated)\n");
8188 left = 0;
8189 } else {
8190 left -= entry.len();
8191 }
8192 body.push_str(&entry);
8193 }
8194 if !facts.stat.trim().is_empty() {
8195 body.push_str(&format!(
8196 "\n## Diff stat\n\n```\n{}\n```\n",
8197 markdown_safe(facts.stat.trim())
8198 ));
8199 }
8200 } else {
8201 match &state.reviewed_commits {
8202 Some(subjects) => {
8203 for s in subjects {
8204 body.push_str(&format!("- {}\n", s.trim()));
8205 }
8206 }
8207 None => {
8208 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
8211 body.push_str(rest.trim());
8212 body.push('\n');
8213 }
8214 }
8215 }
8216 } else {
8217 let task = state.instruction.trim();
8218 let task = if task.is_empty() {
8219 "(empty task)"
8220 } else {
8221 task
8222 };
8223 body.push_str(&format!(
8224 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
8225 task.replace("</details>", "</details>")
8226 ));
8227 }
8228
8229 body.push_str(&format!(
8230 "\n---\nmagi:run/{} magi:candidate-{}\n",
8231 state.id,
8232 winner.to_ascii_lowercase()
8233 ));
8234
8235 let id = crate::scrub::Identity::current();
8238 PrMessage {
8239 title: crate::scrub::scrub(&title, &id),
8240 body: crate::scrub::scrub(&body, &id),
8241 }
8242}
8243
8244fn seeded_instruction(state: &RunState) -> String {
8248 match refs::describe(&state.seeds) {
8249 Some(facts) => format!(
8250 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
8251 Candidates start from the unmerged branch named above, when there \
8252 is one, and carry any unmerged commit named by sha as a \
8253 cherry-pick. Check that this is what the task meant before \
8254 building on it.",
8255 state.instruction
8256 ),
8257 None => state.instruction.clone(),
8258 }
8259}
8260
8261async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
8265 let base = &state.base_branch;
8266 let mut against = base.clone();
8267 if mode == MergeMode::Pr {
8268 let remote = &state.config.merge.remote;
8269 let tracking = format!("{remote}/{base}");
8270 let fetched = git::fetch(repo, remote, base).await;
8271 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
8272 against = tracking;
8273 }
8274 }
8275 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
8276}
8277
8278fn empty_candidate_detail(state: &RunState, base: &str) -> String {
8281 let mut detail = format!(
8282 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
8283 nothing to open a pull request for"
8284 );
8285 match refs::describe(&state.seeds) {
8286 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
8287 None => detail.push_str(
8288 "\nThe task names no existing branch or commit; if it means to land work \
8289 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
8290 ),
8291 }
8292 detail
8293}
8294
8295#[derive(Debug, PartialEq, Eq)]
8298enum PrPlan {
8299 Create,
8300 Adopt { url: String, title: String },
8301 Stop(String),
8302}
8303
8304fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
8307 match found {
8308 Ok(land::OpenPr::None) => PrPlan::Create,
8309 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
8310 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
8311 "several open pull requests exist for this branch, not picking one: {}",
8312 urls.join(" ")
8313 )),
8314 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
8315 }
8316}
8317
8318async fn gh_pr_create(
8320 cwd: &Path,
8321 base: &str,
8322 head: &str,
8323 title: &str,
8324 body: &str,
8325) -> Result<String> {
8326 let out = tokio::process::Command::new("gh")
8327 .args([
8328 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
8329 ])
8330 .current_dir(cwd)
8331 .quiet()
8332 .stdin(std::process::Stdio::null())
8333 .output()
8334 .await
8335 .context("spawn gh")?;
8336 if out.status.success() {
8337 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
8338 } else {
8339 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
8340 }
8341}
8342
8343pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
8352 let repo = state.repo.clone();
8353 let root = state.worktree_root();
8354 let winner = state.tally.as_ref().map(|t| t.winner);
8355 let mut removed = Vec::new();
8356
8357 for i in 0..state.candidates.len() {
8358 let c = state.candidates[i].clone();
8359 let is_winner = Some(c.label) == winner;
8360 if is_winner && !drop_winner {
8361 continue;
8362 }
8363 if c.worktree.exists() {
8364 git::worktree_remove(&repo, &c.worktree).await.ok();
8365 removed.push(c.worktree.to_string_lossy().into_owned());
8366 }
8367 let handed_over = state.released_branches.contains(&c.branch);
8370 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
8371 git::branch_delete(&repo, &c.branch).await.ok();
8372 removed.push(c.branch.clone());
8373 }
8374 state.candidates[i].folded = true;
8375 }
8376
8377 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
8378 let path = name.path();
8379 let keep = !drop_winner
8380 && winner.is_some_and(|w| {
8381 path.file_name()
8382 .is_some_and(|n| n == format!("cand-{w}").as_str())
8383 });
8384 if keep {
8385 continue;
8386 }
8387 git::worktree_remove(&repo, &path).await.ok();
8388 removed.push(path.to_string_lossy().into_owned());
8389 }
8390
8391 remove_if_empty(&root);
8400
8401 if state.enabled_worktree_config && drop_winner {
8402 git::release_worktree_config(&repo).await.ok();
8406 state.enabled_worktree_config = false;
8407 }
8408 state.save_under(home)?;
8409 Ok(removed)
8410}
8411
8412fn remove_if_empty(dir: &Path) {
8423 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
8424 std::fs::remove_dir(dir).ok();
8425 }
8426}
8427
8428pub fn worst_open(state: &RunState) -> Option<Severity> {
8430 state
8431 .reviews
8432 .last()?
8433 .reviews
8434 .iter()
8435 .flat_map(|r| r.findings.iter())
8436 .map(|f| f.severity)
8437 .max()
8438}
8439
8440#[cfg(test)]
8441mod tests {
8442 #[test]
8443 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
8444 let leaked = vec!["chore(deps): update a crate".to_owned()];
8445 let own = "fix(daemon): apply a chosen action";
8446 assert!(should_retitle("", own, &leaked));
8447 assert!(should_retitle(
8448 &format!("{REVIEW_PROMPT_OPENING} `x`"),
8449 own,
8450 &leaked
8451 ));
8452 assert!(should_retitle(
8453 "chore: land candidate A of run 1",
8454 own,
8455 &leaked
8456 ));
8457 assert!(should_retitle(
8458 "magi: candidate A (uncommitted work)",
8459 own,
8460 &leaked
8461 ));
8462 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
8463 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
8464 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
8465 }
8466
8467 #[test]
8468 fn pr_merge_plan_creates_adopts_or_stops() {
8469 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
8470 assert_eq!(
8471 pr_merge_plan(Ok(land::OpenPr::One {
8472 url: "u".into(),
8473 title: "t".into()
8474 })),
8475 PrPlan::Adopt {
8476 url: "u".into(),
8477 title: "t".into()
8478 }
8479 );
8480 let PrPlan::Stop(many) =
8481 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
8482 else {
8483 panic!("many must stop");
8484 };
8485 assert!(many.contains('a') && many.contains('b'));
8486 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
8487 panic!("a failed lookup must stop");
8488 };
8489 assert!(err.contains("bad token"));
8490 }
8491
8492 use super::*;
8493 use crate::run::GateStatus;
8494 use std::collections::BTreeMap;
8495 use std::time::Duration;
8496
8497 fn conductor() -> AgentSpec {
8498 AgentSpec {
8499 id: "conductor".to_owned(),
8500 kind: crate::config::AgentKind::Command,
8501 model: None,
8502 command: vec!["true".to_owned()],
8503 extra_args: Vec::new(),
8504 env: BTreeMap::new(),
8505 prompt_delivery: None,
8506 }
8507 }
8508
8509 fn spec(id: &str) -> AgentSpec {
8510 AgentSpec {
8511 id: id.to_owned(),
8512 kind: crate::config::AgentKind::Command,
8513 model: None,
8514 command: vec!["true".to_owned()],
8515 extra_args: Vec::new(),
8516 env: BTreeMap::new(),
8517 prompt_delivery: None,
8518 }
8519 }
8520
8521 fn ids(xs: &[&str]) -> BTreeSet<String> {
8522 xs.iter().map(|s| (*s).to_owned()).collect()
8523 }
8524
8525 #[test]
8526 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
8527 let roster = [spec("a"), spec("b"), spec("c")];
8528 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
8529 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8530 }
8531
8532 #[test]
8533 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
8534 let roster = [spec("a"), spec("b"), spec("c")];
8535 let failed = ids(&["a", "b", "c"]);
8536 let mut tried = ids(&["b"]);
8538 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8539 assert_eq!(first.id, "a");
8540 tried.insert(first.id.clone());
8541 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8542 assert_eq!(second.id, "c");
8543 tried.insert(second.id.clone());
8544 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
8545 }
8546
8547 #[test]
8548 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
8549 let roster = [spec("a"), spec("b")];
8550 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
8551 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8552 }
8553
8554 #[test]
8555 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
8556 let roster = [spec("a"), spec("b"), spec("c")];
8557 let h = SeatHistory {
8558 failed: ids(&["a"]),
8559 last_ok: Some("b".to_owned()),
8560 last_fail: None,
8561 };
8562 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
8563 let gone = SeatHistory {
8565 last_ok: Some("zzz".to_owned()),
8566 ..h.clone()
8567 };
8568 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
8569 let failed = SeatHistory {
8570 failed: ids(&["a", "b"]),
8571 last_ok: Some("b".to_owned()),
8572 last_fail: None,
8573 };
8574 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
8575 }
8576
8577 #[test]
8578 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
8579 let first = SeatState::new("review-1", "alpha", 7);
8580 let next = handover_seat("review-1", "gamma", 7);
8581 assert_ne!(first.claude_session, next.claude_session);
8582 }
8583
8584 #[test]
8585 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
8586 let mut state = state_with_summary("x", "y");
8587 let a = handover_seat("review-1", "beta", state.next_seat_seed());
8588 let b = handover_seat("review-1", "beta", state.next_seat_seed());
8589 assert_ne!(a.claude_session, b.claude_session);
8590 }
8591
8592 #[test]
8593 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
8594 let roster = [spec("a"), spec("b")];
8595 let h = SeatHistory {
8596 failed: ids(&["a", "b"]),
8597 ..SeatHistory::default()
8598 };
8599 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
8600 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
8601 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
8602 }
8603
8604 #[test]
8611 fn failure_signature_ignores_numbers_and_paths() {
8612 assert_eq!(
8613 failure_signature("exited with Some(2) and no usable output"),
8614 failure_signature("exited with Some(137) and no usable output")
8615 );
8616 assert_eq!(
8617 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
8618 failure_signature("cannot open /var/x.txt: denied")
8619 );
8620 assert_ne!(failure_signature("boom"), failure_signature("bang"));
8621 }
8622
8623 #[test]
8624 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
8625 let other = FailClass::Other("x".into());
8626 assert!(should_hand_over(None, &FailClass::Quota));
8627 assert!(should_hand_over(Some(&other), &FailClass::Quota));
8628 assert!(should_hand_over(
8629 Some(&FailClass::Timeout),
8630 &FailClass::Timeout
8631 ));
8632 assert!(should_hand_over(None, &other));
8633 assert!(!should_hand_over(Some(&other), &other));
8634 assert!(should_hand_over(
8635 Some(&other),
8636 &FailClass::Other("y".into())
8637 ));
8638 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
8640 assert!(should_hand_over(Some(&FailClass::Quota), &other));
8641 }
8642
8643 #[test]
8644 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
8645 let a = SeatState::new("judge-1", "alpha", 7);
8646 let b = handover_seat("judge-1", "beta", 7);
8647 assert_ne!(a.claude_session, b.claude_session);
8648 assert_eq!(b.turns, 0);
8649 }
8650
8651 #[test]
8652 fn a_timeout_is_classified_apart_from_other_failures() {
8653 assert_eq!(
8654 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
8655 Some(FailClass::Timeout)
8656 );
8657 assert!(matches!(
8658 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
8659 Some(FailClass::Other(_))
8660 ));
8661 }
8662
8663 #[test]
8664 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
8665 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8666 let tried = BTreeSet::from(["beta".to_owned()]);
8667 let next = next_untried_in_roster(&roster, 1, &tried);
8670 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
8671 }
8672
8673 #[test]
8674 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
8675 let roster = vec![spec("alpha"), spec("beta")];
8676 let tried = BTreeSet::from(["beta".to_owned()]);
8677 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8681 }
8682
8683 #[test]
8684 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
8685 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8686 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
8687 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8691 }
8692
8693 #[test]
8694 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
8695 let roster = vec![spec("a"), spec("a"), spec("b")];
8696 let tried = BTreeSet::from(["a".to_owned()]);
8697 let next = next_untried_in_roster(&roster, 0, &tried);
8698 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8699 }
8700
8701 #[test]
8702 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
8703 let roster = vec![spec("a"), spec("b")];
8704 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
8705 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
8706 }
8707
8708 #[test]
8709 fn remove_if_empty_only_ever_takes_a_bare_directory() {
8710 let dir = tempfile::tempdir().unwrap();
8711 let bay = dir.path().join("ffff");
8712
8713 remove_if_empty(&bay);
8715 assert!(!bay.exists());
8716
8717 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
8720 remove_if_empty(&bay);
8721 assert!(bay.exists(), "non-empty directory must survive");
8722
8723 std::fs::remove_dir(bay.join("cand-A")).unwrap();
8725 remove_if_empty(&bay);
8726 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
8727 }
8728
8729 #[test]
8738 fn a_full_panel_that_found_nothing_is_clean() {
8739 assert!(round_is_clean(
8740 0,
8741 true,
8742 2,
8743 2,
8744 0,
8745 IncompleteReviewPolicy::Block
8746 ));
8747 }
8748
8749 #[test]
8750 fn a_missing_seat_is_never_clean_under_the_default_policy() {
8751 assert!(!round_is_clean(
8752 0,
8753 true,
8754 1,
8755 2,
8756 0,
8757 IncompleteReviewPolicy::Block
8758 ));
8759 }
8760
8761 #[test]
8762 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
8763 assert!(!round_is_clean(
8764 1,
8765 true,
8766 1,
8767 2,
8768 0,
8769 IncompleteReviewPolicy::Warn
8770 ));
8771 }
8772
8773 #[test]
8774 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
8775 assert!(round_is_clean(
8776 0,
8777 true,
8778 1,
8779 2,
8780 0,
8781 IncompleteReviewPolicy::Warn
8782 ));
8783 }
8784
8785 #[test]
8786 fn a_full_panel_with_an_open_finding_is_not_clean() {
8787 assert!(!round_is_clean(
8788 1,
8789 true,
8790 2,
8791 2,
8792 0,
8793 IncompleteReviewPolicy::Block
8794 ));
8795 }
8796
8797 #[test]
8798 fn a_full_panel_with_a_red_e2e_is_not_clean() {
8799 assert!(!round_is_clean(
8800 0,
8801 false,
8802 2,
8803 2,
8804 0,
8805 IncompleteReviewPolicy::Block
8806 ));
8807 }
8808
8809 #[test]
8816 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
8817 assert!(round_is_clean(
8820 0,
8821 true,
8822 1,
8823 2,
8824 1,
8825 IncompleteReviewPolicy::Block
8826 ));
8827 }
8828
8829 #[test]
8830 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
8831 assert!(!round_is_clean(
8834 0,
8835 true,
8836 1,
8837 2,
8838 0,
8839 IncompleteReviewPolicy::Block
8840 ));
8841 }
8842
8843 #[test]
8844 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
8845 assert!(!round_is_clean(
8846 1,
8847 true,
8848 1,
8849 2,
8850 1,
8851 IncompleteReviewPolicy::Block
8852 ));
8853 assert!(!round_is_clean(
8854 0,
8855 false,
8856 1,
8857 2,
8858 1,
8859 IncompleteReviewPolicy::Block
8860 ));
8861 }
8862
8863 #[test]
8864 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
8865 assert!(!round_is_clean(
8869 0,
8870 true,
8871 0,
8872 2,
8873 2,
8874 IncompleteReviewPolicy::Block
8875 ));
8876 }
8877
8878 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
8879 CommandOutcome {
8880 command: "test".to_owned(),
8881 code,
8882 output_tail: String::new(),
8883 duration_ms: 0,
8884 resource_blocked,
8885 }
8886 }
8887
8888 #[test]
8889 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
8890 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
8891 assert!(
8892 !verify_inconclusive(&[outcome(Some(1), false)]),
8893 "an ordinary failure is still evidence about the patch"
8894 );
8895 assert!(verify_inconclusive(&[outcome(None, true)]));
8896 assert!(
8897 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
8898 "one inconclusive outcome taints the whole batch"
8899 );
8900 assert!(!verify_inconclusive(&[]));
8901 }
8902
8903 #[tokio::test]
8904 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
8905 let calls = std::sync::atomic::AtomicUsize::new(0);
8909 let started = Instant::now();
8910 wait_for_pids_with(
8911 &[123],
8912 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
8913 Duration::from_millis(5),
8914 Duration::from_secs(5),
8915 )
8916 .await;
8917 assert!(
8918 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
8919 "must keep checking rather than deciding on the first answer"
8920 );
8921 assert!(
8922 started.elapsed() < Duration::from_secs(1),
8923 "must return the moment it is confirmed dead, not wait out the ceiling"
8924 );
8925 }
8926
8927 #[tokio::test]
8928 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
8929 let started = Instant::now();
8930 wait_for_pids_with(
8931 &[123],
8932 |_| true, Duration::from_millis(5),
8934 Duration::from_millis(30),
8935 )
8936 .await;
8937 let elapsed = started.elapsed();
8938 assert!(
8939 elapsed >= Duration::from_millis(30),
8940 "must not give up before its own ceiling: {elapsed:?}"
8941 );
8942 assert!(
8943 elapsed < Duration::from_secs(1),
8944 "must not wait past its own ceiling either: {elapsed:?}"
8945 );
8946 }
8947
8948 #[tokio::test]
8949 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
8950 let started = Instant::now();
8951 wait_for_pids_with(
8952 &[],
8953 |_| true,
8954 Duration::from_secs(5),
8955 Duration::from_secs(5),
8956 )
8957 .await;
8958 assert!(
8959 started.elapsed() < Duration::from_millis(200),
8960 "an empty pid list has nothing to confirm"
8961 );
8962 }
8963
8964 fn review_round(
8970 clean: bool,
8971 blocking: usize,
8972 answered: usize,
8973 expected: usize,
8974 progressed: bool,
8975 e2e_ok: bool,
8976 ) -> ReviewRound {
8977 ReviewRound {
8978 round: 1,
8979 head: "h".to_owned(),
8980 verified_head: None,
8981 verified_at: None,
8982 reviews: Vec::new(),
8983 e2e: vec![CommandOutcome {
8984 command: "test".to_owned(),
8985 code: Some(if e2e_ok { 0 } else { 1 }),
8986 output_tail: String::new(),
8987 duration_ms: 0,
8988 resource_blocked: false,
8989 }],
8990 verify_retried: false,
8991 e2e_deferred: false,
8992 e2e_defer_reason: None,
8993 fix: None,
8994 blocking,
8995 answered,
8996 expected,
8997 clean,
8998 progressed,
8999 vote_split: false,
9000 reconsideration: Vec::new(),
9001 verdict: None,
9002 }
9003 }
9004
9005 #[test]
9006 fn review_conclusion_is_none_when_nothing_has_run() {
9007 assert_eq!(review_conclusion(&[], 3), None);
9008 }
9009
9010 #[test]
9011 fn review_conclusion_is_none_while_rounds_remain() {
9012 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
9013 assert_eq!(review_conclusion(&rounds, 3), None);
9014 }
9015
9016 #[test]
9017 fn review_conclusion_is_gating_once_a_round_is_clean() {
9018 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
9019 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
9020 }
9021
9022 #[test]
9023 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
9024 let rounds = vec![
9025 review_round(false, 1, 2, 2, true, true),
9026 review_round(false, 1, 2, 2, true, true),
9027 ];
9028 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
9029 }
9030
9031 #[test]
9032 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
9033 let rounds = vec![
9034 review_round(false, 1, 2, 2, true, true),
9035 review_round(false, 1, 2, 2, true, false),
9036 ];
9037 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
9038 }
9039
9040 #[test]
9041 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
9042 let mut blocked = review_round(false, 1, 2, 2, true, false);
9049 blocked.e2e[0].resource_blocked = true;
9050 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
9051 assert_eq!(review_conclusion(&rounds, 2), None);
9052 }
9053
9054 #[test]
9055 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
9056 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
9058 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
9059 }
9060
9061 #[test]
9062 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
9063 let rounds = vec![
9064 review_round(false, 1, 2, 2, false, true),
9065 review_round(false, 1, 2, 2, false, true),
9066 ];
9067 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
9068 }
9069
9070 fn secs(n: u64) -> Duration {
9071 Duration::from_secs(n)
9072 }
9073
9074 fn init_repo(dir: &Path) {
9077 let run = |args: &[&str]| {
9078 let out = std::process::Command::new("git")
9079 .args(args)
9080 .current_dir(dir)
9081 .quiet()
9082 .output()
9083 .expect("spawn git");
9084 assert!(
9085 out.status.success(),
9086 "git {args:?} failed: {}",
9087 String::from_utf8_lossy(&out.stderr)
9088 );
9089 };
9090 run(&["init", "-b", "main"]);
9091 run(&["config", "user.name", "magi test"]);
9092 run(&["config", "user.email", "magi@example.com"]);
9093 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
9094 run(&["add", "-A"]);
9095 run(&["commit", "-m", "init"]);
9096 }
9097
9098 fn ask_test_home() {
9106 crate::run::pin_test_home();
9107 }
9108
9109 fn runner_at(status: RunStatus) -> Runner {
9112 let mut state = RunState::new(
9113 PathBuf::from("/nonexistent/repo"),
9114 "main".to_owned(),
9115 "deadbeef".to_owned(),
9116 "task".to_owned(),
9117 Config::default(),
9118 );
9119 state.status = status;
9120 Runner {
9121 state,
9122 roles: ResolvedRoles {
9123 implementers: Vec::new(),
9124 judges: Vec::new(),
9125 reviewers: Vec::new(),
9126 fixer: None,
9127 conductor: conductor(),
9128 implementer_roster: Vec::new(),
9129 judge_roster: Vec::new(),
9130 reviewer_roster: Vec::new(),
9131 },
9132 sem: Arc::new(Semaphore::new(1)),
9133 pause: Pause::new(),
9134 interrupt: Pause::new(),
9135 }
9136 }
9137
9138 #[test]
9142 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
9143 crate::run::pin_test_home();
9144 let mut runner = runner_at(RunStatus::Implementing);
9145 let interrupt = Pause::new();
9146 runner.watch_interrupt(interrupt.clone());
9147
9148 interrupt.park_because("task a1b2 asked to run first");
9149
9150 assert!(runner.park_here().expect("park_here"));
9151 assert!(runner.state.parked);
9152 let last = runner.state.events.last().expect("a park event");
9153 assert_eq!(last.node, "park");
9154 assert!(
9155 last.message.contains("task a1b2 asked to run first"),
9156 "expected the interrupt reason in {:?}",
9157 last.message
9158 );
9159 }
9160
9161 #[test]
9169 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
9170 crate::run::pin_test_home();
9171 let mut runner = runner_at(RunStatus::Implementing);
9172 let shutdown = Pause::new();
9173 runner.on_pause(shutdown.clone());
9174 let interrupt = Pause::new();
9175 runner.watch_interrupt(interrupt.clone());
9176
9177 assert!(!runner.park_here().expect("park_here"));
9179 assert!(!runner.state.parked);
9180
9181 interrupt.park_because("test");
9183 assert!(!shutdown.parked());
9184 assert!(runner.park_here().expect("park_here"));
9185 }
9186
9187 #[tokio::test]
9201 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
9202 crate::run::pin_test_home();
9203 let mut runner = runner_at(RunStatus::Implementing);
9204 let interrupt = Pause::new();
9205 runner.watch_interrupt(interrupt.clone());
9206
9207 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
9208 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
9209
9210 let node = async move {
9214 started_tx.send(()).expect("send started");
9215 finish_rx.await.expect("recv finish");
9216 "node finished"
9217 };
9218
9219 let interrupter = async move {
9220 started_rx.await.expect("recv started");
9221 interrupt.park_because("higher-priority task waiting");
9223 tokio::task::yield_now().await;
9227 finish_tx.send(()).expect("send finish");
9228 };
9229
9230 let (node_result, ()) = tokio::join!(node, interrupter);
9231 assert_eq!(
9232 node_result, "node finished",
9233 "the in-flight call ran to completion"
9234 );
9235
9236 assert!(runner.park_here().expect("park_here"));
9239 assert!(runner.state.parked);
9240 }
9241
9242 #[test]
9248 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
9249 crate::run::pin_test_home();
9250 let mut runner = runner_at(RunStatus::Judging);
9251 runner.state.config.agents = vec![conductor()];
9255 runner.state.candidates = vec![Candidate {
9256 index: 0,
9257 label: 'A',
9258 agent: "alpha".to_owned(),
9259 branch: "magi/x/A".to_owned(),
9260 worktree: PathBuf::from("/nonexistent/worktree"),
9261 summary: "did the thing".to_owned(),
9262 stat: "1 file changed".to_owned(),
9263 files: 1,
9264 commits: 1,
9265 empty: false,
9266 failed: None,
9267 verified_noop: None,
9268 duration_ms: 1234,
9269 folded: false,
9270 }];
9271 let run_id = runner.state.id.clone();
9272
9273 let interrupt = Pause::new();
9274 runner.watch_interrupt(interrupt.clone());
9275 interrupt.park_because("task c3d4 asked to run first");
9276 assert!(runner.park_here().expect("park_here"));
9277
9278 let resumed = Runner::resume(&run_id).expect("resume");
9279 assert_eq!(resumed.state.candidates.len(), 1);
9280 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
9281 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
9282 assert_eq!(resumed.state.status, runner.state.status);
9283 assert!(
9284 resumed.state.parked,
9285 "still parked until `execute` actually walks the graph again"
9286 );
9287 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
9288 }
9289
9290 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
9292 let mut q = ask::Question::new(
9293 run.to_owned(),
9294 "implement".to_owned(),
9295 "impl-A".to_owned(),
9296 "Which storage backend should the cache use?".to_owned(),
9297 String::new(),
9298 vec!["SQLite".to_owned(), "Redis".to_owned()],
9299 );
9300 store.put(&mut q).unwrap();
9301 q
9302 }
9303
9304 #[test]
9305 fn a_failed_runs_open_question_is_abandoned() {
9306 ask_test_home();
9307 let store = ask::Questions::open();
9308 let mut runner = runner_at(RunStatus::Failed);
9309 let run = runner.state.id.clone();
9310 let q = ask_open_question(&store, &run);
9311
9312 runner.settle_questions();
9313
9314 let back = store.get(&q.id).unwrap();
9315 assert!(
9316 !back.status.open(),
9317 "the seat that asked died with the run; nobody is left to read an answer"
9318 );
9319 assert!(
9320 back.detail.contains(&run) && back.detail.contains("failed"),
9321 "the reason names what the run became, not just that it is gone: {}",
9322 back.detail
9323 );
9324 }
9325
9326 #[test]
9327 fn a_merged_runs_open_question_is_abandoned_too() {
9328 ask_test_home();
9329 let store = ask::Questions::open();
9330 for status in [RunStatus::Merged, RunStatus::Ready] {
9333 let mut runner = runner_at(status);
9334 let run = runner.state.id.clone();
9335 let q = ask_open_question(&store, &run);
9336
9337 runner.settle_questions();
9338
9339 let back = store.get(&q.id).unwrap();
9340 assert!(
9341 !back.status.open(),
9342 "{status:?} run's question must not outlive the run"
9343 );
9344 }
9345 }
9346
9347 #[test]
9348 fn a_still_resumable_runs_open_question_is_left_alone() {
9349 ask_test_home();
9350 let store = ask::Questions::open();
9351 for status in [RunStatus::Blocked, RunStatus::Stalled] {
9357 let mut runner = runner_at(status);
9358 let run = runner.state.id.clone();
9359 let q = ask_open_question(&store, &run);
9360
9361 runner.settle_questions();
9362
9363 let back = store.get(&q.id).unwrap();
9364 assert!(
9365 back.status.open(),
9366 "{status:?} is still alive; the question must still be waiting"
9367 );
9368 }
9369 }
9370
9371 #[test]
9372 fn settle_questions_never_touches_an_already_answered_question() {
9373 ask_test_home();
9374 let store = ask::Questions::open();
9375 let mut runner = runner_at(RunStatus::Failed);
9376 let run = runner.state.id.clone();
9377 let mut q = ask_open_question(&store, &run);
9378 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
9379 .unwrap();
9380 store.put(&mut q).unwrap();
9381
9382 runner.settle_questions();
9387 runner.settle_questions();
9388
9389 let back = store.get(&q.id).unwrap();
9390 assert_eq!(
9391 back.status,
9392 ask::QuestionStatus::Answered,
9393 "a real answer is a decision on record, never overwritten by a sweep"
9394 );
9395 }
9396
9397 #[tokio::test]
9408 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
9409 crate::run::pin_test_home();
9410 let tmp = tempfile::tempdir().expect("tempdir");
9411 let repo = tmp.path().join("repo");
9412 std::fs::create_dir_all(&repo).unwrap();
9413 init_repo(&repo);
9414
9415 let mut config = Config::default();
9416 config.graph.worktree_root = Some(tmp.path().join("wt"));
9417
9418 let mut state = RunState::new(
9419 repo.clone(),
9420 "main".to_owned(),
9421 "deadbeef".to_owned(),
9422 "task".to_owned(),
9423 config,
9424 );
9425 let root = state.worktree_root();
9426 let wt_a = root.join("cand-A");
9427 let wt_b = root.join("cand-B");
9428 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
9429 .await
9430 .expect("worktree A");
9431 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
9432 .await
9433 .expect("worktree B");
9434
9435 state.candidates = vec![
9436 Candidate {
9437 index: 0,
9438 label: 'A',
9439 agent: "alpha".to_owned(),
9440 branch: "magi/x/A".to_owned(),
9441 worktree: wt_a.clone(),
9442 summary: String::new(),
9443 stat: String::new(),
9444 files: 0,
9445 commits: 0,
9446 empty: false,
9447 failed: None,
9448 verified_noop: None,
9449 duration_ms: 0,
9450 folded: false,
9451 },
9452 Candidate {
9453 index: 1,
9454 label: 'B',
9455 agent: "beta".to_owned(),
9456 branch: "magi/x/B".to_owned(),
9457 worktree: wt_b.clone(),
9458 summary: String::new(),
9459 stat: String::new(),
9460 files: 0,
9461 commits: 0,
9462 empty: false,
9463 failed: None,
9464 verified_noop: None,
9465 duration_ms: 0,
9466 folded: false,
9467 },
9468 ];
9469 state.tally = Some(Tally {
9470 first_choice: BTreeMap::from([('A', 1)]),
9471 borda: BTreeMap::new(),
9472 winner: 'A',
9473 rankings: 1,
9474 unanimous_initial: true,
9475 deliberated: false,
9476 changed_votes: 0,
9477 unanimous_final: true,
9478 tie_break: None,
9479 judges: 1,
9480 present: 1,
9481 quorum: 1,
9482 met_quorum: true,
9483 uncontested: None,
9484 });
9485 state.status = RunStatus::Ready;
9486
9487 fold_run(&mut state, false, &crate::run::home())
9488 .await
9489 .expect("fold_run");
9490
9491 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
9492 assert!(
9493 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9494 "the unmerged winner's branch survives"
9495 );
9496 assert!(
9497 !state.candidates[0].folded,
9498 "the winner is not marked folded"
9499 );
9500
9501 assert!(!wt_b.exists(), "the loser's worktree is removed");
9502 assert!(
9503 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
9504 "the loser's branch is removed"
9505 );
9506 assert!(state.candidates[1].folded, "the loser is marked folded");
9507 }
9508
9509 #[tokio::test]
9512 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
9513 let tmp = tempfile::tempdir().expect("tempdir");
9514 let repo = tmp.path().join("repo");
9515 std::fs::create_dir_all(&repo).unwrap();
9516 init_repo(&repo);
9517 let home = tmp.path().join("home");
9518
9519 let mut config = Config::default();
9520 config.graph.worktree_root = Some(tmp.path().join("wt"));
9521 let mut state = RunState::new(
9522 repo.clone(),
9523 "main".to_owned(),
9524 "deadbeef".to_owned(),
9525 "task".to_owned(),
9526 config,
9527 );
9528 git::git(&repo, &["branch", "magi/x/A", "main"])
9530 .await
9531 .expect("branch");
9532 state.candidates = vec![Candidate {
9533 index: 0,
9534 label: 'A',
9535 agent: "alpha".to_owned(),
9536 branch: "magi/x/A".to_owned(),
9537 worktree: state.worktree_root().join("cand-A"),
9538 summary: String::new(),
9539 stat: String::new(),
9540 files: 0,
9541 commits: 0,
9542 empty: false,
9543 failed: None,
9544 verified_noop: None,
9545 duration_ms: 0,
9546 folded: true,
9547 }];
9548 state.released_to = Some("20260901-000000-new1".to_owned());
9549 state.released_branches = vec!["magi/x/A".to_owned()];
9550
9551 fold_run(&mut state, true, &home).await.expect("fold_run");
9552
9553 assert!(
9554 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9555 "the handed-over branch survives a fold"
9556 );
9557 }
9558
9559 #[tokio::test]
9563 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
9564 let tmp = tempfile::tempdir().expect("tempdir");
9565 let repo = tmp.path().join("repo");
9566 std::fs::create_dir_all(&repo).unwrap();
9567 init_repo(&repo);
9568 let run = |args: &[&str]| {
9569 let out = std::process::Command::new("git")
9570 .quiet()
9571 .args(args)
9572 .current_dir(&repo)
9573 .output()
9574 .expect("spawn git");
9575 assert!(out.status.success(), "git {args:?}");
9576 };
9577 run(&["branch", "magi/x/A"]);
9578 run(&["checkout", "-q", "-b", "magi/x/B"]);
9579 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
9580 run(&["add", "-A"]);
9581 run(&["commit", "-q", "-m", "work"]);
9582 run(&["checkout", "-q", "main"]);
9583
9584 let mut state = RunState::new(
9585 repo.clone(),
9586 "main".to_owned(),
9587 "deadbeef".to_owned(),
9588 "task".to_owned(),
9589 Config::default(),
9590 );
9591 state.seeds = vec![refs::Seed {
9592 token: "magi/27b2/A".to_owned(),
9593 kind: refs::SeedKind::Unresolved,
9594 sha: String::new(),
9595 branch: true,
9596 detail: "no branch or commit named magi/27b2/A".to_owned(),
9597 }];
9598
9599 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
9600 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
9601 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
9602 let detail = empty_candidate_detail(&state, "main");
9603 assert!(detail.starts_with("empty candidate"), "{detail}");
9604 assert!(detail.contains("magi/27b2/A"), "{detail}");
9605 }
9606
9607 #[tokio::test]
9616 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
9617 let tmp = tempfile::tempdir().expect("tempdir");
9618 let repo = tmp.path().join("repo");
9619 std::fs::create_dir_all(&repo).unwrap();
9620 init_repo(&repo);
9621
9622 let mut config = Config::default();
9623 config.merge.mode = MergeMode::Local;
9624
9625 let mut state = RunState::new(
9626 repo.clone(),
9627 "main".to_owned(),
9628 "deadbeef".to_owned(),
9629 "task".to_owned(),
9630 config,
9631 );
9632 state.candidates = vec![Candidate {
9633 index: 0,
9634 label: 'A',
9635 agent: "alpha".to_owned(),
9636 branch: "does-not-exist".to_owned(),
9637 worktree: repo.clone(),
9638 summary: String::new(),
9639 stat: String::new(),
9640 files: 0,
9641 commits: 0,
9642 empty: false,
9643 failed: None,
9644 verified_noop: None,
9645 duration_ms: 0,
9646 folded: false,
9647 }];
9648 state.tally = Some(Tally {
9649 first_choice: BTreeMap::from([('A', 1)]),
9650 borda: BTreeMap::new(),
9651 winner: 'A',
9652 rankings: 1,
9653 unanimous_initial: true,
9654 deliberated: false,
9655 changed_votes: 0,
9656 unanimous_final: true,
9657 tie_break: None,
9658 judges: 0,
9659 present: 0,
9660 quorum: 0,
9661 met_quorum: true,
9662 uncontested: Some("only candidate A produced a change".to_owned()),
9663 });
9664 state.reviews = vec![ReviewRound {
9665 round: 1,
9666 head: "deadbeef".to_owned(),
9667 verified_head: None,
9668 verified_at: None,
9669 reviews: Vec::new(),
9670 e2e: Vec::new(),
9671 fix: None,
9672 blocking: 0,
9673 answered: 0,
9674 expected: 0,
9675 clean: true,
9676 verify_retried: false,
9677 e2e_deferred: false,
9678 e2e_defer_reason: None,
9679 progressed: false,
9680 vote_split: false,
9681 reconsideration: Vec::new(),
9682 verdict: None,
9683 }];
9684 state.gate = vec![CommandOutcome {
9685 command: "test".to_owned(),
9686 code: Some(0),
9687 output_tail: String::new(),
9688 duration_ms: 0,
9689 resource_blocked: false,
9690 }];
9691 state.gate_ran = true;
9692 state.status = RunStatus::Ready;
9697 state.merge = Some(MergeOutcome {
9698 mode: MergeMode::Local,
9699 ok: false,
9700 detail: "already concluded".to_owned(),
9701 empty: false,
9702 });
9703
9704 let mut runner = Runner {
9705 state,
9706 roles: ResolvedRoles {
9707 implementers: Vec::new(),
9708 judges: Vec::new(),
9709 reviewers: Vec::new(),
9710 fixer: None,
9711 conductor: conductor(),
9712 implementer_roster: Vec::new(),
9713 judge_roster: Vec::new(),
9714 reviewer_roster: Vec::new(),
9715 },
9716 sem: Arc::new(Semaphore::new(1)),
9717 pause: Pause::new(),
9718 interrupt: Pause::new(),
9719 };
9720
9721 runner.merge().await.expect("merge");
9722
9723 assert_eq!(
9724 runner.state.status,
9725 RunStatus::Ready,
9726 "a concluded run's status must not change on reentry"
9727 );
9728 assert_eq!(
9729 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
9730 Some("already concluded"),
9731 "merge must not run again once the node already recorded an outcome"
9732 );
9733 }
9734
9735 #[tokio::test]
9744 async fn merge_refuses_a_gate_that_has_not_actually_run() {
9745 let tmp = tempfile::tempdir().expect("tempdir");
9746 let repo = tmp.path().join("repo");
9747 std::fs::create_dir_all(&repo).unwrap();
9748 init_repo(&repo);
9749
9750 let mut config = Config::default();
9751 config.merge.mode = MergeMode::Local;
9752
9753 let mut state = RunState::new(
9754 repo.clone(),
9755 "main".to_owned(),
9756 "deadbeef".to_owned(),
9757 "task".to_owned(),
9758 config,
9759 );
9760 state.candidates = vec![Candidate {
9761 index: 0,
9762 label: 'A',
9763 agent: "alpha".to_owned(),
9764 branch: "does-not-exist".to_owned(),
9765 worktree: repo.clone(),
9766 summary: String::new(),
9767 stat: String::new(),
9768 files: 0,
9769 commits: 0,
9770 empty: false,
9771 failed: None,
9772 verified_noop: None,
9773 duration_ms: 0,
9774 folded: false,
9775 }];
9776 state.tally = Some(Tally {
9777 first_choice: BTreeMap::from([('A', 1)]),
9778 borda: BTreeMap::new(),
9779 winner: 'A',
9780 rankings: 1,
9781 unanimous_initial: true,
9782 deliberated: false,
9783 changed_votes: 0,
9784 unanimous_final: true,
9785 tie_break: None,
9786 judges: 0,
9787 present: 0,
9788 quorum: 0,
9789 met_quorum: true,
9790 uncontested: Some("only candidate A produced a change".to_owned()),
9791 });
9792 state.reviews = vec![ReviewRound {
9793 round: 1,
9794 head: "deadbeef".to_owned(),
9795 verified_head: None,
9796 verified_at: None,
9797 reviews: Vec::new(),
9798 e2e: Vec::new(),
9799 fix: None,
9800 blocking: 0,
9801 answered: 0,
9802 expected: 0,
9803 clean: true,
9804 verify_retried: false,
9805 e2e_deferred: false,
9806 e2e_defer_reason: None,
9807 progressed: false,
9808 vote_split: false,
9809 reconsideration: Vec::new(),
9810 verdict: None,
9811 }];
9812 state.gate = Vec::new();
9814 state.gate_ran = false;
9815 state.status = RunStatus::Gating;
9816
9817 let mut runner = Runner {
9818 state,
9819 roles: ResolvedRoles {
9820 implementers: Vec::new(),
9821 judges: Vec::new(),
9822 reviewers: Vec::new(),
9823 fixer: None,
9824 conductor: conductor(),
9825 implementer_roster: Vec::new(),
9826 judge_roster: Vec::new(),
9827 reviewer_roster: Vec::new(),
9828 },
9829 sem: Arc::new(Semaphore::new(1)),
9830 pause: Pause::new(),
9831 interrupt: Pause::new(),
9832 };
9833
9834 runner.merge().await.expect("merge");
9835
9836 assert!(
9837 runner.state.merge.is_none(),
9838 "an empty gate must never be read as a passing one: {:?}",
9839 runner.state.merge
9840 );
9841 }
9842
9843 #[tokio::test]
9850 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
9851 let tmp = tempfile::tempdir().expect("tempdir");
9852 let repo = tmp.path().join("repo");
9853 std::fs::create_dir_all(&repo).unwrap();
9854 init_repo(&repo);
9855
9856 let config = Config::default();
9858
9859 let mut state = RunState::new(
9860 repo.clone(),
9861 "main".to_owned(),
9862 "deadbeef".to_owned(),
9863 "task".to_owned(),
9864 config,
9865 );
9866 state.candidates = vec![Candidate {
9867 index: 0,
9868 label: 'A',
9869 agent: "alpha".to_owned(),
9870 branch: "does-not-exist".to_owned(),
9871 worktree: repo.clone(),
9872 summary: String::new(),
9873 stat: String::new(),
9874 files: 0,
9875 commits: 0,
9876 empty: false,
9877 failed: None,
9878 verified_noop: None,
9879 duration_ms: 0,
9880 folded: false,
9881 }];
9882 state.tally = Some(Tally {
9883 first_choice: BTreeMap::from([('A', 1)]),
9884 borda: BTreeMap::new(),
9885 winner: 'A',
9886 rankings: 1,
9887 unanimous_initial: true,
9888 deliberated: false,
9889 changed_votes: 0,
9890 unanimous_final: true,
9891 tie_break: None,
9892 judges: 0,
9893 present: 0,
9894 quorum: 0,
9895 met_quorum: true,
9896 uncontested: Some("only candidate A produced a change".to_owned()),
9897 });
9898 state.reviews = vec![ReviewRound {
9899 round: 1,
9900 head: "deadbeef".to_owned(),
9901 verified_head: None,
9902 verified_at: None,
9903 reviews: Vec::new(),
9904 e2e: Vec::new(),
9905 fix: None,
9906 blocking: 0,
9907 answered: 0,
9908 expected: 0,
9909 clean: true,
9910 verify_retried: false,
9911 e2e_deferred: false,
9912 e2e_defer_reason: None,
9913 progressed: false,
9914 vote_split: false,
9915 reconsideration: Vec::new(),
9916 verdict: None,
9917 }];
9918
9919 let mut runner = Runner {
9920 state,
9921 roles: ResolvedRoles {
9922 implementers: Vec::new(),
9923 judges: Vec::new(),
9924 reviewers: Vec::new(),
9925 fixer: None,
9926 conductor: conductor(),
9927 implementer_roster: Vec::new(),
9928 judge_roster: Vec::new(),
9929 reviewer_roster: Vec::new(),
9930 },
9931 sem: Arc::new(Semaphore::new(1)),
9932 pause: Pause::new(),
9933 interrupt: Pause::new(),
9934 };
9935
9936 runner.gate().await.expect("gate");
9937 assert!(
9938 runner.state.gate_ran,
9939 "zero configured commands is still a real attempt, not an unrun gate"
9940 );
9941 assert!(runner.state.gate.is_empty());
9942 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
9943 assert_ne!(
9944 runner.state.status,
9945 RunStatus::Blocked,
9946 "a gate with nothing to check must not read as failed"
9947 );
9948
9949 runner.merge().await.expect("merge");
9950 assert_eq!(
9951 runner.state.status,
9952 RunStatus::Ready,
9953 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
9954 );
9955 }
9956
9957 #[tokio::test]
9968 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
9969 crate::run::pin_test_home();
9970 let home = crate::run::home();
9971
9972 let tmp = tempfile::tempdir().expect("tempdir");
9973 let repo = tmp.path().join("repo");
9974 std::fs::create_dir_all(&repo).unwrap();
9975 init_repo(&repo);
9976 let cache_dir = tmp.path().join("target");
9979
9980 let mut config = Config::default();
9981 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
9982 config.graph.timeout_verify = Some(2);
9985
9986 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9987 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9988 .expect("no io error acquiring directly")
9989 {
9990 crate::cache::AcquireOutcome::Acquired(g) => g,
9991 crate::cache::AcquireOutcome::Busy(b) => {
9992 panic!("expected the direct acquire to win the lease first: {b:?}")
9993 }
9994 };
9995
9996 let mut state = RunState::new(
9997 repo.clone(),
9998 "main".to_owned(),
9999 "deadbeef".to_owned(),
10000 "task".to_owned(),
10001 config,
10002 );
10003 state.candidates = vec![Candidate {
10004 index: 0,
10005 label: 'A',
10006 agent: "alpha".to_owned(),
10007 branch: "does-not-exist".to_owned(),
10008 worktree: repo.clone(),
10009 summary: String::new(),
10010 stat: String::new(),
10011 files: 0,
10012 commits: 0,
10013 empty: false,
10014 failed: None,
10015 verified_noop: None,
10016 duration_ms: 0,
10017 folded: false,
10018 }];
10019 state.tally = Some(Tally {
10020 first_choice: BTreeMap::from([('A', 1)]),
10021 borda: BTreeMap::new(),
10022 winner: 'A',
10023 rankings: 1,
10024 unanimous_initial: true,
10025 deliberated: false,
10026 changed_votes: 0,
10027 unanimous_final: true,
10028 tie_break: None,
10029 judges: 0,
10030 present: 0,
10031 quorum: 0,
10032 met_quorum: true,
10033 uncontested: Some("only candidate A produced a change".to_owned()),
10034 });
10035 state.reviews = vec![ReviewRound {
10036 round: 1,
10037 head: "deadbeef".to_owned(),
10038 verified_head: None,
10039 verified_at: None,
10040 reviews: Vec::new(),
10041 e2e: Vec::new(),
10042 fix: None,
10043 blocking: 0,
10044 answered: 0,
10045 expected: 0,
10046 clean: true,
10047 verify_retried: false,
10048 e2e_deferred: false,
10049 e2e_defer_reason: None,
10050 progressed: false,
10051 vote_split: false,
10052 reconsideration: Vec::new(),
10053 verdict: None,
10054 }];
10055
10056 let mut runner = Runner {
10057 state,
10058 roles: ResolvedRoles {
10059 implementers: Vec::new(),
10060 judges: Vec::new(),
10061 reviewers: Vec::new(),
10062 fixer: None,
10063 conductor: conductor(),
10064 implementer_roster: Vec::new(),
10065 judge_roster: Vec::new(),
10066 reviewer_roster: Vec::new(),
10067 },
10068 sem: Arc::new(Semaphore::new(1)),
10069 pause: Pause::new(),
10070 interrupt: Pause::new(),
10071 };
10072
10073 let started = std::time::Instant::now();
10074 runner.gate().await.expect("gate");
10075 assert!(
10076 started.elapsed() < Duration::from_secs(1),
10077 "a gate with nothing to run must never wait on a lease it never needed"
10078 );
10079 assert!(
10080 runner.state.gate_ran,
10081 "zero commands is still a real, immediate attempt"
10082 );
10083 assert!(runner.state.gate.is_empty());
10084 assert_ne!(
10085 runner.state.status,
10086 RunStatus::Blocked,
10087 "must not read as resource-blocked on a lease it never asked for"
10088 );
10089 }
10090
10091 #[tokio::test]
10101 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
10102 crate::run::pin_test_home();
10103
10104 let tmp = tempfile::tempdir().expect("tempdir");
10105 let repo = tmp.path().join("repo");
10106 std::fs::create_dir_all(&repo).unwrap();
10107 init_repo(&repo);
10108
10109 let mut config = Config::default();
10110 config.verify.gate = vec![
10111 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
10112 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
10113 .to_owned(),
10114 ];
10115
10116 let mut state = RunState::new(
10117 repo.clone(),
10118 "main".to_owned(),
10119 "deadbeef".to_owned(),
10120 "task".to_owned(),
10121 config,
10122 );
10123 let run_id = state.id.clone();
10124 state.candidates = vec![Candidate {
10125 index: 0,
10126 label: 'A',
10127 agent: "alpha".to_owned(),
10128 branch: "does-not-exist".to_owned(),
10129 worktree: repo.clone(),
10130 summary: String::new(),
10131 stat: String::new(),
10132 files: 0,
10133 commits: 0,
10134 empty: false,
10135 failed: None,
10136 verified_noop: None,
10137 duration_ms: 0,
10138 folded: false,
10139 }];
10140 state.tally = Some(Tally {
10141 first_choice: BTreeMap::from([('A', 1)]),
10142 borda: BTreeMap::new(),
10143 winner: 'A',
10144 rankings: 1,
10145 unanimous_initial: true,
10146 deliberated: false,
10147 changed_votes: 0,
10148 unanimous_final: true,
10149 tie_break: None,
10150 judges: 0,
10151 present: 0,
10152 quorum: 0,
10153 met_quorum: true,
10154 uncontested: Some("only candidate A produced a change".to_owned()),
10155 });
10156 state.reviews = vec![ReviewRound {
10157 round: 1,
10158 head: "deadbeef".to_owned(),
10159 verified_head: None,
10160 verified_at: None,
10161 reviews: Vec::new(),
10162 e2e: Vec::new(),
10163 fix: None,
10164 blocking: 0,
10165 answered: 0,
10166 expected: 0,
10167 clean: true,
10168 verify_retried: false,
10169 e2e_deferred: false,
10170 e2e_defer_reason: None,
10171 progressed: false,
10172 vote_split: false,
10173 reconsideration: Vec::new(),
10174 verdict: None,
10175 }];
10176
10177 let mut runner = Runner {
10178 state,
10179 roles: ResolvedRoles {
10180 implementers: Vec::new(),
10181 judges: Vec::new(),
10182 reviewers: Vec::new(),
10183 fixer: None,
10184 conductor: conductor(),
10185 implementer_roster: Vec::new(),
10186 judge_roster: Vec::new(),
10187 reviewer_roster: Vec::new(),
10188 },
10189 sem: Arc::new(Semaphore::new(1)),
10190 pause: Pause::new(),
10191 interrupt: Pause::new(),
10192 };
10193
10194 let started_marker = repo.join("started.marker");
10195 let release_marker = repo.join("release.marker");
10196 let poller = tokio::spawn(async move {
10197 for _ in 0..100 {
10202 if started_marker.exists()
10203 && let Ok(s) = crate::run::RunState::load(&run_id)
10204 && let Some(a) = s.active.get("gate")
10205 {
10206 std::fs::write(&release_marker, b"go").expect("release marker");
10207 return Some(a.clone());
10208 }
10209 tokio::time::sleep(Duration::from_millis(50)).await;
10210 }
10211 None
10212 });
10213
10214 runner.gate().await.expect("gate");
10215 let captured = poller.await.expect("poller task");
10216 let captured = captured.expect(
10217 "the poller never saw a `gate` task entry in run.json while the command was \
10218 still blocked on its own release marker",
10219 );
10220
10221 assert_eq!(captured.task.as_deref(), Some("gate"));
10222 assert_eq!(captured.node, "gate");
10223 assert_eq!(captured.index, Some(1));
10224 assert_eq!(captured.total, Some(1));
10225 assert!(
10226 captured
10227 .command
10228 .as_deref()
10229 .is_some_and(|c| c.contains("started.marker")),
10230 "{captured:?}"
10231 );
10232
10233 assert!(
10234 runner.state.active.is_empty(),
10235 "the entry must be cleared once the command actually finished: {:?}",
10236 runner.state.active
10237 );
10238 assert!(runner.state.gate_ran);
10239 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
10240 }
10241
10242 #[tokio::test]
10245 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
10246 use crate::verdict::{Finding, ReviewVote, Severity};
10247 crate::run::pin_test_home();
10248 let tmp = tempfile::tempdir().expect("tempdir");
10249 let repo = tmp.path().join("repo");
10250 std::fs::create_dir_all(&repo).unwrap();
10251 init_repo(&repo);
10252
10253 for (severity, vote, expect) in [
10254 (Severity::Major, ReviewVote::Reject, true),
10255 (Severity::Minor, ReviewVote::Reject, false),
10256 (Severity::Major, ReviewVote::Approve, false),
10257 ] {
10258 let mut round = review_round(false, 1, 1, 1, false, true);
10259 round.reviews = vec![ReviewRecord {
10260 reviewer: 1,
10261 agent: "alpha".to_owned(),
10262 summary: String::new(),
10263 findings: vec![Finding {
10264 id: "R1-1-1".to_owned(),
10265 severity,
10266 file: None,
10267 line: None,
10268 title: "t".to_owned(),
10269 detail: String::new(),
10270 }],
10271 vote: Some(vote),
10272 failed: None,
10273 duration_ms: 0,
10274 attempts: 0,
10275 }];
10276 let mut state = RunState::new(
10277 repo.clone(),
10278 "main".to_owned(),
10279 "deadbeef".to_owned(),
10280 "task".to_owned(),
10281 Config::default(),
10282 );
10283 state.reviews = vec![round];
10284 let mut runner = Runner {
10285 state,
10286 roles: ResolvedRoles {
10287 implementers: Vec::new(),
10288 judges: Vec::new(),
10289 reviewers: Vec::new(),
10290 fixer: None,
10291 conductor: conductor(),
10292 implementer_roster: Vec::new(),
10293 judge_roster: Vec::new(),
10294 reviewer_roster: Vec::new(),
10295 },
10296 sem: Arc::new(Semaphore::new(1)),
10297 pause: Pause::new(),
10298 interrupt: Pause::new(),
10299 };
10300 let shell = runner.state.config.shell();
10301 runner
10302 .stop_reviewing("round budget spent", &shell, &repo)
10303 .await
10304 .expect("stop_reviewing");
10305 assert_eq!(runner.state.status, RunStatus::Gating);
10306 assert_eq!(
10307 runner.state.contested_handoff.is_some(),
10308 expect,
10309 "{severity:?} + {vote:?}"
10310 );
10311 }
10312 }
10313
10314 #[tokio::test]
10327 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
10328 crate::run::pin_test_home();
10329 let home = crate::run::home();
10330
10331 let tmp = tempfile::tempdir().expect("tempdir");
10332 let repo = tmp.path().join("repo");
10333 std::fs::create_dir_all(&repo).unwrap();
10334 init_repo(&repo);
10335 let head = crate::git::rev_parse(&repo, "HEAD")
10336 .await
10337 .expect("rev-parse");
10338 let cache_dir = tmp.path().join("target");
10341
10342 let mut config = Config::default();
10343 config.verify.e2e = vec![format!(
10344 "CARGO_TARGET_DIR='{}' test -f README.md",
10345 cache_dir.display()
10346 )];
10347 config.graph.review_rounds = 1;
10348 config.graph.timeout_verify = Some(2);
10351
10352 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10353 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10354 .expect("no io error acquiring directly")
10355 {
10356 crate::cache::AcquireOutcome::Acquired(g) => g,
10357 crate::cache::AcquireOutcome::Busy(b) => {
10358 panic!("expected the direct acquire to win the lease first: {b:?}")
10359 }
10360 };
10361
10362 let mut state = RunState::new(
10363 repo.clone(),
10364 "main".to_owned(),
10365 head.clone(),
10366 "task".to_owned(),
10367 config,
10368 );
10369 state.candidates = vec![Candidate {
10370 index: 0,
10371 label: 'A',
10372 agent: "alpha".to_owned(),
10373 branch: "does-not-exist".to_owned(),
10374 worktree: repo.clone(),
10375 summary: String::new(),
10376 stat: String::new(),
10377 files: 0,
10378 commits: 0,
10379 empty: false,
10380 failed: None,
10381 verified_noop: None,
10382 duration_ms: 0,
10383 folded: false,
10384 }];
10385 state.tally = Some(Tally {
10386 first_choice: BTreeMap::from([('A', 1)]),
10387 borda: BTreeMap::new(),
10388 winner: 'A',
10389 rankings: 1,
10390 unanimous_initial: true,
10391 deliberated: false,
10392 changed_votes: 0,
10393 unanimous_final: true,
10394 tie_break: None,
10395 judges: 0,
10396 present: 0,
10397 quorum: 0,
10398 met_quorum: true,
10399 uncontested: Some("only candidate A produced a change".to_owned()),
10400 });
10401 state.reviews = vec![ReviewRound {
10405 round: 1,
10406 head: head.clone(),
10407 verified_head: None,
10408 verified_at: None,
10409 reviews: Vec::new(),
10410 e2e: Vec::new(),
10411 fix: None,
10412 blocking: 1,
10413 answered: 1,
10414 expected: 1,
10415 clean: false,
10416 verify_retried: false,
10417 e2e_deferred: true,
10418 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
10419 progressed: false,
10420 vote_split: false,
10421 reconsideration: Vec::new(),
10422 verdict: None,
10423 }];
10424
10425 let mut runner = Runner {
10426 state,
10427 roles: ResolvedRoles {
10428 implementers: Vec::new(),
10429 judges: Vec::new(),
10430 reviewers: Vec::new(),
10431 fixer: None,
10432 conductor: conductor(),
10433 implementer_roster: Vec::new(),
10434 judge_roster: Vec::new(),
10435 reviewer_roster: Vec::new(),
10436 },
10437 sem: Arc::new(Semaphore::new(1)),
10438 pause: Pause::new(),
10439 interrupt: Pause::new(),
10440 };
10441
10442 let shell = runner.state.config.shell();
10443 runner
10444 .stop_reviewing("round budget spent", &shell, &repo)
10445 .await
10446 .expect("stop_reviewing");
10447
10448 let last = runner.state.reviews.last().expect("round record");
10449 assert_eq!(
10450 last.e2e_status(),
10451 E2eStatus::ResourceBlocked,
10452 "the shared cache is still held; the attempt must read as blocked, not deferred or \
10453 failed: {last:?}"
10454 );
10455 assert_eq!(
10456 last.verified_head.as_deref(),
10457 Some(head.as_str()),
10458 "which commit this attempt targeted is known even though nothing finished checking \
10459 it"
10460 );
10461 let first_attempt_at = last
10462 .verified_at
10463 .expect("when this attempt ran is known too");
10464 assert_ne!(
10465 runner.state.status,
10466 RunStatus::Blocked,
10467 "contention is evidence about the machine, not the patch — it must not settle the \
10468 run as blocked: {:?}",
10469 runner.state.status
10470 );
10471 assert!(
10472 !runner
10473 .state
10474 .events
10475 .iter()
10476 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
10477 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
10478 runner.state.events
10479 );
10480
10481 runner
10486 .stop_reviewing("round budget spent", &shell, &repo)
10487 .await
10488 .expect("stop_reviewing retry");
10489 assert_eq!(
10490 runner.state.reviews.len(),
10491 1,
10492 "no new round was started: {:?}",
10493 runner.state.reviews
10494 );
10495 let last = runner.state.reviews.last().expect("round record");
10496 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
10497 assert!(
10498 last.verified_at.expect("still known") > first_attempt_at,
10499 "a second reentry must be a fresh attempt, not a stale copy of the first"
10500 );
10501 assert_ne!(runner.state.status, RunStatus::Blocked);
10502
10503 held.release();
10504 }
10505
10506 #[tokio::test]
10518 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
10519 crate::run::pin_test_home();
10520 let home = crate::run::home();
10521
10522 let tmp = tempfile::tempdir().expect("tempdir");
10523 let repo = tmp.path().join("repo");
10524 std::fs::create_dir_all(&repo).unwrap();
10525 init_repo(&repo);
10526 let head = crate::git::rev_parse(&repo, "HEAD")
10527 .await
10528 .expect("rev-parse");
10529 let cache_dir = tmp.path().join("target");
10530
10531 let mut config = Config::default();
10532 config.verify.e2e = vec![format!(
10533 "CARGO_TARGET_DIR='{}' test -f README.md",
10534 cache_dir.display()
10535 )];
10536 config.graph.review_rounds = 1;
10537 config.graph.timeout_verify = Some(2);
10538
10539 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10540 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10541 .expect("no io error acquiring directly")
10542 {
10543 crate::cache::AcquireOutcome::Acquired(g) => g,
10544 crate::cache::AcquireOutcome::Busy(b) => {
10545 panic!("expected the direct acquire to win the lease first: {b:?}")
10546 }
10547 };
10548
10549 let mut state = RunState::new(
10550 repo.clone(),
10551 "main".to_owned(),
10552 head.clone(),
10553 "task".to_owned(),
10554 config,
10555 );
10556 state.candidates = vec![Candidate {
10557 index: 0,
10558 label: 'A',
10559 agent: "alpha".to_owned(),
10560 branch: "does-not-exist".to_owned(),
10561 worktree: repo.clone(),
10562 summary: String::new(),
10563 stat: String::new(),
10564 files: 0,
10565 commits: 0,
10566 empty: false,
10567 failed: None,
10568 verified_noop: None,
10569 duration_ms: 0,
10570 folded: false,
10571 }];
10572 state.tally = Some(Tally {
10573 first_choice: BTreeMap::from([('A', 1)]),
10574 borda: BTreeMap::new(),
10575 winner: 'A',
10576 rankings: 1,
10577 unanimous_initial: true,
10578 deliberated: false,
10579 changed_votes: 0,
10580 unanimous_final: true,
10581 tie_break: None,
10582 judges: 0,
10583 present: 0,
10584 quorum: 0,
10585 met_quorum: true,
10586 uncontested: Some("only candidate A produced a change".to_owned()),
10587 });
10588 state.reviews = vec![ReviewRound {
10592 round: 1,
10593 head: head.clone(),
10594 verified_head: Some(head.clone()),
10595 verified_at: Some(jiff::Timestamp::now()),
10596 reviews: Vec::new(),
10597 e2e: vec![CommandOutcome {
10598 command: format!(
10599 "CARGO_TARGET_DIR='{}' test -f README.md",
10600 cache_dir.display()
10601 ),
10602 code: None,
10603 output_tail: "waiting for the shared build cache".to_owned(),
10604 duration_ms: 0,
10605 resource_blocked: true,
10606 }],
10607 fix: None,
10608 blocking: 1,
10609 answered: 1,
10610 expected: 1,
10611 clean: false,
10612 verify_retried: false,
10613 e2e_deferred: false,
10614 e2e_defer_reason: None,
10615 progressed: false,
10616 vote_split: false,
10617 reconsideration: Vec::new(),
10618 verdict: None,
10619 }];
10620
10621 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
10622 let mut runner = Runner {
10623 state,
10624 roles: ResolvedRoles {
10625 implementers: Vec::new(),
10626 judges: Vec::new(),
10627 reviewers: Vec::new(),
10628 fixer: None,
10629 conductor: conductor(),
10630 implementer_roster: Vec::new(),
10631 judge_roster: Vec::new(),
10632 reviewer_roster: Vec::new(),
10633 },
10634 sem: Arc::new(Semaphore::new(1)),
10635 pause: Pause::new(),
10636 interrupt: Pause::new(),
10637 };
10638
10639 runner.review_loop().await.expect("review_loop");
10644
10645 assert_eq!(
10646 runner.state.reviews.len(),
10647 1,
10648 "no new round was started on top of the unresolved one: {:?}",
10649 runner.state.reviews
10650 );
10651 let last = &runner.state.reviews[0];
10652 assert_eq!(
10653 last.e2e_status(),
10654 E2eStatus::ResourceBlocked,
10655 "still contended: {last:?}"
10656 );
10657 assert!(
10658 last.verified_at.expect("still known") > first_attempt_at,
10659 "review_loop must have actually retried the check, not left it exactly as found"
10660 );
10661 assert_ne!(
10662 runner.state.status,
10663 RunStatus::Blocked,
10664 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
10665 runner.state.status
10666 );
10667
10668 held.release();
10669 }
10670
10671 #[tokio::test]
10672 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
10673 crate::run::pin_test_home();
10674 let tmp = tempfile::tempdir().expect("tempdir");
10675 let repo = tmp.path().join("repo");
10676 std::fs::create_dir_all(&repo).unwrap();
10677 init_repo(&repo);
10678
10679 let mut config = Config::default();
10680 config.merge.mode = MergeMode::Pr;
10681 config.graph.land = true;
10682 config.graph.land_approval = false;
10683
10684 let mut state = RunState::new(
10685 repo.clone(),
10686 "main".to_owned(),
10687 "deadbeef".to_owned(),
10688 "task".to_owned(),
10689 config,
10690 );
10691 state.candidates = vec![Candidate {
10692 index: 0,
10693 label: 'A',
10694 agent: "alpha".to_owned(),
10695 branch: "does-not-exist".to_owned(),
10696 worktree: repo.clone(),
10697 summary: String::new(),
10698 stat: String::new(),
10699 files: 0,
10700 commits: 0,
10701 empty: false,
10702 failed: None,
10703 verified_noop: None,
10704 duration_ms: 0,
10705 folded: false,
10706 }];
10707 state.tally = Some(Tally {
10708 first_choice: BTreeMap::from([('A', 1)]),
10709 borda: BTreeMap::new(),
10710 winner: 'A',
10711 rankings: 1,
10712 unanimous_initial: true,
10713 deliberated: false,
10714 changed_votes: 0,
10715 unanimous_final: true,
10716 tie_break: None,
10717 judges: 0,
10718 present: 0,
10719 quorum: 0,
10720 met_quorum: true,
10721 uncontested: Some("only candidate A produced a change".to_owned()),
10722 });
10723 state.reviews = vec![ReviewRound {
10724 round: 1,
10725 head: "deadbeef".to_owned(),
10726 verified_head: None,
10727 verified_at: None,
10728 reviews: Vec::new(),
10729 e2e: Vec::new(),
10730 fix: None,
10731 blocking: 0,
10732 answered: 0,
10733 expected: 0,
10734 clean: true,
10735 verify_retried: false,
10736 e2e_deferred: false,
10737 e2e_defer_reason: None,
10738 progressed: false,
10739 vote_split: false,
10740 reconsideration: Vec::new(),
10741 verdict: None,
10742 }];
10743 state.gate = vec![CommandOutcome {
10744 command: "test".to_owned(),
10745 code: Some(0),
10746 output_tail: String::new(),
10747 duration_ms: 0,
10748 resource_blocked: false,
10749 }];
10750 state.gate_ran = true;
10751 state.status = RunStatus::Landing;
10755 state.merge = Some(MergeOutcome {
10756 mode: MergeMode::Pr,
10757 ok: true,
10758 detail: "https://example.invalid/x/y/pull/1".to_owned(),
10759 empty: false,
10760 });
10761
10762 ask_test_home();
10766 let store = ask::Questions::open();
10767 let q = ask_open_question(&store, &state.id);
10768
10769 let mut runner = Runner {
10770 state,
10771 roles: ResolvedRoles {
10772 implementers: Vec::new(),
10773 judges: Vec::new(),
10774 reviewers: Vec::new(),
10775 fixer: None,
10776 conductor: conductor(),
10777 implementer_roster: Vec::new(),
10778 judge_roster: Vec::new(),
10779 reviewer_roster: Vec::new(),
10780 },
10781 sem: Arc::new(Semaphore::new(1)),
10782 pause: Pause::new(),
10783 interrupt: Pause::new(),
10784 };
10785
10786 runner.execute().await.expect("execute");
10791
10792 assert_eq!(
10793 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10794 Some("https://example.invalid/x/y/pull/1"),
10795 "reentry must not push again or open a second pull request over the \
10796 one `land` is already watching"
10797 );
10798 assert_ne!(
10799 runner.state.status,
10800 RunStatus::Landing,
10801 "land could not actually reach the fake pull request, so it must \
10802 have given up rather than left the run silently parked forever"
10803 );
10804 assert_eq!(runner.state.status, RunStatus::Blocked);
10808 assert!(
10809 store.get(&q.id).unwrap().status.open(),
10810 "Blocked is still alive; settle_questions must have been a no-op here"
10811 );
10812 }
10813
10814 fn state_with_round(round: ReviewRound) -> RunState {
10815 let mut s = RunState::new(
10816 PathBuf::from("/repo"),
10817 "main".to_owned(),
10818 "abc1234".to_owned(),
10819 "add retries".to_owned(),
10820 Config::default(),
10821 );
10822 s.reviews = vec![round];
10823 s
10824 }
10825
10826 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
10827 crate::verdict::Finding {
10828 id: id.to_owned(),
10829 severity,
10830 file: None,
10831 line: None,
10832 title: title.to_owned(),
10833 detail: String::new(),
10834 }
10835 }
10836
10837 #[test]
10838 fn pr_body_names_open_findings_and_declined_ones() {
10839 let round = ReviewRound {
10840 round: 2,
10841 head: "deadbee".to_owned(),
10842 verified_head: None,
10843 verified_at: None,
10844 reviews: vec![ReviewRecord {
10845 attempts: 0,
10846 reviewer: 1,
10847 agent: "alpha".to_owned(),
10848 summary: String::new(),
10849 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
10850 vote: None,
10851 failed: None,
10852 duration_ms: 0,
10853 }],
10854 e2e: vec![CommandOutcome {
10855 command: "cargo test".to_owned(),
10856 code: Some(0),
10857 output_tail: String::new(),
10858 duration_ms: 0,
10859 resource_blocked: false,
10860 }],
10861 verify_retried: false,
10862 e2e_deferred: false,
10863 e2e_defer_reason: None,
10864 fix: Some(FixRecord {
10865 agent: "alpha".to_owned(),
10866 addressed: Vec::new(),
10867 rejected: vec![crate::verdict::Rejection {
10868 id: "R1-1-1".to_owned(),
10869 why: "not reachable from any caller".to_owned(),
10870 }],
10871 notes: String::new(),
10872 committed: true,
10873 failed: None,
10874 duration_ms: 0,
10875 continuation: None,
10876 }),
10877 blocking: 0,
10878 answered: 1,
10879 expected: 1,
10880 clean: false,
10881 progressed: true,
10882 vote_split: false,
10883 reconsideration: Vec::new(),
10884 verdict: None,
10885 };
10886 let state = state_with_round(round);
10887 let body = pr_message(&state, 'A').body;
10888
10889 assert!(body.contains("add retries"), "the task must still be there");
10890 assert!(body.contains("R2-1-1"), "{body}");
10891 assert!(body.contains("unused import"), "{body}");
10892 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
10893 assert!(
10894 body.contains("not reachable from any caller"),
10895 "the reason it was declined: {body}"
10896 );
10897 }
10898
10899 #[test]
10900 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
10901 let round = ReviewRound {
10902 round: 1,
10903 head: "deadbee".to_owned(),
10904 verified_head: None,
10905 verified_at: None,
10906 reviews: vec![ReviewRecord {
10907 attempts: 0,
10908 reviewer: 1,
10909 agent: "alpha".to_owned(),
10910 summary: String::new(),
10911 findings: Vec::new(),
10912 vote: None,
10913 failed: None,
10914 duration_ms: 0,
10915 }],
10916 e2e: Vec::new(),
10917 verify_retried: false,
10918 e2e_deferred: false,
10919 e2e_defer_reason: None,
10920 fix: None,
10921 blocking: 0,
10922 answered: 1,
10923 expected: 1,
10924 clean: true,
10925 progressed: false,
10926 vote_split: false,
10927 reconsideration: Vec::new(),
10928 verdict: None,
10929 };
10930 let state = state_with_round(round);
10931 let body = pr_message(&state, 'A').body;
10932 assert!(!body.contains("Open review findings"), "{body}");
10933 assert!(!body.contains("Declined"), "{body}");
10934 }
10935
10936 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
10937 let mut state = RunState::new(
10938 PathBuf::from("/repo"),
10939 "main".to_owned(),
10940 "abc1234".to_owned(),
10941 instruction.to_owned(),
10942 Config::default(),
10943 );
10944 state.candidates.push(Candidate {
10945 index: 0,
10946 label: 'A',
10947 agent: "alpha".to_owned(),
10948 branch: "magi/x/A".to_owned(),
10949 worktree: PathBuf::from("/wt"),
10950 summary: summary.to_owned(),
10951 stat: String::new(),
10952 files: 1,
10953 commits: 1,
10954 empty: false,
10955 failed: None,
10956 verified_noop: None,
10957 folded: false,
10958 duration_ms: 0,
10959 });
10960 state
10961 }
10962
10963 fn review_state(subjects: &[&str]) -> RunState {
10964 let mut state = state_with_summary(
10965 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
10966 "",
10967 );
10968 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
10969 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
10970 state
10971 }
10972
10973 #[tokio::test]
10976 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
10977 ask_test_home();
10978 let tmp = tempfile::tempdir().unwrap();
10979 let repo = tmp.path().join("repo");
10980 std::fs::create_dir_all(&repo).unwrap();
10981 init_repo(&repo);
10982 let origin = tmp.path().join("origin.git");
10983 let g = |dir: &Path, args: &[&str]| {
10984 let out = std::process::Command::new("git")
10985 .args(args)
10986 .current_dir(dir)
10987 .quiet()
10988 .output()
10989 .expect("spawn git");
10990 assert!(
10991 out.status.success(),
10992 "git {args:?}: {}",
10993 String::from_utf8_lossy(&out.stderr)
10994 );
10995 };
10996 g(
10997 tmp.path(),
10998 &[
10999 "clone",
11000 "--bare",
11001 "-q",
11002 repo.to_str().unwrap(),
11003 origin.to_str().unwrap(),
11004 ],
11005 );
11006 g(
11007 &repo,
11008 &["remote", "add", "origin", origin.to_str().unwrap()],
11009 );
11010 let c1 = git::rev_parse(&repo, "main").await.unwrap();
11011
11012 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
11014 g(&repo, &["add", "-A"]);
11015 g(
11016 &repo,
11017 &["commit", "-q", "-m", "chore(deps): update a crate"],
11018 );
11019 g(&repo, &["push", "-q", "origin", "main"]);
11020 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
11021 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
11022 g(&repo, &["add", "-A"]);
11023 g(
11024 &repo,
11025 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
11026 );
11027 g(&repo, &["checkout", "-q", "main"]);
11028
11029 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
11030 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
11031 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
11033 assert_eq!(fallback, c1);
11034
11035 let mut state = review_state(&[
11036 "chore(deps): update a crate",
11037 "fix(daemon): apply a chosen action",
11038 ]);
11039 state.repo = repo.clone();
11040 state.base_branch = "main".to_owned();
11041 state.base_commit = c1;
11042 refresh_reviewed_commits(&mut state, "feat/own").await;
11043 assert_eq!(
11044 state.reviewed_commits,
11045 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
11046 );
11047 assert_eq!(
11048 review_title(&state).as_deref(),
11049 Some("fix(daemon): apply a chosen action")
11050 );
11051 assert_eq!(
11052 leaked_subjects(&state, "feat/own").await,
11053 Some(vec!["chore(deps): update a crate".to_owned()])
11054 );
11055 state.config.merge.remote = "nowhere".to_owned();
11058 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
11059 }
11060
11061 #[test]
11062 fn pr_message_review_single_commit_uses_its_subject() {
11063 let state = review_state(&["feat(nats): per-role user"]);
11064 let m = pr_message(&state, 'A');
11065 assert_eq!(m.title, "feat(nats): per-role user");
11066 assert!(!m.body.contains("Review the work already"), "{}", m.body);
11067 assert!(m.body.contains("## Commits under review"), "{}", m.body);
11068 }
11069
11070 #[test]
11071 fn pr_message_review_multi_commit_takes_the_oldest() {
11072 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
11073 let m = pr_message(&state, 'A');
11074 assert_eq!(m.title, "feat: the change");
11075 for s in ["feat: the change", "fix: typo", "fix: again"] {
11076 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
11077 }
11078 }
11079
11080 #[test]
11081 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
11082 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
11083 let mut state = review_state(&[first, "fix: later fixup"]);
11084 state.candidates[0].branch = "機能/ログイン".to_owned();
11085 let m = pr_message(&state, 'A');
11086 assert!(
11087 m.title.starts_with("chore: land candidate A of run"),
11088 "{}",
11089 m.title
11090 );
11091 }
11092 }
11093
11094 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
11095 BranchFacts {
11096 commits: commits
11097 .iter()
11098 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
11099 .collect(),
11100 stat: stat.to_owned(),
11101 }
11102 }
11103
11104 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
11105 let mut r = review_round(true, 0, 1, 1, true, true);
11106 r.round = round;
11107 r.fix = notes.map(|n| FixRecord {
11108 agent: "fixer".to_owned(),
11109 addressed: Vec::new(),
11110 rejected: Vec::new(),
11111 notes: n.to_owned(),
11112 committed: true,
11113 failed: None,
11114 duration_ms: 0,
11115 continuation: None,
11116 });
11117 r
11118 }
11119
11120 #[test]
11121 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
11122 let state = review_state(&["ignored"]);
11123 let f = facts(
11124 &[
11125 (
11126 "fix(login): resolve PATH on macOS",
11127 "Login shells skip rc files.",
11128 ),
11129 ("fix: address review", ""),
11130 ],
11131 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
11132 );
11133 let m = pr_message_with(&state, 'A', Some(&f));
11134 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
11135 assert!(
11136 m.body
11137 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
11138 "{}",
11139 m.body
11140 );
11141 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
11142 assert!(m.body.contains("## Diff stat"), "{}", m.body);
11143 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
11144 for banned in [
11145 "Review the work already",
11146 "no task statement",
11147 "Original task",
11148 ] {
11149 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
11150 }
11151 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
11152 }
11153
11154 #[test]
11155 fn pr_message_review_truncates_a_huge_first_commit_body() {
11156 let state = review_state(&["ignored"]);
11157 let f = facts(
11158 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
11159 "s",
11160 );
11161 let m = pr_message_with(&state, 'A', Some(&f));
11162 assert!(m.body.len() < 30_000, "{}", m.body.len());
11163 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
11164 assert!(
11165 m.body.contains("- ... 1 more commit(s)"),
11166 "{}",
11167 m.body.len()
11168 );
11169 assert!(m.body.ends_with("magi:candidate-a\n"));
11170 }
11171
11172 #[test]
11173 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
11174 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
11175 assert_eq!(m.title, "feat: x");
11176 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
11177 assert!(!m.body.contains("Diff stat"), "{}", m.body);
11178 assert!(!m.body.contains("no task statement"), "{}", m.body);
11179 }
11180
11181 #[test]
11182 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
11183 let mut state = review_state(&["日本語の件名"]);
11184 state.candidates[0].branch = "fix/macos-login-path".to_owned();
11185 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
11186 state.candidates[0].branch = "機能/ログイン".to_owned();
11187 assert!(
11188 pr_message(&state, 'A')
11189 .title
11190 .starts_with("chore: land candidate A")
11191 );
11192 }
11193
11194 #[test]
11195 fn pr_message_review_fixes_survive_a_clean_final_round() {
11196 let mut state = review_state(&["feat: x"]);
11197 state.reviews = vec![
11198 round_with_notes(1, Some("handled the PATH case")),
11199 round_with_notes(2, None),
11200 ];
11201 let body = pr_message(&state, 'A').body;
11202 assert!(
11203 body.contains("## Review fixes\n\nhandled the PATH case\n"),
11204 "{body}"
11205 );
11206 assert!(!body.contains("### Round"), "{body}");
11207
11208 state.reviews = vec![
11209 round_with_notes(1, Some("first fix")),
11210 round_with_notes(2, Some("")),
11211 round_with_notes(3, Some("second fix")),
11212 round_with_notes(4, None),
11213 ];
11214 let body = pr_message(&state, 'A').body;
11215 assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
11216 assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
11217 assert!(!body.contains("### Round 2"), "{body}");
11218 }
11219
11220 #[test]
11221 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
11222 let state = state_with_summary(
11223 "add retries to the client",
11224 "TITLE: feat: retries\n\nDid it.",
11225 );
11226 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
11227 assert_eq!(m.title, "feat: retries");
11228 assert!(
11229 m.body.contains("<summary>Original task</summary>"),
11230 "{}",
11231 m.body
11232 );
11233 assert!(!m.body.contains("Commits under review"), "{}", m.body);
11234 assert!(
11235 m.body
11236 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
11237 "{}",
11238 m.body
11239 );
11240 }
11241
11242 #[test]
11243 fn pr_message_review_bounds_a_long_english_subject() {
11244 let long = format!("feat: {}", "word ".repeat(100));
11245 let m = pr_message(&review_state(&[&long]), 'A');
11246 assert!(m.title.starts_with("feat: word"), "{}", m.title);
11247 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11248 }
11249
11250 #[test]
11251 fn pr_message_implementation_run_is_unchanged_by_review_support() {
11252 let state = state_with_summary("add retries\n\ndetails", "- did some things");
11253 let m = pr_message(&state, 'A');
11254 assert_eq!(m.title, "add retries");
11255 assert!(m.body.contains("<summary>Original task</summary>"));
11256 assert!(!m.body.contains("Commits under review"));
11257 assert_eq!(landing_subject_source(&state), state.instruction);
11258 }
11259
11260 #[test]
11261 fn review_run_squash_subject_is_the_change_not_the_prompt() {
11262 let state = review_state(&["feat: the change", "fix: typo"]);
11263 let source = landing_subject_source(&state);
11264 assert_eq!(land::merge_subject("", &source), "feat: the change");
11265 assert_eq!(
11266 land::merge_subject("magi: candidate A (uncommitted work)", &source),
11267 "feat: the change"
11268 );
11269 assert_eq!(
11271 land::merge_subject("feat: renamed by hand", &source),
11272 "feat: renamed by hand"
11273 );
11274 let blank = review_state(&["日本語"]);
11275 assert!(
11276 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
11277 );
11278 }
11279
11280 #[test]
11281 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
11282 let state = review_state(&["feat: the change"]);
11283 let source = landing_subject_source(&state);
11284 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
11285 assert_eq!(
11286 land::merge_subject(landing_title(&state, old), &source),
11287 "feat: the change"
11288 );
11289 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
11290 let task = state_with_summary("add retries", "");
11291 assert_eq!(landing_title(&task, old), old);
11292 }
11293
11294 #[test]
11295 fn pr_message_describes_the_change_not_the_task() {
11296 let state = state_with_summary(
11297 "今回やってほしいこと: results projector を直す",
11298 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
11299 );
11300 let m = pr_message(&state, 'A');
11301 assert_eq!(m.title, "fix(web): batch the runs list reads");
11302 assert!(
11303 m.body.starts_with("## Summary\n\n- reads run.json once"),
11304 "{}",
11305 m.body
11306 );
11307 assert!(!m.body.contains("TITLE:"), "{}", m.body);
11308 let task_at = m.body.find("今回やってほしいこと").unwrap();
11309 let details_at = m.body.find("<details>").unwrap();
11310 assert!(
11311 details_at < task_at,
11312 "the task lives inside <details>: {}",
11313 m.body
11314 );
11315 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
11316 assert!(m.body.contains("magi:candidate-a"));
11317 }
11318
11319 #[test]
11320 fn pr_message_falls_back_to_the_task_without_a_title_line() {
11321 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
11322 let m = pr_message(&state, 'A');
11323 assert_eq!(m.title, "add retries");
11324 assert!(
11325 m.body.contains("## Summary\n\n- did some things"),
11326 "{}",
11327 m.body
11328 );
11329
11330 let none = RunState::new(
11331 PathBuf::from("/repo"),
11332 "main".to_owned(),
11333 "abc1234".to_owned(),
11334 "add retries".to_owned(),
11335 Config::default(),
11336 );
11337 let m = pr_message(&none, 'A');
11338 assert_eq!(m.title, "add retries");
11339 assert!(!m.body.contains("## Summary"), "{}", m.body);
11340 }
11341
11342 #[test]
11343 fn pr_message_refuses_the_candidate_commit_subject() {
11344 for bad in [
11345 "TITLE: magi: candidate A (uncommitted work)",
11346 "TITLE: chore: stuff (uncommitted work)",
11347 "TITLE: ",
11348 ] {
11349 let state = state_with_summary("add retries", bad);
11350 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
11351 }
11352 }
11353
11354 #[test]
11355 fn pr_message_bounds_a_very_long_task_and_title() {
11356 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
11357 let state = state_with_summary(&long, "- nothing");
11358 let m = pr_message(&state, 'A');
11359 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11360 assert!(!m.title.contains('\n'));
11361
11362 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
11363 let m = pr_message(&state, 'A');
11364 assert!(m.title.starts_with("feat: "));
11365 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11366 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
11367 }
11368
11369 fn long_title_of(instruction: &str) -> String {
11370 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
11371 }
11372
11373 #[test]
11374 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
11375 let first = "Make the landing path keep a readable title for long tasks";
11376 let line = format!(
11377 "{first}. {}",
11378 "And then keep going with more words ".repeat(20)
11379 );
11380 let t = long_title_of(&line);
11381 assert_eq!(t, first);
11382 assert!(!t.starts_with("chore: land"));
11383 }
11384
11385 #[test]
11386 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
11387 let line = "word ".repeat(200);
11388 let t = long_title_of(&line);
11389 assert!(t.ends_with("word..."), "{t}");
11390 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
11391 }
11392
11393 #[test]
11394 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
11395 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
11396 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
11397 }
11398 }
11399
11400 #[test]
11401 fn pr_message_title_limit_is_exact() {
11402 let at = "a".repeat(PR_TITLE_MAX);
11403 assert_eq!(long_title_of(&at), at);
11404 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
11405 assert!(over.ends_with("..."), "{over}");
11406 assert_eq!(over.chars().count(), PR_TITLE_MAX);
11407 }
11408
11409 #[test]
11410 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
11411 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
11412 let t = long_title_of(&line);
11413 assert!(t.ends_with("..."), "{t}");
11414 assert!(t.is_ascii(), "{t}");
11415 }
11416
11417 #[test]
11418 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
11419 let line = format!(
11420 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
11421 "plus more filler words ".repeat(20)
11422 );
11423 let t = long_title_of(&line);
11424 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
11425 }
11426
11427 #[test]
11428 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
11429 let line = format!("\n\n# {}", "title words ".repeat(40));
11430 let state = state_with_summary(&line, "- nothing");
11431 let m = pr_message(&state, 'A');
11432 assert!(m.title.starts_with("title words"), "{}", m.title);
11433 assert_eq!(
11434 land::merge_subject(&m.title, &landing_subject_source(&state)),
11435 m.title
11436 );
11437 assert_eq!(
11439 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
11440 "feat: renamed by hand"
11441 );
11442 }
11443
11444 #[test]
11445 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
11446 let mut state = state_with_summary(
11450 "add retries",
11451 "TITLE: fix(web): batch reads\n- reads run.json once",
11452 );
11453 state.config.graph.language = "ja".to_owned();
11454 let m = pr_message(&state, 'A');
11455 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
11456
11457 let task = "今回やってほしいこと: results projector を直す";
11460 let mut state = state_with_summary(task, "- no title line");
11461 state.config.graph.language = "ja".to_owned();
11462 let m = pr_message(&state, 'A');
11463 assert_eq!(
11464 m.title,
11465 format!("chore: land candidate A of run {}", state.id)
11466 );
11467 assert!(
11468 m.body.contains(&format!(
11469 "<summary>Original task</summary>\n\n{task}\n\n</details>"
11470 )),
11471 "{}",
11472 m.body
11473 );
11474 }
11475
11476 #[test]
11477 fn pr_message_scrubs_home_paths_and_addresses() {
11478 let state = state_with_summary(
11479 "fix it in /Users/someone/src/x",
11480 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
11481 );
11482 let m = pr_message(&state, 'A');
11483 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
11484 assert!(!m.body.contains(leak), "{}", m.body);
11485 }
11486 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
11487 }
11488
11489 #[test]
11490 fn pr_message_survives_a_task_that_closes_details() {
11491 let state = state_with_summary("a </details> b", "TITLE: fix: x");
11492 let m = pr_message(&state, 'A');
11493 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
11494 }
11495
11496 #[test]
11497 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
11498 let cmd = manual_merge_command(
11499 MergeStyle::Squash,
11500 Path::new("/repo"),
11501 "b",
11502 "fix: \"quoted\" $(x) `y`\n\nbody",
11503 );
11504 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
11505 }
11506
11507 #[test]
11508 fn manual_merge_command_matches_the_configured_style() {
11509 let repo = Path::new("/repo");
11510 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
11511
11512 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
11513 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
11514
11515 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
11516 assert_eq!(
11517 squash,
11518 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
11519 \"Merge magi run 0832 (candidate A)\""
11520 );
11521
11522 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
11523 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
11524 }
11525
11526 #[test]
11527 fn a_nudge_gets_a_quarter_of_the_budget() {
11528 assert_eq!(retry_budget(secs(1200), true), secs(300));
11530 assert_eq!(retry_budget(secs(3600), true), secs(900));
11531 }
11532
11533 #[test]
11534 fn a_resent_prompt_keeps_the_whole_budget() {
11535 assert_eq!(retry_budget(secs(1200), false), secs(1200));
11538 assert_eq!(retry_budget(secs(60), false), secs(60));
11539 }
11540
11541 #[test]
11542 fn the_floor_never_exceeds_the_original_budget() {
11543 assert_eq!(retry_budget(secs(60), true), secs(60));
11547 assert_eq!(retry_budget(secs(480), true), secs(120));
11548 assert_eq!(retry_budget(secs(0), true), secs(0));
11549 }
11550
11551 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
11552 agent::CommandEvidence {
11553 id: "item1".to_owned(),
11554 description: "cargo test".to_owned(),
11555 exit_code,
11556 result_summary: String::new(),
11557 source: "codex".to_owned(),
11558 }
11559 }
11560
11561 #[test]
11562 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
11563 assert!(!has_unconfirmed_command(&[]));
11567 }
11568
11569 #[test]
11570 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
11571 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
11575 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
11576 assert!(!has_unconfirmed_command(&[
11577 evidence(Some(0)),
11578 evidence(Some(101))
11579 ]));
11580 }
11581
11582 #[test]
11583 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
11584 assert!(has_unconfirmed_command(&[
11585 evidence(Some(0)),
11586 evidence(None)
11587 ]));
11588 }
11589
11590 #[test]
11591 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
11592 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11593 assert_eq!(
11594 verified_noop_claim(true, &[], text).as_deref(),
11595 Some("already fixed by b32cfc4, on main.")
11596 );
11597 }
11598
11599 #[test]
11600 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
11601 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11604 assert!(verified_noop_claim(false, &[], text).is_none());
11605 }
11606
11607 #[test]
11608 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
11609 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11610 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
11611 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
11613 }
11614
11615 #[test]
11616 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
11617 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
11618 }
11619
11620 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
11623 runner.state.candidates = shape
11624 .iter()
11625 .enumerate()
11626 .map(|(i, &(empty, verified))| Candidate {
11627 index: i,
11628 label: (b'A' + i as u8) as char,
11629 agent: "sonnet".to_owned(),
11630 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
11631 worktree: PathBuf::from(format!("/wt/{i}")),
11632 summary: String::new(),
11633 stat: String::new(),
11634 files: 0,
11635 commits: 0,
11636 empty,
11637 failed: None,
11638 verified_noop: verified.map(str::to_owned),
11639 duration_ms: 0,
11640 folded: false,
11641 })
11642 .collect();
11643 }
11644
11645 #[test]
11646 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
11647 ask_test_home();
11648 let mut runner = runner_at(RunStatus::Implementing);
11649 set_candidates(
11650 &mut runner,
11651 &[
11652 (true, Some("already on main at b32cfc4")),
11653 (true, Some("same fix, see the existing test")),
11654 ],
11655 );
11656
11657 runner
11658 .after_implement()
11659 .expect("a verified no-op is not an error");
11660
11661 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
11662 }
11663
11664 #[test]
11665 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
11666 ask_test_home();
11667 let mut runner = runner_at(RunStatus::Implementing);
11668 set_candidates(
11672 &mut runner,
11673 &[(true, Some("already on main at b32cfc4")), (true, None)],
11674 );
11675
11676 let err = runner
11677 .after_implement()
11678 .expect_err("an unverified empty candidate must still fail the run");
11679
11680 assert!(
11681 err.to_string().contains("no candidate produced a change"),
11682 "{err}"
11683 );
11684 assert_eq!(runner.state.status, RunStatus::Failed);
11685 }
11686
11687 #[test]
11688 fn after_implement_still_fails_an_ordinary_all_empty_run() {
11689 ask_test_home();
11690 let mut runner = runner_at(RunStatus::Implementing);
11691 set_candidates(&mut runner, &[(true, None), (true, None)]);
11692
11693 let err = runner
11694 .after_implement()
11695 .expect_err("no candidate declared anything; this is an ordinary failure");
11696
11697 assert!(
11698 err.to_string().contains("no candidate produced a change"),
11699 "{err}"
11700 );
11701 assert_eq!(runner.state.status, RunStatus::Failed);
11702 }
11703}