1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::land;
41use crate::proc::Quiet as _;
42use crate::prompt::{
43 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
44};
45use crate::queue;
46use crate::refs;
47use crate::run::{
48 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
49 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord, Handover,
50 JobRecord, JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome,
51 OperatorFixRequest, Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState,
52 RunStatus, SeatHistory, Tally, VoteRecord, tail, write_artifact,
53};
54use crate::verdict::{
55 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
56 ReviewVote, Severity,
57};
58
59const OUTPUT_TAIL: usize = 8_000;
61
62const EVENT_OUTPUT_TAIL: usize = 2_000;
65
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
69
70const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
87
88pub(crate) const STAGNANT_LIMIT: usize = 2;
102
103const BASE_SYNC_ROUNDS: usize = 4;
116
117const MAX_FIX_CONTINUATIONS: usize = 2;
133
134#[derive(Clone)]
140struct SeatJob {
141 spec: AgentSpec,
142 seat: SeatState,
143 cwd: PathBuf,
144 prompt: String,
145 timeout: Duration,
146 allow_write: bool,
147 sessions: bool,
148 artifacts: PathBuf,
149 stem: String,
150 handover: Option<String>,
155}
156
157enum AgentOutcome {
169 Ok(AgentOutput),
171 Quota(AgentOutput),
173 Dropped(AgentOutput),
176 Failed(String),
178}
179
180#[derive(Debug, Clone, Default)]
207pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
208
209impl Pause {
210 #[must_use]
212 pub fn new() -> Self {
213 Self::default()
214 }
215
216 pub fn park(&self) {
218 self.0.store(true, Ordering::SeqCst);
219 }
220
221 pub fn park_because(&self, reason: impl Into<String>) {
227 let mut reason_guard = self
228 .1
229 .lock()
230 .unwrap_or_else(std::sync::PoisonError::into_inner);
231 if reason_guard.is_none() {
232 *reason_guard = Some(reason.into());
233 }
234 drop(reason_guard);
235 self.park();
236 }
237
238 #[must_use]
240 pub fn parked(&self) -> bool {
241 self.0.load(Ordering::SeqCst)
242 }
243
244 #[must_use]
246 pub fn reason(&self) -> Option<String> {
247 self.1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner)
250 .clone()
251 }
252}
253
254pub struct Runner {
256 pub state: RunState,
258 roles: ResolvedRoles,
259 sem: Arc<Semaphore>,
260 pause: Pause,
264 interrupt: Pause,
270}
271
272async fn review_base(
277 repo: &Path,
278 remote: &str,
279 base_branch: &str,
280 base_commit: &str,
281 branch: &str,
282) -> String {
283 review_base_checked(repo, remote, base_branch, base_commit, branch)
284 .await
285 .0
286}
287
288async fn review_base_checked(
293 repo: &Path,
294 remote: &str,
295 base_branch: &str,
296 base_commit: &str,
297 branch: &str,
298) -> (String, bool) {
299 let tracking = format!("{remote}/{base_branch}");
300 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
301 if git::rev_exists(repo, &tracking).await
302 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
303 && !mb.is_empty()
304 {
305 return (mb, fresh);
306 }
307 let mb = git::merge_base(repo, base_commit, branch)
308 .await
309 .ok()
310 .filter(|mb| !mb.is_empty())
311 .unwrap_or_else(|| base_commit.to_owned());
312 (mb, false)
313}
314
315pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
319 if !is_review_run(state) {
320 return;
321 }
322 let base = review_base(
323 &state.repo,
324 &state.config.merge.remote,
325 &state.base_branch,
326 &state.base_commit,
327 branch,
328 )
329 .await;
330 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
331 && !subjects.is_empty()
332 && state.reviewed_commits.as_ref() != Some(&subjects)
333 {
334 state.reviewed_commits = Some(subjects);
335 state.save().ok();
336 }
337}
338
339async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
343 let (base, trusted) = review_base_checked(
344 &state.repo,
345 &state.config.merge.remote,
346 &state.base_branch,
347 &state.base_commit,
348 branch,
349 )
350 .await;
351 if !trusted {
352 return None;
353 }
354 git::subjects(&state.repo, &state.base_commit, &base)
355 .await
356 .ok()
357}
358
359fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
363 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
364 if is_leak(computed) {
365 return false;
366 }
367 let cur = current.trim();
368 cur.is_empty()
369 || cur.starts_with(REVIEW_PROMPT_OPENING)
370 || cur.starts_with("chore: land candidate")
371 || cur.starts_with("magi: candidate")
372 || is_leak(cur)
373}
374
375async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
404 let tracking = format!("{remote}/{branch}");
405 let fetched = git::fetch(repo, remote, branch).await;
406 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
407 let local_exists = git::branch_exists(repo, branch).await?;
408 if !fresh {
409 if !local_exists {
410 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
411 }
412 tracing::warn!(
413 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
414 );
415 return Ok(());
416 }
417 let remote_sha = git::rev_parse(repo, &tracking).await?;
418 if !local_exists {
419 git::git(repo, &["branch", branch, &tracking]).await?;
420 return Ok(());
421 }
422 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
423 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
424 return Ok(());
425 }
426 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
427 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
433 .await?
434 {
435 crate::reconcile::Reconciliation::Pushed => {
436 tracing::warn!(
437 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
438 short(&local_sha),
439 short(&remote_sha)
440 );
441 return Ok(());
442 }
443 crate::reconcile::Reconciliation::Placeholder => {}
444 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
445 }
446 }
447 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
448 if !out.ok() {
449 bail!(
450 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
451 short(&local_sha),
452 short(&remote_sha),
453 out.stderr
454 );
455 }
456 tracing::warn!(
457 "local `{branch}` was stale: fast-forwarded {} -> {}",
458 short(&local_sha),
459 short(&remote_sha)
460 );
461 Ok(())
462}
463
464async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
465 let tracking = format!("{remote}/{base_branch}");
466 let fetched = git::fetch(repo, remote, base_branch).await;
467 if let Ok(out) = &fetched
468 && out.ok()
469 && git::rev_exists(repo, &tracking).await
470 {
471 return git::rev_parse(repo, &tracking).await;
472 }
473 let why = match &fetched {
474 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
475 Ok(_) => format!("{remote} has no {base_branch}"),
476 Err(e) => e.to_string(),
477 };
478 bail!(
481 "cannot read {tracking} ({why}); refusing to branch off the local \
482 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
483 base / remote in magi.toml"
484 )
485}
486
487struct FixClaim {
504 path: PathBuf,
505}
506
507impl FixClaim {
508 fn acquire(dir: &Path) -> Result<Self> {
509 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
510 let path = dir.join("fix.lock");
511 match Self::create(&path) {
512 Ok(claim) => Ok(claim),
513 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
514 if Self::reclaim_if_dead(&path) {
515 Self::create(&path).with_context(|| format!("lock {}", path.display()))
516 } else {
517 bail!(
518 "another `magi fix` is already running for this run ({} exists)",
519 path.display()
520 )
521 }
522 }
523 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
524 }
525 }
526
527 fn create(path: &Path) -> std::io::Result<Self> {
528 let mut f = std::fs::OpenOptions::new()
529 .write(true)
530 .create_new(true)
531 .open(path)?;
532 use std::io::Write as _;
533 writeln!(f, "{}", std::process::id())?;
535 Ok(Self {
536 path: path.to_owned(),
537 })
538 }
539
540 fn reclaim_if_dead(path: &Path) -> bool {
544 let dead = std::fs::read_to_string(path)
545 .ok()
546 .and_then(|body| body.trim().parse::<u32>().ok())
547 .is_some_and(|pid| !crate::proc::pid_alive(pid));
548 dead && std::fs::remove_file(path).is_ok()
549 }
550}
551
552impl Drop for FixClaim {
553 fn drop(&mut self) {
554 let _ = std::fs::remove_file(&self.path);
555 }
556}
557
558impl Runner {
559 pub async fn start(
561 repo: &Path,
562 instruction: String,
563 config: Config,
564 origin: Origin,
565 ) -> Result<Self> {
566 Self::start_naming(repo, instruction, "", config, origin).await
567 }
568
569 pub async fn start_naming(
573 repo: &Path,
574 instruction: String,
575 also_scan: &str,
576 config: Config,
577 origin: Origin,
578 ) -> Result<Self> {
579 let repo = git::toplevel(repo).await?;
580 let missing = agent::missing_programs(&config.agents);
581 if !missing.is_empty() {
582 bail!(
583 "these agent programs are not on PATH: {}. Fix the roster in \
584 magi.toml or install them.",
585 missing.join(", ")
586 );
587 }
588 let base_branch =
589 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
590 .await?;
591 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
592 let roles = config.resolve_roles()?;
593 let max_parallel = config.graph.max_parallel.max(1);
594 let seeds = refs::resolve(
597 &repo,
598 &base_commit,
599 &config.merge.remote,
600 &format!("{also_scan}\n{instruction}"),
601 )
602 .await;
603 refs::plan(&repo, &seeds).await?;
604 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
605 state.origin = Some(origin);
609 for seed in &seeds {
610 state.event(
611 "seed",
612 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
613 );
614 }
615 state.seeds = seeds;
616 state.event("start", format!("run {} created", state.id));
617 state.save()?;
618 Ok(Self {
619 state,
620 roles,
621 sem: Arc::new(Semaphore::new(max_parallel)),
622 pause: Pause::new(),
623 interrupt: Pause::new(),
624 })
625 }
626
627 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
641 Self::review_taking_over(repo, branch, config, None, origin).await
642 }
643
644 pub async fn review_taking_over(
651 repo: &Path,
652 branch: &str,
653 config: Config,
654 takeover: Option<crate::handover::Takeover>,
655 origin: Origin,
656 ) -> Result<Self> {
657 let repo = git::toplevel(repo).await?;
658 let missing = agent::missing_programs(&config.agents);
659 if !missing.is_empty() {
660 bail!(
661 "these agent programs are not on PATH: {}. Fix the roster in \
662 magi.toml or install them.",
663 missing.join(", ")
664 );
665 }
666 let base_branch =
667 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
668 .await?;
669 if base_branch == branch {
670 bail!("`{branch}` is the base branch; there is nothing to review against");
671 }
672 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
673
674 let roles = config.resolve_roles()?;
675 let max_parallel = config.graph.max_parallel.max(1);
676 let mut state = RunState::new(
677 repo.clone(),
678 base_branch,
679 base_commit.clone(),
680 String::new(),
681 config,
682 );
683 state.origin = Some(origin);
684
685 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
692 earlier: Vec::new(),
693 home: crate::run::home(),
694 choice: None,
695 });
696 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
697 if let Some(released) = &released {
698 state.event(
699 "release",
700 format!(
701 "took `{branch}` over from run {}: its worktree was released: {}",
702 crate::run::short_of(&released.old_id),
703 released.audit
704 ),
705 );
706 }
707 if let Some(choice) = takeover.choice.as_ref()
711 && let Err(e) =
712 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
713 .await
714 {
715 if let Some(released) = &released {
716 released.restore(&repo, branch).await;
717 }
718 return Err(e.context("applying the owner's answer about the diverged branch"));
719 }
720 let opened =
721 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
722 if opened.is_err()
723 && let Some(released) = &released
724 {
725 released.restore(&repo, branch).await;
726 }
727 opened
728 }
729
730 async fn open_review(
733 repo: &Path,
734 branch: &str,
735 mut state: RunState,
736 roles: ResolvedRoles,
737 max_parallel: usize,
738 base_commit: String,
739 ) -> Result<Self> {
740 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
741 let start = review_base(
744 repo,
745 &state.config.merge.remote,
746 &state.base_branch,
747 &base_commit,
748 branch,
749 )
750 .await;
751 let log = git::log_oneline(repo, &start, branch)
752 .await
753 .unwrap_or_default();
754 let instruction = format!(
755 "Review the work already on branch `{branch}`. There is no task \
756 statement: what the change claims to do is whatever its commits \
757 say.\n\n{}",
758 if log.trim().is_empty() {
759 "(no commit messages)"
760 } else {
761 log.trim()
762 }
763 );
764 state.instruction = instruction;
765 state.reviewed_commits = Some(
766 git::subjects(repo, &start, branch)
767 .await
768 .unwrap_or_default(),
769 );
770
771 let worktree = state.worktree_root().join("under-review");
774 if let Some(parent) = worktree.parent() {
775 tokio::fs::create_dir_all(parent).await.ok();
776 }
777 let path = worktree.to_string_lossy().to_string();
778 git::git(repo, &["worktree", "add", &path, branch])
779 .await
780 .with_context(|| {
781 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
782 })?;
783
784 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
785 .await
786 .unwrap_or(0);
787 if commits == 0 {
788 git::worktree_remove(repo, &worktree).await.ok();
789 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
790 }
791 let files = git::changed_files(&worktree, &base_commit, "HEAD")
792 .await
793 .map(|f| f.len())
794 .unwrap_or(0);
795 if files == 0
796 && let (Ok(head_tree), Ok(base_tree)) = (
797 git::tree_of(&worktree, "HEAD").await,
798 git::tree_of(&worktree, &base_commit).await,
799 )
800 && head_tree == base_tree
801 {
802 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
803 git::worktree_remove(repo, &worktree).await.ok();
804 bail!(
805 "`{branch}` at {} has a tree identical to base {}; this usually means \
806 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
807 against `{}/{branch}`) rather than an empty change",
808 short(&head),
809 short(&base_commit),
810 state.config.merge.remote
811 );
812 }
813 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
814 .await
815 .unwrap_or_default();
816
817 state.candidates.push(Candidate {
818 index: 0,
819 label: 'A',
820 agent: EXISTING_BRANCH.to_owned(),
823 branch: branch.to_owned(),
824 worktree,
825 summary: String::new(),
826 stat,
827 files,
828 commits,
829 empty: false,
830 failed: None,
831 verified_noop: None,
832 duration_ms: 0,
833 folded: false,
834 });
835 state.tally = Some(Tally {
836 first_choice: BTreeMap::from([('A', 0)]),
837 borda: BTreeMap::new(),
838 winner: 'A',
839 rankings: 0,
840 unanimous_initial: false,
841 deliberated: false,
842 changed_votes: 0,
843 unanimous_final: false,
844 tie_break: None,
845 judges: 0,
849 present: 0,
850 quorum: 0,
851 met_quorum: true,
852 uncontested: Some("review-only run: nothing competed".to_owned()),
853 });
854 state.status = RunStatus::Reviewing;
855 state.event(
856 "start",
857 format!(
858 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
859 state.id
860 ),
861 );
862 state.save()?;
863 Ok(Self {
864 state,
865 roles,
866 sem: Arc::new(Semaphore::new(max_parallel)),
867 pause: Pause::new(),
868 interrupt: Pause::new(),
869 })
870 }
871
872 pub fn resume(id: &str) -> Result<Self> {
874 let state = RunState::load(id)?;
875 if let Some(to) = &state.released_to {
876 bail!(
877 "run {} cannot be resumed: its worktree was released to run {}",
878 state.short(),
879 crate::run::short_of(to)
880 );
881 }
882 let roles = state.config.resolve_roles()?;
883 let max_parallel = state.config.graph.max_parallel.max(1);
884 Ok(Self {
885 state,
886 roles,
887 sem: Arc::new(Semaphore::new(max_parallel)),
888 pause: Pause::new(),
889 interrupt: Pause::new(),
890 })
891 }
892
893 pub async fn execute(&mut self) -> Result<()> {
900 let result = self.execute_graph().await;
901 self.mark_driver_exited();
902 let ended = if result.is_err() {
903 Some(crate::notices::run_stopped(&self.state.id, &self.state))
904 } else {
905 crate::notices::run_ended(&self.state)
906 };
907 if let Some(notice) = ended {
908 crate::notices::raise(notice);
909 }
910 result
911 }
912
913 fn mark_driver_exited(&mut self) {
922 self.state.driver_exited = true;
923 let pid = std::process::id();
924 let Ok(mut disk) = RunState::load(&self.state.id) else {
925 return;
926 };
927 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
928 return;
929 }
930 disk.driver_exited = true;
931 if let Err(e) = disk.save() {
932 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
933 }
934 }
935
936 async fn execute_graph(&mut self) -> Result<()> {
937 self.state.parked = false;
942 self.state.clear_active();
949 if let Ok(disk) = RunState::load(&self.state.id)
967 && let Some(to) = &disk.released_to
968 {
969 bail!(
970 "run {} cannot continue: its worktree was released to run {}",
971 self.state.short(),
972 crate::run::short_of(to)
973 );
974 }
975 let pid = std::process::id();
976 self.state.driver_pid = Some(pid);
977 self.state.driver_started_at = crate::proc::process_started_at(pid);
978 self.state.driver_exited = false;
979 self.state.save()?;
980 if self.state.status == RunStatus::Stalled {
993 if self.recover_stall().await? {
994 self.finish_after_tally().await?;
995 } else {
996 self.state.save()?;
998 }
999 return Ok(());
1000 }
1001 if self.state.status == RunStatus::Landing {
1011 self.run_land().await?;
1012 self.settle_questions();
1017 return Ok(());
1018 }
1019 self.prep().await?;
1020 if self.park_here()? {
1021 return Ok(());
1022 }
1023 self.advise().await?;
1024 if self.park_here()? {
1025 return Ok(());
1026 }
1027 self.implement().await?;
1028 if self.park_here()? {
1029 return Ok(());
1030 }
1031 if self.state.status == RunStatus::VerifiedNoop {
1035 return Ok(());
1036 }
1037 self.judge().await?;
1038 if self.park_here()? {
1039 return Ok(());
1040 }
1041 self.deliberate().await?;
1042 if self.park_here()? {
1043 return Ok(());
1044 }
1045 self.vote().await?;
1046 if self.park_here()? {
1047 return Ok(());
1048 }
1049 self.tally()?;
1050 if self.state.status == RunStatus::Stalled {
1055 self.state.save()?;
1059 return Ok(());
1060 }
1061 self.finish_after_tally().await?;
1062 Ok(())
1063 }
1064
1065 fn park_here(&mut self) -> Result<bool> {
1072 if !self.pause.parked() && !self.interrupt.parked() {
1077 return Ok(false);
1078 }
1079 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
1080 Some(reason) => format!(
1081 "parked after `{}` ({reason}) — resume to carry on from here",
1082 self.state.status.as_str()
1083 ),
1084 None => format!(
1085 "parked after `{}` — resume to carry on from here",
1086 self.state.status.as_str()
1087 ),
1088 };
1089 self.state.event("park", why);
1090 self.state.parked = true;
1091 self.state.save()?;
1092 Ok(true)
1093 }
1094
1095 pub fn on_pause(&mut self, pause: Pause) {
1097 self.pause = pause;
1098 }
1099
1100 pub fn watch_interrupt(&mut self, pause: Pause) {
1106 self.interrupt = pause;
1107 }
1108
1109 fn settle_questions(&mut self) {
1129 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1130 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1131 }
1132 }
1133
1134 async fn finish_after_tally(&mut self) -> Result<()> {
1137 self.fold_losers().await?;
1138 self.sync_to_base().await?;
1143 if self.state.status == RunStatus::AlreadyInBase {
1144 return Ok(());
1145 }
1146 self.review_loop().await?;
1147 self.sync_to_base().await?;
1148 if self.state.status == RunStatus::AlreadyInBase {
1149 return Ok(());
1150 }
1151 self.gate().await?;
1152 self.merge().await?;
1153 self.state.save()?;
1154 Ok(())
1155 }
1156
1157 async fn prep(&mut self) -> Result<()> {
1160 if !self.state.candidates.is_empty() {
1161 return Ok(());
1162 }
1163 self.state.status = RunStatus::Prep;
1164 let repo = self.state.repo.clone();
1165 let base = self.state.base_commit.clone();
1166 let plan = refs::plan(&repo, &self.state.seeds).await?;
1167 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1168 let root = self.state.worktree_root();
1169 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1170
1171 let hooks_dir = self.state.dir().join("hooks");
1174 if self.state.config.blind.commit_msg_hook {
1175 std::fs::create_dir_all(&hooks_dir)
1176 .with_context(|| format!("create {}", hooks_dir.display()))?;
1177 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1178 let path = hooks_dir.join("commit-msg");
1179 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1180 make_executable(&path)?;
1181 git::acquire_worktree_config(&repo).await?;
1189 self.state.enabled_worktree_config = true;
1190 }
1191
1192 for (index, (spec, label)) in self
1193 .roles
1194 .implementers
1195 .clone()
1196 .into_iter()
1197 .zip(labels)
1198 .enumerate()
1199 {
1200 let branch = self.state.branch_for(label);
1201 let worktree = root.join(format!("cand-{label}"));
1202 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1203 if self.state.config.blind.commit_msg_hook {
1204 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1205 }
1206 git::local_exclude(&worktree, "/.magi/").await?;
1207 for pick in &plan.picks {
1208 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1209 self.state.status = RunStatus::Blocked;
1210 self.state
1211 .event("prep", format!("cannot apply referenced commit: {e}"));
1212 self.state.save()?;
1213 return Err(e);
1214 }
1215 }
1216 self.state.candidates.push(Candidate {
1217 index,
1218 label,
1219 agent: spec.id.clone(),
1220 branch,
1221 worktree,
1222 summary: String::new(),
1223 stat: String::new(),
1224 files: 0,
1225 commits: 0,
1226 empty: false,
1227 failed: None,
1228 verified_noop: None,
1229 duration_ms: 0,
1230 folded: false,
1231 });
1232 }
1233
1234 for j in 1..=self.roles.judges.len() {
1235 let wt = root.join(format!("judge-{j}"));
1236 if !wt.exists() {
1237 git::worktree_add_detached(&repo, &wt, &base).await?;
1238 }
1239 }
1240
1241 if self.state.config.graph.advise {
1249 for k in 1..=self.state.config.graph.advisors {
1250 let wt = root.join(format!("advisor-{k}"));
1251 if !wt.exists() {
1252 git::worktree_add_detached(&repo, &wt, &base).await?;
1253 }
1254 }
1255 }
1256
1257 let authors: Vec<&str> = self
1262 .roles
1263 .implementers
1264 .iter()
1265 .map(|a| a.id.as_str())
1266 .collect();
1267 let overlap: Vec<String> = self
1268 .roles
1269 .judges
1270 .iter()
1271 .enumerate()
1272 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1273 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1274 .collect();
1275 if !overlap.is_empty() {
1276 let note = format!(
1277 "{} also authored a candidate; blind, but the panel is less \
1278 independent than {} distinct agents would be",
1279 overlap.join(", "),
1280 self.roles.judges.len()
1281 );
1282 self.state.event("prep", note);
1283 }
1284
1285 self.state.event(
1286 "prep",
1287 format!(
1288 "{} candidates, {} judges, base {} ({})",
1289 self.state.candidates.len(),
1290 self.roles.judges.len(),
1291 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1292 self.state.base_branch
1293 ),
1294 );
1295 self.state.status = RunStatus::Implementing;
1296 self.state.save()?;
1297 Ok(())
1298 }
1299
1300 async fn advise(&mut self) -> Result<()> {
1333 let implement_untouched = self
1334 .state
1335 .candidates
1336 .iter()
1337 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1338 if !self.state.config.graph.advise || self.state.advise_attempted {
1339 return Ok(());
1340 }
1341 if !implement_untouched {
1342 self.state.event(
1343 "advise",
1344 "skipping the design-deliberation stage: at least one \
1345 candidate already shows implementation progress, so this \
1346 run is past the point the stage exists to run before"
1347 .to_owned(),
1348 );
1349 self.state.advise_attempted = true;
1350 self.state.save()?;
1351 return Ok(());
1352 }
1353 let run_id = self.state.id.clone();
1354 let prompts = self.state.config.prompts.clone();
1355 let instruction = self.state.instruction.clone();
1356 let language = self.state.config.graph.language.clone();
1357 let root = self.state.worktree_root();
1358 let n = self.state.config.graph.advisors;
1359 let where_recorded = self.state.dir().join("run.json");
1360
1361 let seats = match self.state.config.advisors() {
1362 Ok(seats) if !seats.is_empty() => seats,
1363 Ok(_) => {
1364 self.state.event(
1365 "advise",
1366 format!(
1367 "[graph] advisors is 0; skipping the design-deliberation \
1368 stage and continuing without a synthesis brief (see {})",
1369 where_recorded.display()
1370 ),
1371 );
1372 self.state.advise_attempted = true;
1373 self.state.save()?;
1374 return Ok(());
1375 }
1376 Err(e) => {
1377 self.state.event(
1378 "advise",
1379 format!(
1380 "could not resolve advisor seats ({e:#}); continuing \
1381 without a design-deliberation brief (see {})",
1382 where_recorded.display()
1383 ),
1384 );
1385 self.state.advise_attempted = true;
1386 self.state.save()?;
1387 return Ok(());
1388 }
1389 };
1390
1391 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1392 let artifacts = agent::artifacts_dir(&self.state.dir());
1393 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1394
1395 let mut jobs = Vec::new();
1396 for (i, spec) in seats.iter().cloned().enumerate() {
1397 let seat_key = format!("advisor-{}", i + 1);
1398 let seat = self.seat(&seat_key, &spec.id);
1399 jobs.push(SeatJob {
1400 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1401 spec,
1402 seat,
1403 cwd: worktrees[i % worktrees.len()].clone(),
1404 timeout,
1405 allow_write: false,
1406 sessions: false,
1407 artifacts: artifacts.clone(),
1408 stem: seat_key,
1409 handover: None,
1410 });
1411 }
1412
1413 self.state.event(
1414 "advise",
1415 format!(
1416 "{} advisor seat(s) sketching a design in parallel",
1417 jobs.len()
1418 ),
1419 );
1420 let mut quota_losses = Vec::new();
1421 let cache = self.state.config.cache_dir();
1422 let ctx = WaveCtx {
1423 carry_seats: false,
1424 run: &run_id,
1425 node: "advise",
1426 prompts: &prompts,
1427 cache: cache.as_deref(),
1428 round: None,
1429 };
1430 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1431 let results = ask_json_wave::<Proposal>(
1432 jobs,
1433 Arc::clone(&self.sem),
1434 self.state.config.graph.retries,
1435 &advisor_roster,
1436 &ctx,
1437 &mut quota_losses,
1438 &mut self.state,
1439 &|p: &Proposal| p.validate(),
1440 )
1441 .await;
1442 self.state.quota.extend(quota_losses);
1443
1444 let mut records = Vec::with_capacity(results.len());
1445 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1446 let agent_id = seat.agent.clone();
1447 self.state.seats.insert(seat.key.clone(), seat);
1448 match res {
1449 Ok((proposal, out)) => {
1450 self.state
1451 .event("advise", format!("advisor-{} proposed a design", i + 1));
1452 records.push(advise::AdvisorRecord::proposed(
1453 i + 1,
1454 agent_id,
1455 proposal,
1456 out.duration_ms,
1457 ));
1458 }
1459 Err(e) => {
1460 self.state.event(
1461 "advise",
1462 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1463 );
1464 records.push(advise::AdvisorRecord::failed(
1465 i + 1,
1466 agent_id,
1467 e.to_string(),
1468 ));
1469 }
1470 }
1471 }
1472
1473 let mut advice = advise::Advice {
1474 records,
1475 synthesis: None,
1476 };
1477 if advice.proposals().is_empty() {
1478 self.state.event(
1479 "advise",
1480 "no advisor produced a usable proposal; continuing without a \
1481 synthesis brief"
1482 .to_owned(),
1483 );
1484 } else {
1485 match self
1486 .synthesize_brief(
1487 &advice,
1488 &instruction,
1489 &language,
1490 &worktrees[0],
1491 &artifacts,
1492 &run_id,
1493 &prompts,
1494 cache.as_deref(),
1495 )
1496 .await
1497 {
1498 Ok(Some(text)) => {
1499 self.state.event(
1500 "advise",
1501 "synthesized a design brief for the implementer".to_owned(),
1502 );
1503 advice.synthesis = Some(text);
1504 }
1505 Ok(None) => {
1506 self.state.event(
1507 "advise",
1508 "the synthesis seat produced nothing usable; continuing \
1509 without a design brief"
1510 .to_owned(),
1511 );
1512 }
1513 Err(e) => {
1514 self.state.event(
1515 "advise",
1516 format!("could not synthesize a design brief: {e:#}"),
1517 );
1518 }
1519 }
1520 }
1521 advise::apply_reflection(&mut advice);
1522
1523 self.state.advice = Some(advice);
1524 self.state.advise_attempted = true;
1525 self.state.save()?;
1526 Ok(())
1527 }
1528
1529 #[allow(clippy::too_many_arguments)]
1541 async fn synthesize_brief(
1542 &mut self,
1543 advice: &advise::Advice,
1544 instruction: &str,
1545 language: &str,
1546 cwd: &Path,
1547 artifacts: &Path,
1548 run_id: &str,
1549 prompts: &Prompts,
1550 cache: Option<&Path>,
1551 ) -> Result<Option<String>> {
1552 let chain = agent::pick_chain(
1553 &self.state.config.agents,
1554 self.state.config.roles.synthesizer.as_ref(),
1555 &agent::installed,
1556 "synthesizer",
1557 )?;
1558 let proposals = advice.proposals();
1559 let mut prompt = prompt::with_overlay(
1560 prompt::synthesize_brief(instruction, &proposals, language),
1561 prompts.overlay("advise"),
1562 );
1563 if cache.is_some() {
1564 prompt.push('\n');
1569 prompt.push_str(&prompt::build_cache_note("advise", false));
1570 }
1571 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1572 let mut last = None;
1577 for (n, spec) in chain.iter().enumerate() {
1578 if n > 0 {
1579 self.state
1580 .event("advise", format!("synthesis falling back to {}", spec.id));
1581 }
1582 let mut seat = self.seat("advise-synthesis", &spec.id);
1583 let outcome = agent::invoke(
1584 spec,
1585 &mut seat,
1586 &Invocation {
1587 cwd,
1588 prompt: &prompt,
1589 timeout,
1590 allow_write: false,
1591 unsandboxed: false,
1592 sessions: false,
1593 artifacts,
1594 stem: &if n == 0 {
1595 "advise-synthesis".to_owned()
1596 } else {
1597 format!("advise-synthesis-{}", spec.id)
1598 },
1599 run: run_id,
1600 node: "advise",
1601 cache_dir: None,
1602 attachments: &[],
1603 writable: &[],
1604 },
1605 )
1606 .await;
1607 if outcome.is_ok() {
1608 self.state.seats.insert(seat.key.clone(), seat);
1609 }
1610 let advance = agent::chain_advances(&outcome);
1611 last = Some(outcome);
1612 if !advance {
1613 break;
1614 }
1615 }
1616 let out = last.expect("a chain holds at least one agent")?;
1619 if !out.usable() {
1620 return Ok(None);
1621 }
1622 let text =
1623 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1624 Ok((!text.trim().is_empty()).then_some(text))
1625 }
1626
1627 async fn implement(&mut self) -> Result<()> {
1630 let run_id = self.state.id.clone();
1635 let prompts = self.state.config.prompts.clone();
1636 let todo: Vec<usize> = self
1637 .state
1638 .candidates
1639 .iter()
1640 .enumerate()
1641 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1642 .map(|(i, _)| i)
1643 .collect();
1644 if todo.is_empty() {
1645 return self.after_implement();
1646 }
1647 self.state.status = RunStatus::Implementing;
1648
1649 let language = self.state.config.graph.language.clone();
1650 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1651 let sessions = self.state.config.graph.sessions;
1652 let artifacts = agent::artifacts_dir(&self.state.dir());
1653 let brief = self
1657 .state
1658 .advice
1659 .as_ref()
1660 .and_then(|a| a.synthesis.as_deref())
1661 .map(str::to_owned);
1662 let attachments = self.state.attachments.clone();
1663
1664 let mut jobs = Vec::new();
1665 for &i in &todo {
1666 let (index, label, worktree) = {
1667 let c = &self.state.candidates[i];
1668 (c.index, c.label, c.worktree.clone())
1669 };
1670 let spec = self.roles.implementers[index].clone();
1671 let seat_key = format!("impl-{label}");
1672 let seat = self.seat(&seat_key, &spec.id);
1673 let instruction = seeded_instruction(&self.state);
1674 jobs.push(SeatJob {
1675 spec,
1676 seat,
1677 prompt: prompt::implement(
1678 &instruction,
1679 &worktree.to_string_lossy(),
1680 &language,
1681 brief.as_deref(),
1682 &attachments,
1683 ),
1684 cwd: worktree,
1685 timeout,
1686 allow_write: true,
1687 sessions,
1688 artifacts: artifacts.clone(),
1689 stem: format!("impl-{label}"),
1690 handover: None,
1691 });
1692 }
1693
1694 self.state.event(
1695 "implement",
1696 format!("{} candidates in parallel", jobs.len()),
1697 );
1698 let mut sent = jobs.clone();
1704 let cache = self.state.config.cache_dir();
1705 let ctx = WaveCtx {
1706 carry_seats: false,
1707 run: &run_id,
1708 node: "implement",
1709 prompts: &prompts,
1710 cache: cache.as_deref(),
1711 round: None,
1712 };
1713 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1714 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1715 .await;
1716 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1717 .await;
1718 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1719 .await;
1720
1721 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1722 let seat_key = seat.key.clone();
1723 let agent = seat.agent.clone();
1732 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1733 self.state.seats.insert(seat.key.clone(), seat);
1734 let label = self.state.candidates[i].label;
1735 let worktree = self.state.candidates[i].worktree.clone();
1736 let base = self.state.base_commit.clone();
1737
1738 let (summary, duration, failed, verified_claim) = match out {
1739 AgentOutcome::Ok(o) => {
1740 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1741 let failed = (!o.usable()).then(|| {
1742 if o.timed_out {
1743 "agent timed out".to_owned()
1744 } else {
1745 format!("agent exited with {:?}", o.exit_code)
1746 }
1747 });
1748 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1749 (text, o.duration_ms, failed, verified_claim)
1750 }
1751 AgentOutcome::Dropped(o) => {
1757 let why = o
1758 .dropped
1759 .as_ref()
1760 .map(|d| d.why.as_str())
1761 .unwrap_or("the CLI ended the stream without delivering its answer");
1762 (
1763 String::new(),
1764 o.duration_ms,
1765 Some(format!("the CLI dropped the stream ({why})")),
1766 None,
1767 )
1768 }
1769 AgentOutcome::Quota(o) => {
1770 self.state.quota.push(QuotaLoss {
1771 seat: seat_key,
1772 node: "implement".to_owned(),
1773 at: Timestamp::now(),
1774 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1775 });
1776 (
1777 String::new(),
1778 o.duration_ms,
1779 Some("rate limited (quota); produced no change".to_owned()),
1780 None,
1781 )
1782 }
1783 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1784 };
1785
1786 let rescued = match git::rescue_commit(
1789 &worktree,
1790 &format!("magi: candidate {label} (uncommitted work)"),
1791 )
1792 .await
1793 {
1794 Ok(r) => {
1795 self.state.note_withheld("implement", &r.withheld);
1796 r.committed
1797 }
1798 Err(_) => false,
1799 };
1800 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1801 .await
1802 .unwrap_or(0);
1803 let patch = git::diff(&worktree, &base, "HEAD")
1804 .await
1805 .unwrap_or_default();
1806 let stat = git::diff_stat(&worktree, &base, "HEAD")
1807 .await
1808 .unwrap_or_default();
1809 let files = git::changed_files(&worktree, &base, "HEAD")
1810 .await
1811 .map(|f| f.len())
1812 .unwrap_or(0);
1813 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1814
1815 let c = &mut self.state.candidates[i];
1816 if !exhausted_the_fallback_chain {
1817 c.agent = agent;
1818 }
1819 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1820 c.stat = stat;
1821 c.files = files;
1822 c.commits = commits;
1823 c.duration_ms = duration;
1824 c.empty = commits == 0 || patch.trim().is_empty();
1825 c.failed = match failed {
1828 Some(_) if c.empty => failed,
1829 _ => None,
1830 };
1831 c.verified_noop = if c.empty { verified_claim } else { None };
1836 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1837 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1838 (None, true, Some(_), _) => {
1839 format!("candidate {label}: no change produced (agent-verified no-op)")
1840 }
1841 (None, true, None, _) => format!("candidate {label}: no change produced"),
1842 (None, false, _, true) => {
1843 format!(
1844 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1845 )
1846 }
1847 (None, false, _, false) => {
1848 format!("candidate {label}: {files} files, {commits} commits")
1849 }
1850 };
1851 self.state.event("implement", note);
1852 self.state.save()?;
1853 }
1854
1855 self.after_implement()
1856 }
1857
1858 async fn resume_undelivered(
1886 &mut self,
1887 results: &mut [(usize, SeatState, AgentOutcome)],
1888 sent: &[SeatJob],
1889 prompts: &Prompts,
1890 run_id: &str,
1891 ) {
1892 for (wi, seat, out) in results.iter_mut() {
1893 let Some(dropped) = (match &*out {
1894 AgentOutcome::Dropped(o) => o.dropped.clone(),
1895 _ => None,
1896 }) else {
1897 continue;
1898 };
1899 let Some(job) = sent.get(*wi) else { continue };
1900 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1902 self.state.event(
1903 "implement",
1904 format!(
1905 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1906 work is in the tree",
1907 seat.key, dropped.output_tokens, dropped.why
1908 ),
1909 );
1910 continue;
1911 }
1912 if !has_context(&job.spec, seat, job.sessions) {
1920 self.state.event(
1921 "implement",
1922 format!(
1923 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1924 is no session left to resume",
1925 seat.key, dropped.output_tokens, dropped.why
1926 ),
1927 );
1928 continue;
1929 }
1930 self.state.event(
1931 "implement",
1932 format!(
1933 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1934 conversation",
1935 seat.key, dropped.output_tokens, dropped.why
1936 ),
1937 );
1938 let mut retry = job.clone();
1939 retry.seat = seat.clone();
1940 retry.prompt = prompt::resume_after_drop(&dropped.why);
1941 retry.timeout = retry_budget(job.timeout, true);
1942 retry.stem = format!("{}-resume", job.stem);
1943 let cache = self.state.config.cache_dir();
1944 let ctx = WaveCtx {
1945 carry_seats: false,
1946 run: run_id,
1947 node: "implement",
1948 prompts,
1949 cache: cache.as_deref(),
1950 round: None,
1951 };
1952 let (resumed_seat, resumed) =
1953 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1954 *seat = resumed_seat;
1955 *out = resumed;
1956 }
1957 }
1958
1959 async fn resume_seat_handovers(
2024 &mut self,
2025 results: &mut [(usize, SeatState, AgentOutcome)],
2026 sent: &mut [SeatJob],
2027 prompts: &Prompts,
2028 run_id: &str,
2029 ) {
2030 let instruction = seeded_instruction(&self.state);
2031 let language = self.state.config.graph.language.clone();
2032 let brief = self
2033 .state
2034 .advice
2035 .as_ref()
2036 .and_then(|a| a.synthesis.as_deref())
2037 .map(str::to_owned);
2038 let attachments = self.state.attachments.clone();
2039 for (wi, seat, out) in results.iter_mut() {
2040 let others: BTreeSet<String> = sent
2043 .iter()
2044 .enumerate()
2045 .filter(|(j, _)| j != wi)
2046 .map(|(_, j)| j.spec.id.clone())
2047 .collect();
2048 let Some(job) = sent.get_mut(*wi) else {
2049 continue;
2050 };
2051 let start = self
2056 .roles
2057 .implementer_roster
2058 .iter()
2059 .position(|s| s.id == job.spec.id)
2060 .unwrap_or(0);
2061 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2062 let mut fallback_attempt = 0usize;
2063 let mut prev: Option<FailClass> = None;
2064 while let Some(cur) = FailClass::of(&*out) {
2065 if !should_hand_over(prev.as_ref(), &cur) {
2066 break;
2067 }
2068 let Some(next) =
2069 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2070 .cloned()
2071 else {
2072 break;
2073 };
2074 tried.insert(next.id.clone());
2075 fallback_attempt += 1;
2076
2077 if let Ok(r) = git::rescue_commit(
2078 &job.cwd,
2079 &format!(
2080 "magi: candidate {} (uncommitted work before {} fallback)",
2081 seat.key,
2082 if cur == FailClass::Quota {
2083 "quota"
2084 } else {
2085 "handover"
2086 }
2087 ),
2088 )
2089 .await
2090 {
2091 self.state.note_withheld("implement", &r.withheld);
2092 }
2093
2094 record_handover(
2095 &mut self.state,
2096 "implement",
2097 &seat.key,
2098 &seat.agent,
2099 &next.id,
2100 &cur,
2101 &fail_reason(&*out),
2102 );
2103 prev = Some(cur.clone());
2104
2105 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2106 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2107 job.spec = next.clone();
2115 let mut retry = job.clone();
2116 retry.seat = new_seat;
2117 retry.prompt = prompt::implement(
2118 &instruction,
2119 &job.cwd.to_string_lossy(),
2120 &language,
2121 brief.as_deref(),
2122 &attachments,
2123 );
2124 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2125 let cache = self.state.config.cache_dir();
2126 let ctx = WaveCtx {
2127 carry_seats: false,
2128 run: run_id,
2129 node: "implement",
2130 prompts,
2131 cache: cache.as_deref(),
2132 round: None,
2133 };
2134 let (fallback_seat, fallback_out) = run_one(
2135 retry,
2136 Arc::clone(&self.sem),
2137 &ctx,
2138 &mut self.state,
2139 fallback_attempt,
2140 )
2141 .await;
2142 *seat = fallback_seat;
2143 *out = fallback_out;
2144 }
2145 }
2146 }
2147
2148 async fn resume_unconfirmed_commands(
2172 &mut self,
2173 results: &mut [(usize, SeatState, AgentOutcome)],
2174 sent: &[SeatJob],
2175 prompts: &Prompts,
2176 run_id: &str,
2177 ) {
2178 for (wi, seat, out) in results.iter_mut() {
2179 let AgentOutcome::Ok(o) = &*out else {
2180 continue;
2181 };
2182 if !has_unconfirmed_command(&o.commands) {
2183 continue;
2184 }
2185 let Some(job) = sent.get(*wi) else { continue };
2186 if !has_context(&job.spec, seat, job.sessions) {
2187 self.state.event(
2188 "implement",
2189 format!(
2190 "{}: the reply named a command whose own CLI never confirmed the exit \
2191 status of, but there is no session left to resume",
2192 seat.key
2193 ),
2194 );
2195 continue;
2196 }
2197 self.state.event(
2198 "implement",
2199 format!(
2200 "{}: the reply named a command whose own CLI never confirmed the exit \
2201 status of; resuming the conversation",
2202 seat.key
2203 ),
2204 );
2205 let mut retry = job.clone();
2206 retry.seat = seat.clone();
2207 retry.prompt = prompt::resume_incomplete(
2208 "a command in your last reply had no confirmed exit status",
2209 );
2210 retry.timeout = retry_budget(job.timeout, true);
2211 retry.stem = format!("{}-confirm", job.stem);
2212 let cache = self.state.config.cache_dir();
2213 let ctx = WaveCtx {
2214 carry_seats: false,
2215 run: run_id,
2216 node: "implement",
2217 prompts,
2218 cache: cache.as_deref(),
2219 round: None,
2220 };
2221 let (resumed_seat, resumed) =
2222 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2223 *seat = resumed_seat;
2224 *out = resumed;
2225 }
2226 }
2227
2228 async fn continue_fix_report(
2249 &mut self,
2250 mut seat: SeatState,
2251 parse_err: String,
2252 job: &SeatJob,
2253 prompts: &Prompts,
2254 run_id: &str,
2255 round: usize,
2256 ) -> (
2257 SeatState,
2258 Option<FixReport>,
2259 Option<String>,
2260 ContinuationRecord,
2261 ) {
2262 let mut last_err = parse_err;
2263 let mut cumulative_wait_ms = 0u64;
2264 let mut attempts = 0usize;
2265 loop {
2266 if !has_context(&job.spec, &seat, job.sessions) {
2267 self.state.event(
2268 "fix",
2269 format!(
2270 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2271 session left to resume into"
2272 ),
2273 );
2274 let outcome = if attempts == 0 {
2275 ContinuationOutcome::NoSession
2276 } else {
2277 ContinuationOutcome::Exhausted
2278 };
2279 return (
2280 seat,
2281 None,
2282 Some(format!("unparsable fix report: {last_err}")),
2283 ContinuationRecord {
2284 attempts,
2285 cumulative_wait_ms,
2286 outcome,
2287 },
2288 );
2289 }
2290 if attempts >= MAX_FIX_CONTINUATIONS {
2291 self.state.event(
2292 "fix",
2293 format!(
2294 "round {round}: fixer's reply still had no adoption report after \
2295 {attempts} continuation(s) ({last_err}); giving up"
2296 ),
2297 );
2298 return (
2299 seat,
2300 None,
2301 Some(format!(
2302 "unparsable fix report after {attempts} continuation(s): {last_err}"
2303 )),
2304 ContinuationRecord {
2305 attempts,
2306 cumulative_wait_ms,
2307 outcome: ContinuationOutcome::Exhausted,
2308 },
2309 );
2310 }
2311 attempts += 1;
2312 self.state.event(
2313 "fix",
2314 format!(
2315 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2316 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2317 ),
2318 );
2319 let mut retry = job.clone();
2320 retry.seat = seat.clone();
2321 retry.prompt = prompt::resume_incomplete(&last_err);
2322 retry.timeout = retry_budget(job.timeout, true);
2323 retry.stem = format!("{}-continue{attempts}", job.stem);
2324 let cache = self.state.config.cache_dir();
2325 let ctx = WaveCtx {
2326 carry_seats: false,
2327 run: run_id,
2328 node: "fix",
2329 prompts,
2330 cache: cache.as_deref(),
2331 round: Some(round),
2332 };
2333 let (resumed_seat, resumed_out) = run_one(
2334 retry,
2335 Arc::clone(&self.sem),
2336 &ctx,
2337 &mut self.state,
2338 attempts,
2339 )
2340 .await;
2341 seat = resumed_seat;
2342 match resumed_out {
2343 AgentOutcome::Ok(o) => {
2344 cumulative_wait_ms += o.duration_ms;
2345 match verdict::extract_json::<FixReport>(&o.text) {
2346 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2347 self.state.event(
2348 "fix",
2349 format!(
2350 "round {round}: fixer's adoption report recovered after \
2351 {attempts} continuation(s)"
2352 ),
2353 );
2354 return (
2355 seat,
2356 Some(report),
2357 None,
2358 ContinuationRecord {
2359 attempts,
2360 cumulative_wait_ms,
2361 outcome: ContinuationOutcome::Resumed,
2362 },
2363 );
2364 }
2365 Ok(_) => {
2373 last_err = "the reply parsed, but it reported a command whose own CLI \
2374 never confirmed an exit status"
2375 .to_owned();
2376 }
2377 Err(e) => last_err = e.to_string(),
2378 }
2379 }
2380 AgentOutcome::Quota(o) => {
2381 cumulative_wait_ms += o.duration_ms;
2382 self.state.quota.push(QuotaLoss {
2383 seat: seat.key.clone(),
2384 node: "fix".to_owned(),
2385 at: Timestamp::now(),
2386 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2387 });
2388 self.state.event(
2389 "fix",
2390 format!(
2391 "round {round}: continuation rate limited (quota); not retrying now"
2392 ),
2393 );
2394 return (
2395 seat,
2396 None,
2397 Some("rate limited (quota) while recovering the fix report".to_owned()),
2398 ContinuationRecord {
2399 attempts,
2400 cumulative_wait_ms,
2401 outcome: ContinuationOutcome::QuotaLost,
2402 },
2403 );
2404 }
2405 AgentOutcome::Dropped(o) => {
2406 cumulative_wait_ms += o.duration_ms;
2407 let why = o
2408 .dropped
2409 .as_ref()
2410 .map(|d| d.why.as_str())
2411 .unwrap_or("the CLI ended the stream without delivering its answer");
2412 last_err = format!("the CLI dropped the stream ({why})");
2413 }
2414 AgentOutcome::Failed(e) => last_err = e,
2415 }
2416 }
2417 }
2418
2419 fn after_implement(&mut self) -> Result<()> {
2420 if self.state.leaks.is_empty() {
2422 let cfg = self.state.config.blind.clone();
2423 let mut leaks = Vec::new();
2424 for c in &self.state.candidates {
2425 let Some(patch) =
2426 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2427 else {
2428 continue;
2429 };
2430 leaks.extend(blind::scan(
2431 &format!("candidate {} patch", c.label),
2432 &patch,
2433 &cfg.vendor_tokens,
2434 ));
2435 }
2436 if !leaks.is_empty() {
2437 let summary = leaks
2438 .iter()
2439 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2440 .collect::<Vec<_>>()
2441 .join(", ");
2442 match cfg.on_leak {
2443 LeakPolicy::Fail => {
2444 self.state.status = RunStatus::Failed;
2445 self.state
2446 .event("blind", format!("vendor text in a patch: {summary}"));
2447 self.state.leaks = leaks;
2448 self.state.save()?;
2449 self.settle_questions();
2450 bail!(
2451 "blind.on_leak = \"fail\" and vendor text reached a \
2452 judged patch: {summary}"
2453 );
2454 }
2455 LeakPolicy::Redact => self.state.event(
2456 "blind",
2457 format!("redacting vendor text for judging: {summary}"),
2458 ),
2459 LeakPolicy::Warn => self.state.event(
2460 "blind",
2461 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2462 ),
2463 }
2464 self.state.leaks = leaks;
2465 }
2466 }
2467
2468 if self.state.viable().is_empty() {
2469 if self.state.all_candidates_verified_noop() {
2470 self.state.status = RunStatus::VerifiedNoop;
2481 self.state.save()?;
2482 self.settle_questions();
2483 return Ok(());
2484 }
2485 self.state.status = RunStatus::Failed;
2486 self.state.save()?;
2487 self.settle_questions();
2488 bail!("no candidate produced a change; nothing to judge");
2489 }
2490 self.state.status = RunStatus::Judging;
2491 self.state.save()?;
2492 Ok(())
2493 }
2494
2495 async fn judge(&mut self) -> Result<()> {
2498 let run_id = self.state.id.clone();
2503 let prompts = self.state.config.prompts.clone();
2504 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2505 return Ok(());
2506 }
2507 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2508 if viable.len() == 1 {
2509 self.state.judge_skipped = true;
2516 self.state.event(
2517 "judge",
2518 format!(
2519 "only candidate {} produced a change; judging skipped",
2520 viable[0].label
2521 ),
2522 );
2523 self.state.save()?;
2524 return Ok(());
2525 }
2526 self.state.status = RunStatus::Judging;
2527
2528 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2529 let language = self.state.config.graph.language.clone();
2530 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2531 let sessions = self.state.config.graph.sessions;
2532 let artifacts = agent::artifacts_dir(&self.state.dir());
2533 let root = self.state.worktree_root();
2534 let base_short = short(&self.state.base_commit);
2535
2536 let mut jobs = Vec::new();
2537 let mut orders = Vec::new();
2538 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2539 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2540 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2541 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2542 let seat_key = format!("judge-{}", j + 1);
2543 let seat = self.seat(&seat_key, &spec.id);
2544 jobs.push(SeatJob {
2545 prompt: prompt::judge(
2546 &self.state.instruction,
2547 &views,
2548 self.roles.judges.len(),
2549 &base_short,
2550 &language,
2551 ),
2552 spec,
2553 seat,
2554 cwd: root.join(format!("judge-{}", j + 1)),
2555 timeout,
2556 allow_write: false,
2557 sessions,
2558 artifacts: artifacts.clone(),
2559 stem: format!("judge-{}", j + 1),
2560 handover: None,
2561 });
2562 }
2563
2564 self.state.event(
2565 "judge",
2566 format!(
2567 "{} judges ranking {} candidates blind",
2568 jobs.len(),
2569 viable.len()
2570 ),
2571 );
2572 let labels_for_check = labels.clone();
2573 let mut quota_losses = Vec::new();
2574 let cache = self.state.config.cache_dir();
2575 let ctx = WaveCtx {
2576 carry_seats: false,
2577 run: &run_id,
2578 node: "judge",
2579 prompts: &prompts,
2580 cache: cache.as_deref(),
2581 round: None,
2582 };
2583 let results = ask_json_wave::<Ranking>(
2584 jobs,
2585 Arc::clone(&self.sem),
2586 self.state.config.graph.retries,
2587 &self.roles.judge_roster,
2588 &ctx,
2589 &mut quota_losses,
2590 &mut self.state,
2591 &move |r: &Ranking| r.validate(&labels_for_check),
2592 )
2593 .await;
2594 self.state.quota.extend(quota_losses);
2595
2596 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2597 let agent_id = seat.agent.clone();
2598 self.state.seats.insert(seat.key.clone(), seat);
2599 let mut record = Judgement {
2600 judge: j + 1,
2601 seat: format!("judge-{}", j + 1),
2602 agent: agent_id,
2603 ranking: Vec::new(),
2604 reasons: BTreeMap::new(),
2605 confidence: None,
2606 order: orders[j].clone(),
2607 failed: None,
2608 duration_ms: 0,
2609 };
2610 match res {
2611 Ok((ranking, out)) => {
2612 record.ranking = ranking.normalized();
2613 record.reasons = ranking.reasons;
2614 record.confidence = ranking.confidence;
2615 record.duration_ms = out.duration_ms;
2616 self.state.event(
2617 "judge",
2618 format!(
2619 "judge {} ranked {}",
2620 j + 1,
2621 record.ranking.iter().collect::<String>()
2622 ),
2623 );
2624 }
2625 Err(e) => {
2626 record.failed = Some(e.to_string());
2627 self.state
2628 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2629 }
2630 }
2631 self.state.judgements.push(record);
2632 self.state.save()?;
2633 }
2634 Ok(())
2635 }
2636
2637 async fn deliberate(&mut self) -> Result<()> {
2640 let run_id = self.state.id.clone();
2645 let prompts = self.state.config.prompts.clone();
2646 if !self.state.deliberation.is_empty() {
2647 return Ok(());
2648 }
2649 let tops: Vec<char> = self
2650 .state
2651 .judgements
2652 .iter()
2653 .filter_map(|j| j.ranking.first().copied())
2654 .collect();
2655 let rounds = self.state.config.graph.deliberate_rounds;
2656 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2657 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2658 self.state.event(
2659 "deliberate",
2660 format!("judges agreed on {} outright; no deliberation", tops[0]),
2661 );
2662 }
2663 self.state.status = RunStatus::Voting;
2664 self.state.save()?;
2665 return Ok(());
2666 }
2667
2668 self.state.status = RunStatus::Deliberating;
2669 self.state.event(
2670 "deliberate",
2671 format!(
2672 "split: first choices were {} — opening {rounds} round(s)",
2673 tops.iter().collect::<String>()
2674 ),
2675 );
2676
2677 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2678 let language = self.state.config.graph.language.clone();
2679 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2680 let sessions = self.state.config.graph.sessions;
2681 let artifacts = agent::artifacts_dir(&self.state.dir());
2682 let root = self.state.worktree_root();
2683 let base_short = short(&self.state.base_commit);
2684
2685 for round in 1..=rounds {
2689 let mut turns: Vec<DeliberationTurn> = Vec::new();
2690 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2691 if self.state.judgements[j].failed.is_some() {
2692 continue;
2693 }
2694 let seat_key = format!("judge-{}", j + 1);
2695 let spec = self.occupant(&seat_key, spec);
2696 let mut seat = self.seat(&seat_key, &spec.id);
2697 let transcript = self.transcript(&turns, j);
2698 let build = |context: Option<&str>| {
2699 prompt::deliberate(
2700 &self.state.instruction,
2701 context,
2702 &transcript,
2703 round,
2704 rounds,
2705 &language,
2706 )
2707 };
2708 let block = self.candidate_block(&viable, &base_short);
2709 let full = build(Some(&block));
2710 let text = if has_context(&spec, &seat, sessions) {
2711 build(None)
2712 } else {
2713 full.clone()
2714 };
2715 let job = SeatJob {
2716 spec,
2717 seat: seat.clone(),
2718 prompt: text,
2719 cwd: root.join(format!("judge-{}", j + 1)),
2720 timeout,
2721 allow_write: false,
2722 sessions,
2723 artifacts: artifacts.clone(),
2724 stem: format!("delib-{round}-judge-{}", j + 1),
2725 handover: Some(full),
2726 };
2727 let cache = self.state.config.cache_dir();
2728 let ctx = WaveCtx {
2729 carry_seats: false,
2730 run: &run_id,
2731 node: "deliberate",
2732 prompts: &prompts,
2733 cache: cache.as_deref(),
2734 round: None,
2735 };
2736 let mut losses = Vec::new();
2740 let mut results = ask_wave_with::<String>(
2741 vec![job],
2742 Arc::clone(&self.sem),
2743 0,
2744 &self.roles.judge_roster,
2745 &ctx,
2746 &mut losses,
2747 &mut self.state,
2748 &|text: &str| {
2749 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
2750 },
2751 )
2752 .await;
2753 self.state.quota.extend(losses);
2754 let (updated, res, _) = results.pop().expect("one job in, one result out");
2755 seat = updated;
2756 let agent_id = seat.agent.clone();
2757 self.state.seats.insert(seat.key.clone(), seat);
2758 let body = match res {
2759 Ok((body, _)) => body,
2760 Err(e) => {
2763 self.state
2764 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2765 continue;
2766 }
2767 };
2768 let tentative = verdict::extract_json::<Position>(&body)
2769 .ok()
2770 .and_then(|p| p.tentative)
2771 .and_then(|s| s.trim().chars().next())
2772 .map(|c| c.to_ascii_uppercase());
2773 self.state.event(
2774 "deliberate",
2775 format!(
2776 "round {round}: judge {} now favours {}",
2777 j + 1,
2778 tentative.map_or("—".to_owned(), |c| c.to_string())
2779 ),
2780 );
2781 turns.push(DeliberationTurn {
2782 judge: j + 1,
2783 agent: agent_id,
2784 body: blind::sanitize_prose(&body, &self.state.config.blind),
2785 tentative,
2786 });
2787 }
2788 self.state
2789 .deliberation
2790 .push(DeliberationRound { round, turns });
2791 self.state.save()?;
2792 }
2793
2794 self.state.status = RunStatus::Voting;
2795 self.state.save()?;
2796 Ok(())
2797 }
2798
2799 async fn vote(&mut self) -> Result<()> {
2802 let run_id = self.state.id.clone();
2807 let prompts = self.state.config.prompts.clone();
2808 if !self.state.votes.is_empty() {
2809 return Ok(());
2810 }
2811 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2812 if viable.len() == 1 {
2813 return Ok(());
2814 }
2815 self.state.status = RunStatus::Voting;
2816
2817 let language = self.state.config.graph.language.clone();
2818 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2819 let sessions = self.state.config.graph.sessions;
2820 let artifacts = agent::artifacts_dir(&self.state.dir());
2821 let root = self.state.worktree_root();
2822 let base_short = short(&self.state.base_commit);
2823 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2824
2825 let mut jobs = Vec::new();
2826 let mut seats_at = Vec::new();
2827 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2828 if self
2829 .state
2830 .judgements
2831 .get(j)
2832 .is_some_and(|r| r.failed.is_some())
2833 {
2834 continue;
2835 }
2836 let seat_key = format!("judge-{}", j + 1);
2837 let spec = self.occupant(&seat_key, spec);
2838 let seat = self.seat(&seat_key, &spec.id);
2839 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
2840 let text = if has_context(&spec, &seat, sessions) {
2841 prompt::final_vote(&viable, &language)
2842 } else {
2843 full.clone()
2844 };
2845 jobs.push(SeatJob {
2846 spec,
2847 seat,
2848 prompt: text,
2849 cwd: root.join(format!("judge-{}", j + 1)),
2850 timeout,
2851 allow_write: false,
2852 sessions,
2853 artifacts: artifacts.clone(),
2854 stem: format!("vote-judge-{}", j + 1),
2855 handover: Some(full),
2856 });
2857 seats_at.push(j);
2858 }
2859
2860 self.state.event(
2861 "vote",
2862 format!(
2863 "collecting {} final votes one by one, privately",
2864 jobs.len()
2865 ),
2866 );
2867 let allowed = viable.clone();
2868 let mut quota_losses = Vec::new();
2869 let cache = self.state.config.cache_dir();
2870 let ctx = WaveCtx {
2871 carry_seats: false,
2872 run: &run_id,
2873 node: "vote",
2874 prompts: &prompts,
2875 cache: cache.as_deref(),
2876 round: None,
2877 };
2878 let results = ask_json_wave::<FinalVote>(
2879 jobs,
2880 Arc::clone(&self.sem),
2881 self.state.config.graph.retries,
2882 &self.roles.judge_roster,
2883 &ctx,
2884 &mut quota_losses,
2885 &mut self.state,
2886 &move |v: &FinalVote| match v.label() {
2887 Some(c) if allowed.contains(&c) => Ok(()),
2888 other => bail!("vote {other:?} is not one of {allowed:?}"),
2889 },
2890 )
2891 .await;
2892 self.state.quota.extend(quota_losses);
2893
2894 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2895 let agent_id = seat.agent.clone();
2896 self.state.seats.insert(seat.key.clone(), seat);
2897 let initial = self
2898 .state
2899 .judgements
2900 .get(j)
2901 .and_then(|r| r.ranking.first().copied());
2902 let mut record = VoteRecord {
2903 judge: j + 1,
2904 agent: agent_id,
2905 vote: None,
2906 reason: String::new(),
2907 changed: false,
2908 };
2909 match res {
2910 Ok((v, _)) => {
2911 record.vote = v.label();
2912 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2913 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2914 self.state.event(
2915 "vote",
2916 format!(
2917 "judge {} voted {}{}",
2918 j + 1,
2919 record.vote.unwrap_or('?'),
2920 if record.changed { " (changed)" } else { "" }
2921 ),
2922 );
2923 }
2924 Err(e) => {
2925 self.state
2926 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2927 }
2928 }
2929 self.state.votes.push(record);
2930 self.state.save()?;
2931 }
2932 Ok(())
2933 }
2934
2935 fn tally(&mut self) -> Result<()> {
2938 if self.state.tally.is_some() {
2939 return Ok(());
2940 }
2941 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2942 let tops: Vec<char> = self
2943 .state
2944 .judgements
2945 .iter()
2946 .filter_map(|j| j.ranking.first().copied())
2947 .collect();
2948 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2949
2950 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2953 let mut cast: Vec<char> = Vec::new();
2954 for (i, j) in self.state.judgements.iter().enumerate() {
2955 let vote = self
2956 .state
2957 .votes
2958 .iter()
2959 .find(|v| v.judge == i + 1)
2960 .and_then(|v| v.vote)
2961 .or_else(|| j.ranking.first().copied());
2962 if let Some(v) = vote {
2963 *first_choice.entry(v).or_insert(0) += 1;
2964 cast.push(v);
2965 }
2966 }
2967
2968 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2969 for j in &self.state.judgements {
2970 let n = j.ranking.len();
2971 for (pos, label) in j.ranking.iter().enumerate() {
2972 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2973 }
2974 }
2975
2976 let best = first_choice.values().copied().max().unwrap_or(0);
2977 let mut leaders: Vec<char> = first_choice
2978 .iter()
2979 .filter(|(_, v)| **v == best)
2980 .map(|(k, _)| *k)
2981 .collect();
2982 let mut tie_break = None;
2983 if leaders.len() > 1 {
2984 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2985 let borda_leaders: Vec<char> = leaders
2986 .iter()
2987 .copied()
2988 .filter(|l| borda[l] == top_borda)
2989 .collect();
2990 tie_break = Some(if borda_leaders.len() == 1 {
2991 format!(
2992 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2993 leaders.len()
2994 )
2995 } else {
2996 format!(
2997 "{} way tie on both first-choice votes and Borda points, broken by label order",
2998 leaders.len()
2999 )
3000 });
3001 leaders = borda_leaders;
3002 leaders.sort_unstable();
3003 }
3004 let winner = *leaders
3005 .first()
3006 .or(viable.first())
3007 .context("no candidate to declare a winner from")?;
3008
3009 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3010 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3011 let deliberated = !self.state.deliberation.is_empty();
3012
3013 let quota_seats: std::collections::BTreeSet<&str> =
3017 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3018 let mut present = 0usize;
3019 for (i, j) in self.state.judgements.iter().enumerate() {
3020 if quota_seats.contains(j.seat.as_str()) {
3021 continue;
3022 }
3023 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3024 let voted = self
3025 .state
3026 .votes
3027 .iter()
3028 .any(|v| v.judge == i + 1 && v.vote.is_some());
3029 if ranked || voted {
3030 present += 1;
3031 }
3032 }
3033 let needs_quorum = viable.len() > 1;
3039 let judges_total = if needs_quorum {
3040 self.roles.judges.len()
3041 } else {
3042 0
3043 };
3044 let quorum = if needs_quorum {
3045 judges_total / 2 + 1
3046 } else {
3047 0
3048 };
3049 let met_quorum = !needs_quorum || present >= quorum;
3050 let uncontested = (!needs_quorum).then(|| {
3051 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3052 });
3053
3054 self.state.event(
3055 "tally",
3056 match &uncontested {
3057 Some(reason) => format!("winner {winner} — {reason}"),
3058 None => format!(
3059 "winner {winner} — votes {} | initial {} | {} changed | \
3060 {present}/{judges_total} judges{}",
3061 first_choice
3062 .iter()
3063 .map(|(k, v)| format!("{k}:{v}"))
3064 .collect::<Vec<_>>()
3065 .join(" "),
3066 if unanimous_initial {
3067 "unanimous"
3068 } else {
3069 "split"
3070 },
3071 changed_votes,
3072 if met_quorum {
3073 String::new()
3074 } else {
3075 format!(" — below quorum ({quorum} required)")
3076 },
3077 ),
3078 },
3079 );
3080 if !met_quorum {
3081 self.state.event(
3082 "stall",
3083 format!(
3084 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3085 the run stops here, resumable"
3086 ),
3087 );
3088 }
3089 self.state.tally = Some(Tally {
3090 first_choice,
3091 borda,
3092 winner,
3093 rankings: tops.len(),
3094 unanimous_initial,
3095 deliberated,
3096 changed_votes,
3097 unanimous_final,
3098 tie_break,
3099 judges: judges_total,
3100 present,
3101 quorum,
3102 met_quorum,
3103 uncontested,
3104 });
3105 self.state.status = if met_quorum {
3106 RunStatus::Reviewing
3107 } else {
3108 RunStatus::Stalled
3109 };
3110 self.state.save()?;
3111 Ok(())
3112 }
3113
3114 #[allow(clippy::too_many_lines)]
3135 async fn recover_stall(&mut self) -> Result<bool> {
3136 let run_id = self.state.id.clone();
3141 let prompts = self.state.config.prompts.clone();
3142 let quota_seats: BTreeSet<&str> =
3147 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3148 let absent: Vec<String> = self
3149 .state
3150 .judgements
3151 .iter()
3152 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3153 .map(|j| j.seat.clone())
3154 .collect();
3155 if absent.is_empty() {
3156 return Ok(false);
3157 }
3158 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3159 if viable.len() <= 1 {
3160 return Ok(false);
3161 }
3162 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3163 let language = self.state.config.graph.language.clone();
3164 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3165 let sessions = self.state.config.graph.sessions;
3166 let artifacts = agent::artifacts_dir(&self.state.dir());
3167 let root = self.state.worktree_root();
3168 let base_short = short(&self.state.base_commit);
3169 let candidates: Vec<Candidate> = viable.clone();
3170
3171 let mut positions: Vec<usize> = absent
3173 .iter()
3174 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3175 .collect();
3176 if positions.is_empty() {
3177 return Ok(false);
3178 }
3179 positions.sort_unstable();
3180 positions.dedup();
3181
3182 let mut judge_jobs = Vec::new();
3184 for &j in &positions {
3185 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3186 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3187 let seat_key = format!("judge-{}", j + 1);
3188 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3189 let seat = self.seat(&seat_key, &spec.id);
3190 judge_jobs.push(SeatJob {
3191 spec,
3192 seat,
3193 prompt: prompt::judge(
3194 &self.state.instruction,
3195 &views,
3196 self.roles.judges.len(),
3197 &base_short,
3198 &language,
3199 ),
3200 cwd: root.join(seat_key),
3201 timeout,
3202 allow_write: false,
3203 sessions,
3204 artifacts: artifacts.clone(),
3205 stem: format!("judge-{}-recover", j + 1),
3206 handover: None,
3207 });
3208 }
3209
3210 let labels_for_check = labels.clone();
3211 let mut judge_losses = Vec::new();
3212 let retries = self.state.config.graph.retries;
3213 let cache = self.state.config.cache_dir();
3214 let ctx = WaveCtx {
3215 carry_seats: false,
3216 run: &run_id,
3217 node: "judge",
3218 prompts: &prompts,
3219 cache: cache.as_deref(),
3220 round: None,
3221 };
3222 let results = ask_json_wave::<Ranking>(
3223 judge_jobs,
3224 Arc::clone(&self.sem),
3225 retries,
3226 &self.roles.judge_roster,
3227 &ctx,
3228 &mut judge_losses,
3229 &mut self.state,
3230 &move |r: &Ranking| r.validate(&labels_for_check),
3231 )
3232 .await;
3233
3234 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3236 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3237 let agent_id = seat.agent.clone();
3238 self.state.seats.insert(seat.key.clone(), seat);
3239 let record = &mut self.state.judgements[j];
3240 match res {
3241 Ok((ranking, out)) => {
3242 record.agent = agent_id;
3243 record.ranking = ranking.normalized();
3244 record.reasons = ranking.reasons;
3245 record.confidence = ranking.confidence;
3246 record.failed = None;
3247 record.duration_ms = out.duration_ms;
3248 recovered.insert(j);
3249 self.state.event(
3250 "recover",
3251 format!("judge {} ranked again after the limit", j + 1),
3252 );
3253 }
3254 Err(e) => {
3255 self.state
3256 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3257 }
3258 }
3259 }
3260
3261 let mut vote_jobs = Vec::new();
3263 let mut vote_pos: Vec<usize> = Vec::new();
3264 for &j in &recovered {
3265 let seat_key = format!("judge-{}", j + 1);
3266 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3267 let seat = self.seat(&seat_key, &spec.id);
3268 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3269 let text = if has_context(&spec, &seat, sessions) {
3270 prompt::final_vote(&labels, &language)
3271 } else {
3272 full.clone()
3273 };
3274 vote_jobs.push(SeatJob {
3275 spec,
3276 seat,
3277 prompt: text,
3278 cwd: root.join(seat_key),
3279 timeout,
3280 allow_write: false,
3281 sessions,
3282 artifacts: artifacts.clone(),
3283 stem: format!("vote-judge-{}-recover", j + 1),
3284 handover: Some(full),
3285 });
3286 vote_pos.push(j);
3287 }
3288 let allowed = labels.clone();
3289 let mut vote_losses = Vec::new();
3290 let vote_retries = self.state.config.graph.retries;
3291 let vote_cache = self.state.config.cache_dir();
3292 let ctx = WaveCtx {
3293 carry_seats: false,
3294 run: &run_id,
3295 node: "vote",
3296 prompts: &prompts,
3297 cache: vote_cache.as_deref(),
3298 round: None,
3299 };
3300 let votes = ask_json_wave::<FinalVote>(
3301 vote_jobs,
3302 Arc::clone(&self.sem),
3303 vote_retries,
3304 &self.roles.judge_roster,
3305 &ctx,
3306 &mut vote_losses,
3307 &mut self.state,
3308 &move |v: &FinalVote| match v.label() {
3309 Some(c) if allowed.contains(&c) => Ok(()),
3310 other => bail!("vote {other:?} is not one of {allowed:?}"),
3311 },
3312 )
3313 .await;
3314 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3315 let agent_id = seat.agent.clone();
3316 self.state.seats.insert(seat.key.clone(), seat);
3317 match res {
3318 Ok((v, _)) => {
3319 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3320 rec.vote = v.label();
3321 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3322 } else {
3323 self.state.votes.push(VoteRecord {
3324 judge: j + 1,
3325 agent: agent_id,
3326 vote: v.label(),
3327 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3328 changed: false,
3329 });
3330 }
3331 self.state.event(
3332 "recover",
3333 format!("judge {} voted again after the limit", j + 1),
3334 );
3335 }
3336 Err(e) => {
3337 self.state
3338 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3339 }
3340 }
3341 }
3342
3343 let recovered_keys: BTreeSet<String> = recovered
3347 .iter()
3348 .map(|&j| format!("judge-{}", j + 1))
3349 .collect();
3350 self.state
3351 .quota
3352 .retain(|q| !recovered_keys.contains(&q.seat));
3353 for loss in judge_losses.into_iter().chain(vote_losses) {
3357 if recovered_keys.contains(&loss.seat) {
3358 continue;
3359 }
3360 self.state.quota.retain(|q| q.seat != loss.seat);
3361 self.state.quota.push(loss);
3362 }
3363
3364 self.state.tally = None;
3366 self.tally()?;
3367 Ok(self
3368 .state
3369 .tally
3370 .as_ref()
3371 .map(|t| t.met_quorum)
3372 .unwrap_or(false))
3373 }
3374
3375 async fn fold_losers(&mut self) -> Result<()> {
3378 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3379 return Ok(());
3380 };
3381 let repo = self.state.repo.clone();
3382 let mut folded = Vec::new();
3383 for i in 0..self.state.candidates.len() {
3384 let c = &self.state.candidates[i];
3385 if c.label == winner || c.folded {
3386 continue;
3387 }
3388 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3389 git::worktree_remove(&repo, &wt).await.ok();
3390 git::branch_delete(&repo, &branch).await.ok();
3391 self.state.candidates[i].folded = true;
3392 folded.push(label.to_string());
3393 }
3394 let root = self.state.worktree_root();
3396 for j in 1..=self.roles.judges.len() {
3397 let wt = root.join(format!("judge-{j}"));
3398 if wt.exists() {
3399 git::worktree_remove(&repo, &wt).await.ok();
3400 }
3401 }
3402 if self.state.config.graph.advise {
3405 for k in 1..=self.state.config.graph.advisors {
3406 let wt = root.join(format!("advisor-{k}"));
3407 if wt.exists() {
3408 git::worktree_remove(&repo, &wt).await.ok();
3409 }
3410 }
3411 }
3412 if !folded.is_empty() {
3413 self.state
3414 .event("fold", format!("folded candidates {}", folded.join(", ")));
3415 self.state.save()?;
3416 }
3417 Ok(())
3418 }
3419
3420 async fn sync_to_base(&mut self) -> Result<()> {
3460 if self.state.status == RunStatus::AlreadyInBase {
3461 return Ok(());
3462 }
3463 let conflicted = self
3464 .state
3465 .base_sync
3466 .as_ref()
3467 .is_some_and(|s| s.conflict.is_some());
3468 let Some(winner) = self.state.winner().cloned() else {
3469 return Ok(());
3470 };
3471
3472 let repo = self.state.repo.clone();
3473 let remote = self.state.config.merge.remote.clone();
3474 let base_branch = self.state.base_branch.clone();
3475 let tracking = format!("{remote}/{base_branch}");
3476
3477 git::fetch(&repo, &remote, &base_branch).await.ok();
3478 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3483 return Ok(());
3484 };
3485
3486 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3487 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3488 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3489
3490 if (behind > 0 || conflicted || head == tip)
3499 && self
3500 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3501 .await?
3502 {
3503 return Ok(());
3504 }
3505 if conflicted {
3506 return Ok(());
3507 }
3508
3509 if behind == 0 {
3510 if let Some(from) = self
3519 .state
3520 .rebase_fixes
3521 .iter()
3522 .rev()
3523 .find_map(|r| r.from.clone())
3524 && from != head
3525 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3526 .await
3527 .is_ok_and(|o| o.ok())
3528 {
3529 git::sync_to_head(&winner.worktree).await?;
3530 }
3531 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3537 if let Some(why) = &conflict {
3538 self.state.status = RunStatus::Blocked;
3539 self.state.event("land", why.clone());
3540 }
3541 self.state.base_sync = Some(BaseSync {
3542 tip,
3543 behind: 0,
3544 attempts,
3545 conflict,
3546 already_in: None,
3547 });
3548 self.state.save()?;
3549 return Ok(());
3550 }
3551
3552 if attempts >= BASE_SYNC_ROUNDS {
3553 let why = format!(
3554 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3555 rebase(s); rebasing again would only race it",
3556 winner.branch
3557 );
3558 self.state.status = RunStatus::Blocked;
3559 self.state.base_sync = Some(BaseSync {
3560 tip,
3561 behind,
3562 attempts,
3563 conflict: Some(why.clone()),
3564 already_in: None,
3565 });
3566 self.state.event("land", why);
3567 self.state.save()?;
3568 return Ok(());
3569 }
3570
3571 self.state.event(
3572 "land",
3573 format!(
3574 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3575 winner.branch
3576 ),
3577 );
3578 self.state.save()?;
3579
3580 let branch_tracking = format!("{remote}/{}", winner.branch);
3586 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3587 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3588 git::rev_parse(&repo, &branch_tracking).await.ok()
3589 } else {
3590 None
3591 };
3592 if let Some(theirs) = &remote_tip
3596 && !git::is_ancestor(&repo, theirs, &head).await
3597 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3598 .await
3599 .is_ok_and(|missing| missing.is_empty())
3600 {
3601 let why = format!(
3602 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3603 them",
3604 short(theirs),
3605 winner.branch
3606 );
3607 self.state.status = RunStatus::Blocked;
3608 self.state.base_sync = Some(BaseSync {
3609 tip,
3610 behind,
3611 attempts,
3612 conflict: Some(why.clone()),
3613 already_in: None,
3614 });
3615 self.state.event("land", why);
3616 self.state.save()?;
3617 return Ok(());
3618 }
3619
3620 let scratch = self.state.dir().join("base-sync");
3621 let rebased = match crate::rebase::rebase_with_fixer(
3622 &mut self.state,
3623 &scratch,
3624 &winner.branch,
3625 &tracking,
3626 )
3627 .await
3628 {
3629 Ok(crate::rebase::Rebased::Applied) => Ok(None),
3630 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3631 Err(e) => Err(e),
3632 };
3633 let attempts = attempts + 1;
3634 match rebased {
3635 Ok(None) => {
3636 git::sync_to_head(&winner.worktree).await?;
3640 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
3641 let mut conflict = None;
3642 if let Some(pinned) = &remote_tip {
3643 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3644 match pushed {
3645 Ok(o) if o.ok() => self.state.event(
3646 "land",
3647 format!("pushed rebased {} to {remote}", winner.branch),
3648 ),
3649 Ok(o) => {
3650 conflict = Some(format!(
3651 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
3652 winner.branch,
3653 short(pinned),
3654 o.stderr.chars().take(600).collect::<String>()
3655 ));
3656 }
3657 Err(e) => {
3658 conflict = Some(format!(
3659 "rebased {} locally but could not push it: {e:#}",
3660 winner.branch
3661 ));
3662 }
3663 }
3664 }
3665 if let Some(why) = &conflict {
3666 self.state.status = RunStatus::Blocked;
3667 self.state.event("land", why.clone());
3668 }
3669 self.state.base_sync = Some(BaseSync {
3670 tip: tip.clone(),
3671 behind: 0,
3672 attempts,
3673 conflict,
3674 already_in: None,
3675 });
3676 self.state
3677 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3678 }
3679 Ok(Some(conflict)) => {
3680 let why = format!(
3681 "{} conflicts with {tracking} and did not rebase: {}",
3682 winner.branch,
3683 conflict.chars().take(600).collect::<String>()
3684 );
3685 self.state.status = RunStatus::Blocked;
3686 self.state.base_sync = Some(BaseSync {
3687 tip,
3688 behind,
3689 attempts,
3690 conflict: Some(why.clone()),
3691 already_in: None,
3692 });
3693 self.state.event("land", why);
3694 }
3695 Err(e) => {
3696 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3697 self.state.status = RunStatus::Blocked;
3698 self.state.base_sync = Some(BaseSync {
3699 tip,
3700 behind,
3701 attempts,
3702 conflict: Some(why.clone()),
3703 already_in: None,
3704 });
3705 self.state.event("land", why);
3706 }
3707 }
3708 self.state.save()?;
3709 Ok(())
3710 }
3711
3712 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
3718 let repo = self.state.repo.clone();
3719 let remote = self.state.config.merge.remote.clone();
3720 let fetched = git::fetch(&repo, &remote, branch).await;
3721 if !matches!(&fetched, Ok(o) if o.ok()) {
3722 return None;
3723 }
3724 let branch_tracking = format!("{remote}/{branch}");
3725 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
3726 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
3727 return None;
3728 }
3729 if !crate::reconcile::origin_missing(&repo, head, &theirs)
3730 .await
3731 .is_ok_and(|missing| missing.is_empty())
3732 {
3733 return Some(format!(
3734 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
3735 them",
3736 short(&theirs)
3737 ));
3738 }
3739 match git::push_pinned(&repo, &remote, branch, &theirs).await {
3740 Ok(o) if o.ok() => {
3741 self.state
3742 .event("land", format!("pushed rebased {branch} to {remote}"));
3743 None
3744 }
3745 Ok(o) => Some(format!(
3746 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
3747 someone may have pushed): {}",
3748 short(&theirs),
3749 o.stderr.chars().take(600).collect::<String>()
3750 )),
3751 Err(e) => Some(format!(
3752 "{branch} is rebased locally but could not be pushed: {e:#}"
3753 )),
3754 }
3755 }
3756
3757 async fn settle_already_in(
3774 &mut self,
3775 branch: &str,
3776 tip: &str,
3777 head: &str,
3778 attempts: usize,
3779 behind: usize,
3780 ) -> Result<bool> {
3781 let repo = self.state.repo.clone();
3782 let remote = self.state.config.merge.remote.clone();
3783 let start = self.state.base_commit.clone();
3784 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3785 Ok(Some(e)) => e,
3786 Ok(None) => return Ok(false),
3787 Err(e) => {
3788 tracing::warn!("already-in-base check for {branch}: {e:#}");
3789 return Ok(false);
3790 }
3791 };
3792 let mut verified = vec![head.to_owned()];
3793 let fetched = git::fetch(&repo, &remote, branch).await;
3794 if matches!(&fetched, Ok(o) if o.ok())
3795 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3796 && theirs != head
3797 {
3798 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3799 Ok(Some(_)) => verified.push(theirs),
3800 _ => return Ok(false),
3801 }
3802 }
3803 let base_branch = self.state.base_branch.clone();
3804 let message = format!(
3805 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3806 land",
3807 evidence.names(),
3808 evidence.proof.as_str()
3809 );
3810 let closed =
3811 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3812 match closed {
3813 Ok(Ok(url)) => self
3814 .state
3815 .event("land", format!("closed {url}: superseded on {base_branch}")),
3816 Ok(Err(why)) => self
3817 .state
3818 .event("land", format!("did not close a pull request: {why}")),
3819 Err(e) => {
3820 let why = format!(
3821 "{branch} is already in {remote}/{base_branch}, but its pull request could \
3822 not be closed ({e:#}); resume to retry"
3823 );
3824 self.state.status = RunStatus::Blocked;
3825 self.state.base_sync = Some(BaseSync {
3826 tip: tip.to_owned(),
3827 behind,
3828 attempts,
3829 conflict: Some(why.clone()),
3830 already_in: None,
3831 });
3832 self.state.event("land", why);
3833 self.state.save()?;
3834 return Ok(true);
3835 }
3836 }
3837 self.state.status = RunStatus::AlreadyInBase;
3838 self.state.base_sync = Some(BaseSync {
3839 tip: tip.to_owned(),
3840 behind,
3841 attempts,
3842 conflict: None,
3843 already_in: Some(evidence),
3844 });
3845 self.state.event("land", message);
3846 self.state.save()?;
3847 self.settle_questions();
3848 Ok(true)
3849 }
3850
3851 fn landing_base(&self) -> String {
3861 self.state
3862 .base_sync
3863 .as_ref()
3864 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3865 }
3866
3867 pub async fn fix_selected(
3900 &mut self,
3901 ids: &[String],
3902 reason: &str,
3903 allow_stale: bool,
3904 ) -> Result<()> {
3905 let reason = reason.trim();
3906 if reason.is_empty() {
3907 bail!("a fix request needs a reason — that is the operator's own record of why");
3908 }
3909 if ids.is_empty() {
3910 bail!("no finding id given");
3911 }
3912 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3913 bail!(
3914 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3915 has already concluded — can be given a targeted fix. A run still \
3916 in progress should simply be resumed; a `merged` run's branch has \
3917 already landed, so its answer is a fresh `magi review <branch>`, \
3918 not reopening this run's own record",
3919 self.state.id,
3920 self.state.status.as_str()
3921 );
3922 }
3923 let Some(winner) = self.state.winner().cloned() else {
3924 bail!("run {} has no winning candidate to fix", self.state.id);
3925 };
3926 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3927 bail!(
3928 "branch `{}` no longer exists; this run cannot be extended",
3929 winner.branch
3930 );
3931 }
3932 let home = crate::run::home();
3933 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3934 bail!(
3935 "run {} is currently being worked on by another magi process",
3936 self.state.id
3937 );
3938 }
3939 let _claim = FixClaim::acquire(&self.state.dir())?;
3945
3946 let mut seen = BTreeSet::new();
3950 let mut findings = Vec::new();
3951 let mut missing = Vec::new();
3952 for id in ids {
3953 if !seen.insert(id.clone()) {
3954 continue;
3955 }
3956 match self.state.finding(id) {
3957 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3958 id: f.id.clone(),
3959 severity: f.severity,
3960 reviewer_vote: rec.vote,
3961 round: round.round,
3962 round_head: round.head.clone(),
3963 reviewer: rec.reviewer,
3964 agent: rec.agent.clone(),
3965 file: f.file.clone(),
3966 line: f.line,
3967 title: f.title.clone(),
3968 detail: f.detail.clone(),
3969 outcome: OperatorFixOutcome::Pending,
3970 }),
3971 None => missing.push(id.clone()),
3972 }
3973 }
3974 if !missing.is_empty() {
3975 bail!(
3976 "unknown finding id(s): {}; nothing was changed",
3977 missing.join(", ")
3978 );
3979 }
3980
3981 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3982 let stale_details: Vec<(String, String)> = findings
3983 .iter()
3984 .filter(|f| f.round_head != head_at_request)
3985 .map(|f| (f.id.clone(), f.round_head.clone()))
3986 .collect();
3987 let stale = !stale_details.is_empty();
3988 if stale && !allow_stale {
3989 bail!(
3990 "the branch has moved since some finding(s) were raised — {} — now \
3991 at {}; pass --allow-stale to fix anyway, or re-run review first",
3992 stale_details
3993 .iter()
3994 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3995 .collect::<Vec<_>>()
3996 .join(", "),
3997 short(&head_at_request)
3998 );
3999 }
4000
4001 let request = OperatorFixRequest {
4002 requested_at: Timestamp::now(),
4003 reason: reason.to_owned(),
4004 findings,
4005 head_at_request: head_at_request.clone(),
4006 allow_stale,
4007 stale,
4008 fix: None,
4009 result_head: None,
4010 follow_up_review_run: None,
4011 };
4012 self.state.event(
4013 "fix",
4014 format!(
4015 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4016 request.findings.len(),
4017 request
4018 .findings
4019 .iter()
4020 .map(|f| f.id.as_str())
4021 .collect::<Vec<_>>()
4022 .join(", "),
4023 ),
4024 );
4025 self.state.operator_fixes.push(request);
4032 self.state.save()?;
4033 let request_index = self.state.operator_fixes.len() - 1;
4034
4035 if winner.worktree.exists() {
4044 let dirty = git::git(
4047 &winner.worktree,
4048 &["status", "--porcelain", "--untracked-files=all"],
4049 )
4050 .await?;
4051 let only_withheld = dirty.lines().all(|l| {
4052 l.strip_prefix("?? ")
4053 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4054 });
4055 if !only_withheld {
4056 bail!(
4057 "`{}` has uncommitted changes; refusing to touch it — commit or \
4058 discard them first",
4059 winner.worktree.display()
4060 );
4061 }
4062 git::worktree_remove(&self.state.repo, &winner.worktree)
4063 .await
4064 .ok();
4065 }
4066 let fix_worktree = self.state.worktree_root().join("operator-fix");
4067 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
4068 git::git(
4069 &self.state.repo,
4070 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
4071 )
4072 .await
4073 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
4074 if !git::is_clean(&fix_worktree).await? {
4075 git::worktree_remove(&self.state.repo, &fix_worktree)
4076 .await
4077 .ok();
4078 bail!(
4079 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
4080 winner.branch
4081 );
4082 }
4083
4084 let run_id = self.state.id.clone();
4085 let prompts = self.state.config.prompts.clone();
4086 let language = self.state.config.graph.language.clone();
4087 let sessions = self.state.config.graph.sessions;
4088 let artifacts = agent::artifacts_dir(&self.state.dir());
4089 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
4090 .findings
4091 .iter()
4092 .map(|f| Finding {
4093 id: f.id.clone(),
4094 severity: f.severity,
4095 file: f.file.clone(),
4096 line: f.line,
4097 title: f.title.clone(),
4098 detail: f.detail.clone(),
4099 })
4100 .collect();
4101 let fix_prompt = prompt::operator_fix(
4102 &self.state.instruction,
4103 &finding_list,
4104 reason,
4105 &stale_details,
4106 &head_at_request,
4107 &language,
4108 );
4109 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4110 let (job, seat, out) = self
4111 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
4112 prompt: fix_prompt.clone(),
4113 spec,
4114 seat,
4115 cwd: fix_worktree.clone(),
4116 timeout,
4117 allow_write: true,
4118 sessions,
4119 artifacts: artifacts.clone(),
4120 stem: "operator-fix".to_owned(),
4121 handover: None,
4122 })
4123 .await;
4124 let agent_id = seat.agent.clone();
4125
4126 let mut fix = FixRecord {
4127 agent: agent_id,
4128 addressed: Vec::new(),
4129 rejected: Vec::new(),
4130 notes: String::new(),
4131 committed: false,
4132 failed: None,
4133 duration_ms: 0,
4134 continuation: None,
4135 };
4136 let mut final_seat = seat.clone();
4137 match out {
4138 AgentOutcome::Ok(o) => {
4139 fix.duration_ms = o.duration_ms;
4140 let parsed = verdict::extract_json::<FixReport>(&o.text);
4141 let incomplete_reason = match &parsed {
4142 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4143 "the reply parsed, but it reported a command whose own CLI \
4144 never confirmed an exit status"
4145 .to_owned(),
4146 ),
4147 Ok(_) => None,
4148 Err(e) => Some(e.to_string()),
4149 };
4150 match incomplete_reason {
4151 None => {
4152 let report = parsed.expect("checked Ok above");
4153 fix.addressed = report.addressed;
4154 fix.rejected = report.rejected;
4155 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4156 }
4157 Some(reason) => {
4158 let (resumed_seat, resolved, failure, cont) = self
4159 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4160 .await;
4161 fix.duration_ms += cont.cumulative_wait_ms;
4162 fix.continuation = Some(cont);
4163 final_seat = resumed_seat;
4164 match resolved {
4165 Some(report) => {
4166 fix.addressed = report.addressed;
4167 fix.rejected = report.rejected;
4168 fix.notes =
4169 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4170 }
4171 None => fix.failed = failure,
4172 }
4173 }
4174 }
4175 }
4176 AgentOutcome::Dropped(o) => {
4177 fix.duration_ms = o.duration_ms;
4178 let why = o
4179 .dropped
4180 .as_ref()
4181 .map(|d| d.why.as_str())
4182 .unwrap_or("the CLI ended the stream without delivering its answer");
4183 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4184 }
4185 AgentOutcome::Quota(o) => {
4186 self.state.quota.push(QuotaLoss {
4187 seat: final_seat.key.clone(),
4188 node: "fix".to_owned(),
4189 at: Timestamp::now(),
4190 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4191 });
4192 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4193 }
4194 AgentOutcome::Failed(e) => fix.failed = Some(e),
4195 }
4196 if fix.continuation.is_none() {
4197 fix.continuation = Some(ContinuationRecord::not_needed());
4198 }
4199 self.state.seats.insert(final_seat.key.clone(), final_seat);
4200
4201 let rescue_message = format!(
4202 "magi: operator-selected fix ({}) (uncommitted work)",
4203 self.state.operator_fixes[request_index]
4204 .findings
4205 .iter()
4206 .map(|f| f.id.as_str())
4207 .collect::<Vec<_>>()
4208 .join(", ")
4209 );
4210 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4211 self.state.note_withheld("fix", &r.withheld);
4212 }
4213 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4214 fix.committed = after != head_at_request;
4215 git::worktree_remove(&self.state.repo, &fix_worktree)
4216 .await
4217 .ok();
4218
4219 self.state.event(
4220 "fix",
4221 match &fix.failed {
4222 Some(reason) => format!(
4223 "operator fix: adoption report was lost ({reason}); {}",
4224 if fix.committed {
4225 "committed"
4226 } else {
4227 "NO new commit"
4228 }
4229 ),
4230 None => format!(
4231 "operator fix: {} addressed, {} rejected, {}",
4232 fix.addressed.len(),
4233 fix.rejected.len(),
4234 if fix.committed {
4235 "committed"
4236 } else {
4237 "NO new commit"
4238 }
4239 ),
4240 },
4241 );
4242
4243 for f in &mut self.state.operator_fixes[request_index].findings {
4250 f.outcome = if fix.failed.is_some() {
4251 OperatorFixOutcome::Unreported
4252 } else if fix.addressed.contains(&f.id) {
4253 OperatorFixOutcome::Addressed
4254 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4255 OperatorFixOutcome::Rejected { why: r.why.clone() }
4256 } else {
4257 OperatorFixOutcome::Unreported
4258 };
4259 }
4260
4261 let committed = fix.committed;
4262 if committed {
4263 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4264 }
4265 self.state.operator_fixes[request_index].fix = Some(fix);
4266 self.state.save()?;
4269
4270 if committed {
4271 self.state.event(
4272 "fix",
4273 format!(
4274 "operator fix committed {}; opening a follow-up review-only run",
4275 short(&after)
4276 ),
4277 );
4278 let origin =
4281 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4282 match Self::review(
4283 &self.state.repo,
4284 &winner.branch,
4285 self.state.config.clone(),
4286 origin,
4287 )
4288 .await
4289 {
4290 Ok(mut follow_up) => {
4291 follow_up.state.event(
4292 "start",
4293 format!(
4294 "requested by an operator fix on run {} for finding(s) {}",
4295 self.state.id,
4296 self.state.operator_fixes[request_index]
4297 .findings
4298 .iter()
4299 .map(|f| f.id.as_str())
4300 .collect::<Vec<_>>()
4301 .join(", "),
4302 ),
4303 );
4304 follow_up.state.save()?;
4305 let follow_up_id = follow_up.state.id.clone();
4306 let adopted = match crate::direct::adopt(
4309 &follow_up.state,
4310 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
4311 ) {
4312 Ok(a) => a,
4313 Err(e) => {
4314 self.state.event(
4317 "fix",
4318 format!(
4319 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
4320 ),
4321 );
4322 self.state.save()?;
4323 return Ok(());
4324 }
4325 };
4326 let executed = follow_up.execute().await;
4327 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
4328 if let Some(a) = adopted {
4329 a.finish(&follow_up.state, executed);
4330 }
4331 if let Some(e) = failure {
4332 self.state.event(
4333 "fix",
4334 format!(
4335 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4336 ),
4337 );
4338 }
4339 self.state.operator_fixes[request_index].follow_up_review_run =
4340 Some(follow_up_id);
4341 }
4342 Err(e) => {
4343 self.state.event(
4344 "fix",
4345 format!("committed the fix but could not open a follow-up review: {e:#}"),
4346 );
4347 }
4348 }
4349 self.state.save()?;
4350 }
4351
4352 Ok(())
4353 }
4354
4355 async fn ask_fixer(
4375 &mut self,
4376 winner: &Candidate,
4377 node: &'static str,
4378 round: Option<usize>,
4379 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
4380 ) -> (SeatJob, SeatState, AgentOutcome) {
4381 let run_id = self.state.id.clone();
4382 let prompts = self.state.config.prompts.clone();
4383 let cache = self.state.config.cache_dir();
4384 let attempts = fixer::attempts(&self.state, &self.roles, winner);
4385 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
4386 let mut last = None;
4387 for (i, (spec, key)) in attempts.into_iter().enumerate() {
4388 let seat = self.seat(&key, &spec.id);
4389 let mut job = build(spec.clone(), seat);
4390 if i > 0 {
4391 job.stem = format!("{}-{}", job.stem, spec.id);
4392 }
4393 let ctx = WaveCtx {
4394 carry_seats: false,
4395 run: &run_id,
4396 node,
4397 prompts: &prompts,
4398 cache: cache.as_deref(),
4399 round,
4400 };
4401 let (seat, out) =
4402 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4403 let advances = match &out {
4404 AgentOutcome::Ok(o) => agent::output_advances(o),
4405 _ => true,
4406 };
4407 if let Some(next) = ids.get(i + 1)
4408 && advances
4409 {
4410 self.state.seats.insert(seat.key.clone(), seat.clone());
4411 let class =
4412 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
4413 let reason = match &out {
4414 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
4415 other => fail_reason(other),
4416 };
4417 record_handover(
4418 &mut self.state,
4419 node,
4420 &seat.key,
4421 &spec.id,
4422 next,
4423 &class,
4424 &reason,
4425 );
4426 continue;
4427 }
4428 last = Some((job, seat, out));
4429 break;
4430 }
4431 last.expect("the fixer chain always has an entry")
4432 }
4433
4434 async fn review_loop(&mut self) -> Result<()> {
4435 if self
4440 .state
4441 .base_sync
4442 .as_ref()
4443 .is_some_and(|s| s.conflict.is_some())
4444 {
4445 return Ok(());
4446 }
4447 let run_id = self.state.id.clone();
4452 let prompts = self.state.config.prompts.clone();
4453 let Some(winner) = self.state.winner().cloned() else {
4454 return Ok(());
4455 };
4456 let max_rounds = self.state.config.graph.review_rounds;
4457 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4467 if status == RunStatus::Gating {
4470 self.record_contested_handoff();
4471 }
4472 self.state.status = status;
4473 self.state.save()?;
4474 return Ok(());
4475 }
4476 self.state.status = RunStatus::Reviewing;
4477 if self
4487 .state
4488 .reviews
4489 .last()
4490 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4491 {
4492 let shell = self.state.config.shell();
4493 return self
4494 .stop_reviewing(
4495 "the last round's own verification never resolved",
4496 &shell,
4497 &winner.worktree,
4498 )
4499 .await;
4500 }
4501
4502 let repo = self.state.repo.clone();
4503 let root = self.state.worktree_root();
4504 let language = self.state.config.graph.language.clone();
4505 let sessions = self.state.config.graph.sessions;
4506 let artifacts = agent::artifacts_dir(&self.state.dir());
4507 let base = self.landing_base();
4508 let base_short = short(&base);
4509 let reviewers = self.roles.reviewers.clone();
4510 let shell = self.state.config.shell();
4511
4512 for round in (self.state.reviews.len() + 1)..=max_rounds {
4513 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4514 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4515 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4516 let prev_verification = self
4525 .state
4526 .reviews
4527 .last()
4528 .and_then(|r| r.verification_summary(&head));
4529
4530 let mut jobs = Vec::new();
4534 for (r, spec) in reviewers.iter().cloned().enumerate() {
4535 let wt = root.join(format!("review-{}", r + 1));
4536 if wt.exists() {
4537 git::reset_detached(&wt, &head).await?;
4538 } else {
4539 git::worktree_add_detached(&repo, &wt, &head).await?;
4540 }
4541 let seat_key = format!("review-{}", r + 1);
4542 let spec = pick_start_spec(
4545 &self.roles.reviewer_roster,
4546 spec,
4547 self.state.seat_history.get(&seat_key),
4548 );
4549 let seat = self.seat(&seat_key, &spec.id);
4550 jobs.push(SeatJob {
4551 prompt: prompt::review(&prompt::ReviewCtx {
4552 instruction: &self.state.instruction,
4553 branch: &winner.branch,
4554 base_short: &base_short,
4555 stat: &stat,
4556 patch: &patch,
4557 verification: prev_verification.as_ref(),
4558 reviewers: reviewers.len(),
4559 round,
4560 rounds: max_rounds,
4561 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4564 lens: Lens::for_seat(r),
4565 language: &language,
4566 }),
4567 spec,
4568 seat,
4569 cwd: wt,
4570 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4571 allow_write: false,
4572 sessions,
4573 artifacts: artifacts.clone(),
4574 stem: format!("review-{round}-{}", r + 1),
4575 handover: None,
4576 });
4577 }
4578
4579 self.state.event(
4580 "review",
4581 format!(
4582 "round {round}: {} reviewers on {}",
4583 jobs.len(),
4584 short(&head)
4585 ),
4586 );
4587 let mut quota_losses = Vec::new();
4588 let review_retries = self.state.config.graph.retries;
4589 let review_cache = self.state.config.cache_dir();
4590 let ctx = WaveCtx {
4591 carry_seats: true,
4592 run: &run_id,
4593 node: "review",
4594 prompts: &prompts,
4595 cache: review_cache.as_deref(),
4596 round: Some(round),
4597 };
4598 let results = ask_json_wave::<Review>(
4599 jobs,
4600 Arc::clone(&self.sem),
4601 review_retries,
4602 &self.roles.reviewer_roster,
4603 &ctx,
4604 &mut quota_losses,
4605 &mut self.state,
4606 &|_: &Review| Ok(()),
4607 )
4608 .await;
4609 let round_quota_missing = quota_losses.len();
4613 self.state.quota.extend(quota_losses);
4614
4615 let mut records = Vec::new();
4616 let mut all_findings = Vec::new();
4617 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4618 let agent_id = seat.agent.clone();
4619 self.state.seats.insert(seat.key.clone(), seat);
4620 let mut record = ReviewRecord {
4621 reviewer: r + 1,
4622 agent: agent_id,
4623 summary: String::new(),
4624 findings: Vec::new(),
4625 vote: None,
4626 failed: None,
4627 duration_ms: 0,
4628 attempts,
4634 };
4635 match res {
4636 Ok((review, out)) => {
4637 record.summary =
4645 blind::sanitize_prose(&review.summary, &self.state.config.blind);
4646 record.vote = Some(review.vote);
4647 record.duration_ms = out.duration_ms;
4648 for (n, mut f) in review.findings.into_iter().enumerate() {
4649 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4652 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4653 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4654 f.file = f
4660 .file
4661 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4662 all_findings.push(f.clone());
4663 record.findings.push(f);
4664 }
4665 self.state.event(
4666 "review",
4667 format!(
4668 "round {round}: reviewer {} voted {} with {} finding(s)",
4669 r + 1,
4670 review.vote.label(),
4671 record.findings.len()
4672 ),
4673 );
4674 }
4675 Err(e) => {
4676 record.failed = Some(e.to_string());
4677 self.state.event(
4678 "review",
4679 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4680 );
4681 }
4682 }
4683 records.push(record);
4684 }
4685
4686 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4693 let vote_split =
4694 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4695 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4696 if vote_split {
4697 self.state.event(
4698 "review",
4699 format!(
4700 "round {round}: votes split ({}) — one round of reconsideration",
4701 initial_votes
4702 .iter()
4703 .map(|v| v.label())
4704 .collect::<Vec<_>>()
4705 .join(", ")
4706 ),
4707 );
4708 let panel: Vec<ReviewSeatReport<'_>> = records
4711 .iter()
4712 .filter_map(|r| {
4713 r.vote.map(|vote| ReviewSeatReport {
4714 reviewer: r.reviewer,
4715 vote,
4716 summary: &r.summary,
4717 findings: &r.findings,
4718 })
4719 })
4720 .collect();
4721
4722 let mut jobs = Vec::new();
4723 let mut seats_at = Vec::new();
4724 for (r, spec) in reviewers.iter().cloned().enumerate() {
4725 if records[r].vote.is_none() {
4729 continue;
4730 }
4731 let wt = root.join(format!("review-{}", r + 1));
4732 let seat_key = format!("review-{}", r + 1);
4733 let spec = self.occupant(&seat_key, spec);
4734 let seat = self.seat(&seat_key, &spec.id);
4735 let build = |with_patch: bool| {
4743 prompt::review_reconsider(&ReviewReconsiderCtx {
4744 instruction: &self.state.instruction,
4745 reviewer: r + 1,
4746 lens: Lens::for_seat(r),
4747 panel: &panel,
4748 patch: with_patch.then_some(ReviewPatch {
4749 branch: &winner.branch,
4750 base_short: &base_short,
4751 stat: &stat,
4752 patch: &patch,
4753 }),
4754 round,
4755 rounds: max_rounds,
4756 language: &language,
4757 })
4758 };
4759 let full = build(true);
4760 let prompt = if has_context(&spec, &seat, sessions) {
4761 build(false)
4762 } else {
4763 full.clone()
4764 };
4765 jobs.push(SeatJob {
4766 prompt,
4767 spec,
4768 seat,
4769 cwd: wt,
4770 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4771 allow_write: false,
4772 sessions,
4773 artifacts: artifacts.clone(),
4774 stem: format!("review-{round}-reconsider-{}", r + 1),
4775 handover: Some(full),
4776 });
4777 seats_at.push(r);
4778 }
4779
4780 let mut recon_quota_losses = Vec::new();
4781 let recon_cache = self.state.config.cache_dir();
4782 let recon_ctx = WaveCtx {
4783 carry_seats: true,
4784 run: &run_id,
4785 node: "review",
4786 prompts: &prompts,
4787 cache: recon_cache.as_deref(),
4788 round: Some(round),
4789 };
4790 let recon_results = ask_json_wave::<ReviewRevote>(
4791 jobs,
4792 Arc::clone(&self.sem),
4793 review_retries,
4794 &self.roles.reviewer_roster,
4795 &recon_ctx,
4796 &mut recon_quota_losses,
4797 &mut self.state,
4798 &|_: &ReviewRevote| Ok(()),
4799 )
4800 .await;
4801 self.state.quota.extend(recon_quota_losses);
4802
4803 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4804 let agent_id = seat.agent.clone();
4805 self.state.seats.insert(seat.key.clone(), seat);
4806 let mut rec = ReviewRevoteRecord {
4807 reviewer: r + 1,
4808 agent: agent_id,
4809 vote: None,
4810 reason: String::new(),
4811 failed: None,
4812 };
4813 match res {
4814 Ok((rv, _)) => {
4815 rec.vote = Some(rv.vote);
4816 rec.reason =
4817 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4818 self.state.event(
4819 "review",
4820 format!(
4821 "round {round}: reviewer {} revoted {}",
4822 r + 1,
4823 rv.vote.label()
4824 ),
4825 );
4826 }
4827 Err(e) => {
4828 rec.failed = Some(e.to_string());
4829 self.state.event(
4830 "review",
4831 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4832 );
4833 }
4834 }
4835 reconsideration.push(rec);
4836 }
4837 } else if initial_votes.len() > 1 {
4838 self.state.event(
4839 "review",
4840 format!(
4841 "round {round}: votes agreed ({}) — no reconsideration",
4842 initial_votes[0].label()
4843 ),
4844 );
4845 }
4846
4847 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4848 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4849 let defer_e2e =
4860 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4861 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4862 let reason =
4863 format!("{blocking} blocking finding(s) already required a fix this round");
4864 self.state.event(
4865 "verify",
4866 format!(
4867 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4868 {}); it will run once a round has none left",
4869 short(&head)
4870 ),
4871 );
4872 (Vec::new(), false, true, Some(reason))
4873 } else {
4874 let e2e_commands = self.state.config.verify.e2e.clone();
4875 let cache_dir = self.state.config.cache_dir();
4876 let context = format!("round {round}");
4877 let (e2e, verify_retried) = with_cache_lease(
4878 &mut self.state,
4879 cache_dir.as_deref(),
4880 "e2e",
4881 "e2e",
4882 &winner.worktree,
4883 &head,
4884 verify_timeout,
4885 &context,
4886 |state, budget| {
4887 let shell = shell.clone();
4888 let e2e_commands = e2e_commands.clone();
4889 let worktree = winner.worktree.clone();
4890 let context = context.clone();
4891 async move {
4892 run_e2e_with_retry(
4893 state,
4894 &shell,
4895 &e2e_commands,
4896 &worktree,
4897 budget,
4898 &context,
4899 )
4900 .await
4901 }
4902 },
4903 )
4904 .await;
4905 (e2e, verify_retried, false, None)
4906 };
4907
4908 let expected = records.len();
4909 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4910 let incomplete = answered < expected;
4911 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4912 let policy = self.state.config.graph.incomplete_review;
4913 let clean = round_is_clean(
4914 blocking,
4915 e2e_ok,
4916 answered,
4917 expected,
4918 round_quota_missing,
4919 policy,
4920 );
4921
4922 let mut round_record = ReviewRound {
4923 round,
4924 head: head.clone(),
4925 verified_head: None,
4926 verified_at: None,
4927 reviews: records,
4928 e2e,
4929 verify_retried,
4930 e2e_deferred,
4931 e2e_defer_reason,
4932 fix: None,
4933 blocking,
4934 answered,
4935 expected,
4936 clean,
4937 progressed: false,
4938 vote_split,
4939 reconsideration,
4940 verdict: None,
4941 };
4942 round_record.verdict = ReviewVote::worst(
4946 round_record
4947 .final_votes()
4948 .into_iter()
4949 .map(|(_, _, vote)| vote),
4950 );
4951 if !matches!(
4962 round_record.e2e_status(),
4963 E2eStatus::Deferred | E2eStatus::NotConfigured
4964 ) {
4965 round_record.verified_head = Some(head.clone());
4966 round_record.verified_at = Some(Timestamp::now());
4967 }
4968 let this_round_verification = round_record.verification_summary(&head);
4969
4970 if incomplete {
4971 let missing: Vec<String> = round_record
4972 .reviews
4973 .iter()
4974 .filter(|r| r.failed.is_some())
4975 .map(|r| format!("review-{}", r.reviewer))
4976 .collect();
4977 self.state.event(
4978 "review",
4979 format!(
4980 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4981 missing.join(", ")
4982 ),
4983 );
4984 }
4985
4986 if clean {
4987 self.state.event(
4988 "review",
4989 if incomplete && policy == IncompleteReviewPolicy::Warn {
4990 format!(
4991 "round {round}: clean (warn policy, incomplete panel) — no \
4992 blocking findings from the seats that answered, verification green"
4993 )
4994 } else if incomplete {
4995 format!(
4996 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4997 quorum) — no blocking findings from the seats that answered, \
4998 verification green",
4999 expected - answered
5000 )
5001 } else {
5002 format!("round {round}: clean — no blocking findings, verification green")
5003 },
5004 );
5005 self.state.reviews.push(round_record);
5006 self.state.status = RunStatus::Gating;
5007 self.state.save()?;
5008 return Ok(());
5009 }
5010
5011 if incomplete && blocking == 0 && e2e_ok {
5019 self.state.reviews.push(round_record);
5020 self.state.save()?;
5021 if round == max_rounds {
5022 self.state.status = RunStatus::Blocked;
5023 self.state.event(
5024 "review",
5025 format!(
5026 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
5027 refusing to call it clean",
5028 expected - answered
5029 ),
5030 );
5031 return Ok(());
5032 }
5033 continue;
5034 }
5035
5036 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
5048 self.state.reviews.push(round_record);
5049 return self
5050 .stop_reviewing(
5051 "the round's own verification could not run",
5052 &shell,
5053 &winner.worktree,
5054 )
5055 .await;
5056 }
5057
5058 if round == max_rounds {
5059 self.state.reviews.push(round_record);
5060 return self
5061 .stop_reviewing(
5062 &format!(
5063 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
5064 ),
5065 &shell,
5066 &winner.worktree,
5067 )
5068 .await;
5069 }
5070
5071 let blocking_findings: Vec<_> = all_findings
5074 .iter()
5075 .filter(|f| f.severity.blocks())
5076 .cloned()
5077 .collect();
5078 let fix_prompt = prompt::fix(
5079 &self.state.instruction,
5080 &blocking_findings,
5081 this_round_verification.as_ref(),
5082 round,
5083 max_rounds,
5084 &language,
5085 );
5086 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5087 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5088 let (job, seat, out) = self
5089 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
5090 prompt: fix_prompt.clone(),
5091 spec,
5092 seat,
5093 cwd: winner.worktree.clone(),
5094 timeout,
5095 allow_write: true,
5096 sessions,
5097 artifacts: artifacts.clone(),
5098 stem: format!("fix-{round}"),
5099 handover: None,
5100 })
5101 .await;
5102 let agent_id = seat.agent.clone();
5103
5104 let mut fix = FixRecord {
5105 agent: agent_id,
5106 addressed: Vec::new(),
5107 rejected: Vec::new(),
5108 notes: String::new(),
5109 committed: false,
5110 failed: None,
5111 duration_ms: 0,
5112 continuation: None,
5113 };
5114 let mut continuation = ContinuationRecord::not_needed();
5115 let mut final_seat = seat.clone();
5116 match out {
5117 AgentOutcome::Ok(o) => {
5118 fix.duration_ms = o.duration_ms;
5119 let parsed = verdict::extract_json::<FixReport>(&o.text);
5120 let incomplete_reason = match &parsed {
5127 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5128 "the reply parsed, but it reported a command whose own CLI never \
5129 confirmed an exit status"
5130 .to_owned(),
5131 ),
5132 Ok(_) => None,
5133 Err(e) => Some(e.to_string()),
5134 };
5135 match incomplete_reason {
5136 None => {
5137 let report = parsed.expect("checked Ok above");
5138 fix.addressed = report.addressed;
5139 fix.rejected = report.rejected;
5140 fix.notes =
5141 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5142 }
5143 Some(reason) => {
5144 let (resumed_seat, resolved, failure, cont) = self
5145 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
5146 .await;
5147 fix.duration_ms += cont.cumulative_wait_ms;
5148 continuation = cont;
5149 final_seat = resumed_seat;
5150 match resolved {
5151 Some(report) => {
5152 fix.addressed = report.addressed;
5153 fix.rejected = report.rejected;
5154 fix.notes = blind::sanitize_prose(
5155 &report.notes,
5156 &self.state.config.blind,
5157 );
5158 }
5159 None => fix.failed = failure,
5160 }
5161 }
5162 }
5163 }
5164 AgentOutcome::Dropped(o) => {
5166 fix.duration_ms = o.duration_ms;
5167 let why = o
5168 .dropped
5169 .as_ref()
5170 .map(|d| d.why.as_str())
5171 .unwrap_or("the CLI ended the stream without delivering its answer");
5172 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5173 }
5174 AgentOutcome::Quota(o) => {
5175 self.state.quota.push(QuotaLoss {
5176 seat: final_seat.key.clone(),
5177 node: "fix".to_owned(),
5178 at: Timestamp::now(),
5179 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5180 });
5181 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5182 }
5183 AgentOutcome::Failed(e) => fix.failed = Some(e),
5184 }
5185 fix.continuation = Some(continuation);
5186 self.state.seats.insert(final_seat.key.clone(), final_seat);
5187 if let Ok(r) = git::rescue_commit(
5188 &winner.worktree,
5189 &format!("magi: review round {round} fixes (uncommitted work)"),
5190 )
5191 .await
5192 {
5193 self.state.note_withheld("fix", &r.withheld);
5194 }
5195 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5196 fix.committed = after != before;
5197 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5205 let progressed = diff_after != patch;
5206 let commit_note = if fix.committed {
5207 "committed"
5208 } else {
5209 "NO new commit"
5210 };
5211 let tree_note = if progressed {
5212 "changed vs base"
5213 } else {
5214 "unchanged vs base"
5215 };
5216 self.state.event(
5217 "fix",
5218 match &fix.failed {
5219 Some(reason) => {
5225 format!(
5226 "round {round}: fixer's adoption report was lost ({reason}); \
5227 {commit_note}, tree {tree_note}"
5228 )
5229 }
5230 None => format!(
5231 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5232 {tree_note}{}",
5233 fix.addressed.len(),
5234 fix.rejected.len(),
5235 if continuation.outcome == ContinuationOutcome::Resumed {
5236 format!(
5237 " (adoption report recovered after {} continuation(s))",
5238 continuation.attempts
5239 )
5240 } else {
5241 String::new()
5242 },
5243 ),
5244 },
5245 );
5246 round_record.fix = Some(fix);
5247 round_record.progressed = progressed;
5248 self.state.reviews.push(round_record);
5249 self.state.save()?;
5250
5251 if matches!(
5264 continuation.outcome,
5265 ContinuationOutcome::Exhausted
5266 | ContinuationOutcome::QuotaLost
5267 | ContinuationOutcome::NoSession
5268 ) {
5269 return self
5270 .stop_reviewing(
5271 "the fixer's adoption report never came back, even after resuming its \
5272 own seat; refusing to start another round against the same worktree \
5273 while that is unresolved",
5274 &shell,
5275 &winner.worktree,
5276 )
5277 .await;
5278 }
5279
5280 let streak = self
5281 .state
5282 .reviews
5283 .iter()
5284 .rev()
5285 .take_while(|r| !r.progressed)
5286 .count();
5287 if streak >= STAGNANT_LIMIT {
5288 return self
5289 .stop_reviewing(
5290 &format!(
5291 "the tree has not moved against base for {streak} round(s) in a row"
5292 ),
5293 &shell,
5294 &winner.worktree,
5295 )
5296 .await;
5297 }
5298 }
5299 Ok(())
5300 }
5301
5302 fn record_contested_handoff(&mut self) {
5336 if self.state.contested_handoff.is_some() {
5337 return;
5338 }
5339 let Some(contested) = self
5340 .state
5341 .reviews
5342 .last()
5343 .and_then(ReviewRound::contested_handoff)
5344 else {
5345 return;
5346 };
5347 self.state.event(
5348 "review",
5349 format!(
5350 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5351 wait for the owner's approval",
5352 contested.findings.len(),
5353 contested.rejecters.len()
5354 ),
5355 );
5356 self.state.contested_handoff = Some(contested);
5357 }
5358
5359 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5360 let round_idx = self.state.reviews.len() - 1;
5361 let needs_catchup_run = matches!(
5369 self.state.reviews[round_idx].e2e_status(),
5370 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5371 );
5372 if needs_catchup_run {
5373 let round = self.state.reviews[round_idx].round;
5374 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5375 let commands = self.state.config.verify.e2e.clone();
5376 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5377 let cache_dir = self.state.config.cache_dir();
5378 let context = format!(
5379 "round {round}: verification unresolved, catching up before the final decision"
5380 );
5381 let (outcomes, verify_retried) = with_cache_lease(
5382 &mut self.state,
5383 cache_dir.as_deref(),
5384 "e2e",
5385 "e2e",
5386 worktree,
5387 &attempted_head,
5388 timeout,
5389 &context,
5390 |state, budget| {
5391 let shell = shell.to_vec();
5392 let commands = commands.clone();
5393 let context = context.clone();
5394 async move {
5395 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5396 .await
5397 }
5398 },
5399 )
5400 .await;
5401 let last = &mut self.state.reviews[round_idx];
5402 last.e2e = outcomes;
5403 last.verify_retried = verify_retried;
5404 last.verified_head = Some(attempted_head);
5411 last.verified_at = Some(Timestamp::now());
5412 if verify_inconclusive(&last.e2e) {
5413 self.state.save()?;
5420 return Ok(());
5421 }
5422 last.e2e_deferred = false;
5423 }
5424 let last = &self.state.reviews[round_idx];
5425 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5426
5427 match last.e2e_status() {
5428 E2eStatus::Failed => {
5429 let red: Vec<String> = last
5430 .e2e
5431 .iter()
5432 .filter(|o| !o.ok())
5433 .map(|o| {
5434 format!(
5435 "`{}` -> {:?}\n{}",
5436 o.command,
5437 o.code,
5438 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5439 )
5440 })
5441 .collect();
5442 self.state
5443 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5444 self.state.status = RunStatus::Blocked;
5445 }
5446 E2eStatus::ResourceBlocked => {
5451 self.state.event(
5452 "review",
5453 format!(
5454 "{why}; e2e could not run (shared build cache unavailable); not \
5455 deciding yet"
5456 ),
5457 );
5458 }
5459 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5460 self.state.event(
5461 "review",
5462 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5463 );
5464 self.record_contested_handoff();
5465 self.state.status = RunStatus::Gating;
5466 }
5467 }
5468 self.state.save()?;
5469 Ok(())
5470 }
5471
5472 async fn gate(&mut self) -> Result<()> {
5475 if self.state.status == RunStatus::Failed
5487 || self
5488 .state
5489 .base_sync
5490 .as_ref()
5491 .is_some_and(|s| s.conflict.is_some())
5492 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5493 != Some(RunStatus::Gating)
5494 {
5495 return Ok(());
5496 }
5497 if self.state.gate_ran {
5498 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5509 self.state.status = RunStatus::Blocked;
5510 self.state.save()?;
5511 }
5512 return Ok(());
5513 }
5514 let Some(winner) = self.state.winner().cloned() else {
5515 return Ok(());
5516 };
5517 self.state.status = RunStatus::Gating;
5518 let mut outcomes = self.run_gate(&winner).await?;
5519 loop {
5520 if verify_inconclusive(&outcomes) {
5531 self.state.save()?;
5532 return Ok(());
5533 }
5534 if outcomes.iter().all(CommandOutcome::ok) {
5535 break;
5536 }
5537 match self.gate_fix_round(&winner, &outcomes).await? {
5538 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5539 GateFix::Stop => break,
5540 GateFix::Defer => {
5541 self.state.save()?;
5542 return Ok(());
5543 }
5544 }
5545 }
5546 let passed = outcomes.iter().all(CommandOutcome::ok);
5547 self.state.gate = outcomes;
5548 self.state.gate_ran = true;
5549 if !passed {
5550 self.state.status = RunStatus::Blocked;
5551 let spent = self.state.gate_fixes.len();
5552 self.state.event(
5553 "gate",
5554 if spent == 0 {
5555 "gate failed; not merging".to_owned()
5556 } else {
5557 format!("gate failed after {spent} gate-fix round(s); not merging")
5558 },
5559 );
5560 }
5561 self.state.save()?;
5562 Ok(())
5563 }
5564
5565 async fn run_pre_gate(&mut self, winner: &Candidate) {
5575 let commands = self.state.config.verify.pre_gate.clone();
5576 if commands.is_empty() {
5577 return;
5578 }
5579 let shell = self.state.config.shell();
5580 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5581 let (outcomes, _) = run_commands(
5582 &mut self.state,
5583 "pre_gate",
5584 "pre_gate",
5585 0,
5586 &shell,
5587 &commands,
5588 &winner.worktree,
5589 timeout,
5590 )
5591 .await;
5592 for o in &outcomes {
5593 if !o.ok() {
5594 tracing::warn!(
5595 "pre_gate `{}` failed ({:?}); the gate decides",
5596 o.command,
5597 o.code
5598 );
5599 }
5600 self.state.event(
5601 "pre_gate",
5602 format!(
5603 "`{}` -> {}",
5604 o.command,
5605 if o.ok() {
5606 "pass".to_owned()
5607 } else {
5608 format!(
5609 "FAIL ({:?})\n{}",
5610 o.code,
5611 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5612 )
5613 }
5614 ),
5615 );
5616 }
5617 self.state.pre_gate = outcomes;
5618 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5619 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5620 Ok(head) => {
5621 self.state
5622 .event("pre_gate", format!("committed mechanical fixes ({head})"));
5623 self.state.pre_gate_commit = Some(head);
5624 }
5625 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5626 },
5627 Ok(false) => {}
5628 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5629 }
5630 if let Err(e) = self.state.save() {
5631 tracing::warn!("could not persist the pre_gate record: {e:#}");
5632 }
5633 }
5634
5635 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5638 self.run_pre_gate(winner).await;
5639 let shell = self.state.config.shell();
5640 let gate_commands = self.state.config.verify.gate.clone();
5641 let outcomes = if gate_commands.is_empty() {
5650 Vec::new()
5651 } else {
5652 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5653 let cache_dir = self.state.config.cache_dir();
5654 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5655 let (outcomes, _) = with_cache_lease(
5656 &mut self.state,
5657 cache_dir.as_deref(),
5658 "gate",
5659 "gate",
5660 &winner.worktree,
5661 &head,
5662 timeout,
5663 "final gate",
5664 |state, budget| {
5665 let shell = shell.clone();
5666 let gate_commands = gate_commands.clone();
5667 let worktree = winner.worktree.clone();
5668 async move {
5669 let (outcomes, timed_out_pids) = run_commands(
5670 state,
5671 "gate",
5672 "gate",
5673 0,
5674 &shell,
5675 &gate_commands,
5676 &worktree,
5677 budget,
5678 )
5679 .await;
5680 (outcomes, false, timed_out_pids)
5681 }
5682 },
5683 )
5684 .await;
5685 outcomes
5686 };
5687 if outcomes.is_empty() {
5688 self.state.event(
5693 "gate",
5694 "no gate commands configured; nothing to check, passing",
5695 );
5696 }
5697 for o in &outcomes {
5698 self.state.event(
5699 "gate",
5700 format!(
5701 "`{}` -> {}",
5702 o.command,
5703 if o.ok() {
5704 "pass".to_owned()
5705 } else {
5706 format!(
5707 "FAIL ({:?})\n{}",
5708 o.code,
5709 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5710 )
5711 }
5712 ),
5713 );
5714 }
5715 Ok(outcomes)
5716 }
5717
5718 async fn gate_fix_round(
5731 &mut self,
5732 winner: &Candidate,
5733 outcomes: &[CommandOutcome],
5734 ) -> Result<GateFix> {
5735 let cap = self.state.config.graph.gate_fix_rounds;
5736 let spent = self.state.gate_fixes.len();
5737 if spent >= cap {
5738 if cap > 0 {
5739 self.state.event(
5740 "gate",
5741 format!("{spent} gate-fix round(s) spent and the gate still fails"),
5742 );
5743 }
5744 return Ok(GateFix::Stop);
5745 }
5746 if !gate_fixable(outcomes) {
5747 self.state.event(
5748 "gate",
5749 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5750 command or similar); not spending a fix round on it",
5751 );
5752 return Ok(GateFix::Stop);
5753 }
5754 let min_free = self.state.config.disk.min_free_bytes;
5755 if min_free > 0 {
5756 match crate::disk::free_bytes(&winner.worktree) {
5757 Ok(free) if crate::disk::enough_space(free, min_free) => {}
5758 Ok(free) => {
5759 self.state.event(
5760 "gate",
5761 format!(
5762 "only {free} bytes free ({min_free} required by `[disk] \
5763 min_free_bytes`); not spending a fix round on a failure the disk \
5764 may explain"
5765 ),
5766 );
5767 return Ok(GateFix::Stop);
5768 }
5769 Err(e) => {
5770 self.state.event(
5771 "gate",
5772 format!("free disk space could not be measured ({e:#}); no fix round"),
5773 );
5774 return Ok(GateFix::Stop);
5775 }
5776 }
5777 }
5778
5779 let attempt = spent + 1;
5780 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5781 let base = self.landing_base();
5782 let fix_prompt = prompt::gate_fix(
5783 &self.state.instruction,
5784 &failed,
5785 attempt,
5786 cap,
5787 &self.state.config.graph.language,
5788 );
5789 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5790 let sessions = self.state.config.graph.sessions;
5791 let artifacts = agent::artifacts_dir(&self.state.dir());
5792 self.state.event(
5793 "gate",
5794 format!("gate failed; gate-fix round {attempt} of {cap}"),
5795 );
5796 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5797 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5798 let (_, seat, out) = self
5799 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
5800 prompt: fix_prompt.clone(),
5801 spec,
5802 seat,
5803 cwd: winner.worktree.clone(),
5804 timeout,
5805 allow_write: true,
5806 sessions,
5807 artifacts: artifacts.clone(),
5808 stem: format!("gate-fix-{attempt}"),
5809 handover: None,
5810 })
5811 .await;
5812 let mut record = GateFixRecord {
5813 agent: seat.agent.clone(),
5814 failed,
5815 notes: String::new(),
5816 committed: false,
5817 error: None,
5818 };
5819 match out {
5820 AgentOutcome::Ok(o) => {
5821 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5824 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5825 }
5826 }
5827 AgentOutcome::Dropped(_) => {
5828 record.error = Some("the CLI dropped the stream".to_owned());
5829 }
5830 AgentOutcome::Quota(o) => {
5831 self.state.quota.push(QuotaLoss {
5832 seat: seat.key.clone(),
5833 node: "gate-fix".to_owned(),
5834 at: Timestamp::now(),
5835 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5836 });
5837 record.error = Some("rate limited (quota); fixer could not run".to_owned());
5838 }
5839 AgentOutcome::Failed(e) => record.error = Some(e),
5840 }
5841 self.state.seats.insert(seat.key.clone(), seat);
5842 if let Ok(r) = git::rescue_commit(
5843 &winner.worktree,
5844 &format!("magi: gate fix {attempt} (uncommitted work)"),
5845 )
5846 .await
5847 {
5848 self.state.note_withheld("gate-fix", &r.withheld);
5849 }
5850 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5851 record.committed = after != before;
5852 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5853 let note = record.error.clone();
5854 self.state.gate_fixes.push(record);
5855 self.state.save()?;
5856 if !changed {
5857 self.state.event(
5858 "gate",
5859 match note {
5860 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5861 None => format!("gate-fix round {attempt}: the tree did not change"),
5862 },
5863 );
5864 return Ok(GateFix::Stop);
5865 }
5866 self.state.event(
5867 "gate",
5868 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5869 );
5870
5871 let commands = self.state.config.verify.e2e.clone();
5872 if !commands.is_empty() {
5873 let shell = self.state.config.shell();
5874 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5875 let cache_dir = self.state.config.cache_dir();
5876 let context = format!("gate-fix round {attempt}");
5877 let (e2e, _) = with_cache_lease(
5878 &mut self.state,
5879 cache_dir.as_deref(),
5880 "e2e",
5881 "e2e",
5882 &winner.worktree,
5883 &after,
5884 timeout,
5885 &context,
5886 |state, budget| {
5887 let shell = shell.clone();
5888 let commands = commands.clone();
5889 let context = context.clone();
5890 let worktree = winner.worktree.clone();
5891 async move {
5892 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5893 .await
5894 }
5895 },
5896 )
5897 .await;
5898 if verify_inconclusive(&e2e) {
5899 return Ok(GateFix::Defer);
5900 }
5901 if e2e.iter().any(|o| !o.ok()) {
5902 self.state.event(
5903 "gate",
5904 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5905 );
5906 return Ok(GateFix::Stop);
5907 }
5908 }
5909 Ok(GateFix::Retry)
5910 }
5911
5912 async fn guarded_pr_message(
5916 &mut self,
5917 winner: &Candidate,
5918 facts: Option<&BranchFacts>,
5919 posting: bool,
5920 ) -> PrMessage {
5921 let mut pr = pr_message_raw(&self.state, winner.label, facts);
5922 if !posting {
5923 let identity = crate::scrub::Identity::current();
5924 return PrMessage {
5925 title: crate::scrub::scrub(&pr.title, &identity),
5926 body: crate::scrub::scrub(&pr.body, &identity),
5927 };
5928 }
5929 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
5930 let mut violations = crate::github_text::check_with(&pr.title, &pr.body, decision);
5931 let identity = crate::scrub::Identity::current();
5932 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
5933 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
5934 && !violations.contains(&crate::github_text::Violation::SensitiveData)
5935 {
5936 violations.push(crate::github_text::Violation::SensitiveData);
5937 }
5938 violations.retain(|v| *v != crate::github_text::Violation::SensitiveData);
5942 if self.state.config.graph.github_text_guard && !violations.is_empty() {
5943 self.state.event(
5944 "github-text",
5945 format!("description rejected: {violations:?}; requesting one rewrite"),
5946 );
5947 let mut rewritten = false;
5948 if let Some(spec) = self
5949 .state
5950 .config
5951 .agents
5952 .iter()
5953 .find(|a| a.id == winner.agent)
5954 .cloned()
5955 {
5956 let key = format!("impl-{}", winner.label);
5957 let seat = self.seat(&key, &spec.id);
5958 let prompt = format!(
5959 "Rewrite only the following pull request description. The posting gate reported {violations:?}. Write English prose and remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the complete Markdown body. Preserve the magi run marker.\n\nTITLE: {}\n{}",
5960 pr.title, pr.body
5961 );
5962 let job = SeatJob {
5963 spec,
5964 seat,
5965 cwd: winner.worktree.clone(),
5966 prompt,
5967 timeout: retry_budget(
5968 Duration::from_secs(self.state.config.graph.timeout_implement),
5969 true,
5970 ),
5971 allow_write: false,
5972 sessions: self.state.config.graph.sessions,
5973 artifacts: agent::artifacts_dir(&self.state.dir()),
5974 stem: "github-text-rewrite".to_owned(),
5975 handover: None,
5976 };
5977 let prompts = self.state.config.prompts.clone();
5978 let cache = self.state.config.cache_dir();
5979 let run = self.state.id.clone();
5980 let ctx = WaveCtx {
5981 run: &run,
5982 node: "github-text",
5983 prompts: &prompts,
5984 cache: cache.as_deref(),
5985 round: None,
5986 carry_seats: false,
5987 };
5988 let (seat, outcome) =
5989 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
5990 self.state.seats.insert(seat.key.clone(), seat);
5991 if let AgentOutcome::Ok(output) = outcome
5992 && let Some(title) = summary_title(&output.text)
5993 {
5994 let mut body = summary_without_title(&output.text);
5995 let rewrite = PrMessage {
5996 title: title.clone(),
5997 body: body.clone(),
5998 };
5999 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
6000 if !body.trim().is_empty()
6001 && crate::github_text::check_with(&title, &body, redecision).is_empty()
6002 {
6003 decision = redecision;
6004 let marker = format!("magi:run/{}", self.state.id);
6005 if !body.contains(&marker) {
6006 body.push_str(&format!("\n\n{marker}"));
6007 }
6008 pr = PrMessage { title, body };
6009 rewritten = true;
6010 self.state
6011 .event("github-text", "description rewrite passed");
6012 }
6013 }
6014 }
6015 if !rewritten {
6016 self.state.event(
6017 "github-text",
6018 "description rewrite unavailable or rejected; using neutral body",
6019 );
6020 pr = PrMessage {
6021 title: pr.title,
6022 body: format!(
6023 "{}\n\nmagi:run/{}",
6024 crate::github_text::NEUTRAL_BODY,
6025 self.state.id
6026 ),
6027 };
6028 decision = decision.map(|d| crate::github_text::LanguageDecision {
6031 body_english: true,
6032 ..d
6033 });
6034 }
6035 }
6036 let (title, body) =
6037 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
6038 PrMessage { title, body }
6039 }
6040
6041 async fn judge_pr_language(
6044 &mut self,
6045 cwd: &Path,
6046 pr: &PrMessage,
6047 ) -> Option<crate::github_text::LanguageDecision> {
6048 self.state.config.roles.language_judge.as_ref()?;
6049 let decision =
6050 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
6051 self.state.event(
6052 "github-text",
6053 if decision.is_some() {
6054 "language decided by the language judge"
6055 } else {
6056 "language judge unavailable; using built-in heuristics"
6057 },
6058 );
6059 decision
6060 }
6061
6062 async fn merge(&mut self) -> Result<()> {
6063 if self
6078 .state
6079 .base_sync
6080 .as_ref()
6081 .is_some_and(|s| s.conflict.is_some())
6082 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6083 != Some(RunStatus::Gating)
6084 || !self.state.gate_status().ok()
6093 {
6094 return Ok(());
6095 }
6096 if self.state.merge.is_some() {
6105 return Ok(());
6106 }
6107 let Some(winner) = self.state.winner().cloned() else {
6108 return Ok(());
6109 };
6110 let repo = self.state.repo.clone();
6111 let base = self.state.base_branch.clone();
6112 let mode = self.state.config.merge.mode;
6113 let style = self.state.config.merge.style;
6114 let facts = if is_review_run(&self.state) {
6115 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
6116 let start = review_base(
6117 &repo,
6118 &self.state.config.merge.remote,
6119 &base,
6120 &self.state.base_commit,
6121 &winner.branch,
6122 )
6123 .await;
6124 branch_facts(&repo, &start, &winner.branch).await
6125 } else {
6126 None
6127 };
6128 let leaked = if is_review_run(&self.state) {
6131 leaked_subjects(&self.state, &winner.branch).await
6132 } else {
6133 Some(Vec::new())
6134 };
6135 let pr = self
6136 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
6137 .await;
6138 let message = pr.commit_message();
6139
6140 let outcome = match mode {
6141 MergeMode::None => MergeOutcome {
6142 mode,
6143 ok: true,
6144 detail: manual_merge_command(style, &repo, &winner.branch, &message),
6145 empty: false,
6146 },
6147 MergeMode::Pr | MergeMode::Local
6148 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
6149 {
6150 MergeOutcome {
6151 mode,
6152 ok: false,
6153 detail: empty_candidate_detail(&self.state, &base),
6154 empty: true,
6155 }
6156 }
6157 MergeMode::Local => {
6158 let on = git::current_branch(&repo).await?;
6159 if on.as_deref() != Some(base.as_str()) {
6160 MergeOutcome {
6161 mode,
6162 ok: false,
6163 detail: format!(
6164 "{} has {} checked out, not the base branch {base}",
6165 repo.display(),
6166 on.unwrap_or_else(|| "a detached HEAD".to_owned())
6167 ),
6168 empty: false,
6169 }
6170 } else if !git::is_clean(&repo).await? {
6171 MergeOutcome {
6172 mode,
6173 ok: false,
6174 detail: format!("{} is dirty; refusing to merge", repo.display()),
6175 empty: false,
6176 }
6177 } else {
6178 let out = match style {
6179 MergeStyle::Merge => {
6180 git::merge_no_ff(&repo, &winner.branch, &message).await?
6181 }
6182 MergeStyle::Squash => {
6183 git::merge_squash(&repo, &winner.branch, &message).await?
6184 }
6185 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
6186 };
6187 MergeOutcome {
6188 mode,
6189 ok: out.ok(),
6190 detail: if out.ok() { out.stdout } else { out.stderr },
6191 empty: false,
6192 }
6193 }
6194 }
6195 MergeMode::Pr => {
6196 let remote = self.state.config.merge.remote.clone();
6197 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
6198 if !pushed.ok() {
6199 MergeOutcome {
6200 mode,
6201 ok: false,
6202 detail: pushed.stderr,
6203 empty: false,
6204 }
6205 } else {
6206 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
6212 let out = match pr_merge_plan(found) {
6213 PrPlan::Create => {
6214 gh_pr_create(
6215 &winner.worktree,
6216 &base,
6217 &winner.branch,
6218 &pr.title,
6219 &pr.body,
6220 )
6221 .await
6222 }
6223 PrPlan::Adopt { url, title } => {
6224 self.state
6225 .event("merge", format!("Pr: adopted open pull request {url}"));
6226 if title != pr.title
6227 && (!is_review_run(&self.state)
6228 || leaked
6229 .as_deref()
6230 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
6231 && let Err(e) = land::set_pr_title(
6232 &mut self.state,
6233 &winner.worktree,
6234 &url,
6235 &pr.title,
6236 )
6237 .await
6238 {
6239 tracing::warn!("could not refresh title of {url}: {e:#}");
6240 self.state
6241 .event("merge", format!("Pr: title refresh failed: {e:#}"));
6242 }
6243 Ok(url)
6244 }
6245 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
6246 };
6247 match out {
6248 Ok(url) => MergeOutcome {
6249 mode,
6250 ok: true,
6251 detail: url,
6252 empty: false,
6253 },
6254 Err(e) => MergeOutcome {
6255 mode,
6256 ok: false,
6257 detail: e.to_string(),
6258 empty: false,
6259 },
6260 }
6261 }
6262 }
6263 };
6264
6265 self.state.status = match (mode, outcome.ok) {
6266 (MergeMode::None, _) => RunStatus::Ready,
6267 (_, true) => RunStatus::Merged,
6268 (_, false) => RunStatus::Blocked,
6269 };
6270 self.state.event(
6271 "merge",
6272 format!(
6273 "{:?}: {}",
6274 mode,
6275 outcome.detail.lines().next().unwrap_or("")
6276 ),
6277 );
6278 self.state.merge = Some(outcome);
6279 self.state.save()?;
6280
6281 if self.state.config.graph.land
6287 && mode == MergeMode::Pr
6288 && self.state.status == RunStatus::Merged
6289 {
6290 self.run_land().await?;
6291 }
6292 self.settle_questions();
6297 Ok(())
6298 }
6299
6300 async fn run_land(&mut self) -> Result<()> {
6311 let url = self
6312 .state
6313 .merge
6314 .as_ref()
6315 .map(|m| m.detail.clone())
6316 .unwrap_or_default();
6317 let url = url.lines().next().unwrap_or("").trim().to_owned();
6318 if !url.starts_with("http") {
6319 return Ok(());
6320 }
6321 match land::land(&mut self.state, &url).await {
6324 Ok(pr) if self.state.parked => {
6325 let _ = pr;
6329 }
6330 Ok(pr) => {
6331 self.state.status = match pr.state {
6332 land::PrLifecycle::Merged => RunStatus::Merged,
6333 _ => RunStatus::Blocked,
6334 };
6335 if bump::should_release_bump(self.state.status)
6342 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
6343 {
6344 self.state
6350 .event("bump", format!("release bump skipped: {e:#}"));
6351 }
6352 if self.state.status == RunStatus::Merged {
6355 crate::followup::after_merge(&mut self.state, &pr.url).await;
6356 }
6357 self.state.save()?;
6358 }
6359 Err(e) => {
6360 self.state.status = RunStatus::Blocked;
6361 self.state.event("land", format!("gave up: {e}"));
6362 self.state.save()?;
6363 }
6364 }
6365 Ok(())
6366 }
6367
6368 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
6372 if let Some(existing) = self.state.seats.get(key)
6373 && existing.agent == agent
6374 {
6375 return existing.clone();
6376 }
6377 let fresh = if self.state.seats.contains_key(key) {
6380 handover_seat(key, agent, self.state.next_seat_seed())
6381 } else {
6382 SeatState::new(key, agent, self.state.seed)
6383 };
6384 self.state.seats.insert(key.to_owned(), fresh.clone());
6385 fresh
6386 }
6387
6388 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
6394 match self.state.seats.get(key) {
6395 Some(s) if s.agent != spec.id => {
6396 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
6397 }
6398 _ => spec,
6399 }
6400 }
6401
6402 fn view(&self, c: &Candidate) -> CandidateView {
6404 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
6405 .unwrap_or_default();
6406 let (patch, _) = blind::sanitize_patch(
6407 &format!("candidate {} patch", c.label),
6408 &raw,
6409 &self.state.config.blind,
6410 );
6411 CandidateView {
6412 label: c.label,
6413 branch: c.branch.clone(),
6414 summary: c.summary.clone(),
6415 stat: c.stat.clone(),
6416 patch,
6417 }
6418 }
6419
6420 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
6422 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
6423 prompt::judge(
6424 "(see above)",
6425 &views,
6426 self.roles.judges.len(),
6427 base_short,
6428 "en",
6429 )
6430 }
6431
6432 fn vote_prompt_full(
6438 &self,
6439 j: usize,
6440 labels: &[char],
6441 language: &str,
6442 candidates: &[Candidate],
6443 base_short: &str,
6444 ) -> String {
6445 let mut text = format!(
6446 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
6447 prompt::final_vote(labels, language),
6448 self.state.instruction,
6449 self.candidate_block(candidates, base_short)
6450 );
6451 if let Some(own) = self
6452 .state
6453 .judgements
6454 .get(j)
6455 .filter(|r| !r.ranking.is_empty())
6456 {
6457 let reasons = own
6458 .reasons
6459 .iter()
6460 .map(|(k, v)| format!("- {k}: {v}"))
6461 .collect::<Vec<_>>()
6462 .join("\n");
6463 text.push_str(&format!(
6464 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
6465 own.ranking.iter().collect::<String>(),
6466 if reasons.is_empty() {
6467 ""
6468 } else {
6469 ", because:\n"
6470 }
6471 ));
6472 }
6473 if !self.state.deliberation.is_empty() {
6474 text.push_str("\n# What was argued before this vote\n");
6475 for t in self.transcript(&[], j) {
6476 text.push_str(&format!(
6477 "\n## {}{}\n\n{}\n",
6478 t.who,
6479 if t.is_self { " (you)" } else { "" },
6480 t.body.trim()
6481 ));
6482 }
6483 }
6484 text
6485 }
6486
6487 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6494 let mut turns = Vec::new();
6495 for j in &self.state.judgements {
6496 if j.ranking.is_empty() {
6497 continue;
6498 }
6499 let reasons = j
6500 .reasons
6501 .iter()
6502 .map(|(k, v)| format!("- {k}: {v}"))
6503 .collect::<Vec<_>>()
6504 .join("\n");
6505 turns.push(Turn {
6506 who: format!("Judge {} (opening ranking)", j.judge),
6507 is_self: j.judge == self_idx + 1,
6508 body: format!(
6509 "Ranked {}{}{reasons}",
6510 j.ranking.iter().collect::<String>(),
6511 if reasons.is_empty() {
6512 ""
6513 } else {
6514 ", because:\n"
6515 }
6516 ),
6517 });
6518 }
6519 for t in self
6520 .state
6521 .deliberation
6522 .iter()
6523 .flat_map(|r| r.turns.iter())
6524 .chain(current)
6525 {
6526 turns.push(Turn {
6527 who: format!("Judge {}", t.judge),
6528 is_self: t.judge == self_idx + 1,
6529 body: t.body.clone(),
6530 });
6531 }
6532 turns
6533 }
6534}
6535
6536fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6538 agent::has_session(spec.kind, seat, sessions)
6539}
6540
6541fn next_untried_in_roster<'a>(
6562 roster: &'a [AgentSpec],
6563 start: usize,
6564 tried: &BTreeSet<String>,
6565) -> Option<&'a AgentSpec> {
6566 roster
6567 .get(start + 1..)?
6568 .iter()
6569 .find(|s| !tried.contains(&s.id))
6570}
6571
6572fn pick_successor<'a>(
6593 roster: &'a [AgentSpec],
6594 start: usize,
6595 tried: &BTreeSet<String>,
6596 carried: Option<&BTreeSet<String>>,
6597 others: &BTreeSet<String>,
6598) -> Option<&'a AgentSpec> {
6599 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
6600 roster
6601 .get(start + 1..)
6602 .and_then(|tail| {
6603 tail.iter()
6604 .filter(free)
6605 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
6606 })
6607 .or_else(|| {
6608 carried
6609 .is_some()
6610 .then(|| roster.iter().find(free))
6611 .flatten()
6612 })
6613 .or_else(|| match carried {
6614 Some(c) => next_for_seat(roster, start, tried, c),
6615 None => next_untried_in_roster(roster, start, tried),
6616 })
6617}
6618
6619fn next_for_seat<'a>(
6629 roster: &'a [AgentSpec],
6630 start: usize,
6631 round_tried: &BTreeSet<String>,
6632 carried_failed: &BTreeSet<String>,
6633) -> Option<&'a AgentSpec> {
6634 roster
6635 .get(start + 1..)?
6636 .iter()
6637 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
6638 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
6639}
6640
6641fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
6647 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
6648 return spec;
6649 };
6650 let ok = |id: &str| !h.failed.contains(id);
6651 if let Some(last) = h.last_ok.as_deref()
6652 && ok(last)
6653 && let Some(s) = roster.iter().find(|s| s.id == last)
6654 {
6655 return s.clone();
6656 }
6657 if ok(&spec.id) {
6658 return spec;
6659 }
6660 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
6661 roster
6662 .get(from + 1..)
6663 .into_iter()
6664 .flatten()
6665 .chain(roster.iter())
6666 .find(|s| ok(&s.id))
6667 .cloned()
6668 .unwrap_or(spec)
6669}
6670
6671pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6675 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6676}
6677
6678const TIMED_OUT: &str = "timed out";
6681
6682impl FailClass {
6683 fn of(out: &AgentOutcome) -> Option<Self> {
6685 match out {
6686 AgentOutcome::Ok(_) => None,
6687 AgentOutcome::Quota(_) => Some(Self::Quota),
6688 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6689 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6690 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6691 }
6692 }
6693
6694 fn stem_word(&self) -> &'static str {
6696 match self {
6697 Self::Quota => "quota",
6698 _ => "handover",
6699 }
6700 }
6701}
6702
6703fn failure_signature(msg: &str) -> String {
6707 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6708 let mut out = Vec::new();
6709 for word in line.split_whitespace() {
6710 if word.contains('/') || word.contains('\\') {
6711 out.push("<path>".to_owned());
6712 continue;
6713 }
6714 let mut w = String::new();
6715 let mut in_digits = false;
6716 for c in word.chars() {
6717 if c.is_ascii_digit() {
6718 if !in_digits {
6719 w.push('#');
6720 }
6721 in_digits = true;
6722 } else {
6723 in_digits = false;
6724 w.push(c);
6725 }
6726 }
6727 out.push(w);
6728 }
6729 out.join(" ").chars().take(120).collect()
6730}
6731
6732fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6739 match cur {
6740 FailClass::Quota | FailClass::Timeout => true,
6741 FailClass::Other(_) => prev != Some(cur),
6742 }
6743}
6744
6745fn fail_reason(out: &AgentOutcome) -> String {
6747 match out {
6748 AgentOutcome::Ok(_) => String::new(),
6749 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6750 AgentOutcome::Dropped(o) => format!(
6751 "the CLI dropped the stream ({})",
6752 o.dropped
6753 .as_ref()
6754 .map(|d| d.why.as_str())
6755 .unwrap_or("it ended without delivering its answer")
6756 ),
6757 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6758 }
6759}
6760
6761pub(crate) fn record_handover(
6764 state: &mut RunState,
6765 node: &str,
6766 seat: &str,
6767 from: &str,
6768 to: &str,
6769 class: &FailClass,
6770 reason: &str,
6771) {
6772 let message = if *class == FailClass::Quota {
6773 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6774 } else {
6775 format!("{seat}: handed over {from} -> {to} ({reason})")
6776 };
6777 state.event(node, message);
6778 state.handovers.push(Handover {
6779 at: Timestamp::now(),
6780 node: node.to_owned(),
6781 seat: seat.to_owned(),
6782 from: from.to_owned(),
6783 to: to.to_owned(),
6784 reason: reason.to_owned(),
6785 });
6786}
6787
6788fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6802 commands.iter().any(|c| c.exit_code.is_none())
6803}
6804
6805fn verified_noop_claim(
6818 usable: bool,
6819 commands: &[agent::CommandEvidence],
6820 text: &str,
6821) -> Option<String> {
6822 (usable && !has_unconfirmed_command(commands))
6823 .then(|| verdict::verified_noop(text))
6824 .flatten()
6825}
6826
6827fn short(commit: &str) -> String {
6828 commit.chars().take(7).collect()
6829}
6830
6831fn make_executable(path: &Path) -> Result<()> {
6832 #[cfg(unix)]
6833 {
6834 use std::os::unix::fs::PermissionsExt as _;
6835 let mut perms = std::fs::metadata(path)?.permissions();
6836 perms.set_mode(0o755);
6837 std::fs::set_permissions(path, perms)?;
6838 }
6839 #[cfg(not(unix))]
6840 {
6841 let _ = path;
6842 }
6843 Ok(())
6844}
6845
6846struct WaveCtx<'a> {
6853 run: &'a str,
6856 node: &'a str,
6858 prompts: &'a Prompts,
6859 cache: Option<&'a Path>,
6861 round: Option<usize>,
6864 carry_seats: bool,
6868}
6869
6870async fn run_one(
6872 job: SeatJob,
6873 sem: Arc<Semaphore>,
6874 ctx: &WaveCtx<'_>,
6875 state: &mut RunState,
6876 attempt: usize,
6877) -> (SeatState, AgentOutcome) {
6878 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6879 .await
6880 .pop()
6881 .expect("one job in, one result out");
6882 (seat, out)
6883}
6884
6885async fn wave(
6891 jobs: Vec<SeatJob>,
6892 sem: Arc<Semaphore>,
6893 ctx: &WaveCtx<'_>,
6894 state: &mut RunState,
6895 attempt: usize,
6896) -> Vec<(usize, SeatState, AgentOutcome)> {
6897 let WaveCtx {
6898 run,
6899 node,
6900 prompts,
6901 cache,
6902 round,
6903 carry_seats: _,
6904 } = *ctx;
6905 for job in &jobs {
6906 state.seat_started(node, &job.seat.key, job.timeout, attempt);
6907 }
6908 if let Err(e) = state.save() {
6909 tracing::warn!("could not persist in-progress seats: {e:#}");
6914 }
6915 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6931 let wait_started = Instant::now();
6932 let cache_guard = if let Some(cache_dir) = cache {
6933 if jobs_had_a_writer {
6934 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6935 let budget = jobs
6936 .iter()
6937 .map(|j| j.timeout)
6938 .max()
6939 .unwrap_or(Duration::from_secs(60));
6940 acquire_cache_lease(state, cache_dir, &owner, budget, node)
6941 .await
6942 .ok()
6943 } else {
6944 None
6945 }
6946 } else {
6947 None
6948 };
6949 let waited_for_lease = wait_started.elapsed();
6956 let mut set = tokio::task::JoinSet::new();
6957 let overlay = prompts.overlay(node);
6958 for (i, mut job) in jobs.into_iter().enumerate() {
6959 job.timeout = job.timeout.saturating_sub(waited_for_lease);
6960 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6961 if cache.is_some() {
6962 job.prompt.push('\n');
6963 job.prompt
6964 .push_str(&prompt::build_cache_note(node, job.allow_write));
6965 }
6966 let sem = Arc::clone(&sem);
6967 let run = run.to_owned();
6968 let node = node.to_owned();
6969 let attachments = if node == "implement" {
6972 state.attachments.clone()
6973 } else {
6974 Vec::new()
6975 };
6976 let cache = cache
6987 .filter(|_| job.allow_write && cache_guard.is_some())
6988 .map(Path::to_path_buf);
6989 set.spawn(async move {
6990 let _permit = sem.acquire().await;
6991 let mut seat = job.seat;
6992 let out = agent::invoke(
6993 &job.spec,
6994 &mut seat,
6995 &Invocation {
6996 cwd: &job.cwd,
6997 prompt: &job.prompt,
6998 timeout: job.timeout,
6999 allow_write: job.allow_write,
7000 unsandboxed: false,
7001 sessions: job.sessions,
7002 artifacts: &job.artifacts,
7003 stem: &job.stem,
7004 run: &run,
7005 node: &node,
7006 cache_dir: cache.as_deref(),
7007 attachments: &attachments,
7008 writable: &[],
7009 },
7010 )
7011 .await;
7012 let out = match out {
7013 Ok(o) if o.usable() => AgentOutcome::Ok(o),
7014 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
7015 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
7023 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
7024 Ok(o) => AgentOutcome::Failed(format!(
7025 "exited with {:?} and no usable output",
7026 o.exit_code
7027 )),
7028 Err(e) => AgentOutcome::Failed(e.to_string()),
7029 };
7030 (i, seat, out)
7031 });
7032 }
7033 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
7034 while let Some(joined) = set.join_next().await {
7035 let (i, seat, out) = match joined {
7036 Ok(v) => v,
7037 Err(e) => {
7041 tracing::error!("agent task panicked: {e}");
7042 continue;
7043 }
7044 };
7045 state.seat_finished(&seat.key);
7046 record_jobs(state, node, round, &seat.key, &out);
7047 if let Err(e) = state.save() {
7048 tracing::warn!("could not persist a seat's completion: {e:#}");
7049 }
7050 if collected.len() <= i {
7051 collected.resize_with(i + 1, || None);
7052 }
7053 collected[i] = Some((i, seat, out));
7054 }
7055 if state
7061 .active
7062 .values()
7063 .any(|a| a.node == node && a.attempt == attempt)
7064 {
7065 state
7066 .active
7067 .retain(|_, a| !(a.node == node && a.attempt == attempt));
7068 if let Err(e) = state.save() {
7069 tracing::warn!("could not persist the end of a wave: {e:#}");
7070 }
7071 }
7072 if let Some(cache_dir) = cache
7079 && jobs_had_a_writer
7080 {
7081 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
7082 }
7083 if let Some(guard) = cache_guard {
7084 guard.release();
7085 }
7086 collected.into_iter().flatten().collect()
7087}
7088
7089fn record_jobs(
7100 state: &mut RunState,
7101 node: &str,
7102 round: Option<usize>,
7103 seat: &str,
7104 out: &AgentOutcome,
7105) {
7106 let commands: &[agent::CommandEvidence] = match out {
7107 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
7108 AgentOutcome::Failed(_) => &[],
7109 };
7110 let checked_at = Timestamp::now();
7111 for c in commands {
7112 state.jobs.push(JobRecord {
7113 node: node.to_owned(),
7114 round,
7115 seat: seat.to_owned(),
7116 id: c.id.clone(),
7117 description: c.description.clone(),
7118 checked_at,
7119 status: match c.exit_code {
7120 Some(0) => JobStatus::Completed,
7121 Some(_) => JobStatus::Failed,
7122 None => JobStatus::Unknown,
7123 },
7124 exit_code: c.exit_code,
7125 result_summary: c.result_summary.clone(),
7126 source: c.source.clone(),
7127 });
7128 }
7129}
7130
7131fn round_is_clean(
7152 blocking: usize,
7153 e2e_ok: bool,
7154 answered: usize,
7155 expected: usize,
7156 quota_missing: usize,
7157 policy: IncompleteReviewPolicy,
7158) -> bool {
7159 if blocking != 0 || !e2e_ok {
7160 return false;
7161 }
7162 if answered == expected || policy == IncompleteReviewPolicy::Warn {
7163 return true;
7164 }
7165 answered > 0 && expected - answered <= quota_missing
7166}
7167
7168fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
7192 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
7193 return Some(RunStatus::Gating);
7194 }
7195 let last = reviews.last()?;
7196 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
7197 if reviews.len() < max_rounds && !stagnant {
7198 return None;
7199 }
7200 if last.incomplete() && last.blocking == 0 {
7201 return Some(RunStatus::Blocked);
7202 }
7203 if last.e2e_status() == E2eStatus::ResourceBlocked {
7204 return None;
7205 }
7206 Some(if last.e2e.iter().all(CommandOutcome::ok) {
7207 RunStatus::Gating
7208 } else {
7209 RunStatus::Blocked
7210 })
7211}
7212
7213pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
7228 if nudged {
7229 (full / 4).max(Duration::from_secs(120)).min(full)
7230 } else {
7231 full
7232 }
7233}
7234
7235#[allow(clippy::too_many_arguments)]
7264async fn ask_json_wave<T>(
7265 jobs: Vec<SeatJob>,
7266 sem: Arc<Semaphore>,
7267 retries: usize,
7268 roster: &[AgentSpec],
7269 ctx: &WaveCtx<'_>,
7270 losses: &mut Vec<QuotaLoss>,
7271 state: &mut RunState,
7272 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
7273) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7274where
7275 T: serde::de::DeserializeOwned + Send + 'static,
7276{
7277 ask_wave_with(
7278 jobs,
7279 sem,
7280 retries,
7281 roster,
7282 ctx,
7283 losses,
7284 state,
7285 &|text: &str| {
7286 let v = verdict::extract_json::<T>(text)?;
7287 validate(&v)?;
7288 Ok(v)
7289 },
7290 )
7291 .await
7292}
7293
7294#[allow(clippy::too_many_arguments)]
7303async fn ask_wave_with<T>(
7304 jobs: Vec<SeatJob>,
7305 sem: Arc<Semaphore>,
7306 retries: usize,
7307 roster: &[AgentSpec],
7308 ctx: &WaveCtx<'_>,
7309 losses: &mut Vec<QuotaLoss>,
7310 state: &mut RunState,
7311 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
7312) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7313where
7314 T: Send + 'static,
7315{
7316 let n = jobs.len();
7317 let originals: Vec<SeatJob> = jobs;
7318 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
7319 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
7320 let mut nudges: Vec<usize> = vec![0; n];
7327 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
7331 let mut tried: Vec<BTreeSet<String>> = specs
7332 .iter()
7333 .map(|s| BTreeSet::from([s.id.clone()]))
7334 .collect();
7335 let starts: Vec<usize> = specs
7336 .iter()
7337 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
7338 .collect();
7339 let carry = ctx.carry_seats && !roster.is_empty();
7340 let carried: Vec<BTreeSet<String>> = originals
7343 .iter()
7344 .map(|j| {
7345 state
7346 .seat_history
7347 .get(&j.seat.key)
7348 .filter(|_| carry)
7349 .map(|h| h.failed.clone())
7350 .unwrap_or_default()
7351 })
7352 .collect();
7353 let mut prev: Vec<Option<FailClass>> = originals
7354 .iter()
7355 .map(|j| {
7356 state
7357 .seat_history
7358 .get(&j.seat.key)
7359 .filter(|_| carry)
7360 .and_then(|h| h.last_fail.clone())
7361 })
7362 .collect();
7363 let next_agent =
7364 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
7365 let others: BTreeSet<String> = specs
7366 .iter()
7367 .enumerate()
7368 .filter(|(j, _)| *j != i)
7369 .map(|(_, s)| s.id.clone())
7370 .collect();
7371 pick_successor(
7372 roster,
7373 starts[i],
7374 tried,
7375 carry.then(|| &carried[i]),
7376 &others,
7377 )
7378 .cloned()
7379 };
7380 let mut fresh: Vec<Option<String>> = vec![None; n];
7381 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
7382 let mut pending: Vec<usize> = (0..n).collect();
7383
7384 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
7389 for round in 0..max_rounds {
7390 if pending.is_empty() {
7391 break;
7392 }
7393 let mut batch = Vec::with_capacity(pending.len());
7394 let mut renudged: Vec<&str> = Vec::new();
7395 for &i in &pending {
7396 let src = &originals[i];
7397 let full: &str = match &src.handover {
7402 Some(h) if specs[i].id != src.spec.id => h,
7403 _ => &src.prompt,
7404 };
7405 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
7408 (
7409 full.to_owned(),
7410 src.timeout,
7411 format!("{}-{word}-{}", src.stem, specs[i].id),
7412 )
7413 } else if nudges[i] == 0 {
7414 (full.to_owned(), src.timeout, src.stem.clone())
7415 } else {
7416 renudged.push(src.seat.key.as_str());
7417 let why = done[i]
7418 .as_ref()
7419 .and_then(|r| r.as_ref().err().map(ToString::to_string))
7420 .unwrap_or_else(|| "no parsable answer".to_owned());
7421 let nudge = prompt::nudge(&why);
7422 let nudged = has_context(&specs[i], &seats[i], src.sessions);
7423 let prompt = if nudged {
7424 nudge
7425 } else {
7426 format!("{full}\n\n---\n\n{nudge}")
7427 };
7428 (
7429 prompt,
7430 retry_budget(src.timeout, nudged),
7431 format!("{}-retry{}", src.stem, nudges[i]),
7432 )
7433 };
7434 batch.push(SeatJob {
7435 spec: specs[i].clone(),
7436 seat: seats[i].clone(),
7437 cwd: src.cwd.clone(),
7438 prompt,
7439 timeout,
7440 allow_write: src.allow_write,
7441 sessions: src.sessions,
7442 artifacts: src.artifacts.clone(),
7443 stem,
7444 handover: None,
7445 });
7446 }
7447
7448 if !renudged.is_empty() {
7449 state.event(
7450 ctx.node,
7451 format!("retry {round}: re-asking {}", renudged.join(", ")),
7452 );
7453 }
7454 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
7455 let mut still = Vec::new();
7456 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
7457 seats[i] = seat;
7458 let class = FailClass::of(&out);
7459 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
7462 && nudges[i] < retries
7463 && !roster.is_empty();
7464 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
7465 let next = should_hand_over(prev[i].as_ref(), &cur)
7466 .then(|| next_agent(i, &tried[i], &specs))
7467 .flatten();
7468 if carry {
7469 let h = state
7470 .seat_history
7471 .entry(originals[i].seat.key.clone())
7472 .or_default();
7473 h.failed.insert(specs[i].id.clone());
7474 h.last_fail = Some(cur.clone());
7475 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
7476 h.last_ok = None;
7477 }
7478 if let Err(e) = state.save() {
7481 tracing::warn!("could not persist a seat's failure history: {e:#}");
7482 }
7483 }
7484 if let Some(next) = next {
7485 record_handover(
7486 state,
7487 ctx.node,
7488 &originals[i].seat.key,
7489 &specs[i].id,
7490 &next.id,
7491 &cur,
7492 &fail_reason(&out),
7493 );
7494 tried[i].insert(next.id.clone());
7495 prev[i] = Some(cur.clone());
7496 seats[i] =
7497 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
7498 specs[i] = next;
7499 fresh[i] = Some(cur.stem_word().to_owned());
7500 nudges[i] = 0;
7501 done[i] = Some(Err(anyhow::anyhow!(
7502 "handed over after: {}",
7503 fail_reason(&out)
7504 )));
7505 still.push(i);
7506 continue;
7507 }
7508 }
7509 if carry
7510 && !nudge_first
7511 && let Some(cur) = class.clone()
7512 {
7513 let h = state
7515 .seat_history
7516 .entry(originals[i].seat.key.clone())
7517 .or_default();
7518 h.failed.insert(specs[i].id.clone());
7519 h.last_fail = Some(cur);
7520 }
7521 let parsed = match out {
7522 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
7523 AgentOutcome::Quota(o) => {
7524 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
7525 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
7526 }
7527 AgentOutcome::Dropped(o) => {
7531 let why = o
7532 .dropped
7533 .as_ref()
7534 .map(|d| d.why.as_str())
7535 .unwrap_or("the CLI ended the stream without delivering its answer");
7536 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
7537 }
7538 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
7539 };
7540 let quota = class == Some(FailClass::Quota);
7541 let failed = parsed.is_err();
7542 done[i] = Some(parsed);
7543 if carry && !failed {
7544 let h = state
7545 .seat_history
7546 .entry(originals[i].seat.key.clone())
7547 .or_default();
7548 h.failed.remove(&specs[i].id);
7549 h.last_ok = Some(specs[i].id.clone());
7550 h.last_fail = None;
7551 }
7552 let agent_failure = class.is_some()
7560 && !nudge_first
7561 && !roster.is_empty()
7562 && next_agent(i, &tried[i], &specs).is_some();
7563 if failed && !quota && !agent_failure && nudges[i] < retries {
7564 nudges[i] += 1;
7565 still.push(i);
7566 }
7567 }
7568 pending = still;
7569 }
7570
7571 for (i, q) in last_quota.into_iter().enumerate() {
7574 if let Some(reset) = q {
7575 losses.push(QuotaLoss {
7576 seat: originals[i].seat.key.clone(),
7577 node: ctx.node.to_owned(),
7578 at: Timestamp::now(),
7579 reset,
7580 });
7581 }
7582 }
7583
7584 seats
7585 .into_iter()
7586 .zip(done)
7587 .zip(nudges)
7588 .map(|((seat, res), attempts)| {
7589 (
7590 seat,
7591 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
7592 attempts,
7593 )
7594 })
7595 .collect()
7596}
7597
7598async fn acquire_cache_lease(
7611 state: &mut RunState,
7612 cache_dir: &Path,
7613 owner: &crate::cache::Owner,
7614 budget: Duration,
7615 context: &str,
7616) -> Result<crate::cache::Guard> {
7617 let home = crate::run::home();
7618 let started = Instant::now();
7619 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
7620 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
7621 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
7622 Err(e) => {
7623 state.event(
7624 "verify",
7625 format!("{context}: could not check the shared build cache: {e:#}"),
7626 );
7627 if let Err(e2) = state.save() {
7628 tracing::warn!("could not persist a cache-check failure: {e2:#}");
7629 }
7630 return Err(e);
7631 }
7632 };
7633 state.event(
7634 "verify",
7635 format!(
7636 "{context}: waiting for the shared build cache at {} ({})",
7637 cache_dir.display(),
7638 busy.describe()
7639 ),
7640 );
7641 if let Err(e) = state.save() {
7642 tracing::warn!("could not persist a cache wait: {e:#}");
7643 }
7644 let remaining = budget.saturating_sub(started.elapsed());
7645 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
7646 Ok(g) => Ok(g),
7647 Err(e) => {
7648 state.event("verify", format!("{context}: {e:#}"));
7649 if let Err(e2) = state.save() {
7650 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
7651 }
7652 Err(e)
7653 }
7654 }
7655}
7656
7657#[allow(clippy::too_many_arguments)]
7678async fn with_cache_lease<'s, F, Fut>(
7679 state: &'s mut RunState,
7680 cache_dir: Option<&Path>,
7681 node: &str,
7682 seat: &str,
7683 worktree: &Path,
7684 head: &str,
7685 budget: Duration,
7686 context: &str,
7687 body: F,
7688) -> (Vec<CommandOutcome>, bool)
7689where
7690 F: FnOnce(&'s mut RunState, Duration) -> Fut,
7691 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
7692{
7693 let Some(cache_dir) = cache_dir else {
7694 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
7695 return (outcomes, retried);
7696 };
7697 let home = crate::run::home();
7698 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
7699 let started = Instant::now();
7700 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
7701 Ok(g) => g,
7702 Err(e) => {
7703 return (
7704 vec![CommandOutcome {
7705 command: "(waiting for the shared build cache)".to_owned(),
7706 code: None,
7707 output_tail: e.to_string(),
7708 duration_ms: started.elapsed().as_millis() as u64,
7709 resource_blocked: true,
7710 }],
7711 false,
7712 );
7713 }
7714 };
7715 let identity = crate::cache::Identity::new(worktree, head);
7716 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7717 state.event(
7725 "verify",
7726 format!(
7727 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7728 worktree.display(),
7729 short(head)
7730 ),
7731 );
7732 guard.release();
7733 return (
7734 vec![CommandOutcome {
7735 command: "(confirming the shared build cache is fresh)".to_owned(),
7736 code: None,
7737 output_tail: e.to_string(),
7738 duration_ms: started.elapsed().as_millis() as u64,
7739 resource_blocked: true,
7740 }],
7741 false,
7742 );
7743 }
7744 let remaining = budget.saturating_sub(started.elapsed());
7745 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7746 if !timed_out_pids.is_empty() {
7751 wait_for_timed_out_children_to_die(&timed_out_pids).await;
7752 }
7753 guard.release();
7754 (outcomes, retried)
7755}
7756
7757async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7769 wait_for_pids_with(
7770 pids,
7771 crate::proc::pid_alive,
7772 LEASE_RELEASE_POLL,
7773 LEASE_RELEASE_MAX_WAIT,
7774 )
7775 .await;
7776}
7777
7778async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7784 pids: &[u32],
7785 alive: F,
7786 poll: Duration,
7787 max_wait: Duration,
7788) {
7789 let deadline = Instant::now() + max_wait;
7790 loop {
7791 if pids.iter().all(|&pid| !alive(pid)) {
7792 return;
7793 }
7794 if Instant::now() >= deadline {
7795 return;
7796 }
7797 tokio::time::sleep(poll).await;
7798 }
7799}
7800
7801fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7808 outcomes.iter().any(|o| o.resource_blocked)
7809}
7810
7811enum GateFix {
7813 Retry,
7815 Stop,
7818 Defer,
7821}
7822
7823fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7831 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7832 red.peek().is_some()
7833 && red.all(|o| {
7834 !o.resource_blocked
7835 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7836 && !o.output_tail.trim().is_empty()
7837 })
7838}
7839
7840fn e2e_outcome_label(o: &CommandOutcome) -> String {
7844 if o.ok() {
7845 return "pass".to_owned();
7846 }
7847 let reason = if o.build_failed() {
7848 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7849 } else {
7850 format!("FAIL ({:?})", o.code)
7851 };
7852 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7853}
7854
7855async fn run_e2e_with_retry(
7863 state: &mut RunState,
7864 shell: &[String],
7865 commands: &[String],
7866 worktree: &Path,
7867 timeout: Duration,
7868 context: &str,
7869) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7870 let (mut e2e, mut timed_out_pids) = run_commands(
7871 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7872 )
7873 .await;
7874 for o in &e2e {
7875 state.event(
7876 "verify",
7877 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7878 );
7879 }
7880 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7884 if verify_retried {
7885 state.event(
7886 "verify",
7887 format!(
7888 "{context}: verify could not build/link, not a test result — retrying once \
7889 before concluding"
7890 ),
7891 );
7892 let retried = run_commands(
7893 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7894 )
7895 .await;
7896 e2e = retried.0;
7897 timed_out_pids.extend(retried.1);
7900 for o in &e2e {
7901 state.event(
7902 "verify",
7903 format!(
7904 "{context}: retry `{}` -> {}",
7905 o.command,
7906 e2e_outcome_label(o)
7907 ),
7908 );
7909 }
7910 }
7911 (e2e, verify_retried, timed_out_pids)
7912}
7913
7914#[allow(clippy::too_many_arguments)]
7930async fn run_commands(
7931 state: &mut RunState,
7932 node: &str,
7933 task: &str,
7934 attempt: usize,
7935 shell: &[String],
7936 commands: &[String],
7937 cwd: &Path,
7938 timeout: Duration,
7939) -> (Vec<CommandOutcome>, Vec<u32>) {
7940 if commands.is_empty() {
7941 return (Vec::new(), Vec::new());
7946 }
7947 let mut out = Vec::new();
7948 let mut timed_out_pids = Vec::new();
7949 let total = commands.len();
7950 for (idx, command) in commands.iter().enumerate() {
7951 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7952 if let Err(e) = state.save() {
7953 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7954 }
7955 let started = Instant::now();
7956 let mut cmd = tokio::process::Command::new(&shell[0]);
7957 cmd.quiet();
7958 cmd.args(&shell[1..])
7959 .arg(command)
7960 .current_dir(cwd)
7961 .stdin(std::process::Stdio::null())
7962 .stdout(std::process::Stdio::piped())
7963 .stderr(std::process::Stdio::piped())
7964 .kill_on_drop(true);
7965 let spawned = cmd.spawn();
7966 let (code, body) = match spawned {
7967 Ok(child) => {
7968 let pid = child.id();
7973 match tokio::time::timeout(timeout, child.wait_with_output()).await {
7974 Ok(Ok(o)) => {
7975 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7976 body.push_str(&String::from_utf8_lossy(&o.stderr));
7977 (o.status.code(), body)
7978 }
7979 Ok(Err(e)) => (None, format!("failed to run: {e}")),
7980 Err(_) => {
7981 if let Some(pid) = pid {
7982 timed_out_pids.push(pid);
7983 }
7984 (None, format!("timed out after {}s", timeout.as_secs()))
7985 }
7986 }
7987 }
7988 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7989 };
7990 out.push(CommandOutcome {
7991 command: command.clone(),
7992 code,
7993 output_tail: tail(&body, OUTPUT_TAIL),
7994 duration_ms: started.elapsed().as_millis() as u64,
7995 resource_blocked: false,
7996 });
7997 }
7998 state.task_finished(task);
7999 if let Err(e) = state.save() {
8000 tracing::warn!("could not persist the end of {task}: {e:#}");
8001 }
8002 (out, timed_out_pids)
8003}
8004
8005fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
8017 let repo = repo.display();
8018 match style {
8019 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
8020 MergeStyle::Squash => {
8021 let subject = message
8024 .lines()
8025 .next()
8026 .unwrap_or(branch)
8027 .replace(['\\', '"', '$', '`'], "");
8028 format!(
8029 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
8030 )
8031 }
8032 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
8033 }
8034}
8035
8036const PR_TITLE_MAX: usize = 240;
8047
8048fn english_title(text: &str, max: usize) -> Option<String> {
8059 let line = queue::first_line(text)?;
8060 let chars: Vec<char> = line.chars().collect();
8061 let (kept, mark) = if chars.len() <= max {
8062 (line.to_owned(), "")
8063 } else if let Some(end) = sentence_end(&chars, max) {
8064 (chars[..end].iter().collect::<String>(), "")
8065 } else {
8066 let room = max.saturating_sub(3);
8067 let cut = if chars[room].is_whitespace() {
8070 room
8071 } else {
8072 chars[..room]
8073 .iter()
8074 .rposition(|c| c.is_whitespace())
8075 .unwrap_or(room)
8076 };
8077 let head: String = chars[..cut].iter().collect();
8078 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
8079 (head.to_owned(), "...")
8080 };
8081 if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
8082 return None;
8083 }
8084 Some(format!("{kept}{mark}"))
8085}
8086
8087fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
8091 const MIN: usize = 20;
8092 for i in MIN..max.min(chars.len()) {
8093 if !matches!(chars[i], '.' | '!' | '?') {
8094 continue;
8095 }
8096 let Some(&next) = chars.get(i + 1) else {
8097 continue;
8098 };
8099 if !next.is_whitespace() {
8100 continue;
8101 }
8102 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
8103 if after.is_some_and(|c| c.is_ascii_lowercase()) {
8104 continue;
8105 }
8106 let word: String = chars[..i]
8107 .iter()
8108 .rev()
8109 .take_while(|c| !c.is_whitespace())
8110 .collect::<Vec<_>>()
8111 .into_iter()
8112 .rev()
8113 .collect();
8114 let word = word.to_ascii_lowercase();
8115 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
8116 continue;
8117 }
8118 let end = chars[..i]
8119 .iter()
8120 .rposition(|c| !c.is_whitespace())
8121 .map_or(i, |p| p + 1);
8122 return Some(end);
8123 }
8124 None
8125}
8126
8127struct PrMessage {
8132 title: String,
8133 body: String,
8134}
8135
8136impl PrMessage {
8137 fn commit_message(&self) -> String {
8141 format!("{}\n\n{}", self.title, self.body)
8142 }
8143}
8144
8145fn title_marker(line: &str) -> Option<&str> {
8147 let line = line.trim();
8148 let head = line.get(..6)?;
8149 head.eq_ignore_ascii_case("title:")
8150 .then(|| line[6..].trim())
8151}
8152
8153fn summary_title(summary: &str) -> Option<String> {
8158 let first = summary.lines().find(|l| !l.trim().is_empty())?;
8159 let raw = title_marker(first)?;
8160 if raw.is_empty() {
8161 return None;
8162 }
8163 let title = queue::title_from(raw, PR_TITLE_MAX);
8164 let lower = title.to_ascii_lowercase();
8165 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
8166 return None;
8167 }
8168 Some(title)
8169}
8170
8171const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
8173
8174const EXISTING_BRANCH: &str = "(existing branch)";
8176
8177fn is_review_run(state: &RunState) -> bool {
8181 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
8182}
8183
8184fn review_title(state: &RunState) -> Option<String> {
8189 english_subject(state.reviewed_commits.as_ref()?.first()?)
8190}
8191
8192fn english_subject(raw: &str) -> Option<String> {
8195 let raw = raw.trim();
8196 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
8197 return None;
8198 }
8199 let title = queue::title_from(raw, PR_TITLE_MAX);
8200 let lower = title.to_ascii_lowercase();
8201 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
8202 return None;
8203 }
8204 Some(title)
8205}
8206
8207#[derive(Debug, Clone, PartialEq, Eq)]
8210struct BranchFacts {
8211 commits: Vec<(String, String)>,
8213 stat: String,
8215}
8216
8217const STAT_FILE_LINES: usize = 25;
8219const COMMITS_MAX_CHARS: usize = 20_000;
8221
8222async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
8225 let commits = git::commit_log(repo, base, branch).await.ok()?;
8226 if commits.is_empty() {
8227 return None;
8228 }
8229 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
8230 let lines: Vec<&str> = stat.lines().collect();
8231 let stat = if lines.len() > STAT_FILE_LINES + 1 {
8232 let omitted = lines.len() - 1 - STAT_FILE_LINES;
8233 let more = format!(" ... {omitted} more file(s)");
8234 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
8235 kept.push(&more);
8236 kept.push(lines[lines.len() - 1]);
8237 kept.join("\n")
8238 } else {
8239 lines.join("\n")
8240 };
8241 Some(BranchFacts { commits, stat })
8242}
8243
8244fn markdown_safe(text: &str) -> String {
8247 text.replace("</details>", "</details>")
8248 .replace("\x60\x60\x60", "~~~")
8249}
8250
8251fn neutral_title(state: &RunState, winner: char) -> String {
8252 format!(
8253 "chore: land candidate {} of run {}",
8254 winner.to_ascii_uppercase(),
8255 state.id
8256 )
8257}
8258
8259pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
8265 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
8266 ""
8267 } else {
8268 pr_title
8269 }
8270}
8271
8272pub fn landing_subject_source(state: &RunState) -> String {
8276 if is_review_run(state) {
8277 let winner = state.candidates.first().map_or('A', |c| c.label);
8278 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
8279 }
8280 state.instruction.clone()
8281}
8282
8283fn summary_without_title(summary: &str) -> String {
8286 let mut lines = summary.trim().lines().peekable();
8287 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
8288 lines.next();
8289 }
8290 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
8291}
8292
8293#[cfg(test)]
8307fn pr_message(state: &RunState, winner: char) -> PrMessage {
8308 pr_message_with(state, winner, None)
8309}
8310
8311#[cfg(test)]
8314fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8315 let raw = pr_message_raw(state, winner, facts);
8316 let id = crate::scrub::Identity::current();
8317 PrMessage {
8318 title: crate::scrub::scrub(&raw.title, &id),
8319 body: crate::scrub::scrub(&raw.body, &id),
8320 }
8321}
8322
8323fn pr_message_raw(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8324 let summary = state
8325 .candidates
8326 .iter()
8327 .find(|c| c.label == winner)
8328 .map(|c| c.summary.as_str())
8329 .unwrap_or_default();
8330 let review = is_review_run(state);
8333 let title = if review {
8334 facts
8335 .and_then(|f| english_subject(&f.commits.first()?.0))
8336 .or_else(|| review_title(state))
8337 .or_else(|| {
8338 state
8339 .candidates
8340 .iter()
8341 .find(|c| c.label == winner)
8342 .filter(|c| !c.branch.starts_with("magi/"))
8343 .and_then(|c| english_subject(&c.branch))
8344 })
8345 .unwrap_or_else(|| neutral_title(state, winner))
8346 } else {
8347 summary_title(summary).unwrap_or_else(|| {
8348 english_title(&state.instruction, PR_TITLE_MAX)
8349 .unwrap_or_else(|| neutral_title(state, winner))
8350 })
8351 };
8352
8353 let mut body = String::new();
8354 let what = summary_without_title(summary);
8355 if !what.is_empty() {
8356 body.push_str("## Summary\n\n");
8357 body.push_str(&what);
8358 body.push_str("\n\n");
8359 }
8360
8361 let notes: Vec<(usize, &str)> = state
8364 .reviews
8365 .iter()
8366 .filter_map(|r| {
8367 let n = r.fix.as_ref()?.notes.trim();
8368 (!n.is_empty()).then_some((r.round, n))
8369 })
8370 .collect();
8371 if !notes.is_empty() {
8372 body.push_str("## Review fixes\n\n");
8373 if let [(_, only)] = notes.as_slice() {
8374 body.push_str(only);
8375 body.push_str("\n\n");
8376 } else {
8377 for (round, n) in ¬es {
8378 body.push_str(&format!("### Round {round}\n\n{n}\n\n"));
8379 }
8380 }
8381 }
8382 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
8383
8384 let open = state.open_findings();
8385 if !open.is_empty() {
8386 body.push_str("## Open review findings\n\n");
8387 for f in &open {
8388 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
8389 }
8390 body.push('\n');
8391 }
8392
8393 if let Some(fix) = fix
8394 && !fix.rejected.is_empty()
8395 {
8396 body.push_str("## Declined by the fixer\n\n");
8397 for r in &fix.rejected {
8398 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
8399 }
8400 body.push('\n');
8401 }
8402
8403 if review {
8404 body.push_str("## Commits under review\n\n");
8406 if let Some(facts) = facts {
8407 let mut left = COMMITS_MAX_CHARS;
8408 for (i, (subject, text)) in facts.commits.iter().enumerate() {
8409 let mut entry = format!("- {}\n", markdown_safe(subject));
8410 for l in markdown_safe(text).lines() {
8411 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
8412 }
8413 if left == 0 {
8414 body.push_str(&format!(
8415 "- ... {} more commit(s)\n",
8416 facts.commits.len() - i
8417 ));
8418 break;
8419 }
8420 if entry.len() > left {
8421 let mut end = left;
8424 while !entry.is_char_boundary(end) {
8425 end -= 1;
8426 }
8427 entry.truncate(end);
8428 entry.push_str("\n ... (truncated)\n");
8429 left = 0;
8430 } else {
8431 left -= entry.len();
8432 }
8433 body.push_str(&entry);
8434 }
8435 if !facts.stat.trim().is_empty() {
8436 body.push_str(&format!(
8437 "\n## Diff stat\n\n```\n{}\n```\n",
8438 markdown_safe(facts.stat.trim())
8439 ));
8440 }
8441 } else {
8442 match &state.reviewed_commits {
8443 Some(subjects) => {
8444 for s in subjects {
8445 body.push_str(&format!("- {}\n", s.trim()));
8446 }
8447 }
8448 None => {
8449 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
8452 body.push_str(rest.trim());
8453 body.push('\n');
8454 }
8455 }
8456 }
8457 } else {
8458 let task = state.instruction.trim();
8459 let task = if task.is_empty() {
8460 "(empty task)"
8461 } else {
8462 task
8463 };
8464 body.push_str(&format!(
8465 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
8466 task.replace("</details>", "</details>")
8467 ));
8468 }
8469
8470 body.push_str(&format!(
8471 "\n---\nmagi:run/{} magi:candidate-{}\n",
8472 state.id,
8473 winner.to_ascii_lowercase()
8474 ));
8475
8476 PrMessage { title, body }
8477}
8478
8479fn seeded_instruction(state: &RunState) -> String {
8483 match refs::describe(&state.seeds) {
8484 Some(facts) => format!(
8485 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
8486 Candidates start from the unmerged branch named above, when there \
8487 is one, and carry any unmerged commit named by sha as a \
8488 cherry-pick. Check that this is what the task meant before \
8489 building on it.",
8490 state.instruction
8491 ),
8492 None => state.instruction.clone(),
8493 }
8494}
8495
8496async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
8500 let base = &state.base_branch;
8501 let mut against = base.clone();
8502 if mode == MergeMode::Pr {
8503 let remote = &state.config.merge.remote;
8507 let tracking = format!("{remote}/{base}");
8508 let fetched = git::fetch(repo, remote, base).await;
8509 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
8510 against = tracking;
8511 } else {
8512 return false;
8513 }
8514 }
8515 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
8516}
8517
8518fn empty_candidate_detail(state: &RunState, base: &str) -> String {
8521 let mut detail = format!(
8522 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
8523 nothing to open a pull request for"
8524 );
8525 match refs::describe(&state.seeds) {
8526 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
8527 None => detail.push_str(
8528 "\nThe task names no existing branch or commit; if it means to land work \
8529 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
8530 ),
8531 }
8532 detail
8533}
8534
8535#[derive(Debug, PartialEq, Eq)]
8538enum PrPlan {
8539 Create,
8540 Adopt { url: String, title: String },
8541 Stop(String),
8542}
8543
8544fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
8547 match found {
8548 Ok(land::OpenPr::None) => PrPlan::Create,
8549 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
8550 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
8551 "several open pull requests exist for this branch, not picking one: {}",
8552 urls.join(" ")
8553 )),
8554 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
8555 }
8556}
8557
8558async fn gh_pr_create(
8560 cwd: &Path,
8561 base: &str,
8562 head: &str,
8563 title: &str,
8564 body: &str,
8565) -> Result<String> {
8566 let out = tokio::process::Command::new("gh")
8567 .args([
8568 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
8569 ])
8570 .current_dir(cwd)
8571 .quiet()
8572 .stdin(std::process::Stdio::null())
8573 .output()
8574 .await
8575 .context("spawn gh")?;
8576 if out.status.success() {
8577 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
8578 } else {
8579 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
8580 }
8581}
8582
8583pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
8592 let repo = state.repo.clone();
8593 let root = state.worktree_root();
8594 let winner = state.tally.as_ref().map(|t| t.winner);
8595 let mut removed = Vec::new();
8596
8597 for i in 0..state.candidates.len() {
8598 let c = state.candidates[i].clone();
8599 let is_winner = Some(c.label) == winner;
8600 if is_winner && !drop_winner {
8601 continue;
8602 }
8603 if c.worktree.exists() {
8604 git::worktree_remove(&repo, &c.worktree).await.ok();
8605 removed.push(c.worktree.to_string_lossy().into_owned());
8606 }
8607 let handed_over = state.released_branches.contains(&c.branch);
8610 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
8611 git::branch_delete(&repo, &c.branch).await.ok();
8612 removed.push(c.branch.clone());
8613 }
8614 state.candidates[i].folded = true;
8615 }
8616
8617 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
8618 let path = name.path();
8619 let keep = !drop_winner
8620 && winner.is_some_and(|w| {
8621 path.file_name()
8622 .is_some_and(|n| n == format!("cand-{w}").as_str())
8623 });
8624 if keep {
8625 continue;
8626 }
8627 git::worktree_remove(&repo, &path).await.ok();
8628 removed.push(path.to_string_lossy().into_owned());
8629 }
8630
8631 remove_if_empty(&root);
8640
8641 if state.enabled_worktree_config && drop_winner {
8642 git::release_worktree_config(&repo).await.ok();
8646 state.enabled_worktree_config = false;
8647 }
8648 state.save_under(home)?;
8649 Ok(removed)
8650}
8651
8652fn remove_if_empty(dir: &Path) {
8663 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
8664 std::fs::remove_dir(dir).ok();
8665 }
8666}
8667
8668pub fn worst_open(state: &RunState) -> Option<Severity> {
8670 state
8671 .reviews
8672 .last()?
8673 .reviews
8674 .iter()
8675 .flat_map(|r| r.findings.iter())
8676 .map(|f| f.severity)
8677 .max()
8678}
8679
8680#[cfg(test)]
8681mod tests {
8682 #[test]
8683 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
8684 let leaked = vec!["chore(deps): update a crate".to_owned()];
8685 let own = "fix(daemon): apply a chosen action";
8686 assert!(should_retitle("", own, &leaked));
8687 assert!(should_retitle(
8688 &format!("{REVIEW_PROMPT_OPENING} `x`"),
8689 own,
8690 &leaked
8691 ));
8692 assert!(should_retitle(
8693 "chore: land candidate A of run 1",
8694 own,
8695 &leaked
8696 ));
8697 assert!(should_retitle(
8698 "magi: candidate A (uncommitted work)",
8699 own,
8700 &leaked
8701 ));
8702 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
8703 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
8704 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
8705 }
8706
8707 #[test]
8708 fn pr_merge_plan_creates_adopts_or_stops() {
8709 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
8710 assert_eq!(
8711 pr_merge_plan(Ok(land::OpenPr::One {
8712 url: "u".into(),
8713 title: "t".into()
8714 })),
8715 PrPlan::Adopt {
8716 url: "u".into(),
8717 title: "t".into()
8718 }
8719 );
8720 let PrPlan::Stop(many) =
8721 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
8722 else {
8723 panic!("many must stop");
8724 };
8725 assert!(many.contains('a') && many.contains('b'));
8726 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
8727 panic!("a failed lookup must stop");
8728 };
8729 assert!(err.contains("bad token"));
8730 }
8731
8732 use super::*;
8733 use crate::run::GateStatus;
8734 use std::collections::BTreeMap;
8735 use std::time::Duration;
8736
8737 fn conductor() -> AgentSpec {
8738 AgentSpec {
8739 id: "conductor".to_owned(),
8740 kind: crate::config::AgentKind::Command,
8741 model: None,
8742 command: vec!["true".to_owned()],
8743 extra_args: Vec::new(),
8744 env: BTreeMap::new(),
8745 prompt_delivery: None,
8746 }
8747 }
8748
8749 fn spec(id: &str) -> AgentSpec {
8750 AgentSpec {
8751 id: id.to_owned(),
8752 kind: crate::config::AgentKind::Command,
8753 model: None,
8754 command: vec!["true".to_owned()],
8755 extra_args: Vec::new(),
8756 env: BTreeMap::new(),
8757 prompt_delivery: None,
8758 }
8759 }
8760
8761 fn ids(xs: &[&str]) -> BTreeSet<String> {
8762 xs.iter().map(|s| (*s).to_owned()).collect()
8763 }
8764
8765 #[test]
8766 fn pick_successor_skips_an_agent_another_seat_holds() {
8767 let roster = [spec("a"), spec("b"), spec("c")];
8769 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
8770 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8771 }
8772
8773 #[test]
8774 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
8775 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
8777 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
8778 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8779 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
8780 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
8781 }
8782
8783 #[test]
8784 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
8785 let roster = [spec("a"), spec("b")];
8786 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
8787 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8788 let carried = ids(&["b"]);
8789 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
8790 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8791 }
8792
8793 #[test]
8794 fn pick_successor_returns_none_without_any_untried_successor() {
8795 let roster = [spec("a"), spec("b")];
8796 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
8797 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
8798 let carried = ids(&["a"]);
8799 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
8800 }
8801
8802 #[test]
8803 fn pick_successor_rescue_avoids_another_seats_occupant() {
8804 let roster = [spec("a"), spec("b"), spec("c")];
8806 let carried = ids(&["b", "c"]);
8807 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
8808 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8809 }
8810
8811 #[test]
8812 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
8813 let roster = [spec("a"), spec("b"), spec("c")];
8814 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
8815 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8816 }
8817
8818 #[test]
8819 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
8820 let roster = [spec("a"), spec("b"), spec("c")];
8821 let failed = ids(&["a", "b", "c"]);
8822 let mut tried = ids(&["b"]);
8824 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8825 assert_eq!(first.id, "a");
8826 tried.insert(first.id.clone());
8827 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8828 assert_eq!(second.id, "c");
8829 tried.insert(second.id.clone());
8830 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
8831 }
8832
8833 #[test]
8834 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
8835 let roster = [spec("a"), spec("b")];
8836 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
8837 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8838 }
8839
8840 #[test]
8841 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
8842 let roster = [spec("a"), spec("b"), spec("c")];
8843 let h = SeatHistory {
8844 failed: ids(&["a"]),
8845 last_ok: Some("b".to_owned()),
8846 last_fail: None,
8847 };
8848 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
8849 let gone = SeatHistory {
8851 last_ok: Some("zzz".to_owned()),
8852 ..h.clone()
8853 };
8854 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
8855 let failed = SeatHistory {
8856 failed: ids(&["a", "b"]),
8857 last_ok: Some("b".to_owned()),
8858 last_fail: None,
8859 };
8860 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
8861 }
8862
8863 #[test]
8864 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
8865 let first = SeatState::new("review-1", "alpha", 7);
8866 let next = handover_seat("review-1", "gamma", 7);
8867 assert_ne!(first.claude_session, next.claude_session);
8868 }
8869
8870 #[test]
8871 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
8872 let mut state = state_with_summary("x", "y");
8873 let a = handover_seat("review-1", "beta", state.next_seat_seed());
8874 let b = handover_seat("review-1", "beta", state.next_seat_seed());
8875 assert_ne!(a.claude_session, b.claude_session);
8876 }
8877
8878 #[test]
8879 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
8880 let roster = [spec("a"), spec("b")];
8881 let h = SeatHistory {
8882 failed: ids(&["a", "b"]),
8883 ..SeatHistory::default()
8884 };
8885 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
8886 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
8887 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
8888 }
8889
8890 #[test]
8897 fn failure_signature_ignores_numbers_and_paths() {
8898 assert_eq!(
8899 failure_signature("exited with Some(2) and no usable output"),
8900 failure_signature("exited with Some(137) and no usable output")
8901 );
8902 assert_eq!(
8903 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
8904 failure_signature("cannot open /var/x.txt: denied")
8905 );
8906 assert_ne!(failure_signature("boom"), failure_signature("bang"));
8907 }
8908
8909 #[test]
8910 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
8911 let other = FailClass::Other("x".into());
8912 assert!(should_hand_over(None, &FailClass::Quota));
8913 assert!(should_hand_over(Some(&other), &FailClass::Quota));
8914 assert!(should_hand_over(
8915 Some(&FailClass::Timeout),
8916 &FailClass::Timeout
8917 ));
8918 assert!(should_hand_over(None, &other));
8919 assert!(!should_hand_over(Some(&other), &other));
8920 assert!(should_hand_over(
8921 Some(&other),
8922 &FailClass::Other("y".into())
8923 ));
8924 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
8926 assert!(should_hand_over(Some(&FailClass::Quota), &other));
8927 }
8928
8929 #[test]
8930 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
8931 let a = SeatState::new("judge-1", "alpha", 7);
8932 let b = handover_seat("judge-1", "beta", 7);
8933 assert_ne!(a.claude_session, b.claude_session);
8934 assert_eq!(b.turns, 0);
8935 }
8936
8937 #[test]
8938 fn a_timeout_is_classified_apart_from_other_failures() {
8939 assert_eq!(
8940 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
8941 Some(FailClass::Timeout)
8942 );
8943 assert!(matches!(
8944 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
8945 Some(FailClass::Other(_))
8946 ));
8947 }
8948
8949 #[test]
8950 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
8951 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8952 let tried = BTreeSet::from(["beta".to_owned()]);
8953 let next = next_untried_in_roster(&roster, 1, &tried);
8956 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
8957 }
8958
8959 #[test]
8960 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
8961 let roster = vec![spec("alpha"), spec("beta")];
8962 let tried = BTreeSet::from(["beta".to_owned()]);
8963 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8967 }
8968
8969 #[test]
8970 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
8971 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8972 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
8973 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8977 }
8978
8979 #[test]
8980 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
8981 let roster = vec![spec("a"), spec("a"), spec("b")];
8982 let tried = BTreeSet::from(["a".to_owned()]);
8983 let next = next_untried_in_roster(&roster, 0, &tried);
8984 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8985 }
8986
8987 #[test]
8988 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
8989 let roster = vec![spec("a"), spec("b")];
8990 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
8991 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
8992 }
8993
8994 #[test]
8995 fn remove_if_empty_only_ever_takes_a_bare_directory() {
8996 let dir = tempfile::tempdir().unwrap();
8997 let bay = dir.path().join("ffff");
8998
8999 remove_if_empty(&bay);
9001 assert!(!bay.exists());
9002
9003 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
9006 remove_if_empty(&bay);
9007 assert!(bay.exists(), "non-empty directory must survive");
9008
9009 std::fs::remove_dir(bay.join("cand-A")).unwrap();
9011 remove_if_empty(&bay);
9012 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
9013 }
9014
9015 #[test]
9024 fn a_full_panel_that_found_nothing_is_clean() {
9025 assert!(round_is_clean(
9026 0,
9027 true,
9028 2,
9029 2,
9030 0,
9031 IncompleteReviewPolicy::Block
9032 ));
9033 }
9034
9035 #[test]
9036 fn a_missing_seat_is_never_clean_under_the_default_policy() {
9037 assert!(!round_is_clean(
9038 0,
9039 true,
9040 1,
9041 2,
9042 0,
9043 IncompleteReviewPolicy::Block
9044 ));
9045 }
9046
9047 #[test]
9048 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
9049 assert!(!round_is_clean(
9050 1,
9051 true,
9052 1,
9053 2,
9054 0,
9055 IncompleteReviewPolicy::Warn
9056 ));
9057 }
9058
9059 #[test]
9060 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
9061 assert!(round_is_clean(
9062 0,
9063 true,
9064 1,
9065 2,
9066 0,
9067 IncompleteReviewPolicy::Warn
9068 ));
9069 }
9070
9071 #[test]
9072 fn a_full_panel_with_an_open_finding_is_not_clean() {
9073 assert!(!round_is_clean(
9074 1,
9075 true,
9076 2,
9077 2,
9078 0,
9079 IncompleteReviewPolicy::Block
9080 ));
9081 }
9082
9083 #[test]
9084 fn a_full_panel_with_a_red_e2e_is_not_clean() {
9085 assert!(!round_is_clean(
9086 0,
9087 false,
9088 2,
9089 2,
9090 0,
9091 IncompleteReviewPolicy::Block
9092 ));
9093 }
9094
9095 #[test]
9102 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
9103 assert!(round_is_clean(
9106 0,
9107 true,
9108 1,
9109 2,
9110 1,
9111 IncompleteReviewPolicy::Block
9112 ));
9113 }
9114
9115 #[test]
9116 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
9117 assert!(!round_is_clean(
9120 0,
9121 true,
9122 1,
9123 2,
9124 0,
9125 IncompleteReviewPolicy::Block
9126 ));
9127 }
9128
9129 #[test]
9130 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
9131 assert!(!round_is_clean(
9132 1,
9133 true,
9134 1,
9135 2,
9136 1,
9137 IncompleteReviewPolicy::Block
9138 ));
9139 assert!(!round_is_clean(
9140 0,
9141 false,
9142 1,
9143 2,
9144 1,
9145 IncompleteReviewPolicy::Block
9146 ));
9147 }
9148
9149 #[test]
9150 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
9151 assert!(!round_is_clean(
9155 0,
9156 true,
9157 0,
9158 2,
9159 2,
9160 IncompleteReviewPolicy::Block
9161 ));
9162 }
9163
9164 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
9165 CommandOutcome {
9166 command: "test".to_owned(),
9167 code,
9168 output_tail: String::new(),
9169 duration_ms: 0,
9170 resource_blocked,
9171 }
9172 }
9173
9174 #[test]
9175 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
9176 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
9177 assert!(
9178 !verify_inconclusive(&[outcome(Some(1), false)]),
9179 "an ordinary failure is still evidence about the patch"
9180 );
9181 assert!(verify_inconclusive(&[outcome(None, true)]));
9182 assert!(
9183 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
9184 "one inconclusive outcome taints the whole batch"
9185 );
9186 assert!(!verify_inconclusive(&[]));
9187 }
9188
9189 #[tokio::test]
9190 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
9191 let calls = std::sync::atomic::AtomicUsize::new(0);
9195 let started = Instant::now();
9196 wait_for_pids_with(
9197 &[123],
9198 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
9199 Duration::from_millis(5),
9200 Duration::from_secs(5),
9201 )
9202 .await;
9203 assert!(
9204 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
9205 "must keep checking rather than deciding on the first answer"
9206 );
9207 assert!(
9208 started.elapsed() < Duration::from_secs(1),
9209 "must return the moment it is confirmed dead, not wait out the ceiling"
9210 );
9211 }
9212
9213 #[tokio::test]
9214 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
9215 let started = Instant::now();
9216 wait_for_pids_with(
9217 &[123],
9218 |_| true, Duration::from_millis(5),
9220 Duration::from_millis(30),
9221 )
9222 .await;
9223 let elapsed = started.elapsed();
9224 assert!(
9225 elapsed >= Duration::from_millis(30),
9226 "must not give up before its own ceiling: {elapsed:?}"
9227 );
9228 assert!(
9229 elapsed < Duration::from_secs(1),
9230 "must not wait past its own ceiling either: {elapsed:?}"
9231 );
9232 }
9233
9234 #[tokio::test]
9235 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
9236 let started = Instant::now();
9237 wait_for_pids_with(
9238 &[],
9239 |_| true,
9240 Duration::from_secs(5),
9241 Duration::from_secs(5),
9242 )
9243 .await;
9244 assert!(
9245 started.elapsed() < Duration::from_millis(200),
9246 "an empty pid list has nothing to confirm"
9247 );
9248 }
9249
9250 fn review_round(
9256 clean: bool,
9257 blocking: usize,
9258 answered: usize,
9259 expected: usize,
9260 progressed: bool,
9261 e2e_ok: bool,
9262 ) -> ReviewRound {
9263 ReviewRound {
9264 round: 1,
9265 head: "h".to_owned(),
9266 verified_head: None,
9267 verified_at: None,
9268 reviews: Vec::new(),
9269 e2e: vec![CommandOutcome {
9270 command: "test".to_owned(),
9271 code: Some(if e2e_ok { 0 } else { 1 }),
9272 output_tail: String::new(),
9273 duration_ms: 0,
9274 resource_blocked: false,
9275 }],
9276 verify_retried: false,
9277 e2e_deferred: false,
9278 e2e_defer_reason: None,
9279 fix: None,
9280 blocking,
9281 answered,
9282 expected,
9283 clean,
9284 progressed,
9285 vote_split: false,
9286 reconsideration: Vec::new(),
9287 verdict: None,
9288 }
9289 }
9290
9291 #[test]
9292 fn review_conclusion_is_none_when_nothing_has_run() {
9293 assert_eq!(review_conclusion(&[], 3), None);
9294 }
9295
9296 #[test]
9297 fn review_conclusion_is_none_while_rounds_remain() {
9298 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
9299 assert_eq!(review_conclusion(&rounds, 3), None);
9300 }
9301
9302 #[test]
9303 fn review_conclusion_is_gating_once_a_round_is_clean() {
9304 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
9305 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
9306 }
9307
9308 #[test]
9309 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
9310 let rounds = vec![
9311 review_round(false, 1, 2, 2, true, true),
9312 review_round(false, 1, 2, 2, true, true),
9313 ];
9314 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
9315 }
9316
9317 #[test]
9318 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
9319 let rounds = vec![
9320 review_round(false, 1, 2, 2, true, true),
9321 review_round(false, 1, 2, 2, true, false),
9322 ];
9323 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
9324 }
9325
9326 #[test]
9327 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
9328 let mut blocked = review_round(false, 1, 2, 2, true, false);
9335 blocked.e2e[0].resource_blocked = true;
9336 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
9337 assert_eq!(review_conclusion(&rounds, 2), None);
9338 }
9339
9340 #[test]
9341 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
9342 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
9344 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
9345 }
9346
9347 #[test]
9348 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
9349 let rounds = vec![
9350 review_round(false, 1, 2, 2, false, true),
9351 review_round(false, 1, 2, 2, false, true),
9352 ];
9353 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
9354 }
9355
9356 fn secs(n: u64) -> Duration {
9357 Duration::from_secs(n)
9358 }
9359
9360 fn init_repo(dir: &Path) {
9363 let run = |args: &[&str]| {
9364 let out = std::process::Command::new("git")
9365 .args(args)
9366 .current_dir(dir)
9367 .quiet()
9368 .output()
9369 .expect("spawn git");
9370 assert!(
9371 out.status.success(),
9372 "git {args:?} failed: {}",
9373 String::from_utf8_lossy(&out.stderr)
9374 );
9375 };
9376 run(&["init", "-b", "main"]);
9377 run(&["config", "user.name", "magi test"]);
9378 run(&["config", "user.email", "magi@example.com"]);
9379 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
9380 run(&["add", "-A"]);
9381 run(&["commit", "-m", "init"]);
9382 }
9383
9384 fn ask_test_home() {
9392 crate::run::pin_test_home();
9393 }
9394
9395 fn runner_at(status: RunStatus) -> Runner {
9398 let mut state = RunState::new(
9399 PathBuf::from("/nonexistent/repo"),
9400 "main".to_owned(),
9401 "deadbeef".to_owned(),
9402 "task".to_owned(),
9403 Config::default(),
9404 );
9405 state.status = status;
9406 Runner {
9407 state,
9408 roles: ResolvedRoles {
9409 implementers: Vec::new(),
9410 judges: Vec::new(),
9411 reviewers: Vec::new(),
9412 fixer: None,
9413 conductor: conductor(),
9414 implementer_roster: Vec::new(),
9415 judge_roster: Vec::new(),
9416 reviewer_roster: Vec::new(),
9417 },
9418 sem: Arc::new(Semaphore::new(1)),
9419 pause: Pause::new(),
9420 interrupt: Pause::new(),
9421 }
9422 }
9423
9424 #[test]
9428 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
9429 crate::run::pin_test_home();
9430 let mut runner = runner_at(RunStatus::Implementing);
9431 let interrupt = Pause::new();
9432 runner.watch_interrupt(interrupt.clone());
9433
9434 interrupt.park_because("task a1b2 asked to run first");
9435
9436 assert!(runner.park_here().expect("park_here"));
9437 assert!(runner.state.parked);
9438 let last = runner.state.events.last().expect("a park event");
9439 assert_eq!(last.node, "park");
9440 assert!(
9441 last.message.contains("task a1b2 asked to run first"),
9442 "expected the interrupt reason in {:?}",
9443 last.message
9444 );
9445 }
9446
9447 #[test]
9455 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
9456 crate::run::pin_test_home();
9457 let mut runner = runner_at(RunStatus::Implementing);
9458 let shutdown = Pause::new();
9459 runner.on_pause(shutdown.clone());
9460 let interrupt = Pause::new();
9461 runner.watch_interrupt(interrupt.clone());
9462
9463 assert!(!runner.park_here().expect("park_here"));
9465 assert!(!runner.state.parked);
9466
9467 interrupt.park_because("test");
9469 assert!(!shutdown.parked());
9470 assert!(runner.park_here().expect("park_here"));
9471 }
9472
9473 #[tokio::test]
9487 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
9488 crate::run::pin_test_home();
9489 let mut runner = runner_at(RunStatus::Implementing);
9490 let interrupt = Pause::new();
9491 runner.watch_interrupt(interrupt.clone());
9492
9493 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
9494 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
9495
9496 let node = async move {
9500 started_tx.send(()).expect("send started");
9501 finish_rx.await.expect("recv finish");
9502 "node finished"
9503 };
9504
9505 let interrupter = async move {
9506 started_rx.await.expect("recv started");
9507 interrupt.park_because("higher-priority task waiting");
9509 tokio::task::yield_now().await;
9513 finish_tx.send(()).expect("send finish");
9514 };
9515
9516 let (node_result, ()) = tokio::join!(node, interrupter);
9517 assert_eq!(
9518 node_result, "node finished",
9519 "the in-flight call ran to completion"
9520 );
9521
9522 assert!(runner.park_here().expect("park_here"));
9525 assert!(runner.state.parked);
9526 }
9527
9528 #[test]
9534 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
9535 crate::run::pin_test_home();
9536 let mut runner = runner_at(RunStatus::Judging);
9537 runner.state.config.agents = vec![conductor()];
9541 runner.state.candidates = vec![Candidate {
9542 index: 0,
9543 label: 'A',
9544 agent: "alpha".to_owned(),
9545 branch: "magi/x/A".to_owned(),
9546 worktree: PathBuf::from("/nonexistent/worktree"),
9547 summary: "did the thing".to_owned(),
9548 stat: "1 file changed".to_owned(),
9549 files: 1,
9550 commits: 1,
9551 empty: false,
9552 failed: None,
9553 verified_noop: None,
9554 duration_ms: 1234,
9555 folded: false,
9556 }];
9557 let run_id = runner.state.id.clone();
9558
9559 let interrupt = Pause::new();
9560 runner.watch_interrupt(interrupt.clone());
9561 interrupt.park_because("task c3d4 asked to run first");
9562 assert!(runner.park_here().expect("park_here"));
9563
9564 let resumed = Runner::resume(&run_id).expect("resume");
9565 assert_eq!(resumed.state.candidates.len(), 1);
9566 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
9567 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
9568 assert_eq!(resumed.state.status, runner.state.status);
9569 assert!(
9570 resumed.state.parked,
9571 "still parked until `execute` actually walks the graph again"
9572 );
9573 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
9574 }
9575
9576 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
9578 let mut q = ask::Question::new(
9579 run.to_owned(),
9580 "implement".to_owned(),
9581 "impl-A".to_owned(),
9582 "Which storage backend should the cache use?".to_owned(),
9583 String::new(),
9584 vec!["SQLite".to_owned(), "Redis".to_owned()],
9585 );
9586 store.put(&mut q).unwrap();
9587 q
9588 }
9589
9590 #[test]
9591 fn a_failed_runs_open_question_is_abandoned() {
9592 ask_test_home();
9593 let store = ask::Questions::open();
9594 let mut runner = runner_at(RunStatus::Failed);
9595 let run = runner.state.id.clone();
9596 let q = ask_open_question(&store, &run);
9597
9598 runner.settle_questions();
9599
9600 let back = store.get(&q.id).unwrap();
9601 assert!(
9602 !back.status.open(),
9603 "the seat that asked died with the run; nobody is left to read an answer"
9604 );
9605 assert!(
9606 back.detail.contains(&run) && back.detail.contains("failed"),
9607 "the reason names what the run became, not just that it is gone: {}",
9608 back.detail
9609 );
9610 }
9611
9612 #[test]
9613 fn a_merged_runs_open_question_is_abandoned_too() {
9614 ask_test_home();
9615 let store = ask::Questions::open();
9616 for status in [RunStatus::Merged, RunStatus::Ready] {
9619 let mut runner = runner_at(status);
9620 let run = runner.state.id.clone();
9621 let q = ask_open_question(&store, &run);
9622
9623 runner.settle_questions();
9624
9625 let back = store.get(&q.id).unwrap();
9626 assert!(
9627 !back.status.open(),
9628 "{status:?} run's question must not outlive the run"
9629 );
9630 }
9631 }
9632
9633 #[test]
9634 fn a_still_resumable_runs_open_question_is_left_alone() {
9635 ask_test_home();
9636 let store = ask::Questions::open();
9637 for status in [RunStatus::Blocked, RunStatus::Stalled] {
9643 let mut runner = runner_at(status);
9644 let run = runner.state.id.clone();
9645 let q = ask_open_question(&store, &run);
9646
9647 runner.settle_questions();
9648
9649 let back = store.get(&q.id).unwrap();
9650 assert!(
9651 back.status.open(),
9652 "{status:?} is still alive; the question must still be waiting"
9653 );
9654 }
9655 }
9656
9657 #[test]
9658 fn settle_questions_never_touches_an_already_answered_question() {
9659 ask_test_home();
9660 let store = ask::Questions::open();
9661 let mut runner = runner_at(RunStatus::Failed);
9662 let run = runner.state.id.clone();
9663 let mut q = ask_open_question(&store, &run);
9664 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
9665 .unwrap();
9666 store.put(&mut q).unwrap();
9667
9668 runner.settle_questions();
9673 runner.settle_questions();
9674
9675 let back = store.get(&q.id).unwrap();
9676 assert_eq!(
9677 back.status,
9678 ask::QuestionStatus::Answered,
9679 "a real answer is a decision on record, never overwritten by a sweep"
9680 );
9681 }
9682
9683 #[tokio::test]
9694 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
9695 crate::run::pin_test_home();
9696 let tmp = tempfile::tempdir().expect("tempdir");
9697 let repo = tmp.path().join("repo");
9698 std::fs::create_dir_all(&repo).unwrap();
9699 init_repo(&repo);
9700
9701 let mut config = Config::default();
9702 config.graph.worktree_root = Some(tmp.path().join("wt"));
9703
9704 let mut state = RunState::new(
9705 repo.clone(),
9706 "main".to_owned(),
9707 "deadbeef".to_owned(),
9708 "task".to_owned(),
9709 config,
9710 );
9711 let root = state.worktree_root();
9712 let wt_a = root.join("cand-A");
9713 let wt_b = root.join("cand-B");
9714 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
9715 .await
9716 .expect("worktree A");
9717 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
9718 .await
9719 .expect("worktree B");
9720
9721 state.candidates = vec![
9722 Candidate {
9723 index: 0,
9724 label: 'A',
9725 agent: "alpha".to_owned(),
9726 branch: "magi/x/A".to_owned(),
9727 worktree: wt_a.clone(),
9728 summary: String::new(),
9729 stat: String::new(),
9730 files: 0,
9731 commits: 0,
9732 empty: false,
9733 failed: None,
9734 verified_noop: None,
9735 duration_ms: 0,
9736 folded: false,
9737 },
9738 Candidate {
9739 index: 1,
9740 label: 'B',
9741 agent: "beta".to_owned(),
9742 branch: "magi/x/B".to_owned(),
9743 worktree: wt_b.clone(),
9744 summary: String::new(),
9745 stat: String::new(),
9746 files: 0,
9747 commits: 0,
9748 empty: false,
9749 failed: None,
9750 verified_noop: None,
9751 duration_ms: 0,
9752 folded: false,
9753 },
9754 ];
9755 state.tally = Some(Tally {
9756 first_choice: BTreeMap::from([('A', 1)]),
9757 borda: BTreeMap::new(),
9758 winner: 'A',
9759 rankings: 1,
9760 unanimous_initial: true,
9761 deliberated: false,
9762 changed_votes: 0,
9763 unanimous_final: true,
9764 tie_break: None,
9765 judges: 1,
9766 present: 1,
9767 quorum: 1,
9768 met_quorum: true,
9769 uncontested: None,
9770 });
9771 state.status = RunStatus::Ready;
9772
9773 fold_run(&mut state, false, &crate::run::home())
9774 .await
9775 .expect("fold_run");
9776
9777 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
9778 assert!(
9779 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9780 "the unmerged winner's branch survives"
9781 );
9782 assert!(
9783 !state.candidates[0].folded,
9784 "the winner is not marked folded"
9785 );
9786
9787 assert!(!wt_b.exists(), "the loser's worktree is removed");
9788 assert!(
9789 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
9790 "the loser's branch is removed"
9791 );
9792 assert!(state.candidates[1].folded, "the loser is marked folded");
9793 }
9794
9795 #[tokio::test]
9798 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
9799 let tmp = tempfile::tempdir().expect("tempdir");
9800 let repo = tmp.path().join("repo");
9801 std::fs::create_dir_all(&repo).unwrap();
9802 init_repo(&repo);
9803 let home = tmp.path().join("home");
9804
9805 let mut config = Config::default();
9806 config.graph.worktree_root = Some(tmp.path().join("wt"));
9807 let mut state = RunState::new(
9808 repo.clone(),
9809 "main".to_owned(),
9810 "deadbeef".to_owned(),
9811 "task".to_owned(),
9812 config,
9813 );
9814 git::git(&repo, &["branch", "magi/x/A", "main"])
9816 .await
9817 .expect("branch");
9818 state.candidates = vec![Candidate {
9819 index: 0,
9820 label: 'A',
9821 agent: "alpha".to_owned(),
9822 branch: "magi/x/A".to_owned(),
9823 worktree: state.worktree_root().join("cand-A"),
9824 summary: String::new(),
9825 stat: String::new(),
9826 files: 0,
9827 commits: 0,
9828 empty: false,
9829 failed: None,
9830 verified_noop: None,
9831 duration_ms: 0,
9832 folded: true,
9833 }];
9834 state.released_to = Some("20260901-000000-new1".to_owned());
9835 state.released_branches = vec!["magi/x/A".to_owned()];
9836
9837 fold_run(&mut state, true, &home).await.expect("fold_run");
9838
9839 assert!(
9840 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9841 "the handed-over branch survives a fold"
9842 );
9843 }
9844
9845 #[tokio::test]
9849 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
9850 let tmp = tempfile::tempdir().expect("tempdir");
9851 let repo = tmp.path().join("repo");
9852 std::fs::create_dir_all(&repo).unwrap();
9853 init_repo(&repo);
9854 let run = |args: &[&str]| {
9855 let out = std::process::Command::new("git")
9856 .quiet()
9857 .args(args)
9858 .current_dir(&repo)
9859 .output()
9860 .expect("spawn git");
9861 assert!(out.status.success(), "git {args:?}");
9862 };
9863 run(&["branch", "magi/x/A"]);
9864 run(&["checkout", "-q", "-b", "magi/x/B"]);
9865 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
9866 run(&["add", "-A"]);
9867 run(&["commit", "-q", "-m", "work"]);
9868 run(&["checkout", "-q", "main"]);
9869
9870 let probe = RunState::new(
9874 repo.clone(),
9875 "main".to_owned(),
9876 "deadbeef".to_owned(),
9877 "task".to_owned(),
9878 Config::default(),
9879 );
9880 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
9881 let bare = tmp.path().join("origin.git");
9882 let out = std::process::Command::new("git")
9883 .quiet()
9884 .args(["init", "-q", "--bare"])
9885 .arg(&bare)
9886 .output()
9887 .expect("spawn git");
9888 assert!(out.status.success(), "git init --bare");
9889 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
9890 run(&["push", "-q", "origin", "main"]);
9891
9892 let mut state = RunState::new(
9893 repo.clone(),
9894 "main".to_owned(),
9895 "deadbeef".to_owned(),
9896 "task".to_owned(),
9897 Config::default(),
9898 );
9899 state.seeds = vec![refs::Seed {
9900 token: "magi/27b2/A".to_owned(),
9901 kind: refs::SeedKind::Unresolved,
9902 sha: String::new(),
9903 branch: true,
9904 detail: "no branch or commit named magi/27b2/A".to_owned(),
9905 }];
9906
9907 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
9908 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
9909 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
9910 let detail = empty_candidate_detail(&state, "main");
9911 assert!(detail.starts_with("empty candidate"), "{detail}");
9912 assert!(detail.contains("magi/27b2/A"), "{detail}");
9913 }
9914
9915 #[tokio::test]
9924 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
9925 let tmp = tempfile::tempdir().expect("tempdir");
9926 let repo = tmp.path().join("repo");
9927 std::fs::create_dir_all(&repo).unwrap();
9928 init_repo(&repo);
9929
9930 let mut config = Config::default();
9931 config.merge.mode = MergeMode::Local;
9932
9933 let mut state = RunState::new(
9934 repo.clone(),
9935 "main".to_owned(),
9936 "deadbeef".to_owned(),
9937 "task".to_owned(),
9938 config,
9939 );
9940 state.candidates = vec![Candidate {
9941 index: 0,
9942 label: 'A',
9943 agent: "alpha".to_owned(),
9944 branch: "does-not-exist".to_owned(),
9945 worktree: repo.clone(),
9946 summary: String::new(),
9947 stat: String::new(),
9948 files: 0,
9949 commits: 0,
9950 empty: false,
9951 failed: None,
9952 verified_noop: None,
9953 duration_ms: 0,
9954 folded: false,
9955 }];
9956 state.tally = Some(Tally {
9957 first_choice: BTreeMap::from([('A', 1)]),
9958 borda: BTreeMap::new(),
9959 winner: 'A',
9960 rankings: 1,
9961 unanimous_initial: true,
9962 deliberated: false,
9963 changed_votes: 0,
9964 unanimous_final: true,
9965 tie_break: None,
9966 judges: 0,
9967 present: 0,
9968 quorum: 0,
9969 met_quorum: true,
9970 uncontested: Some("only candidate A produced a change".to_owned()),
9971 });
9972 state.reviews = vec![ReviewRound {
9973 round: 1,
9974 head: "deadbeef".to_owned(),
9975 verified_head: None,
9976 verified_at: None,
9977 reviews: Vec::new(),
9978 e2e: Vec::new(),
9979 fix: None,
9980 blocking: 0,
9981 answered: 0,
9982 expected: 0,
9983 clean: true,
9984 verify_retried: false,
9985 e2e_deferred: false,
9986 e2e_defer_reason: None,
9987 progressed: false,
9988 vote_split: false,
9989 reconsideration: Vec::new(),
9990 verdict: None,
9991 }];
9992 state.gate = vec![CommandOutcome {
9993 command: "test".to_owned(),
9994 code: Some(0),
9995 output_tail: String::new(),
9996 duration_ms: 0,
9997 resource_blocked: false,
9998 }];
9999 state.gate_ran = true;
10000 state.status = RunStatus::Ready;
10005 state.merge = Some(MergeOutcome {
10006 mode: MergeMode::Local,
10007 ok: false,
10008 detail: "already concluded".to_owned(),
10009 empty: false,
10010 });
10011
10012 let mut runner = Runner {
10013 state,
10014 roles: ResolvedRoles {
10015 implementers: Vec::new(),
10016 judges: Vec::new(),
10017 reviewers: Vec::new(),
10018 fixer: None,
10019 conductor: conductor(),
10020 implementer_roster: Vec::new(),
10021 judge_roster: Vec::new(),
10022 reviewer_roster: Vec::new(),
10023 },
10024 sem: Arc::new(Semaphore::new(1)),
10025 pause: Pause::new(),
10026 interrupt: Pause::new(),
10027 };
10028
10029 runner.merge().await.expect("merge");
10030
10031 assert_eq!(
10032 runner.state.status,
10033 RunStatus::Ready,
10034 "a concluded run's status must not change on reentry"
10035 );
10036 assert_eq!(
10037 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10038 Some("already concluded"),
10039 "merge must not run again once the node already recorded an outcome"
10040 );
10041 }
10042
10043 #[tokio::test]
10052 async fn merge_refuses_a_gate_that_has_not_actually_run() {
10053 let tmp = tempfile::tempdir().expect("tempdir");
10054 let repo = tmp.path().join("repo");
10055 std::fs::create_dir_all(&repo).unwrap();
10056 init_repo(&repo);
10057
10058 let mut config = Config::default();
10059 config.merge.mode = MergeMode::Local;
10060
10061 let mut state = RunState::new(
10062 repo.clone(),
10063 "main".to_owned(),
10064 "deadbeef".to_owned(),
10065 "task".to_owned(),
10066 config,
10067 );
10068 state.candidates = vec![Candidate {
10069 index: 0,
10070 label: 'A',
10071 agent: "alpha".to_owned(),
10072 branch: "does-not-exist".to_owned(),
10073 worktree: repo.clone(),
10074 summary: String::new(),
10075 stat: String::new(),
10076 files: 0,
10077 commits: 0,
10078 empty: false,
10079 failed: None,
10080 verified_noop: None,
10081 duration_ms: 0,
10082 folded: false,
10083 }];
10084 state.tally = Some(Tally {
10085 first_choice: BTreeMap::from([('A', 1)]),
10086 borda: BTreeMap::new(),
10087 winner: 'A',
10088 rankings: 1,
10089 unanimous_initial: true,
10090 deliberated: false,
10091 changed_votes: 0,
10092 unanimous_final: true,
10093 tie_break: None,
10094 judges: 0,
10095 present: 0,
10096 quorum: 0,
10097 met_quorum: true,
10098 uncontested: Some("only candidate A produced a change".to_owned()),
10099 });
10100 state.reviews = vec![ReviewRound {
10101 round: 1,
10102 head: "deadbeef".to_owned(),
10103 verified_head: None,
10104 verified_at: None,
10105 reviews: Vec::new(),
10106 e2e: Vec::new(),
10107 fix: None,
10108 blocking: 0,
10109 answered: 0,
10110 expected: 0,
10111 clean: true,
10112 verify_retried: false,
10113 e2e_deferred: false,
10114 e2e_defer_reason: None,
10115 progressed: false,
10116 vote_split: false,
10117 reconsideration: Vec::new(),
10118 verdict: None,
10119 }];
10120 state.gate = Vec::new();
10122 state.gate_ran = false;
10123 state.status = RunStatus::Gating;
10124
10125 let mut runner = Runner {
10126 state,
10127 roles: ResolvedRoles {
10128 implementers: Vec::new(),
10129 judges: Vec::new(),
10130 reviewers: Vec::new(),
10131 fixer: None,
10132 conductor: conductor(),
10133 implementer_roster: Vec::new(),
10134 judge_roster: Vec::new(),
10135 reviewer_roster: Vec::new(),
10136 },
10137 sem: Arc::new(Semaphore::new(1)),
10138 pause: Pause::new(),
10139 interrupt: Pause::new(),
10140 };
10141
10142 runner.merge().await.expect("merge");
10143
10144 assert!(
10145 runner.state.merge.is_none(),
10146 "an empty gate must never be read as a passing one: {:?}",
10147 runner.state.merge
10148 );
10149 }
10150
10151 #[tokio::test]
10158 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
10159 let tmp = tempfile::tempdir().expect("tempdir");
10160 let repo = tmp.path().join("repo");
10161 std::fs::create_dir_all(&repo).unwrap();
10162 init_repo(&repo);
10163
10164 let config = Config::default();
10166
10167 let mut state = RunState::new(
10168 repo.clone(),
10169 "main".to_owned(),
10170 "deadbeef".to_owned(),
10171 "task".to_owned(),
10172 config,
10173 );
10174 state.candidates = vec![Candidate {
10175 index: 0,
10176 label: 'A',
10177 agent: "alpha".to_owned(),
10178 branch: "does-not-exist".to_owned(),
10179 worktree: repo.clone(),
10180 summary: String::new(),
10181 stat: String::new(),
10182 files: 0,
10183 commits: 0,
10184 empty: false,
10185 failed: None,
10186 verified_noop: None,
10187 duration_ms: 0,
10188 folded: false,
10189 }];
10190 state.tally = Some(Tally {
10191 first_choice: BTreeMap::from([('A', 1)]),
10192 borda: BTreeMap::new(),
10193 winner: 'A',
10194 rankings: 1,
10195 unanimous_initial: true,
10196 deliberated: false,
10197 changed_votes: 0,
10198 unanimous_final: true,
10199 tie_break: None,
10200 judges: 0,
10201 present: 0,
10202 quorum: 0,
10203 met_quorum: true,
10204 uncontested: Some("only candidate A produced a change".to_owned()),
10205 });
10206 state.reviews = vec![ReviewRound {
10207 round: 1,
10208 head: "deadbeef".to_owned(),
10209 verified_head: None,
10210 verified_at: None,
10211 reviews: Vec::new(),
10212 e2e: Vec::new(),
10213 fix: None,
10214 blocking: 0,
10215 answered: 0,
10216 expected: 0,
10217 clean: true,
10218 verify_retried: false,
10219 e2e_deferred: false,
10220 e2e_defer_reason: None,
10221 progressed: false,
10222 vote_split: false,
10223 reconsideration: Vec::new(),
10224 verdict: None,
10225 }];
10226
10227 let mut runner = Runner {
10228 state,
10229 roles: ResolvedRoles {
10230 implementers: Vec::new(),
10231 judges: Vec::new(),
10232 reviewers: Vec::new(),
10233 fixer: None,
10234 conductor: conductor(),
10235 implementer_roster: Vec::new(),
10236 judge_roster: Vec::new(),
10237 reviewer_roster: Vec::new(),
10238 },
10239 sem: Arc::new(Semaphore::new(1)),
10240 pause: Pause::new(),
10241 interrupt: Pause::new(),
10242 };
10243
10244 runner.gate().await.expect("gate");
10245 assert!(
10246 runner.state.gate_ran,
10247 "zero configured commands is still a real attempt, not an unrun gate"
10248 );
10249 assert!(runner.state.gate.is_empty());
10250 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
10251 assert_ne!(
10252 runner.state.status,
10253 RunStatus::Blocked,
10254 "a gate with nothing to check must not read as failed"
10255 );
10256
10257 runner.merge().await.expect("merge");
10258 assert_eq!(
10259 runner.state.status,
10260 RunStatus::Ready,
10261 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
10262 );
10263 }
10264
10265 #[tokio::test]
10276 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
10277 crate::run::pin_test_home();
10278 let home = crate::run::home();
10279
10280 let tmp = tempfile::tempdir().expect("tempdir");
10281 let repo = tmp.path().join("repo");
10282 std::fs::create_dir_all(&repo).unwrap();
10283 init_repo(&repo);
10284 let cache_dir = tmp.path().join("target");
10287
10288 let mut config = Config::default();
10289 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
10290 config.graph.timeout_verify = Some(2);
10293
10294 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10295 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10296 .expect("no io error acquiring directly")
10297 {
10298 crate::cache::AcquireOutcome::Acquired(g) => g,
10299 crate::cache::AcquireOutcome::Busy(b) => {
10300 panic!("expected the direct acquire to win the lease first: {b:?}")
10301 }
10302 };
10303
10304 let mut state = RunState::new(
10305 repo.clone(),
10306 "main".to_owned(),
10307 "deadbeef".to_owned(),
10308 "task".to_owned(),
10309 config,
10310 );
10311 state.candidates = vec![Candidate {
10312 index: 0,
10313 label: 'A',
10314 agent: "alpha".to_owned(),
10315 branch: "does-not-exist".to_owned(),
10316 worktree: repo.clone(),
10317 summary: String::new(),
10318 stat: String::new(),
10319 files: 0,
10320 commits: 0,
10321 empty: false,
10322 failed: None,
10323 verified_noop: None,
10324 duration_ms: 0,
10325 folded: false,
10326 }];
10327 state.tally = Some(Tally {
10328 first_choice: BTreeMap::from([('A', 1)]),
10329 borda: BTreeMap::new(),
10330 winner: 'A',
10331 rankings: 1,
10332 unanimous_initial: true,
10333 deliberated: false,
10334 changed_votes: 0,
10335 unanimous_final: true,
10336 tie_break: None,
10337 judges: 0,
10338 present: 0,
10339 quorum: 0,
10340 met_quorum: true,
10341 uncontested: Some("only candidate A produced a change".to_owned()),
10342 });
10343 state.reviews = vec![ReviewRound {
10344 round: 1,
10345 head: "deadbeef".to_owned(),
10346 verified_head: None,
10347 verified_at: None,
10348 reviews: Vec::new(),
10349 e2e: Vec::new(),
10350 fix: None,
10351 blocking: 0,
10352 answered: 0,
10353 expected: 0,
10354 clean: true,
10355 verify_retried: false,
10356 e2e_deferred: false,
10357 e2e_defer_reason: None,
10358 progressed: false,
10359 vote_split: false,
10360 reconsideration: Vec::new(),
10361 verdict: None,
10362 }];
10363
10364 let mut runner = Runner {
10365 state,
10366 roles: ResolvedRoles {
10367 implementers: Vec::new(),
10368 judges: Vec::new(),
10369 reviewers: Vec::new(),
10370 fixer: None,
10371 conductor: conductor(),
10372 implementer_roster: Vec::new(),
10373 judge_roster: Vec::new(),
10374 reviewer_roster: Vec::new(),
10375 },
10376 sem: Arc::new(Semaphore::new(1)),
10377 pause: Pause::new(),
10378 interrupt: Pause::new(),
10379 };
10380
10381 let started = std::time::Instant::now();
10382 runner.gate().await.expect("gate");
10383 assert!(
10384 started.elapsed() < Duration::from_secs(1),
10385 "a gate with nothing to run must never wait on a lease it never needed"
10386 );
10387 assert!(
10388 runner.state.gate_ran,
10389 "zero commands is still a real, immediate attempt"
10390 );
10391 assert!(runner.state.gate.is_empty());
10392 assert_ne!(
10393 runner.state.status,
10394 RunStatus::Blocked,
10395 "must not read as resource-blocked on a lease it never asked for"
10396 );
10397 }
10398
10399 #[tokio::test]
10409 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
10410 crate::run::pin_test_home();
10411
10412 let tmp = tempfile::tempdir().expect("tempdir");
10413 let repo = tmp.path().join("repo");
10414 std::fs::create_dir_all(&repo).unwrap();
10415 init_repo(&repo);
10416
10417 let mut config = Config::default();
10418 config.verify.gate = vec![
10419 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
10420 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
10421 .to_owned(),
10422 ];
10423
10424 let mut state = RunState::new(
10425 repo.clone(),
10426 "main".to_owned(),
10427 "deadbeef".to_owned(),
10428 "task".to_owned(),
10429 config,
10430 );
10431 let run_id = state.id.clone();
10432 state.candidates = vec![Candidate {
10433 index: 0,
10434 label: 'A',
10435 agent: "alpha".to_owned(),
10436 branch: "does-not-exist".to_owned(),
10437 worktree: repo.clone(),
10438 summary: String::new(),
10439 stat: String::new(),
10440 files: 0,
10441 commits: 0,
10442 empty: false,
10443 failed: None,
10444 verified_noop: None,
10445 duration_ms: 0,
10446 folded: false,
10447 }];
10448 state.tally = Some(Tally {
10449 first_choice: BTreeMap::from([('A', 1)]),
10450 borda: BTreeMap::new(),
10451 winner: 'A',
10452 rankings: 1,
10453 unanimous_initial: true,
10454 deliberated: false,
10455 changed_votes: 0,
10456 unanimous_final: true,
10457 tie_break: None,
10458 judges: 0,
10459 present: 0,
10460 quorum: 0,
10461 met_quorum: true,
10462 uncontested: Some("only candidate A produced a change".to_owned()),
10463 });
10464 state.reviews = vec![ReviewRound {
10465 round: 1,
10466 head: "deadbeef".to_owned(),
10467 verified_head: None,
10468 verified_at: None,
10469 reviews: Vec::new(),
10470 e2e: Vec::new(),
10471 fix: None,
10472 blocking: 0,
10473 answered: 0,
10474 expected: 0,
10475 clean: true,
10476 verify_retried: false,
10477 e2e_deferred: false,
10478 e2e_defer_reason: None,
10479 progressed: false,
10480 vote_split: false,
10481 reconsideration: Vec::new(),
10482 verdict: None,
10483 }];
10484
10485 let mut runner = Runner {
10486 state,
10487 roles: ResolvedRoles {
10488 implementers: Vec::new(),
10489 judges: Vec::new(),
10490 reviewers: Vec::new(),
10491 fixer: None,
10492 conductor: conductor(),
10493 implementer_roster: Vec::new(),
10494 judge_roster: Vec::new(),
10495 reviewer_roster: Vec::new(),
10496 },
10497 sem: Arc::new(Semaphore::new(1)),
10498 pause: Pause::new(),
10499 interrupt: Pause::new(),
10500 };
10501
10502 let started_marker = repo.join("started.marker");
10503 let release_marker = repo.join("release.marker");
10504 let poller = tokio::spawn(async move {
10505 for _ in 0..100 {
10510 if started_marker.exists()
10511 && let Ok(s) = crate::run::RunState::load(&run_id)
10512 && let Some(a) = s.active.get("gate")
10513 {
10514 std::fs::write(&release_marker, b"go").expect("release marker");
10515 return Some(a.clone());
10516 }
10517 tokio::time::sleep(Duration::from_millis(50)).await;
10518 }
10519 None
10520 });
10521
10522 runner.gate().await.expect("gate");
10523 let captured = poller.await.expect("poller task");
10524 let captured = captured.expect(
10525 "the poller never saw a `gate` task entry in run.json while the command was \
10526 still blocked on its own release marker",
10527 );
10528
10529 assert_eq!(captured.task.as_deref(), Some("gate"));
10530 assert_eq!(captured.node, "gate");
10531 assert_eq!(captured.index, Some(1));
10532 assert_eq!(captured.total, Some(1));
10533 assert!(
10534 captured
10535 .command
10536 .as_deref()
10537 .is_some_and(|c| c.contains("started.marker")),
10538 "{captured:?}"
10539 );
10540
10541 assert!(
10542 runner.state.active.is_empty(),
10543 "the entry must be cleared once the command actually finished: {:?}",
10544 runner.state.active
10545 );
10546 assert!(runner.state.gate_ran);
10547 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
10548 }
10549
10550 #[tokio::test]
10553 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
10554 use crate::verdict::{Finding, ReviewVote, Severity};
10555 crate::run::pin_test_home();
10556 let tmp = tempfile::tempdir().expect("tempdir");
10557 let repo = tmp.path().join("repo");
10558 std::fs::create_dir_all(&repo).unwrap();
10559 init_repo(&repo);
10560
10561 for (severity, vote, expect) in [
10562 (Severity::Major, ReviewVote::Reject, true),
10563 (Severity::Minor, ReviewVote::Reject, false),
10564 (Severity::Major, ReviewVote::Approve, false),
10565 ] {
10566 let mut round = review_round(false, 1, 1, 1, false, true);
10567 round.reviews = vec![ReviewRecord {
10568 reviewer: 1,
10569 agent: "alpha".to_owned(),
10570 summary: String::new(),
10571 findings: vec![Finding {
10572 id: "R1-1-1".to_owned(),
10573 severity,
10574 file: None,
10575 line: None,
10576 title: "t".to_owned(),
10577 detail: String::new(),
10578 }],
10579 vote: Some(vote),
10580 failed: None,
10581 duration_ms: 0,
10582 attempts: 0,
10583 }];
10584 let mut state = RunState::new(
10585 repo.clone(),
10586 "main".to_owned(),
10587 "deadbeef".to_owned(),
10588 "task".to_owned(),
10589 Config::default(),
10590 );
10591 state.reviews = vec![round];
10592 let mut runner = Runner {
10593 state,
10594 roles: ResolvedRoles {
10595 implementers: Vec::new(),
10596 judges: Vec::new(),
10597 reviewers: Vec::new(),
10598 fixer: None,
10599 conductor: conductor(),
10600 implementer_roster: Vec::new(),
10601 judge_roster: Vec::new(),
10602 reviewer_roster: Vec::new(),
10603 },
10604 sem: Arc::new(Semaphore::new(1)),
10605 pause: Pause::new(),
10606 interrupt: Pause::new(),
10607 };
10608 let shell = runner.state.config.shell();
10609 runner
10610 .stop_reviewing("round budget spent", &shell, &repo)
10611 .await
10612 .expect("stop_reviewing");
10613 assert_eq!(runner.state.status, RunStatus::Gating);
10614 assert_eq!(
10615 runner.state.contested_handoff.is_some(),
10616 expect,
10617 "{severity:?} + {vote:?}"
10618 );
10619 }
10620 }
10621
10622 #[tokio::test]
10635 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
10636 crate::run::pin_test_home();
10637 let home = crate::run::home();
10638
10639 let tmp = tempfile::tempdir().expect("tempdir");
10640 let repo = tmp.path().join("repo");
10641 std::fs::create_dir_all(&repo).unwrap();
10642 init_repo(&repo);
10643 let head = crate::git::rev_parse(&repo, "HEAD")
10644 .await
10645 .expect("rev-parse");
10646 let cache_dir = tmp.path().join("target");
10649
10650 let mut config = Config::default();
10651 config.verify.e2e = vec![format!(
10652 "CARGO_TARGET_DIR='{}' test -f README.md",
10653 cache_dir.display()
10654 )];
10655 config.graph.review_rounds = 1;
10656 config.graph.timeout_verify = Some(2);
10659
10660 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10661 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10662 .expect("no io error acquiring directly")
10663 {
10664 crate::cache::AcquireOutcome::Acquired(g) => g,
10665 crate::cache::AcquireOutcome::Busy(b) => {
10666 panic!("expected the direct acquire to win the lease first: {b:?}")
10667 }
10668 };
10669
10670 let mut state = RunState::new(
10671 repo.clone(),
10672 "main".to_owned(),
10673 head.clone(),
10674 "task".to_owned(),
10675 config,
10676 );
10677 state.candidates = vec![Candidate {
10678 index: 0,
10679 label: 'A',
10680 agent: "alpha".to_owned(),
10681 branch: "does-not-exist".to_owned(),
10682 worktree: repo.clone(),
10683 summary: String::new(),
10684 stat: String::new(),
10685 files: 0,
10686 commits: 0,
10687 empty: false,
10688 failed: None,
10689 verified_noop: None,
10690 duration_ms: 0,
10691 folded: false,
10692 }];
10693 state.tally = Some(Tally {
10694 first_choice: BTreeMap::from([('A', 1)]),
10695 borda: BTreeMap::new(),
10696 winner: 'A',
10697 rankings: 1,
10698 unanimous_initial: true,
10699 deliberated: false,
10700 changed_votes: 0,
10701 unanimous_final: true,
10702 tie_break: None,
10703 judges: 0,
10704 present: 0,
10705 quorum: 0,
10706 met_quorum: true,
10707 uncontested: Some("only candidate A produced a change".to_owned()),
10708 });
10709 state.reviews = vec![ReviewRound {
10713 round: 1,
10714 head: head.clone(),
10715 verified_head: None,
10716 verified_at: None,
10717 reviews: Vec::new(),
10718 e2e: Vec::new(),
10719 fix: None,
10720 blocking: 1,
10721 answered: 1,
10722 expected: 1,
10723 clean: false,
10724 verify_retried: false,
10725 e2e_deferred: true,
10726 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
10727 progressed: false,
10728 vote_split: false,
10729 reconsideration: Vec::new(),
10730 verdict: None,
10731 }];
10732
10733 let mut runner = Runner {
10734 state,
10735 roles: ResolvedRoles {
10736 implementers: Vec::new(),
10737 judges: Vec::new(),
10738 reviewers: Vec::new(),
10739 fixer: None,
10740 conductor: conductor(),
10741 implementer_roster: Vec::new(),
10742 judge_roster: Vec::new(),
10743 reviewer_roster: Vec::new(),
10744 },
10745 sem: Arc::new(Semaphore::new(1)),
10746 pause: Pause::new(),
10747 interrupt: Pause::new(),
10748 };
10749
10750 let shell = runner.state.config.shell();
10751 runner
10752 .stop_reviewing("round budget spent", &shell, &repo)
10753 .await
10754 .expect("stop_reviewing");
10755
10756 let last = runner.state.reviews.last().expect("round record");
10757 assert_eq!(
10758 last.e2e_status(),
10759 E2eStatus::ResourceBlocked,
10760 "the shared cache is still held; the attempt must read as blocked, not deferred or \
10761 failed: {last:?}"
10762 );
10763 assert_eq!(
10764 last.verified_head.as_deref(),
10765 Some(head.as_str()),
10766 "which commit this attempt targeted is known even though nothing finished checking \
10767 it"
10768 );
10769 let first_attempt_at = last
10770 .verified_at
10771 .expect("when this attempt ran is known too");
10772 assert_ne!(
10773 runner.state.status,
10774 RunStatus::Blocked,
10775 "contention is evidence about the machine, not the patch — it must not settle the \
10776 run as blocked: {:?}",
10777 runner.state.status
10778 );
10779 assert!(
10780 !runner
10781 .state
10782 .events
10783 .iter()
10784 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
10785 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
10786 runner.state.events
10787 );
10788
10789 runner
10794 .stop_reviewing("round budget spent", &shell, &repo)
10795 .await
10796 .expect("stop_reviewing retry");
10797 assert_eq!(
10798 runner.state.reviews.len(),
10799 1,
10800 "no new round was started: {:?}",
10801 runner.state.reviews
10802 );
10803 let last = runner.state.reviews.last().expect("round record");
10804 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
10805 assert!(
10806 last.verified_at.expect("still known") > first_attempt_at,
10807 "a second reentry must be a fresh attempt, not a stale copy of the first"
10808 );
10809 assert_ne!(runner.state.status, RunStatus::Blocked);
10810
10811 held.release();
10812 }
10813
10814 #[tokio::test]
10826 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
10827 crate::run::pin_test_home();
10828 let home = crate::run::home();
10829
10830 let tmp = tempfile::tempdir().expect("tempdir");
10831 let repo = tmp.path().join("repo");
10832 std::fs::create_dir_all(&repo).unwrap();
10833 init_repo(&repo);
10834 let head = crate::git::rev_parse(&repo, "HEAD")
10835 .await
10836 .expect("rev-parse");
10837 let cache_dir = tmp.path().join("target");
10838
10839 let mut config = Config::default();
10840 config.verify.e2e = vec![format!(
10841 "CARGO_TARGET_DIR='{}' test -f README.md",
10842 cache_dir.display()
10843 )];
10844 config.graph.review_rounds = 1;
10845 config.graph.timeout_verify = Some(2);
10846
10847 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10848 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10849 .expect("no io error acquiring directly")
10850 {
10851 crate::cache::AcquireOutcome::Acquired(g) => g,
10852 crate::cache::AcquireOutcome::Busy(b) => {
10853 panic!("expected the direct acquire to win the lease first: {b:?}")
10854 }
10855 };
10856
10857 let mut state = RunState::new(
10858 repo.clone(),
10859 "main".to_owned(),
10860 head.clone(),
10861 "task".to_owned(),
10862 config,
10863 );
10864 state.candidates = vec![Candidate {
10865 index: 0,
10866 label: 'A',
10867 agent: "alpha".to_owned(),
10868 branch: "does-not-exist".to_owned(),
10869 worktree: repo.clone(),
10870 summary: String::new(),
10871 stat: String::new(),
10872 files: 0,
10873 commits: 0,
10874 empty: false,
10875 failed: None,
10876 verified_noop: None,
10877 duration_ms: 0,
10878 folded: false,
10879 }];
10880 state.tally = Some(Tally {
10881 first_choice: BTreeMap::from([('A', 1)]),
10882 borda: BTreeMap::new(),
10883 winner: 'A',
10884 rankings: 1,
10885 unanimous_initial: true,
10886 deliberated: false,
10887 changed_votes: 0,
10888 unanimous_final: true,
10889 tie_break: None,
10890 judges: 0,
10891 present: 0,
10892 quorum: 0,
10893 met_quorum: true,
10894 uncontested: Some("only candidate A produced a change".to_owned()),
10895 });
10896 state.reviews = vec![ReviewRound {
10900 round: 1,
10901 head: head.clone(),
10902 verified_head: Some(head.clone()),
10903 verified_at: Some(jiff::Timestamp::now()),
10904 reviews: Vec::new(),
10905 e2e: vec![CommandOutcome {
10906 command: format!(
10907 "CARGO_TARGET_DIR='{}' test -f README.md",
10908 cache_dir.display()
10909 ),
10910 code: None,
10911 output_tail: "waiting for the shared build cache".to_owned(),
10912 duration_ms: 0,
10913 resource_blocked: true,
10914 }],
10915 fix: None,
10916 blocking: 1,
10917 answered: 1,
10918 expected: 1,
10919 clean: false,
10920 verify_retried: false,
10921 e2e_deferred: false,
10922 e2e_defer_reason: None,
10923 progressed: false,
10924 vote_split: false,
10925 reconsideration: Vec::new(),
10926 verdict: None,
10927 }];
10928
10929 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
10930 let mut runner = Runner {
10931 state,
10932 roles: ResolvedRoles {
10933 implementers: Vec::new(),
10934 judges: Vec::new(),
10935 reviewers: Vec::new(),
10936 fixer: None,
10937 conductor: conductor(),
10938 implementer_roster: Vec::new(),
10939 judge_roster: Vec::new(),
10940 reviewer_roster: Vec::new(),
10941 },
10942 sem: Arc::new(Semaphore::new(1)),
10943 pause: Pause::new(),
10944 interrupt: Pause::new(),
10945 };
10946
10947 runner.review_loop().await.expect("review_loop");
10952
10953 assert_eq!(
10954 runner.state.reviews.len(),
10955 1,
10956 "no new round was started on top of the unresolved one: {:?}",
10957 runner.state.reviews
10958 );
10959 let last = &runner.state.reviews[0];
10960 assert_eq!(
10961 last.e2e_status(),
10962 E2eStatus::ResourceBlocked,
10963 "still contended: {last:?}"
10964 );
10965 assert!(
10966 last.verified_at.expect("still known") > first_attempt_at,
10967 "review_loop must have actually retried the check, not left it exactly as found"
10968 );
10969 assert_ne!(
10970 runner.state.status,
10971 RunStatus::Blocked,
10972 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
10973 runner.state.status
10974 );
10975
10976 held.release();
10977 }
10978
10979 #[tokio::test]
10980 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
10981 crate::run::pin_test_home();
10982 let tmp = tempfile::tempdir().expect("tempdir");
10983 let repo = tmp.path().join("repo");
10984 std::fs::create_dir_all(&repo).unwrap();
10985 init_repo(&repo);
10986
10987 let mut config = Config::default();
10988 config.merge.mode = MergeMode::Pr;
10989 config.graph.land = true;
10990 config.graph.land_approval = false;
10991
10992 let mut state = RunState::new(
10993 repo.clone(),
10994 "main".to_owned(),
10995 "deadbeef".to_owned(),
10996 "task".to_owned(),
10997 config,
10998 );
10999 state.candidates = vec![Candidate {
11000 index: 0,
11001 label: 'A',
11002 agent: "alpha".to_owned(),
11003 branch: "does-not-exist".to_owned(),
11004 worktree: repo.clone(),
11005 summary: String::new(),
11006 stat: String::new(),
11007 files: 0,
11008 commits: 0,
11009 empty: false,
11010 failed: None,
11011 verified_noop: None,
11012 duration_ms: 0,
11013 folded: false,
11014 }];
11015 state.tally = Some(Tally {
11016 first_choice: BTreeMap::from([('A', 1)]),
11017 borda: BTreeMap::new(),
11018 winner: 'A',
11019 rankings: 1,
11020 unanimous_initial: true,
11021 deliberated: false,
11022 changed_votes: 0,
11023 unanimous_final: true,
11024 tie_break: None,
11025 judges: 0,
11026 present: 0,
11027 quorum: 0,
11028 met_quorum: true,
11029 uncontested: Some("only candidate A produced a change".to_owned()),
11030 });
11031 state.reviews = vec![ReviewRound {
11032 round: 1,
11033 head: "deadbeef".to_owned(),
11034 verified_head: None,
11035 verified_at: None,
11036 reviews: Vec::new(),
11037 e2e: Vec::new(),
11038 fix: None,
11039 blocking: 0,
11040 answered: 0,
11041 expected: 0,
11042 clean: true,
11043 verify_retried: false,
11044 e2e_deferred: false,
11045 e2e_defer_reason: None,
11046 progressed: false,
11047 vote_split: false,
11048 reconsideration: Vec::new(),
11049 verdict: None,
11050 }];
11051 state.gate = vec![CommandOutcome {
11052 command: "test".to_owned(),
11053 code: Some(0),
11054 output_tail: String::new(),
11055 duration_ms: 0,
11056 resource_blocked: false,
11057 }];
11058 state.gate_ran = true;
11059 state.status = RunStatus::Landing;
11063 state.merge = Some(MergeOutcome {
11064 mode: MergeMode::Pr,
11065 ok: true,
11066 detail: "https://example.invalid/x/y/pull/1".to_owned(),
11067 empty: false,
11068 });
11069
11070 ask_test_home();
11074 let store = ask::Questions::open();
11075 let q = ask_open_question(&store, &state.id);
11076
11077 let mut runner = Runner {
11078 state,
11079 roles: ResolvedRoles {
11080 implementers: Vec::new(),
11081 judges: Vec::new(),
11082 reviewers: Vec::new(),
11083 fixer: None,
11084 conductor: conductor(),
11085 implementer_roster: Vec::new(),
11086 judge_roster: Vec::new(),
11087 reviewer_roster: Vec::new(),
11088 },
11089 sem: Arc::new(Semaphore::new(1)),
11090 pause: Pause::new(),
11091 interrupt: Pause::new(),
11092 };
11093
11094 runner.execute().await.expect("execute");
11099
11100 assert_eq!(
11101 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
11102 Some("https://example.invalid/x/y/pull/1"),
11103 "reentry must not push again or open a second pull request over the \
11104 one `land` is already watching"
11105 );
11106 assert_ne!(
11107 runner.state.status,
11108 RunStatus::Landing,
11109 "land could not actually reach the fake pull request, so it must \
11110 have given up rather than left the run silently parked forever"
11111 );
11112 assert_eq!(runner.state.status, RunStatus::Blocked);
11116 assert!(
11117 store.get(&q.id).unwrap().status.open(),
11118 "Blocked is still alive; settle_questions must have been a no-op here"
11119 );
11120 }
11121
11122 fn state_with_round(round: ReviewRound) -> RunState {
11123 let mut s = RunState::new(
11124 PathBuf::from("/repo"),
11125 "main".to_owned(),
11126 "abc1234".to_owned(),
11127 "add retries".to_owned(),
11128 Config::default(),
11129 );
11130 s.reviews = vec![round];
11131 s
11132 }
11133
11134 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
11135 crate::verdict::Finding {
11136 id: id.to_owned(),
11137 severity,
11138 file: None,
11139 line: None,
11140 title: title.to_owned(),
11141 detail: String::new(),
11142 }
11143 }
11144
11145 #[test]
11146 fn pr_body_names_open_findings_and_declined_ones() {
11147 let round = ReviewRound {
11148 round: 2,
11149 head: "deadbee".to_owned(),
11150 verified_head: None,
11151 verified_at: None,
11152 reviews: vec![ReviewRecord {
11153 attempts: 0,
11154 reviewer: 1,
11155 agent: "alpha".to_owned(),
11156 summary: String::new(),
11157 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
11158 vote: None,
11159 failed: None,
11160 duration_ms: 0,
11161 }],
11162 e2e: vec![CommandOutcome {
11163 command: "cargo test".to_owned(),
11164 code: Some(0),
11165 output_tail: String::new(),
11166 duration_ms: 0,
11167 resource_blocked: false,
11168 }],
11169 verify_retried: false,
11170 e2e_deferred: false,
11171 e2e_defer_reason: None,
11172 fix: Some(FixRecord {
11173 agent: "alpha".to_owned(),
11174 addressed: Vec::new(),
11175 rejected: vec![crate::verdict::Rejection {
11176 id: "R1-1-1".to_owned(),
11177 why: "not reachable from any caller".to_owned(),
11178 }],
11179 notes: String::new(),
11180 committed: true,
11181 failed: None,
11182 duration_ms: 0,
11183 continuation: None,
11184 }),
11185 blocking: 0,
11186 answered: 1,
11187 expected: 1,
11188 clean: false,
11189 progressed: true,
11190 vote_split: false,
11191 reconsideration: Vec::new(),
11192 verdict: None,
11193 };
11194 let state = state_with_round(round);
11195 let body = pr_message(&state, 'A').body;
11196
11197 assert!(body.contains("add retries"), "the task must still be there");
11198 assert!(body.contains("R2-1-1"), "{body}");
11199 assert!(body.contains("unused import"), "{body}");
11200 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
11201 assert!(
11202 body.contains("not reachable from any caller"),
11203 "the reason it was declined: {body}"
11204 );
11205 }
11206
11207 #[test]
11208 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
11209 let round = ReviewRound {
11210 round: 1,
11211 head: "deadbee".to_owned(),
11212 verified_head: None,
11213 verified_at: None,
11214 reviews: vec![ReviewRecord {
11215 attempts: 0,
11216 reviewer: 1,
11217 agent: "alpha".to_owned(),
11218 summary: String::new(),
11219 findings: Vec::new(),
11220 vote: None,
11221 failed: None,
11222 duration_ms: 0,
11223 }],
11224 e2e: Vec::new(),
11225 verify_retried: false,
11226 e2e_deferred: false,
11227 e2e_defer_reason: None,
11228 fix: None,
11229 blocking: 0,
11230 answered: 1,
11231 expected: 1,
11232 clean: true,
11233 progressed: false,
11234 vote_split: false,
11235 reconsideration: Vec::new(),
11236 verdict: None,
11237 };
11238 let state = state_with_round(round);
11239 let body = pr_message(&state, 'A').body;
11240 assert!(!body.contains("Open review findings"), "{body}");
11241 assert!(!body.contains("Declined"), "{body}");
11242 }
11243
11244 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
11245 let mut state = RunState::new(
11246 PathBuf::from("/repo"),
11247 "main".to_owned(),
11248 "abc1234".to_owned(),
11249 instruction.to_owned(),
11250 Config::default(),
11251 );
11252 state.candidates.push(Candidate {
11253 index: 0,
11254 label: 'A',
11255 agent: "alpha".to_owned(),
11256 branch: "magi/x/A".to_owned(),
11257 worktree: PathBuf::from("/wt"),
11258 summary: summary.to_owned(),
11259 stat: String::new(),
11260 files: 1,
11261 commits: 1,
11262 empty: false,
11263 failed: None,
11264 verified_noop: None,
11265 folded: false,
11266 duration_ms: 0,
11267 });
11268 state
11269 }
11270
11271 fn review_state(subjects: &[&str]) -> RunState {
11272 let mut state = state_with_summary(
11273 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
11274 "",
11275 );
11276 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
11277 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
11278 state
11279 }
11280
11281 #[tokio::test]
11284 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
11285 ask_test_home();
11286 let tmp = tempfile::tempdir().unwrap();
11287 let repo = tmp.path().join("repo");
11288 std::fs::create_dir_all(&repo).unwrap();
11289 init_repo(&repo);
11290 let origin = tmp.path().join("origin.git");
11291 let g = |dir: &Path, args: &[&str]| {
11292 let out = std::process::Command::new("git")
11293 .args(args)
11294 .current_dir(dir)
11295 .quiet()
11296 .output()
11297 .expect("spawn git");
11298 assert!(
11299 out.status.success(),
11300 "git {args:?}: {}",
11301 String::from_utf8_lossy(&out.stderr)
11302 );
11303 };
11304 g(
11305 tmp.path(),
11306 &[
11307 "clone",
11308 "--bare",
11309 "-q",
11310 repo.to_str().unwrap(),
11311 origin.to_str().unwrap(),
11312 ],
11313 );
11314 g(
11315 &repo,
11316 &["remote", "add", "origin", origin.to_str().unwrap()],
11317 );
11318 let c1 = git::rev_parse(&repo, "main").await.unwrap();
11319
11320 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
11322 g(&repo, &["add", "-A"]);
11323 g(
11324 &repo,
11325 &["commit", "-q", "-m", "chore(deps): update a crate"],
11326 );
11327 g(&repo, &["push", "-q", "origin", "main"]);
11328 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
11329 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
11330 g(&repo, &["add", "-A"]);
11331 g(
11332 &repo,
11333 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
11334 );
11335 g(&repo, &["checkout", "-q", "main"]);
11336
11337 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
11338 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
11339 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
11341 assert_eq!(fallback, c1);
11342
11343 let mut state = review_state(&[
11344 "chore(deps): update a crate",
11345 "fix(daemon): apply a chosen action",
11346 ]);
11347 state.repo = repo.clone();
11348 state.base_branch = "main".to_owned();
11349 state.base_commit = c1;
11350 refresh_reviewed_commits(&mut state, "feat/own").await;
11351 assert_eq!(
11352 state.reviewed_commits,
11353 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
11354 );
11355 assert_eq!(
11356 review_title(&state).as_deref(),
11357 Some("fix(daemon): apply a chosen action")
11358 );
11359 assert_eq!(
11360 leaked_subjects(&state, "feat/own").await,
11361 Some(vec!["chore(deps): update a crate".to_owned()])
11362 );
11363 state.config.merge.remote = "nowhere".to_owned();
11366 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
11367 }
11368
11369 #[test]
11370 fn pr_message_review_single_commit_uses_its_subject() {
11371 let state = review_state(&["feat(nats): per-role user"]);
11372 let m = pr_message(&state, 'A');
11373 assert_eq!(m.title, "feat(nats): per-role user");
11374 assert!(!m.body.contains("Review the work already"), "{}", m.body);
11375 assert!(m.body.contains("## Commits under review"), "{}", m.body);
11376 }
11377
11378 #[test]
11379 fn pr_message_review_multi_commit_takes_the_oldest() {
11380 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
11381 let m = pr_message(&state, 'A');
11382 assert_eq!(m.title, "feat: the change");
11383 for s in ["feat: the change", "fix: typo", "fix: again"] {
11384 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
11385 }
11386 }
11387
11388 #[test]
11389 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
11390 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
11391 let mut state = review_state(&[first, "fix: later fixup"]);
11392 state.candidates[0].branch = "機能/ログイン".to_owned();
11393 let m = pr_message(&state, 'A');
11394 assert!(
11395 m.title.starts_with("chore: land candidate A of run"),
11396 "{}",
11397 m.title
11398 );
11399 }
11400 }
11401
11402 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
11403 BranchFacts {
11404 commits: commits
11405 .iter()
11406 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
11407 .collect(),
11408 stat: stat.to_owned(),
11409 }
11410 }
11411
11412 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
11413 let mut r = review_round(true, 0, 1, 1, true, true);
11414 r.round = round;
11415 r.fix = notes.map(|n| FixRecord {
11416 agent: "fixer".to_owned(),
11417 addressed: Vec::new(),
11418 rejected: Vec::new(),
11419 notes: n.to_owned(),
11420 committed: true,
11421 failed: None,
11422 duration_ms: 0,
11423 continuation: None,
11424 });
11425 r
11426 }
11427
11428 #[test]
11429 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
11430 let state = review_state(&["ignored"]);
11431 let f = facts(
11432 &[
11433 (
11434 "fix(login): resolve PATH on macOS",
11435 "Login shells skip rc files.",
11436 ),
11437 ("fix: address review", ""),
11438 ],
11439 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
11440 );
11441 let m = pr_message_with(&state, 'A', Some(&f));
11442 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
11443 assert!(
11444 m.body
11445 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
11446 "{}",
11447 m.body
11448 );
11449 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
11450 assert!(m.body.contains("## Diff stat"), "{}", m.body);
11451 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
11452 for banned in [
11453 "Review the work already",
11454 "no task statement",
11455 "Original task",
11456 ] {
11457 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
11458 }
11459 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
11460 }
11461
11462 #[test]
11463 fn pr_message_review_truncates_a_huge_first_commit_body() {
11464 let state = review_state(&["ignored"]);
11465 let f = facts(
11466 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
11467 "s",
11468 );
11469 let m = pr_message_with(&state, 'A', Some(&f));
11470 assert!(m.body.len() < 30_000, "{}", m.body.len());
11471 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
11472 assert!(
11473 m.body.contains("- ... 1 more commit(s)"),
11474 "{}",
11475 m.body.len()
11476 );
11477 assert!(m.body.ends_with("magi:candidate-a\n"));
11478 }
11479
11480 #[test]
11481 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
11482 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
11483 assert_eq!(m.title, "feat: x");
11484 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
11485 assert!(!m.body.contains("Diff stat"), "{}", m.body);
11486 assert!(!m.body.contains("no task statement"), "{}", m.body);
11487 }
11488
11489 #[test]
11490 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
11491 let mut state = review_state(&["日本語の件名"]);
11492 state.candidates[0].branch = "fix/macos-login-path".to_owned();
11493 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
11494 state.candidates[0].branch = "機能/ログイン".to_owned();
11495 assert!(
11496 pr_message(&state, 'A')
11497 .title
11498 .starts_with("chore: land candidate A")
11499 );
11500 }
11501
11502 #[test]
11503 fn pr_message_review_fixes_survive_a_clean_final_round() {
11504 let mut state = review_state(&["feat: x"]);
11505 state.reviews = vec![
11506 round_with_notes(1, Some("handled the PATH case")),
11507 round_with_notes(2, None),
11508 ];
11509 let body = pr_message(&state, 'A').body;
11510 assert!(
11511 body.contains("## Review fixes\n\nhandled the PATH case\n"),
11512 "{body}"
11513 );
11514 assert!(!body.contains("### Round"), "{body}");
11515
11516 state.reviews = vec![
11517 round_with_notes(1, Some("first fix")),
11518 round_with_notes(2, Some("")),
11519 round_with_notes(3, Some("second fix")),
11520 round_with_notes(4, None),
11521 ];
11522 let body = pr_message(&state, 'A').body;
11523 assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
11524 assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
11525 assert!(!body.contains("### Round 2"), "{body}");
11526 }
11527
11528 #[test]
11529 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
11530 let state = state_with_summary(
11531 "add retries to the client",
11532 "TITLE: feat: retries\n\nDid it.",
11533 );
11534 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
11535 assert_eq!(m.title, "feat: retries");
11536 assert!(
11537 m.body.contains("<summary>Original task</summary>"),
11538 "{}",
11539 m.body
11540 );
11541 assert!(!m.body.contains("Commits under review"), "{}", m.body);
11542 assert!(
11543 m.body
11544 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
11545 "{}",
11546 m.body
11547 );
11548 }
11549
11550 #[test]
11551 fn pr_message_review_bounds_a_long_english_subject() {
11552 let long = format!("feat: {}", "word ".repeat(100));
11553 let m = pr_message(&review_state(&[&long]), 'A');
11554 assert!(m.title.starts_with("feat: word"), "{}", m.title);
11555 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11556 }
11557
11558 #[test]
11559 fn pr_message_implementation_run_is_unchanged_by_review_support() {
11560 let state = state_with_summary("add retries\n\ndetails", "- did some things");
11561 let m = pr_message(&state, 'A');
11562 assert_eq!(m.title, "add retries");
11563 assert!(m.body.contains("<summary>Original task</summary>"));
11564 assert!(!m.body.contains("Commits under review"));
11565 assert_eq!(landing_subject_source(&state), state.instruction);
11566 }
11567
11568 #[test]
11569 fn review_run_squash_subject_is_the_change_not_the_prompt() {
11570 let state = review_state(&["feat: the change", "fix: typo"]);
11571 let source = landing_subject_source(&state);
11572 assert_eq!(land::merge_subject("", &source), "feat: the change");
11573 assert_eq!(
11574 land::merge_subject("magi: candidate A (uncommitted work)", &source),
11575 "feat: the change"
11576 );
11577 assert_eq!(
11579 land::merge_subject("feat: renamed by hand", &source),
11580 "feat: renamed by hand"
11581 );
11582 let blank = review_state(&["日本語"]);
11583 assert!(
11584 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
11585 );
11586 }
11587
11588 #[test]
11589 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
11590 let state = review_state(&["feat: the change"]);
11591 let source = landing_subject_source(&state);
11592 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
11593 assert_eq!(
11594 land::merge_subject(landing_title(&state, old), &source),
11595 "feat: the change"
11596 );
11597 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
11598 let task = state_with_summary("add retries", "");
11599 assert_eq!(landing_title(&task, old), old);
11600 }
11601
11602 #[test]
11603 fn pr_message_describes_the_change_not_the_task() {
11604 let state = state_with_summary(
11605 "今回やってほしいこと: results projector を直す",
11606 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
11607 );
11608 let m = pr_message(&state, 'A');
11609 assert_eq!(m.title, "fix(web): batch the runs list reads");
11610 assert!(
11611 m.body.starts_with("## Summary\n\n- reads run.json once"),
11612 "{}",
11613 m.body
11614 );
11615 assert!(!m.body.contains("TITLE:"), "{}", m.body);
11616 let task_at = m.body.find("今回やってほしいこと").unwrap();
11617 let details_at = m.body.find("<details>").unwrap();
11618 assert!(
11619 details_at < task_at,
11620 "the task lives inside <details>: {}",
11621 m.body
11622 );
11623 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
11624 assert!(m.body.contains("magi:candidate-a"));
11625 }
11626
11627 #[test]
11628 fn pr_message_falls_back_to_the_task_without_a_title_line() {
11629 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
11630 let m = pr_message(&state, 'A');
11631 assert_eq!(m.title, "add retries");
11632 assert!(
11633 m.body.contains("## Summary\n\n- did some things"),
11634 "{}",
11635 m.body
11636 );
11637
11638 let none = RunState::new(
11639 PathBuf::from("/repo"),
11640 "main".to_owned(),
11641 "abc1234".to_owned(),
11642 "add retries".to_owned(),
11643 Config::default(),
11644 );
11645 let m = pr_message(&none, 'A');
11646 assert_eq!(m.title, "add retries");
11647 assert!(!m.body.contains("## Summary"), "{}", m.body);
11648 }
11649
11650 #[test]
11651 fn pr_message_refuses_the_candidate_commit_subject() {
11652 for bad in [
11653 "TITLE: magi: candidate A (uncommitted work)",
11654 "TITLE: chore: stuff (uncommitted work)",
11655 "TITLE: ",
11656 ] {
11657 let state = state_with_summary("add retries", bad);
11658 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
11659 }
11660 }
11661
11662 #[test]
11663 fn pr_message_bounds_a_very_long_task_and_title() {
11664 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
11665 let state = state_with_summary(&long, "- nothing");
11666 let m = pr_message(&state, 'A');
11667 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11668 assert!(!m.title.contains('\n'));
11669
11670 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
11671 let m = pr_message(&state, 'A');
11672 assert!(m.title.starts_with("feat: "));
11673 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11674 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
11675 }
11676
11677 fn long_title_of(instruction: &str) -> String {
11678 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
11679 }
11680
11681 #[test]
11682 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
11683 let first = "Make the landing path keep a readable title for long tasks";
11684 let line = format!(
11685 "{first}. {}",
11686 "And then keep going with more words ".repeat(20)
11687 );
11688 let t = long_title_of(&line);
11689 assert_eq!(t, first);
11690 assert!(!t.starts_with("chore: land"));
11691 }
11692
11693 #[test]
11694 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
11695 let line = "word ".repeat(200);
11696 let t = long_title_of(&line);
11697 assert!(t.ends_with("word..."), "{t}");
11698 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
11699 }
11700
11701 #[test]
11702 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
11703 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
11704 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
11705 }
11706 }
11707
11708 #[test]
11709 fn pr_message_title_limit_is_exact() {
11710 let at = "a".repeat(PR_TITLE_MAX);
11711 assert_eq!(long_title_of(&at), at);
11712 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
11713 assert!(over.ends_with("..."), "{over}");
11714 assert_eq!(over.chars().count(), PR_TITLE_MAX);
11715 }
11716
11717 #[test]
11718 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
11719 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
11720 let t = long_title_of(&line);
11721 assert!(t.ends_with("..."), "{t}");
11722 assert!(t.is_ascii(), "{t}");
11723 }
11724
11725 #[test]
11726 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
11727 let line = format!(
11728 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
11729 "plus more filler words ".repeat(20)
11730 );
11731 let t = long_title_of(&line);
11732 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
11733 }
11734
11735 #[test]
11736 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
11737 let line = format!("\n\n# {}", "title words ".repeat(40));
11738 let state = state_with_summary(&line, "- nothing");
11739 let m = pr_message(&state, 'A');
11740 assert!(m.title.starts_with("title words"), "{}", m.title);
11741 assert_eq!(
11742 land::merge_subject(&m.title, &landing_subject_source(&state)),
11743 m.title
11744 );
11745 assert_eq!(
11747 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
11748 "feat: renamed by hand"
11749 );
11750 }
11751
11752 #[tokio::test]
11753 async fn github_text_rewrite_is_bounded_and_falls_back() {
11754 crate::run::pin_test_home();
11755 for (reply, accepted) in [
11756 (
11757 "TITLE: fix: retries\nAdd retries for failed requests.",
11758 true,
11759 ),
11760 (
11761 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
11762 false,
11763 ),
11764 ("TITLE: fix: retries\nUse token=secret", false),
11765 ] {
11766 let dir = tempfile::tempdir().unwrap();
11767 let mut runner = runner_at(RunStatus::Gating);
11768 runner.state = state_with_summary(
11769 "add retries",
11770 "TITLE: fix: retries\n日本語の説明を書きます。",
11771 );
11772 runner.state.repo = dir.path().to_owned();
11773 runner.state.candidates[0].worktree = dir.path().to_owned();
11774 let mut author = spec("alpha");
11775 author.command = vec![
11776 "sh".into(),
11777 "-c".into(),
11778 "printf '%s' \"$REWRITE_REPLY\"".into(),
11779 ];
11780 author.env.insert("REWRITE_REPLY".into(), reply.into());
11781 runner.state.config.agents = vec![author];
11782 let winner = runner.state.candidates[0].clone();
11783 let message = runner.guarded_pr_message(&winner, None, true).await;
11784 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
11785 assert_eq!(
11786 message.body.contains("Add retries for failed requests."),
11787 accepted
11788 );
11789 assert_eq!(
11790 runner.state.seats["impl-A"].turns, 1,
11791 "only one rewrite invocation"
11792 );
11793 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
11794 if !accepted {
11795 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
11796 assert!(
11797 message
11798 .body
11799 .contains(&format!("magi:run/{}", runner.state.id))
11800 );
11801 }
11802 }
11803 }
11804
11805 #[test]
11806 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
11807 let mut state = state_with_summary(
11811 "add retries",
11812 "TITLE: fix(web): batch reads\n- reads run.json once",
11813 );
11814 state.config.graph.language = "ja".to_owned();
11815 let m = pr_message(&state, 'A');
11816 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
11817 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
11818
11819 let task = "今回やってほしいこと: results projector を直す";
11822 let mut state = state_with_summary(task, "- no title line");
11823 state.config.graph.language = "ja".to_owned();
11824 let m = pr_message(&state, 'A');
11825 assert_eq!(
11826 m.title,
11827 format!("chore: land candidate A of run {}", state.id)
11828 );
11829 assert!(
11830 m.body.contains(&format!(
11831 "<summary>Original task</summary>\n\n{task}\n\n</details>"
11832 )),
11833 "{}",
11834 m.body
11835 );
11836 }
11837
11838 #[test]
11839 fn pr_message_scrubs_home_paths_and_addresses() {
11840 let state = state_with_summary(
11841 "fix it in /Users/someone/src/x",
11842 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
11843 );
11844 let m = pr_message(&state, 'A');
11845 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
11846 assert!(!m.body.contains(leak), "{}", m.body);
11847 }
11848 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
11849 }
11850
11851 #[test]
11852 fn pr_message_survives_a_task_that_closes_details() {
11853 let state = state_with_summary("a </details> b", "TITLE: fix: x");
11854 let m = pr_message(&state, 'A');
11855 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
11856 }
11857
11858 #[test]
11859 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
11860 let cmd = manual_merge_command(
11861 MergeStyle::Squash,
11862 Path::new("/repo"),
11863 "b",
11864 "fix: \"quoted\" $(x) `y`\n\nbody",
11865 );
11866 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
11867 }
11868
11869 #[test]
11870 fn manual_merge_command_matches_the_configured_style() {
11871 let repo = Path::new("/repo");
11872 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
11873
11874 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
11875 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
11876
11877 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
11878 assert_eq!(
11879 squash,
11880 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
11881 \"Merge magi run 0832 (candidate A)\""
11882 );
11883
11884 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
11885 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
11886 }
11887
11888 #[test]
11889 fn a_nudge_gets_a_quarter_of_the_budget() {
11890 assert_eq!(retry_budget(secs(1200), true), secs(300));
11892 assert_eq!(retry_budget(secs(3600), true), secs(900));
11893 }
11894
11895 #[test]
11896 fn a_resent_prompt_keeps_the_whole_budget() {
11897 assert_eq!(retry_budget(secs(1200), false), secs(1200));
11900 assert_eq!(retry_budget(secs(60), false), secs(60));
11901 }
11902
11903 #[test]
11904 fn the_floor_never_exceeds_the_original_budget() {
11905 assert_eq!(retry_budget(secs(60), true), secs(60));
11909 assert_eq!(retry_budget(secs(480), true), secs(120));
11910 assert_eq!(retry_budget(secs(0), true), secs(0));
11911 }
11912
11913 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
11914 agent::CommandEvidence {
11915 id: "item1".to_owned(),
11916 description: "cargo test".to_owned(),
11917 exit_code,
11918 result_summary: String::new(),
11919 source: "codex".to_owned(),
11920 }
11921 }
11922
11923 #[test]
11924 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
11925 assert!(!has_unconfirmed_command(&[]));
11929 }
11930
11931 #[test]
11932 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
11933 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
11937 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
11938 assert!(!has_unconfirmed_command(&[
11939 evidence(Some(0)),
11940 evidence(Some(101))
11941 ]));
11942 }
11943
11944 #[test]
11945 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
11946 assert!(has_unconfirmed_command(&[
11947 evidence(Some(0)),
11948 evidence(None)
11949 ]));
11950 }
11951
11952 #[test]
11953 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
11954 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11955 assert_eq!(
11956 verified_noop_claim(true, &[], text).as_deref(),
11957 Some("already fixed by b32cfc4, on main.")
11958 );
11959 }
11960
11961 #[test]
11962 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
11963 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11966 assert!(verified_noop_claim(false, &[], text).is_none());
11967 }
11968
11969 #[test]
11970 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
11971 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11972 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
11973 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
11975 }
11976
11977 #[test]
11978 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
11979 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
11980 }
11981
11982 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
11985 runner.state.candidates = shape
11986 .iter()
11987 .enumerate()
11988 .map(|(i, &(empty, verified))| Candidate {
11989 index: i,
11990 label: (b'A' + i as u8) as char,
11991 agent: "sonnet".to_owned(),
11992 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
11993 worktree: PathBuf::from(format!("/wt/{i}")),
11994 summary: String::new(),
11995 stat: String::new(),
11996 files: 0,
11997 commits: 0,
11998 empty,
11999 failed: None,
12000 verified_noop: verified.map(str::to_owned),
12001 duration_ms: 0,
12002 folded: false,
12003 })
12004 .collect();
12005 }
12006
12007 #[test]
12008 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
12009 ask_test_home();
12010 let mut runner = runner_at(RunStatus::Implementing);
12011 set_candidates(
12012 &mut runner,
12013 &[
12014 (true, Some("already on main at b32cfc4")),
12015 (true, Some("same fix, see the existing test")),
12016 ],
12017 );
12018
12019 runner
12020 .after_implement()
12021 .expect("a verified no-op is not an error");
12022
12023 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
12024 }
12025
12026 #[test]
12027 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
12028 ask_test_home();
12029 let mut runner = runner_at(RunStatus::Implementing);
12030 set_candidates(
12034 &mut runner,
12035 &[(true, Some("already on main at b32cfc4")), (true, None)],
12036 );
12037
12038 let err = runner
12039 .after_implement()
12040 .expect_err("an unverified empty candidate must still fail the run");
12041
12042 assert!(
12043 err.to_string().contains("no candidate produced a change"),
12044 "{err}"
12045 );
12046 assert_eq!(runner.state.status, RunStatus::Failed);
12047 }
12048
12049 #[test]
12050 fn after_implement_still_fails_an_ordinary_all_empty_run() {
12051 ask_test_home();
12052 let mut runner = runner_at(RunStatus::Implementing);
12053 set_candidates(&mut runner, &[(true, None), (true, None)]);
12054
12055 let err = runner
12056 .after_implement()
12057 .expect_err("no candidate declared anything; this is an ordinary failure");
12058
12059 assert!(
12060 err.to_string().contains("no candidate produced a change"),
12061 "{err}"
12062 );
12063 assert_eq!(runner.state.status, RunStatus::Failed);
12064 }
12065}