1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement, MergeOutcome,
52 OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, QuotaLoss, ReviewRecord,
53 ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory, Tally, VoteRecord, tail,
54 write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181}
182
183#[derive(Debug, Clone, Default)]
210pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
211
212impl Pause {
213 #[must_use]
215 pub fn new() -> Self {
216 Self::default()
217 }
218
219 pub fn park(&self) {
221 self.0.store(true, Ordering::SeqCst);
222 }
223
224 pub fn park_because(&self, reason: impl Into<String>) {
230 let mut reason_guard = self
231 .1
232 .lock()
233 .unwrap_or_else(std::sync::PoisonError::into_inner);
234 if reason_guard.is_none() {
235 *reason_guard = Some(reason.into());
236 }
237 drop(reason_guard);
238 self.park();
239 }
240
241 #[must_use]
243 pub fn parked(&self) -> bool {
244 self.0.load(Ordering::SeqCst)
245 }
246
247 #[must_use]
249 pub fn reason(&self) -> Option<String> {
250 self.1
251 .lock()
252 .unwrap_or_else(std::sync::PoisonError::into_inner)
253 .clone()
254 }
255}
256
257pub struct Runner {
259 pub state: RunState,
261 roles: ResolvedRoles,
262 sem: Arc<Semaphore>,
263 pause: Pause,
267 interrupt: Pause,
273}
274
275async fn review_base(
280 repo: &Path,
281 remote: &str,
282 base_branch: &str,
283 base_commit: &str,
284 branch: &str,
285) -> String {
286 review_base_checked(repo, remote, base_branch, base_commit, branch)
287 .await
288 .0
289}
290
291async fn review_base_checked(
296 repo: &Path,
297 remote: &str,
298 base_branch: &str,
299 base_commit: &str,
300 branch: &str,
301) -> (String, bool) {
302 let tracking = format!("{remote}/{base_branch}");
303 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
304 if git::rev_exists(repo, &tracking).await
305 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
306 && !mb.is_empty()
307 {
308 return (mb, fresh);
309 }
310 let mb = git::merge_base(repo, base_commit, branch)
311 .await
312 .ok()
313 .filter(|mb| !mb.is_empty())
314 .unwrap_or_else(|| base_commit.to_owned());
315 (mb, false)
316}
317
318pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
322 if !is_review_run(state) {
323 return;
324 }
325 let base = review_base(
326 &state.repo,
327 &state.config.merge.remote,
328 &state.base_branch,
329 &state.base_commit,
330 branch,
331 )
332 .await;
333 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
334 && !subjects.is_empty()
335 && state.reviewed_commits.as_ref() != Some(&subjects)
336 {
337 state.reviewed_commits = Some(subjects);
338 state.save().ok();
339 }
340}
341
342async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
346 let (base, trusted) = review_base_checked(
347 &state.repo,
348 &state.config.merge.remote,
349 &state.base_branch,
350 &state.base_commit,
351 branch,
352 )
353 .await;
354 if !trusted {
355 return None;
356 }
357 git::subjects(&state.repo, &state.base_commit, &base)
358 .await
359 .ok()
360}
361
362fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
366 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
367 if is_leak(computed) {
368 return false;
369 }
370 let cur = current.trim();
371 cur.is_empty()
372 || cur.starts_with(REVIEW_PROMPT_OPENING)
373 || cur.starts_with("chore: land candidate")
374 || cur.starts_with("magi: candidate")
375 || is_leak(cur)
376}
377
378async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
407 let tracking = format!("{remote}/{branch}");
408 let fetched = git::fetch(repo, remote, branch).await;
409 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
410 let local_exists = git::branch_exists(repo, branch).await?;
411 if !fresh {
412 if !local_exists {
413 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
414 }
415 tracing::warn!(
416 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
417 );
418 return Ok(());
419 }
420 let remote_sha = git::rev_parse(repo, &tracking).await?;
421 if !local_exists {
422 git::git(repo, &["branch", branch, &tracking]).await?;
423 return Ok(());
424 }
425 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
426 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
427 return Ok(());
428 }
429 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
430 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
436 .await?
437 {
438 crate::reconcile::Reconciliation::Pushed => {
439 tracing::warn!(
440 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
441 short(&local_sha),
442 short(&remote_sha)
443 );
444 return Ok(());
445 }
446 crate::reconcile::Reconciliation::Placeholder => {}
447 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
448 }
449 }
450 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
451 if !out.ok() {
452 bail!(
453 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
454 short(&local_sha),
455 short(&remote_sha),
456 out.stderr
457 );
458 }
459 tracing::warn!(
460 "local `{branch}` was stale: fast-forwarded {} -> {}",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 Ok(())
465}
466
467async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
468 let tracking = format!("{remote}/{base_branch}");
469 let fetched = git::fetch(repo, remote, base_branch).await;
470 if let Ok(out) = &fetched
471 && out.ok()
472 && git::rev_exists(repo, &tracking).await
473 {
474 return git::rev_parse(repo, &tracking).await;
475 }
476 let why = match &fetched {
477 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
478 Ok(_) => format!("{remote} has no {base_branch}"),
479 Err(e) => e.to_string(),
480 };
481 bail!(
484 "cannot read {tracking} ({why}); refusing to branch off the local \
485 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
486 base / remote in magi.toml"
487 )
488}
489
490struct FixClaim {
507 path: PathBuf,
508}
509
510impl FixClaim {
511 fn acquire(dir: &Path) -> Result<Self> {
512 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
513 let path = dir.join("fix.lock");
514 match Self::create(&path) {
515 Ok(claim) => Ok(claim),
516 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
517 if Self::reclaim_if_dead(&path) {
518 Self::create(&path).with_context(|| format!("lock {}", path.display()))
519 } else {
520 bail!(
521 "another `magi fix` is already running for this run ({} exists)",
522 path.display()
523 )
524 }
525 }
526 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
527 }
528 }
529
530 fn create(path: &Path) -> std::io::Result<Self> {
531 let mut f = std::fs::OpenOptions::new()
532 .write(true)
533 .create_new(true)
534 .open(path)?;
535 use std::io::Write as _;
536 writeln!(f, "{}", std::process::id())?;
538 Ok(Self {
539 path: path.to_owned(),
540 })
541 }
542
543 fn reclaim_if_dead(path: &Path) -> bool {
547 let dead = std::fs::read_to_string(path)
548 .ok()
549 .and_then(|body| body.trim().parse::<u32>().ok())
550 .is_some_and(|pid| !crate::proc::pid_alive(pid));
551 dead && std::fs::remove_file(path).is_ok()
552 }
553}
554
555impl Drop for FixClaim {
556 fn drop(&mut self) {
557 let _ = std::fs::remove_file(&self.path);
558 }
559}
560
561impl Runner {
562 pub async fn start(
564 repo: &Path,
565 instruction: String,
566 config: Config,
567 origin: Origin,
568 ) -> Result<Self> {
569 Self::start_naming(repo, instruction, "", config, origin).await
570 }
571
572 pub async fn start_naming(
576 repo: &Path,
577 instruction: String,
578 also_scan: &str,
579 config: Config,
580 origin: Origin,
581 ) -> Result<Self> {
582 let repo = git::toplevel(repo).await?;
583 let missing = agent::missing_programs(&config.agents);
584 if !missing.is_empty() {
585 bail!(
586 "these agent programs are not on PATH: {}. Fix the roster in \
587 magi.toml or install them.",
588 missing.join(", ")
589 );
590 }
591 let base_branch =
592 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
593 .await?;
594 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
595 let roles = config.resolve_roles()?;
596 let max_parallel = config.graph.max_parallel.max(1);
597 let seeds = refs::resolve(
600 &repo,
601 &base_commit,
602 &config.merge.remote,
603 &format!("{also_scan}\n{instruction}"),
604 )
605 .await;
606 refs::plan(&repo, &seeds).await?;
607 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
608 state.origin = Some(origin);
612 for seed in &seeds {
613 state.event(
614 "seed",
615 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
616 );
617 }
618 state.seeds = seeds;
619 state.event("start", format!("run {} created", state.id));
620 state.save()?;
621 Ok(Self {
622 state,
623 roles,
624 sem: Arc::new(Semaphore::new(max_parallel)),
625 pause: Pause::new(),
626 interrupt: Pause::new(),
627 })
628 }
629
630 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
644 Self::review_taking_over(repo, branch, config, None, origin).await
645 }
646
647 pub async fn review_taking_over(
654 repo: &Path,
655 branch: &str,
656 config: Config,
657 takeover: Option<crate::handover::Takeover>,
658 origin: Origin,
659 ) -> Result<Self> {
660 let repo = git::toplevel(repo).await?;
661 let missing = agent::missing_programs(&config.agents);
662 if !missing.is_empty() {
663 bail!(
664 "these agent programs are not on PATH: {}. Fix the roster in \
665 magi.toml or install them.",
666 missing.join(", ")
667 );
668 }
669 let base_branch =
670 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
671 .await?;
672 if base_branch == branch {
673 bail!("`{branch}` is the base branch; there is nothing to review against");
674 }
675 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
676
677 let roles = config.resolve_roles()?;
678 let max_parallel = config.graph.max_parallel.max(1);
679 let mut state = RunState::new(
680 repo.clone(),
681 base_branch,
682 base_commit.clone(),
683 String::new(),
684 config,
685 );
686 state.origin = Some(origin);
687
688 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
695 earlier: Vec::new(),
696 home: crate::run::home(),
697 choice: None,
698 });
699 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
700 if let Some(released) = &released {
701 state.event(
702 "release",
703 format!(
704 "took `{branch}` over from run {}: its worktree was released: {}",
705 crate::run::short_of(&released.old_id),
706 released.audit
707 ),
708 );
709 }
710 if let Some(choice) = takeover.choice.as_ref()
714 && let Err(e) =
715 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
716 .await
717 {
718 if let Some(released) = &released {
719 released.restore(&repo, branch).await;
720 }
721 return Err(e.context("applying the owner's answer about the diverged branch"));
722 }
723 let opened =
724 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
725 if opened.is_err()
726 && let Some(released) = &released
727 {
728 released.restore(&repo, branch).await;
729 }
730 opened
731 }
732
733 async fn open_review(
736 repo: &Path,
737 branch: &str,
738 mut state: RunState,
739 roles: ResolvedRoles,
740 max_parallel: usize,
741 base_commit: String,
742 ) -> Result<Self> {
743 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
744 let start = review_base(
747 repo,
748 &state.config.merge.remote,
749 &state.base_branch,
750 &base_commit,
751 branch,
752 )
753 .await;
754 let log = git::log_oneline(repo, &start, branch)
755 .await
756 .unwrap_or_default();
757 let instruction = format!(
758 "Review the work already on branch `{branch}`. There is no task \
759 statement: what the change claims to do is whatever its commits \
760 say.\n\n{}",
761 if log.trim().is_empty() {
762 "(no commit messages)"
763 } else {
764 log.trim()
765 }
766 );
767 state.instruction = instruction;
768 state.reviewed_commits = Some(
769 git::subjects(repo, &start, branch)
770 .await
771 .unwrap_or_default(),
772 );
773
774 let worktree = state.worktree_root().join("under-review");
777 if let Some(parent) = worktree.parent() {
778 tokio::fs::create_dir_all(parent).await.ok();
779 }
780 let path = worktree.to_string_lossy().to_string();
781 git::git(repo, &["worktree", "add", &path, branch])
782 .await
783 .with_context(|| {
784 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
785 })?;
786
787 let setup = match ensure_setup_config(&mut state, repo).await {
788 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
789 Err(e) => Err(e),
790 };
791 if let Err(e) = setup {
792 git::worktree_remove(repo, &worktree).await.ok();
793 return Err(e);
794 }
795 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
796 .await
797 .unwrap_or(0);
798 if commits == 0 {
799 git::worktree_remove(repo, &worktree).await.ok();
800 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
801 }
802 let files = git::changed_files(&worktree, &base_commit, "HEAD")
803 .await
804 .map(|f| f.len())
805 .unwrap_or(0);
806 if files == 0
807 && let (Ok(head_tree), Ok(base_tree)) = (
808 git::tree_of(&worktree, "HEAD").await,
809 git::tree_of(&worktree, &base_commit).await,
810 )
811 && head_tree == base_tree
812 {
813 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
814 git::worktree_remove(repo, &worktree).await.ok();
815 bail!(
816 "`{branch}` at {} has a tree identical to base {}; this usually means \
817 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
818 against `{}/{branch}`) rather than an empty change",
819 short(&head),
820 short(&base_commit),
821 state.config.merge.remote
822 );
823 }
824 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
825 .await
826 .unwrap_or_default();
827
828 state.candidates.push(Candidate {
829 index: 0,
830 label: 'A',
831 agent: EXISTING_BRANCH.to_owned(),
834 branch: branch.to_owned(),
835 worktree,
836 summary: String::new(),
837 stat,
838 files,
839 commits,
840 empty: false,
841 failed: None,
842 verified_noop: None,
843 duration_ms: 0,
844 folded: false,
845 });
846 state.tally = Some(Tally {
847 first_choice: BTreeMap::from([('A', 0)]),
848 borda: BTreeMap::new(),
849 winner: 'A',
850 rankings: 0,
851 unanimous_initial: false,
852 deliberated: false,
853 changed_votes: 0,
854 unanimous_final: false,
855 tie_break: None,
856 judges: 0,
860 present: 0,
861 quorum: 0,
862 met_quorum: true,
863 uncontested: Some("review-only run: nothing competed".to_owned()),
864 });
865 state.status = RunStatus::Reviewing;
866 state.event(
867 "start",
868 format!(
869 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
870 state.id
871 ),
872 );
873 state.save()?;
874 Ok(Self {
875 state,
876 roles,
877 sem: Arc::new(Semaphore::new(max_parallel)),
878 pause: Pause::new(),
879 interrupt: Pause::new(),
880 })
881 }
882
883 pub fn resume(id: &str) -> Result<Self> {
885 let state = RunState::load(id)?;
886 if let Some(to) = &state.released_to {
887 bail!(
888 "run {} cannot be resumed: its worktree was released to run {}",
889 state.short(),
890 crate::run::short_of(to)
891 );
892 }
893 let roles = state.config.resolve_roles()?;
894 let max_parallel = state.config.graph.max_parallel.max(1);
895 Ok(Self {
896 state,
897 roles,
898 sem: Arc::new(Semaphore::new(max_parallel)),
899 pause: Pause::new(),
900 interrupt: Pause::new(),
901 })
902 }
903
904 pub async fn execute(&mut self) -> Result<()> {
911 let result = self.execute_graph().await;
912 self.mark_driver_exited();
913 let ended = if result.is_err() {
914 Some(crate::notices::run_stopped(&self.state.id, &self.state))
915 } else {
916 crate::notices::run_ended(&self.state)
917 };
918 if let Some(notice) = ended {
919 crate::notices::raise(notice);
920 }
921 result
922 }
923
924 fn mark_driver_exited(&mut self) {
933 self.state.driver_exited = true;
934 let pid = std::process::id();
935 let Ok(mut disk) = RunState::load(&self.state.id) else {
936 return;
937 };
938 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
939 return;
940 }
941 disk.driver_exited = true;
942 if let Err(e) = disk.save() {
943 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
944 }
945 }
946
947 async fn execute_graph(&mut self) -> Result<()> {
948 self.state.parked = false;
953 self.state.clear_active();
960 if let Ok(disk) = RunState::load(&self.state.id)
978 && let Some(to) = &disk.released_to
979 {
980 bail!(
981 "run {} cannot continue: its worktree was released to run {}",
982 self.state.short(),
983 crate::run::short_of(to)
984 );
985 }
986 let pid = std::process::id();
987 self.state.driver_pid = Some(pid);
988 self.state.driver_started_at = crate::proc::process_started_at(pid);
989 self.state.driver_exited = false;
990 self.state.save()?;
991 if self.state.status == RunStatus::Stalled {
1004 if self.recover_stall().await? {
1005 self.finish_after_tally().await?;
1006 } else {
1007 self.state.save()?;
1009 }
1010 return Ok(());
1011 }
1012 if self.state.status == RunStatus::Landing {
1022 self.run_land().await?;
1023 self.settle_questions();
1028 return Ok(());
1029 }
1030 if self.state.github_text.is_some()
1035 && self.state.merge.is_none()
1036 && self.state.status == RunStatus::Gating
1037 {
1038 self.merge().await?;
1039 self.state.save()?;
1040 return Ok(());
1041 }
1042 self.prep().await?;
1043 if self.park_here()? {
1044 return Ok(());
1045 }
1046 self.advise().await?;
1047 if self.park_here()? {
1048 return Ok(());
1049 }
1050 self.implement().await?;
1051 if self.park_here()? {
1052 return Ok(());
1053 }
1054 if self.state.status == RunStatus::VerifiedNoop {
1058 return Ok(());
1059 }
1060 self.judge().await?;
1061 if self.park_here()? {
1062 return Ok(());
1063 }
1064 self.deliberate().await?;
1065 if self.park_here()? {
1066 return Ok(());
1067 }
1068 self.vote().await?;
1069 if self.park_here()? {
1070 return Ok(());
1071 }
1072 self.tally()?;
1073 if self.state.status == RunStatus::Stalled {
1078 self.state.save()?;
1082 return Ok(());
1083 }
1084 self.finish_after_tally().await?;
1085 Ok(())
1086 }
1087
1088 fn park_here(&mut self) -> Result<bool> {
1095 if !self.pause.parked() && !self.interrupt.parked() {
1100 return Ok(false);
1101 }
1102 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
1103 Some(reason) => format!(
1104 "parked after `{}` ({reason}) — resume to carry on from here",
1105 self.state.status.as_str()
1106 ),
1107 None => format!(
1108 "parked after `{}` — resume to carry on from here",
1109 self.state.status.as_str()
1110 ),
1111 };
1112 self.state.event("park", why);
1113 self.state.parked = true;
1114 self.state.save()?;
1115 Ok(true)
1116 }
1117
1118 pub fn on_pause(&mut self, pause: Pause) {
1120 self.pause = pause;
1121 }
1122
1123 pub fn watch_interrupt(&mut self, pause: Pause) {
1129 self.interrupt = pause;
1130 }
1131
1132 fn settle_questions(&mut self) {
1152 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1153 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1154 }
1155 }
1156
1157 async fn finish_after_tally(&mut self) -> Result<()> {
1160 self.graft().await?;
1163 self.fold_losers().await?;
1164 self.sync_to_base().await?;
1169 if self.state.status == RunStatus::AlreadyInBase {
1170 return Ok(());
1171 }
1172 self.review_loop().await?;
1173 self.sync_to_base().await?;
1174 if self.state.status == RunStatus::AlreadyInBase {
1175 return Ok(());
1176 }
1177 self.gate().await?;
1178 self.merge().await?;
1179 self.state.save()?;
1180 Ok(())
1181 }
1182
1183 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1190 let result = match ensure_setup_config(&mut self.state, repo).await {
1191 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1192 Err(e) => Err(e),
1193 };
1194 if let Err(e) = result {
1195 git::worktree_remove(repo, wt).await.ok();
1196 if in_prep {
1197 self.rollback_prep(repo).await;
1198 }
1199 self.state.status = RunStatus::Blocked;
1200 self.state.event(
1201 "setup",
1202 format!("worktree setup failed in {}: {e:#}", wt.display()),
1203 );
1204 self.state.save()?;
1205 return Err(e);
1206 }
1207 Ok(())
1208 }
1209
1210 async fn rollback_prep(&mut self, repo: &Path) {
1212 let root = self.state.worktree_root();
1213 for c in std::mem::take(&mut self.state.candidates) {
1214 git::worktree_remove(repo, &c.worktree).await.ok();
1215 git::branch_delete(repo, &c.branch).await.ok();
1216 }
1217 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1218 for j in 1..=seats {
1219 git::worktree_remove(repo, &root.join(format!("judge-{j}")))
1220 .await
1221 .ok();
1222 }
1223 for k in 1..=self.state.config.graph.advisors {
1224 git::worktree_remove(repo, &root.join(format!("advisor-{k}")))
1225 .await
1226 .ok();
1227 }
1228 }
1229
1230 async fn prep(&mut self) -> Result<()> {
1231 if !self.state.candidates.is_empty() {
1232 return Ok(());
1233 }
1234 self.state.status = RunStatus::Prep;
1235 let repo = self.state.repo.clone();
1236 let base = self.state.base_commit.clone();
1237 let plan = refs::plan(&repo, &self.state.seeds).await?;
1238 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1239 let root = self.state.worktree_root();
1240 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1241
1242 let hooks_dir = self.state.dir().join("hooks");
1245 if self.state.config.blind.commit_msg_hook {
1246 std::fs::create_dir_all(&hooks_dir)
1247 .with_context(|| format!("create {}", hooks_dir.display()))?;
1248 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1249 let path = hooks_dir.join("commit-msg");
1250 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1251 make_executable(&path)?;
1252 git::acquire_worktree_config(&repo).await?;
1260 self.state.enabled_worktree_config = true;
1261 }
1262
1263 ensure_setup_config(&mut self.state, &repo).await?;
1266
1267 for (index, (spec, label)) in self
1268 .roles
1269 .implementers
1270 .clone()
1271 .into_iter()
1272 .zip(labels)
1273 .enumerate()
1274 {
1275 let branch = self.state.branch_for(label);
1276 let worktree = root.join(format!("cand-{label}"));
1277 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1278 if self.state.config.blind.commit_msg_hook {
1279 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1280 }
1281 git::local_exclude(&worktree, "/.magi/").await?;
1282 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1283 git::worktree_remove(&repo, &worktree).await.ok();
1286 git::branch_delete(&repo, &branch).await.ok();
1287 self.rollback_prep(&repo).await;
1288 self.state.status = RunStatus::Blocked;
1289 self.state
1290 .event("prep", format!("worktree setup failed: {e:#}"));
1291 self.state.save()?;
1292 return Err(e);
1293 }
1294 for pick in &plan.picks {
1295 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1296 self.state.status = RunStatus::Blocked;
1297 self.state
1298 .event("prep", format!("cannot apply referenced commit: {e}"));
1299 self.state.save()?;
1300 return Err(e);
1301 }
1302 }
1303 self.state.candidates.push(Candidate {
1304 index,
1305 label,
1306 agent: spec.id.clone(),
1307 branch,
1308 worktree,
1309 summary: String::new(),
1310 stat: String::new(),
1311 files: 0,
1312 commits: 0,
1313 empty: false,
1314 failed: None,
1315 verified_noop: None,
1316 duration_ms: 0,
1317 folded: false,
1318 });
1319 }
1320
1321 for j in 1..=self.roles.judges.len() {
1322 let wt = root.join(format!("judge-{j}"));
1323 if !wt.exists() {
1324 git::worktree_add_detached(&repo, &wt, &base).await?;
1325 self.setup_seat_worktree(&repo, &wt, true).await?;
1326 }
1327 }
1328
1329 if self.state.config.graph.advise {
1337 for k in 1..=self.state.config.graph.advisors {
1338 let wt = root.join(format!("advisor-{k}"));
1339 if !wt.exists() {
1340 git::worktree_add_detached(&repo, &wt, &base).await?;
1341 self.setup_seat_worktree(&repo, &wt, true).await?;
1342 }
1343 }
1344 }
1345
1346 let authors: Vec<&str> = self
1351 .roles
1352 .implementers
1353 .iter()
1354 .map(|a| a.id.as_str())
1355 .collect();
1356 let overlap: Vec<String> = self
1357 .roles
1358 .judges
1359 .iter()
1360 .enumerate()
1361 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1362 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1363 .collect();
1364 if !overlap.is_empty() {
1365 let note = format!(
1366 "{} also authored a candidate; blind, but the panel is less \
1367 independent than {} distinct agents would be",
1368 overlap.join(", "),
1369 self.roles.judges.len()
1370 );
1371 self.state.event("prep", note);
1372 }
1373
1374 self.state.event(
1375 "prep",
1376 format!(
1377 "{} candidates, {} judges, base {} ({})",
1378 self.state.candidates.len(),
1379 self.roles.judges.len(),
1380 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1381 self.state.base_branch
1382 ),
1383 );
1384 self.state.status = RunStatus::Implementing;
1385 self.state.save()?;
1386 Ok(())
1387 }
1388
1389 async fn advise(&mut self) -> Result<()> {
1422 let implement_untouched = self
1423 .state
1424 .candidates
1425 .iter()
1426 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1427 if !self.state.config.graph.advise || self.state.advise_attempted {
1428 return Ok(());
1429 }
1430 if !implement_untouched {
1431 self.state.event(
1432 "advise",
1433 "skipping the design-deliberation stage: at least one \
1434 candidate already shows implementation progress, so this \
1435 run is past the point the stage exists to run before"
1436 .to_owned(),
1437 );
1438 self.state.advise_attempted = true;
1439 self.state.save()?;
1440 return Ok(());
1441 }
1442 let run_id = self.state.id.clone();
1443 let prompts = self.state.config.prompts.clone();
1444 let instruction = self.state.instruction.clone();
1445 let language = self.state.config.graph.language.clone();
1446 let root = self.state.worktree_root();
1447 let n = self.state.config.graph.advisors;
1448 let where_recorded = self.state.dir().join("run.json");
1449
1450 let seats = match self.state.config.advisors() {
1451 Ok(seats) if !seats.is_empty() => seats,
1452 Ok(_) => {
1453 self.state.event(
1454 "advise",
1455 format!(
1456 "[graph] advisors is 0; skipping the design-deliberation \
1457 stage and continuing without a synthesis brief (see {})",
1458 where_recorded.display()
1459 ),
1460 );
1461 self.state.advise_attempted = true;
1462 self.state.save()?;
1463 return Ok(());
1464 }
1465 Err(e) => {
1466 self.state.event(
1467 "advise",
1468 format!(
1469 "could not resolve advisor seats ({e:#}); continuing \
1470 without a design-deliberation brief (see {})",
1471 where_recorded.display()
1472 ),
1473 );
1474 self.state.advise_attempted = true;
1475 self.state.save()?;
1476 return Ok(());
1477 }
1478 };
1479
1480 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1481 let artifacts = agent::artifacts_dir(&self.state.dir());
1482 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1483
1484 let mut jobs = Vec::new();
1485 for (i, spec) in seats.iter().cloned().enumerate() {
1486 let seat_key = format!("advisor-{}", i + 1);
1487 let seat = self.seat(&seat_key, &spec.id);
1488 jobs.push(SeatJob {
1489 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1490 spec,
1491 seat,
1492 cwd: worktrees[i % worktrees.len()].clone(),
1493 timeout,
1494 allow_write: false,
1495 sessions: false,
1496 artifacts: artifacts.clone(),
1497 stem: seat_key,
1498 handover: None,
1499 });
1500 }
1501
1502 self.state.event(
1503 "advise",
1504 format!(
1505 "{} advisor seat(s) sketching a design in parallel",
1506 jobs.len()
1507 ),
1508 );
1509 let mut quota_losses = Vec::new();
1510 let cache = self.state.config.cache_dir();
1511 let ctx = WaveCtx {
1512 carry_seats: false,
1513 run: &run_id,
1514 node: "advise",
1515 prompts: &prompts,
1516 cache: cache.as_deref(),
1517 round: None,
1518 };
1519 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1520 let results = ask_json_wave::<Proposal>(
1521 jobs,
1522 Arc::clone(&self.sem),
1523 self.state.config.graph.retries,
1524 &advisor_roster,
1525 &ctx,
1526 &mut quota_losses,
1527 &mut self.state,
1528 &|p: &Proposal| p.validate(),
1529 )
1530 .await;
1531 self.state.quota.extend(quota_losses);
1532
1533 let mut records = Vec::with_capacity(results.len());
1534 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1535 let agent_id = seat.agent.clone();
1536 self.state.seats.insert(seat.key.clone(), seat);
1537 match res {
1538 Ok((proposal, out)) => {
1539 self.state
1540 .event("advise", format!("advisor-{} proposed a design", i + 1));
1541 records.push(advise::AdvisorRecord::proposed(
1542 i + 1,
1543 agent_id,
1544 proposal,
1545 out.duration_ms,
1546 ));
1547 }
1548 Err(e) => {
1549 self.state.event(
1550 "advise",
1551 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1552 );
1553 records.push(advise::AdvisorRecord::failed(
1554 i + 1,
1555 agent_id,
1556 e.to_string(),
1557 ));
1558 }
1559 }
1560 }
1561
1562 let mut advice = advise::Advice {
1563 records,
1564 synthesis: None,
1565 };
1566 if advice.proposals().is_empty() {
1567 self.state.event(
1568 "advise",
1569 "no advisor produced a usable proposal; continuing without a \
1570 synthesis brief"
1571 .to_owned(),
1572 );
1573 } else {
1574 match self
1575 .synthesize_brief(
1576 &advice,
1577 &instruction,
1578 &language,
1579 &worktrees[0],
1580 &artifacts,
1581 &run_id,
1582 &prompts,
1583 cache.as_deref(),
1584 )
1585 .await
1586 {
1587 Ok(Some(text)) => {
1588 self.state.event(
1589 "advise",
1590 "synthesized a design brief for the implementer".to_owned(),
1591 );
1592 advice.synthesis = Some(text);
1593 }
1594 Ok(None) => {
1595 self.state.event(
1596 "advise",
1597 "the synthesis seat produced nothing usable; continuing \
1598 without a design brief"
1599 .to_owned(),
1600 );
1601 }
1602 Err(e) => {
1603 self.state.event(
1604 "advise",
1605 format!("could not synthesize a design brief: {e:#}"),
1606 );
1607 }
1608 }
1609 }
1610 advise::apply_reflection(&mut advice);
1611
1612 self.state.advice = Some(advice);
1613 self.state.advise_attempted = true;
1614 self.state.save()?;
1615 Ok(())
1616 }
1617
1618 #[allow(clippy::too_many_arguments)]
1630 async fn synthesize_brief(
1631 &mut self,
1632 advice: &advise::Advice,
1633 instruction: &str,
1634 language: &str,
1635 cwd: &Path,
1636 artifacts: &Path,
1637 run_id: &str,
1638 prompts: &Prompts,
1639 cache: Option<&Path>,
1640 ) -> Result<Option<String>> {
1641 let chain = agent::pick_chain(
1642 &self.state.config.agents,
1643 self.state.config.roles.synthesizer.as_ref(),
1644 &agent::installed,
1645 "synthesizer",
1646 )?;
1647 let proposals = advice.proposals();
1648 let mut prompt = prompt::with_overlay(
1649 prompt::synthesize_brief(instruction, &proposals, language),
1650 prompts.overlay("advise"),
1651 );
1652 if cache.is_some() {
1653 prompt.push('\n');
1658 prompt.push_str(&prompt::build_cache_note("advise", false));
1659 }
1660 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1661 let mut last = None;
1666 for (n, spec) in chain.iter().enumerate() {
1667 if n > 0 {
1668 self.state
1669 .event("advise", format!("synthesis falling back to {}", spec.id));
1670 }
1671 let mut seat = self.seat("advise-synthesis", &spec.id);
1672 let outcome = agent::invoke(
1673 spec,
1674 &mut seat,
1675 &Invocation {
1676 cwd,
1677 prompt: &prompt,
1678 timeout,
1679 allow_write: false,
1680 unsandboxed: false,
1681 sessions: false,
1682 artifacts,
1683 stem: &if n == 0 {
1684 "advise-synthesis".to_owned()
1685 } else {
1686 format!("advise-synthesis-{}", spec.id)
1687 },
1688 run: run_id,
1689 node: "advise",
1690 cache_dir: None,
1691 attachments: &[],
1692 writable: &[],
1693 },
1694 )
1695 .await;
1696 if outcome.is_ok() {
1697 self.state.seats.insert(seat.key.clone(), seat);
1698 }
1699 let advance = agent::chain_advances(&outcome);
1700 last = Some(outcome);
1701 if !advance {
1702 break;
1703 }
1704 }
1705 let out = last.expect("a chain holds at least one agent")?;
1708 if !out.usable() {
1709 return Ok(None);
1710 }
1711 let text =
1712 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1713 Ok((!text.trim().is_empty()).then_some(text))
1714 }
1715
1716 async fn implement(&mut self) -> Result<()> {
1719 let run_id = self.state.id.clone();
1724 let prompts = self.state.config.prompts.clone();
1725 let todo: Vec<usize> = self
1726 .state
1727 .candidates
1728 .iter()
1729 .enumerate()
1730 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1731 .map(|(i, _)| i)
1732 .collect();
1733 if todo.is_empty() {
1734 return self.after_implement();
1735 }
1736 self.state.status = RunStatus::Implementing;
1737
1738 let language = self.state.config.graph.language.clone();
1739 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1740 let sessions = self.state.config.graph.sessions;
1741 let artifacts = agent::artifacts_dir(&self.state.dir());
1742 let brief = self
1746 .state
1747 .advice
1748 .as_ref()
1749 .and_then(|a| a.synthesis.as_deref())
1750 .map(str::to_owned);
1751 let attachments = self.state.attachments.clone();
1752
1753 let mut jobs = Vec::new();
1754 for &i in &todo {
1755 let (index, label, worktree) = {
1756 let c = &self.state.candidates[i];
1757 (c.index, c.label, c.worktree.clone())
1758 };
1759 let spec = self.roles.implementers[index].clone();
1760 let seat_key = format!("impl-{label}");
1761 let seat = self.seat(&seat_key, &spec.id);
1762 let instruction = seeded_instruction(&self.state);
1763 jobs.push(SeatJob {
1764 spec,
1765 seat,
1766 prompt: prompt::implement(
1767 &instruction,
1768 &worktree.to_string_lossy(),
1769 &language,
1770 brief.as_deref(),
1771 &attachments,
1772 ),
1773 cwd: worktree,
1774 timeout,
1775 allow_write: true,
1776 sessions,
1777 artifacts: artifacts.clone(),
1778 stem: format!("impl-{label}"),
1779 handover: None,
1780 });
1781 }
1782
1783 self.state.event(
1784 "implement",
1785 format!("{} candidates in parallel", jobs.len()),
1786 );
1787 let mut sent = jobs.clone();
1793 let cache = self.state.config.cache_dir();
1794 let ctx = WaveCtx {
1795 carry_seats: false,
1796 run: &run_id,
1797 node: "implement",
1798 prompts: &prompts,
1799 cache: cache.as_deref(),
1800 round: None,
1801 };
1802 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1803 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1804 .await;
1805 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1806 .await;
1807 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1808 .await;
1809
1810 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1811 let seat_key = seat.key.clone();
1812 let agent = seat.agent.clone();
1821 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1822 self.state.seats.insert(seat.key.clone(), seat);
1823 let label = self.state.candidates[i].label;
1824 let worktree = self.state.candidates[i].worktree.clone();
1825 let base = self.state.base_commit.clone();
1826
1827 let (summary, duration, failed, verified_claim) = match out {
1828 AgentOutcome::Ok(o) => {
1829 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1830 let failed = (!o.usable()).then(|| {
1831 if o.timed_out {
1832 "agent timed out".to_owned()
1833 } else {
1834 format!("agent exited with {:?}", o.exit_code)
1835 }
1836 });
1837 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1838 (text, o.duration_ms, failed, verified_claim)
1839 }
1840 AgentOutcome::Dropped(o) => {
1846 let why = o
1847 .dropped
1848 .as_ref()
1849 .map(|d| d.why.as_str())
1850 .unwrap_or("the CLI ended the stream without delivering its answer");
1851 (
1852 String::new(),
1853 o.duration_ms,
1854 Some(format!("the CLI dropped the stream ({why})")),
1855 None,
1856 )
1857 }
1858 AgentOutcome::Quota(o) => {
1859 self.state.quota.push(QuotaLoss {
1860 seat: seat_key,
1861 node: "implement".to_owned(),
1862 at: Timestamp::now(),
1863 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1864 });
1865 (
1866 String::new(),
1867 o.duration_ms,
1868 Some("rate limited (quota); produced no change".to_owned()),
1869 None,
1870 )
1871 }
1872 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1873 };
1874
1875 let rescued = match git::rescue_commit(
1878 &worktree,
1879 &format!("magi: candidate {label} (uncommitted work)"),
1880 )
1881 .await
1882 {
1883 Ok(r) => {
1884 self.state.note_withheld("implement", &r.withheld);
1885 r.committed
1886 }
1887 Err(_) => false,
1888 };
1889 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1890 .await
1891 .unwrap_or(0);
1892 let patch = git::diff(&worktree, &base, "HEAD")
1893 .await
1894 .unwrap_or_default();
1895 let stat = git::diff_stat(&worktree, &base, "HEAD")
1896 .await
1897 .unwrap_or_default();
1898 let files = git::changed_files(&worktree, &base, "HEAD")
1899 .await
1900 .map(|f| f.len())
1901 .unwrap_or(0);
1902 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1903
1904 let c = &mut self.state.candidates[i];
1905 if !exhausted_the_fallback_chain {
1906 c.agent = agent;
1907 }
1908 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1909 c.stat = stat;
1910 c.files = files;
1911 c.commits = commits;
1912 c.duration_ms = duration;
1913 c.empty = commits == 0 || patch.trim().is_empty();
1914 c.failed = match failed {
1917 Some(_) if c.empty => failed,
1918 _ => None,
1919 };
1920 c.verified_noop = if c.empty { verified_claim } else { None };
1925 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1926 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1927 (None, true, Some(_), _) => {
1928 format!("candidate {label}: no change produced (agent-verified no-op)")
1929 }
1930 (None, true, None, _) => format!("candidate {label}: no change produced"),
1931 (None, false, _, true) => {
1932 format!(
1933 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1934 )
1935 }
1936 (None, false, _, false) => {
1937 format!("candidate {label}: {files} files, {commits} commits")
1938 }
1939 };
1940 self.state.event("implement", note);
1941 self.state.save()?;
1942 }
1943
1944 self.after_implement()
1945 }
1946
1947 async fn resume_undelivered(
1975 &mut self,
1976 results: &mut [(usize, SeatState, AgentOutcome)],
1977 sent: &[SeatJob],
1978 prompts: &Prompts,
1979 run_id: &str,
1980 ) {
1981 for (wi, seat, out) in results.iter_mut() {
1982 let Some(dropped) = (match &*out {
1983 AgentOutcome::Dropped(o) => o.dropped.clone(),
1984 _ => None,
1985 }) else {
1986 continue;
1987 };
1988 let Some(job) = sent.get(*wi) else { continue };
1989 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1991 self.state.event(
1992 "implement",
1993 format!(
1994 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1995 work is in the tree",
1996 seat.key, dropped.output_tokens, dropped.why
1997 ),
1998 );
1999 continue;
2000 }
2001 if !has_context(&job.spec, seat, job.sessions) {
2009 self.state.event(
2010 "implement",
2011 format!(
2012 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2013 is no session left to resume",
2014 seat.key, dropped.output_tokens, dropped.why
2015 ),
2016 );
2017 continue;
2018 }
2019 self.state.event(
2020 "implement",
2021 format!(
2022 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2023 conversation",
2024 seat.key, dropped.output_tokens, dropped.why
2025 ),
2026 );
2027 let mut retry = job.clone();
2028 retry.seat = seat.clone();
2029 retry.prompt = prompt::resume_after_drop(&dropped.why);
2030 retry.timeout = retry_budget(job.timeout, true);
2031 retry.stem = format!("{}-resume", job.stem);
2032 let cache = self.state.config.cache_dir();
2033 let ctx = WaveCtx {
2034 carry_seats: false,
2035 run: run_id,
2036 node: "implement",
2037 prompts,
2038 cache: cache.as_deref(),
2039 round: None,
2040 };
2041 let (resumed_seat, resumed) =
2042 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2043 *seat = resumed_seat;
2044 *out = resumed;
2045 }
2046 }
2047
2048 async fn resume_seat_handovers(
2113 &mut self,
2114 results: &mut [(usize, SeatState, AgentOutcome)],
2115 sent: &mut [SeatJob],
2116 prompts: &Prompts,
2117 run_id: &str,
2118 ) {
2119 let instruction = seeded_instruction(&self.state);
2120 let language = self.state.config.graph.language.clone();
2121 let brief = self
2122 .state
2123 .advice
2124 .as_ref()
2125 .and_then(|a| a.synthesis.as_deref())
2126 .map(str::to_owned);
2127 let attachments = self.state.attachments.clone();
2128 for (wi, seat, out) in results.iter_mut() {
2129 let others: BTreeSet<String> = sent
2132 .iter()
2133 .enumerate()
2134 .filter(|(j, _)| j != wi)
2135 .map(|(_, j)| j.spec.id.clone())
2136 .collect();
2137 let Some(job) = sent.get_mut(*wi) else {
2138 continue;
2139 };
2140 let start = self
2145 .roles
2146 .implementer_roster
2147 .iter()
2148 .position(|s| s.id == job.spec.id)
2149 .unwrap_or(0);
2150 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2151 let mut fallback_attempt = 0usize;
2152 let mut prev: Option<FailClass> = None;
2153 while let Some(cur) = FailClass::of(&*out) {
2154 if !should_hand_over(prev.as_ref(), &cur) {
2155 break;
2156 }
2157 let Some(next) =
2158 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2159 .cloned()
2160 else {
2161 break;
2162 };
2163 tried.insert(next.id.clone());
2164 fallback_attempt += 1;
2165
2166 if let Ok(r) = git::rescue_commit(
2167 &job.cwd,
2168 &format!(
2169 "magi: candidate {} (uncommitted work before {} fallback)",
2170 seat.key,
2171 if cur == FailClass::Quota {
2172 "quota"
2173 } else {
2174 "handover"
2175 }
2176 ),
2177 )
2178 .await
2179 {
2180 self.state.note_withheld("implement", &r.withheld);
2181 }
2182
2183 record_handover(
2184 &mut self.state,
2185 "implement",
2186 &seat.key,
2187 &seat.agent,
2188 &next.id,
2189 &cur,
2190 &fail_reason(&*out),
2191 );
2192 prev = Some(cur.clone());
2193
2194 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2195 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2196 job.spec = next.clone();
2204 let mut retry = job.clone();
2205 retry.seat = new_seat;
2206 retry.prompt = prompt::implement(
2207 &instruction,
2208 &job.cwd.to_string_lossy(),
2209 &language,
2210 brief.as_deref(),
2211 &attachments,
2212 );
2213 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2214 let cache = self.state.config.cache_dir();
2215 let ctx = WaveCtx {
2216 carry_seats: false,
2217 run: run_id,
2218 node: "implement",
2219 prompts,
2220 cache: cache.as_deref(),
2221 round: None,
2222 };
2223 let (fallback_seat, fallback_out) = run_one(
2224 retry,
2225 Arc::clone(&self.sem),
2226 &ctx,
2227 &mut self.state,
2228 fallback_attempt,
2229 )
2230 .await;
2231 *seat = fallback_seat;
2232 *out = fallback_out;
2233 }
2234 }
2235 }
2236
2237 async fn resume_unconfirmed_commands(
2261 &mut self,
2262 results: &mut [(usize, SeatState, AgentOutcome)],
2263 sent: &[SeatJob],
2264 prompts: &Prompts,
2265 run_id: &str,
2266 ) {
2267 for (wi, seat, out) in results.iter_mut() {
2268 let AgentOutcome::Ok(o) = &*out else {
2269 continue;
2270 };
2271 if !has_unconfirmed_command(&o.commands) {
2272 continue;
2273 }
2274 let Some(job) = sent.get(*wi) else { continue };
2275 if !has_context(&job.spec, seat, job.sessions) {
2276 self.state.event(
2277 "implement",
2278 format!(
2279 "{}: the reply named a command whose own CLI never confirmed the exit \
2280 status of, but there is no session left to resume",
2281 seat.key
2282 ),
2283 );
2284 continue;
2285 }
2286 self.state.event(
2287 "implement",
2288 format!(
2289 "{}: the reply named a command whose own CLI never confirmed the exit \
2290 status of; resuming the conversation",
2291 seat.key
2292 ),
2293 );
2294 let mut retry = job.clone();
2295 retry.seat = seat.clone();
2296 retry.prompt = prompt::resume_incomplete(
2297 "a command in your last reply had no confirmed exit status",
2298 );
2299 retry.timeout = retry_budget(job.timeout, true);
2300 retry.stem = format!("{}-confirm", job.stem);
2301 let cache = self.state.config.cache_dir();
2302 let ctx = WaveCtx {
2303 carry_seats: false,
2304 run: run_id,
2305 node: "implement",
2306 prompts,
2307 cache: cache.as_deref(),
2308 round: None,
2309 };
2310 let (resumed_seat, resumed) =
2311 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2312 *seat = resumed_seat;
2313 *out = resumed;
2314 }
2315 }
2316
2317 async fn continue_fix_report(
2338 &mut self,
2339 mut seat: SeatState,
2340 parse_err: String,
2341 job: &SeatJob,
2342 prompts: &Prompts,
2343 run_id: &str,
2344 round: usize,
2345 ) -> (
2346 SeatState,
2347 Option<FixReport>,
2348 Option<String>,
2349 ContinuationRecord,
2350 ) {
2351 let mut last_err = parse_err;
2352 let mut cumulative_wait_ms = 0u64;
2353 let mut attempts = 0usize;
2354 loop {
2355 if !has_context(&job.spec, &seat, job.sessions) {
2356 self.state.event(
2357 "fix",
2358 format!(
2359 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2360 session left to resume into"
2361 ),
2362 );
2363 let outcome = if attempts == 0 {
2364 ContinuationOutcome::NoSession
2365 } else {
2366 ContinuationOutcome::Exhausted
2367 };
2368 return (
2369 seat,
2370 None,
2371 Some(format!("unparsable fix report: {last_err}")),
2372 ContinuationRecord {
2373 attempts,
2374 cumulative_wait_ms,
2375 outcome,
2376 },
2377 );
2378 }
2379 if attempts >= MAX_FIX_CONTINUATIONS {
2380 self.state.event(
2381 "fix",
2382 format!(
2383 "round {round}: fixer's reply still had no adoption report after \
2384 {attempts} continuation(s) ({last_err}); giving up"
2385 ),
2386 );
2387 return (
2388 seat,
2389 None,
2390 Some(format!(
2391 "unparsable fix report after {attempts} continuation(s): {last_err}"
2392 )),
2393 ContinuationRecord {
2394 attempts,
2395 cumulative_wait_ms,
2396 outcome: ContinuationOutcome::Exhausted,
2397 },
2398 );
2399 }
2400 attempts += 1;
2401 self.state.event(
2402 "fix",
2403 format!(
2404 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2405 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2406 ),
2407 );
2408 let mut retry = job.clone();
2409 retry.seat = seat.clone();
2410 retry.prompt = prompt::resume_incomplete(&last_err);
2411 retry.timeout = retry_budget(job.timeout, true);
2412 retry.stem = format!("{}-continue{attempts}", job.stem);
2413 let cache = self.state.config.cache_dir();
2414 let ctx = WaveCtx {
2415 carry_seats: false,
2416 run: run_id,
2417 node: "fix",
2418 prompts,
2419 cache: cache.as_deref(),
2420 round: Some(round),
2421 };
2422 let (resumed_seat, resumed_out) = run_one(
2423 retry,
2424 Arc::clone(&self.sem),
2425 &ctx,
2426 &mut self.state,
2427 attempts,
2428 )
2429 .await;
2430 seat = resumed_seat;
2431 match resumed_out {
2432 AgentOutcome::Ok(o) => {
2433 cumulative_wait_ms += o.duration_ms;
2434 match verdict::extract_json::<FixReport>(&o.text) {
2435 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2436 self.state.event(
2437 "fix",
2438 format!(
2439 "round {round}: fixer's adoption report recovered after \
2440 {attempts} continuation(s)"
2441 ),
2442 );
2443 return (
2444 seat,
2445 Some(report),
2446 None,
2447 ContinuationRecord {
2448 attempts,
2449 cumulative_wait_ms,
2450 outcome: ContinuationOutcome::Resumed,
2451 },
2452 );
2453 }
2454 Ok(_) => {
2462 last_err = "the reply parsed, but it reported a command whose own CLI \
2463 never confirmed an exit status"
2464 .to_owned();
2465 }
2466 Err(e) => last_err = e.to_string(),
2467 }
2468 }
2469 AgentOutcome::Quota(o) => {
2470 cumulative_wait_ms += o.duration_ms;
2471 self.state.quota.push(QuotaLoss {
2472 seat: seat.key.clone(),
2473 node: "fix".to_owned(),
2474 at: Timestamp::now(),
2475 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2476 });
2477 self.state.event(
2478 "fix",
2479 format!(
2480 "round {round}: continuation rate limited (quota); not retrying now"
2481 ),
2482 );
2483 return (
2484 seat,
2485 None,
2486 Some("rate limited (quota) while recovering the fix report".to_owned()),
2487 ContinuationRecord {
2488 attempts,
2489 cumulative_wait_ms,
2490 outcome: ContinuationOutcome::QuotaLost,
2491 },
2492 );
2493 }
2494 AgentOutcome::Dropped(o) => {
2495 cumulative_wait_ms += o.duration_ms;
2496 let why = o
2497 .dropped
2498 .as_ref()
2499 .map(|d| d.why.as_str())
2500 .unwrap_or("the CLI ended the stream without delivering its answer");
2501 last_err = format!("the CLI dropped the stream ({why})");
2502 }
2503 AgentOutcome::Failed(e) => last_err = e,
2504 }
2505 }
2506 }
2507
2508 fn after_implement(&mut self) -> Result<()> {
2509 if self.state.leaks.is_empty() {
2511 let cfg = self.state.config.blind.clone();
2512 let mut leaks = Vec::new();
2513 for c in &self.state.candidates {
2514 let Some(patch) =
2515 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2516 else {
2517 continue;
2518 };
2519 leaks.extend(blind::scan(
2520 &format!("candidate {} patch", c.label),
2521 &patch,
2522 &cfg.vendor_tokens,
2523 ));
2524 }
2525 if !leaks.is_empty() {
2526 let summary = leaks
2527 .iter()
2528 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2529 .collect::<Vec<_>>()
2530 .join(", ");
2531 match cfg.on_leak {
2532 LeakPolicy::Fail => {
2533 self.state.status = RunStatus::Failed;
2534 self.state
2535 .event("blind", format!("vendor text in a patch: {summary}"));
2536 self.state.leaks = leaks;
2537 self.state.save()?;
2538 self.settle_questions();
2539 bail!(
2540 "blind.on_leak = \"fail\" and vendor text reached a \
2541 judged patch: {summary}"
2542 );
2543 }
2544 LeakPolicy::Redact => self.state.event(
2545 "blind",
2546 format!("redacting vendor text for judging: {summary}"),
2547 ),
2548 LeakPolicy::Warn => self.state.event(
2549 "blind",
2550 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2551 ),
2552 }
2553 self.state.leaks = leaks;
2554 }
2555 }
2556
2557 if self.state.viable().is_empty() {
2558 if self.state.all_candidates_verified_noop() {
2559 self.state.status = RunStatus::VerifiedNoop;
2570 self.state.save()?;
2571 self.settle_questions();
2572 return Ok(());
2573 }
2574 self.state.status = RunStatus::Failed;
2575 self.state.save()?;
2576 self.settle_questions();
2577 bail!("no candidate produced a change; nothing to judge");
2578 }
2579 self.state.status = RunStatus::Judging;
2580 self.state.save()?;
2581 Ok(())
2582 }
2583
2584 async fn judge(&mut self) -> Result<()> {
2587 let run_id = self.state.id.clone();
2592 let prompts = self.state.config.prompts.clone();
2593 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2594 return Ok(());
2595 }
2596 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2597 if viable.len() == 1 {
2598 self.state.judge_skipped = true;
2605 self.state.event(
2606 "judge",
2607 format!(
2608 "only candidate {} produced a change; judging skipped",
2609 viable[0].label
2610 ),
2611 );
2612 self.state.save()?;
2613 return Ok(());
2614 }
2615 self.state.status = RunStatus::Judging;
2616
2617 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2618 let language = self.state.config.graph.language.clone();
2619 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2620 let sessions = self.state.config.graph.sessions;
2621 let artifacts = agent::artifacts_dir(&self.state.dir());
2622 let root = self.state.worktree_root();
2623 let base_short = short(&self.state.base_commit);
2624
2625 let mut jobs = Vec::new();
2626 let mut orders = Vec::new();
2627 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2628 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2629 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2630 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2631 let seat_key = format!("judge-{}", j + 1);
2632 let seat = self.seat(&seat_key, &spec.id);
2633 jobs.push(SeatJob {
2634 prompt: prompt::judge(
2635 &self.state.instruction,
2636 &views,
2637 self.roles.judges.len(),
2638 &base_short,
2639 &language,
2640 ),
2641 spec,
2642 seat,
2643 cwd: root.join(format!("judge-{}", j + 1)),
2644 timeout,
2645 allow_write: false,
2646 sessions,
2647 artifacts: artifacts.clone(),
2648 stem: format!("judge-{}", j + 1),
2649 handover: None,
2650 });
2651 }
2652
2653 self.state.event(
2654 "judge",
2655 format!(
2656 "{} judges ranking {} candidates blind",
2657 jobs.len(),
2658 viable.len()
2659 ),
2660 );
2661 let labels_for_check = labels.clone();
2662 let mut quota_losses = Vec::new();
2663 let cache = self.state.config.cache_dir();
2664 let ctx = WaveCtx {
2665 carry_seats: false,
2666 run: &run_id,
2667 node: "judge",
2668 prompts: &prompts,
2669 cache: cache.as_deref(),
2670 round: None,
2671 };
2672 let results = ask_json_wave::<Ranking>(
2673 jobs,
2674 Arc::clone(&self.sem),
2675 self.state.config.graph.retries,
2676 &self.roles.judge_roster,
2677 &ctx,
2678 &mut quota_losses,
2679 &mut self.state,
2680 &move |r: &Ranking| r.validate(&labels_for_check),
2681 )
2682 .await;
2683 self.state.quota.extend(quota_losses);
2684
2685 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2686 let agent_id = seat.agent.clone();
2687 self.state.seats.insert(seat.key.clone(), seat);
2688 let mut record = Judgement {
2689 judge: j + 1,
2690 seat: format!("judge-{}", j + 1),
2691 agent: agent_id,
2692 ranking: Vec::new(),
2693 reasons: BTreeMap::new(),
2694 confidence: None,
2695 strengths: BTreeMap::new(),
2696 order: orders[j].clone(),
2697 failed: None,
2698 duration_ms: 0,
2699 };
2700 match res {
2701 Ok((ranking, out)) => {
2702 record.ranking = ranking.normalized();
2703 record.reasons = ranking.reasons;
2704 record.strengths = ranking.strengths;
2705 record.confidence = ranking.confidence;
2706 record.duration_ms = out.duration_ms;
2707 self.state.event(
2708 "judge",
2709 format!(
2710 "judge {} ranked {}",
2711 j + 1,
2712 record.ranking.iter().collect::<String>()
2713 ),
2714 );
2715 }
2716 Err(e) => {
2717 record.failed = Some(e.to_string());
2718 self.state
2719 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2720 }
2721 }
2722 self.state.judgements.push(record);
2723 self.state.save()?;
2724 }
2725 Ok(())
2726 }
2727
2728 async fn deliberate(&mut self) -> Result<()> {
2731 let run_id = self.state.id.clone();
2736 let prompts = self.state.config.prompts.clone();
2737 if !self.state.deliberation.is_empty() {
2738 return Ok(());
2739 }
2740 let tops: Vec<char> = self
2741 .state
2742 .judgements
2743 .iter()
2744 .filter_map(|j| j.ranking.first().copied())
2745 .collect();
2746 let rounds = self.state.config.graph.deliberate_rounds;
2747 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2748 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2749 self.state.event(
2750 "deliberate",
2751 format!("judges agreed on {} outright; no deliberation", tops[0]),
2752 );
2753 }
2754 self.state.status = RunStatus::Voting;
2755 self.state.save()?;
2756 return Ok(());
2757 }
2758
2759 self.state.status = RunStatus::Deliberating;
2760 self.state.event(
2761 "deliberate",
2762 format!(
2763 "split: first choices were {} — opening {rounds} round(s)",
2764 tops.iter().collect::<String>()
2765 ),
2766 );
2767
2768 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2769 let language = self.state.config.graph.language.clone();
2770 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2771 let sessions = self.state.config.graph.sessions;
2772 let artifacts = agent::artifacts_dir(&self.state.dir());
2773 let root = self.state.worktree_root();
2774 let base_short = short(&self.state.base_commit);
2775
2776 for round in 1..=rounds {
2780 let mut turns: Vec<DeliberationTurn> = Vec::new();
2781 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2782 if self.state.judgements[j].failed.is_some() {
2783 continue;
2784 }
2785 let seat_key = format!("judge-{}", j + 1);
2786 let spec = self.occupant(&seat_key, spec);
2787 let mut seat = self.seat(&seat_key, &spec.id);
2788 let transcript = self.transcript(&turns, j);
2789 let build = |context: Option<&str>| {
2790 prompt::deliberate(
2791 &self.state.instruction,
2792 context,
2793 &transcript,
2794 round,
2795 rounds,
2796 &language,
2797 )
2798 };
2799 let block = self.candidate_block(&viable, &base_short);
2800 let full = build(Some(&block));
2801 let text = if has_context(&spec, &seat, sessions) {
2802 build(None)
2803 } else {
2804 full.clone()
2805 };
2806 let job = SeatJob {
2807 spec,
2808 seat: seat.clone(),
2809 prompt: text,
2810 cwd: root.join(format!("judge-{}", j + 1)),
2811 timeout,
2812 allow_write: false,
2813 sessions,
2814 artifacts: artifacts.clone(),
2815 stem: format!("delib-{round}-judge-{}", j + 1),
2816 handover: Some(full),
2817 };
2818 let cache = self.state.config.cache_dir();
2819 let ctx = WaveCtx {
2820 carry_seats: false,
2821 run: &run_id,
2822 node: "deliberate",
2823 prompts: &prompts,
2824 cache: cache.as_deref(),
2825 round: None,
2826 };
2827 let mut losses = Vec::new();
2831 let mut results = ask_wave_with::<String>(
2832 vec![job],
2833 Arc::clone(&self.sem),
2834 0,
2835 &self.roles.judge_roster,
2836 &ctx,
2837 &mut losses,
2838 &mut self.state,
2839 &|text: &str| {
2840 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
2841 },
2842 )
2843 .await;
2844 self.state.quota.extend(losses);
2845 let (updated, res, _) = results.pop().expect("one job in, one result out");
2846 seat = updated;
2847 let agent_id = seat.agent.clone();
2848 self.state.seats.insert(seat.key.clone(), seat);
2849 let body = match res {
2850 Ok((body, _)) => body,
2851 Err(e) => {
2854 self.state
2855 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2856 continue;
2857 }
2858 };
2859 let tentative = verdict::extract_json::<Position>(&body)
2860 .ok()
2861 .and_then(|p| p.tentative)
2862 .and_then(|s| s.trim().chars().next())
2863 .map(|c| c.to_ascii_uppercase());
2864 self.state.event(
2865 "deliberate",
2866 format!(
2867 "round {round}: judge {} now favours {}",
2868 j + 1,
2869 tentative.map_or("—".to_owned(), |c| c.to_string())
2870 ),
2871 );
2872 turns.push(DeliberationTurn {
2873 judge: j + 1,
2874 agent: agent_id,
2875 body: blind::sanitize_prose(&body, &self.state.config.blind),
2876 tentative,
2877 });
2878 }
2879 self.state
2880 .deliberation
2881 .push(DeliberationRound { round, turns });
2882 self.state.save()?;
2883 }
2884
2885 self.state.status = RunStatus::Voting;
2886 self.state.save()?;
2887 Ok(())
2888 }
2889
2890 async fn vote(&mut self) -> Result<()> {
2893 let run_id = self.state.id.clone();
2898 let prompts = self.state.config.prompts.clone();
2899 if !self.state.votes.is_empty() {
2900 return Ok(());
2901 }
2902 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2903 if viable.len() == 1 {
2904 return Ok(());
2905 }
2906 self.state.status = RunStatus::Voting;
2907
2908 let language = self.state.config.graph.language.clone();
2909 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2910 let sessions = self.state.config.graph.sessions;
2911 let artifacts = agent::artifacts_dir(&self.state.dir());
2912 let root = self.state.worktree_root();
2913 let base_short = short(&self.state.base_commit);
2914 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2915
2916 let mut jobs = Vec::new();
2917 let mut seats_at = Vec::new();
2918 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2919 if self
2920 .state
2921 .judgements
2922 .get(j)
2923 .is_some_and(|r| r.failed.is_some())
2924 {
2925 continue;
2926 }
2927 let seat_key = format!("judge-{}", j + 1);
2928 let spec = self.occupant(&seat_key, spec);
2929 let seat = self.seat(&seat_key, &spec.id);
2930 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
2931 let text = if has_context(&spec, &seat, sessions) {
2932 prompt::final_vote(&viable, &language)
2933 } else {
2934 full.clone()
2935 };
2936 jobs.push(SeatJob {
2937 spec,
2938 seat,
2939 prompt: text,
2940 cwd: root.join(format!("judge-{}", j + 1)),
2941 timeout,
2942 allow_write: false,
2943 sessions,
2944 artifacts: artifacts.clone(),
2945 stem: format!("vote-judge-{}", j + 1),
2946 handover: Some(full),
2947 });
2948 seats_at.push(j);
2949 }
2950
2951 self.state.event(
2952 "vote",
2953 format!(
2954 "collecting {} final votes one by one, privately",
2955 jobs.len()
2956 ),
2957 );
2958 let allowed = viable.clone();
2959 let mut quota_losses = Vec::new();
2960 let cache = self.state.config.cache_dir();
2961 let ctx = WaveCtx {
2962 carry_seats: false,
2963 run: &run_id,
2964 node: "vote",
2965 prompts: &prompts,
2966 cache: cache.as_deref(),
2967 round: None,
2968 };
2969 let results = ask_json_wave::<FinalVote>(
2970 jobs,
2971 Arc::clone(&self.sem),
2972 self.state.config.graph.retries,
2973 &self.roles.judge_roster,
2974 &ctx,
2975 &mut quota_losses,
2976 &mut self.state,
2977 &move |v: &FinalVote| match v.label() {
2978 Some(c) if allowed.contains(&c) => Ok(()),
2979 other => bail!("vote {other:?} is not one of {allowed:?}"),
2980 },
2981 )
2982 .await;
2983 self.state.quota.extend(quota_losses);
2984
2985 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2986 let agent_id = seat.agent.clone();
2987 self.state.seats.insert(seat.key.clone(), seat);
2988 let initial = self
2989 .state
2990 .judgements
2991 .get(j)
2992 .and_then(|r| r.ranking.first().copied());
2993 let mut record = VoteRecord {
2994 judge: j + 1,
2995 agent: agent_id,
2996 vote: None,
2997 reason: String::new(),
2998 changed: false,
2999 };
3000 match res {
3001 Ok((v, _)) => {
3002 record.vote = v.label();
3003 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3004 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3005 self.state.event(
3006 "vote",
3007 format!(
3008 "judge {} voted {}{}",
3009 j + 1,
3010 record.vote.unwrap_or('?'),
3011 if record.changed { " (changed)" } else { "" }
3012 ),
3013 );
3014 }
3015 Err(e) => {
3016 self.state
3017 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3018 }
3019 }
3020 self.state.votes.push(record);
3021 self.state.save()?;
3022 }
3023 Ok(())
3024 }
3025
3026 fn tally(&mut self) -> Result<()> {
3029 if self.state.tally.is_some() {
3030 return Ok(());
3031 }
3032 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3033 let tops: Vec<char> = self
3034 .state
3035 .judgements
3036 .iter()
3037 .filter_map(|j| j.ranking.first().copied())
3038 .collect();
3039 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3040
3041 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3044 let mut cast: Vec<char> = Vec::new();
3045 for (i, j) in self.state.judgements.iter().enumerate() {
3046 let vote = self
3047 .state
3048 .votes
3049 .iter()
3050 .find(|v| v.judge == i + 1)
3051 .and_then(|v| v.vote)
3052 .or_else(|| j.ranking.first().copied());
3053 if let Some(v) = vote {
3054 *first_choice.entry(v).or_insert(0) += 1;
3055 cast.push(v);
3056 }
3057 }
3058
3059 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3060 for j in &self.state.judgements {
3061 let n = j.ranking.len();
3062 for (pos, label) in j.ranking.iter().enumerate() {
3063 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3064 }
3065 }
3066
3067 let best = first_choice.values().copied().max().unwrap_or(0);
3068 let mut leaders: Vec<char> = first_choice
3069 .iter()
3070 .filter(|(_, v)| **v == best)
3071 .map(|(k, _)| *k)
3072 .collect();
3073 let mut tie_break = None;
3074 if leaders.len() > 1 {
3075 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3076 let borda_leaders: Vec<char> = leaders
3077 .iter()
3078 .copied()
3079 .filter(|l| borda[l] == top_borda)
3080 .collect();
3081 tie_break = Some(if borda_leaders.len() == 1 {
3082 format!(
3083 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3084 leaders.len()
3085 )
3086 } else {
3087 format!(
3088 "{} way tie on both first-choice votes and Borda points, broken by label order",
3089 leaders.len()
3090 )
3091 });
3092 leaders = borda_leaders;
3093 leaders.sort_unstable();
3094 }
3095 let winner = *leaders
3096 .first()
3097 .or(viable.first())
3098 .context("no candidate to declare a winner from")?;
3099
3100 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3101 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3102 let deliberated = !self.state.deliberation.is_empty();
3103
3104 let quota_seats: std::collections::BTreeSet<&str> =
3108 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3109 let mut present = 0usize;
3110 for (i, j) in self.state.judgements.iter().enumerate() {
3111 if quota_seats.contains(j.seat.as_str()) {
3112 continue;
3113 }
3114 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3115 let voted = self
3116 .state
3117 .votes
3118 .iter()
3119 .any(|v| v.judge == i + 1 && v.vote.is_some());
3120 if ranked || voted {
3121 present += 1;
3122 }
3123 }
3124 let needs_quorum = viable.len() > 1;
3130 let judges_total = if needs_quorum {
3131 self.roles.judges.len()
3132 } else {
3133 0
3134 };
3135 let quorum = if needs_quorum {
3136 judges_total / 2 + 1
3137 } else {
3138 0
3139 };
3140 let met_quorum = !needs_quorum || present >= quorum;
3141 let uncontested = (!needs_quorum).then(|| {
3142 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3143 });
3144
3145 self.state.event(
3146 "tally",
3147 match &uncontested {
3148 Some(reason) => format!("winner {winner} — {reason}"),
3149 None => format!(
3150 "winner {winner} — votes {} | initial {} | {} changed | \
3151 {present}/{judges_total} judges{}",
3152 first_choice
3153 .iter()
3154 .map(|(k, v)| format!("{k}:{v}"))
3155 .collect::<Vec<_>>()
3156 .join(" "),
3157 if unanimous_initial {
3158 "unanimous"
3159 } else {
3160 "split"
3161 },
3162 changed_votes,
3163 if met_quorum {
3164 String::new()
3165 } else {
3166 format!(" — below quorum ({quorum} required)")
3167 },
3168 ),
3169 },
3170 );
3171 if !met_quorum {
3172 self.state.event(
3173 "stall",
3174 format!(
3175 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3176 the run stops here, resumable"
3177 ),
3178 );
3179 }
3180 self.state.tally = Some(Tally {
3181 first_choice,
3182 borda,
3183 winner,
3184 rankings: tops.len(),
3185 unanimous_initial,
3186 deliberated,
3187 changed_votes,
3188 unanimous_final,
3189 tie_break,
3190 judges: judges_total,
3191 present,
3192 quorum,
3193 met_quorum,
3194 uncontested,
3195 });
3196 self.state.status = if met_quorum {
3197 RunStatus::Reviewing
3198 } else {
3199 RunStatus::Stalled
3200 };
3201 self.state.save()?;
3202 Ok(())
3203 }
3204
3205 #[allow(clippy::too_many_lines)]
3226 async fn recover_stall(&mut self) -> Result<bool> {
3227 let run_id = self.state.id.clone();
3232 let prompts = self.state.config.prompts.clone();
3233 let quota_seats: BTreeSet<&str> =
3238 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3239 let absent: Vec<String> = self
3240 .state
3241 .judgements
3242 .iter()
3243 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3244 .map(|j| j.seat.clone())
3245 .collect();
3246 if absent.is_empty() {
3247 return Ok(false);
3248 }
3249 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3250 if viable.len() <= 1 {
3251 return Ok(false);
3252 }
3253 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3254 let language = self.state.config.graph.language.clone();
3255 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3256 let sessions = self.state.config.graph.sessions;
3257 let artifacts = agent::artifacts_dir(&self.state.dir());
3258 let root = self.state.worktree_root();
3259 let base_short = short(&self.state.base_commit);
3260 let candidates: Vec<Candidate> = viable.clone();
3261
3262 let mut positions: Vec<usize> = absent
3264 .iter()
3265 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3266 .collect();
3267 if positions.is_empty() {
3268 return Ok(false);
3269 }
3270 positions.sort_unstable();
3271 positions.dedup();
3272
3273 let mut judge_jobs = Vec::new();
3275 for &j in &positions {
3276 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3277 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3278 let seat_key = format!("judge-{}", j + 1);
3279 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3280 let seat = self.seat(&seat_key, &spec.id);
3281 judge_jobs.push(SeatJob {
3282 spec,
3283 seat,
3284 prompt: prompt::judge(
3285 &self.state.instruction,
3286 &views,
3287 self.roles.judges.len(),
3288 &base_short,
3289 &language,
3290 ),
3291 cwd: root.join(seat_key),
3292 timeout,
3293 allow_write: false,
3294 sessions,
3295 artifacts: artifacts.clone(),
3296 stem: format!("judge-{}-recover", j + 1),
3297 handover: None,
3298 });
3299 }
3300
3301 let labels_for_check = labels.clone();
3302 let mut judge_losses = Vec::new();
3303 let retries = self.state.config.graph.retries;
3304 let cache = self.state.config.cache_dir();
3305 let ctx = WaveCtx {
3306 carry_seats: false,
3307 run: &run_id,
3308 node: "judge",
3309 prompts: &prompts,
3310 cache: cache.as_deref(),
3311 round: None,
3312 };
3313 let results = ask_json_wave::<Ranking>(
3314 judge_jobs,
3315 Arc::clone(&self.sem),
3316 retries,
3317 &self.roles.judge_roster,
3318 &ctx,
3319 &mut judge_losses,
3320 &mut self.state,
3321 &move |r: &Ranking| r.validate(&labels_for_check),
3322 )
3323 .await;
3324
3325 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3327 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3328 let agent_id = seat.agent.clone();
3329 self.state.seats.insert(seat.key.clone(), seat);
3330 let record = &mut self.state.judgements[j];
3331 match res {
3332 Ok((ranking, out)) => {
3333 record.agent = agent_id;
3334 record.ranking = ranking.normalized();
3335 record.reasons = ranking.reasons;
3336 record.strengths = ranking.strengths;
3337 record.confidence = ranking.confidence;
3338 record.failed = None;
3339 record.duration_ms = out.duration_ms;
3340 recovered.insert(j);
3341 self.state.event(
3342 "recover",
3343 format!("judge {} ranked again after the limit", j + 1),
3344 );
3345 }
3346 Err(e) => {
3347 self.state
3348 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3349 }
3350 }
3351 }
3352
3353 let mut vote_jobs = Vec::new();
3355 let mut vote_pos: Vec<usize> = Vec::new();
3356 for &j in &recovered {
3357 let seat_key = format!("judge-{}", j + 1);
3358 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3359 let seat = self.seat(&seat_key, &spec.id);
3360 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3361 let text = if has_context(&spec, &seat, sessions) {
3362 prompt::final_vote(&labels, &language)
3363 } else {
3364 full.clone()
3365 };
3366 vote_jobs.push(SeatJob {
3367 spec,
3368 seat,
3369 prompt: text,
3370 cwd: root.join(seat_key),
3371 timeout,
3372 allow_write: false,
3373 sessions,
3374 artifacts: artifacts.clone(),
3375 stem: format!("vote-judge-{}-recover", j + 1),
3376 handover: Some(full),
3377 });
3378 vote_pos.push(j);
3379 }
3380 let allowed = labels.clone();
3381 let mut vote_losses = Vec::new();
3382 let vote_retries = self.state.config.graph.retries;
3383 let vote_cache = self.state.config.cache_dir();
3384 let ctx = WaveCtx {
3385 carry_seats: false,
3386 run: &run_id,
3387 node: "vote",
3388 prompts: &prompts,
3389 cache: vote_cache.as_deref(),
3390 round: None,
3391 };
3392 let votes = ask_json_wave::<FinalVote>(
3393 vote_jobs,
3394 Arc::clone(&self.sem),
3395 vote_retries,
3396 &self.roles.judge_roster,
3397 &ctx,
3398 &mut vote_losses,
3399 &mut self.state,
3400 &move |v: &FinalVote| match v.label() {
3401 Some(c) if allowed.contains(&c) => Ok(()),
3402 other => bail!("vote {other:?} is not one of {allowed:?}"),
3403 },
3404 )
3405 .await;
3406 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3407 let agent_id = seat.agent.clone();
3408 self.state.seats.insert(seat.key.clone(), seat);
3409 match res {
3410 Ok((v, _)) => {
3411 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3412 rec.vote = v.label();
3413 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3414 } else {
3415 self.state.votes.push(VoteRecord {
3416 judge: j + 1,
3417 agent: agent_id,
3418 vote: v.label(),
3419 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3420 changed: false,
3421 });
3422 }
3423 self.state.event(
3424 "recover",
3425 format!("judge {} voted again after the limit", j + 1),
3426 );
3427 }
3428 Err(e) => {
3429 self.state
3430 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3431 }
3432 }
3433 }
3434
3435 let recovered_keys: BTreeSet<String> = recovered
3439 .iter()
3440 .map(|&j| format!("judge-{}", j + 1))
3441 .collect();
3442 self.state
3443 .quota
3444 .retain(|q| !recovered_keys.contains(&q.seat));
3445 for loss in judge_losses.into_iter().chain(vote_losses) {
3449 if recovered_keys.contains(&loss.seat) {
3450 continue;
3451 }
3452 self.state.quota.retain(|q| q.seat != loss.seat);
3453 self.state.quota.push(loss);
3454 }
3455
3456 self.state.tally = None;
3458 self.tally()?;
3459 Ok(self
3460 .state
3461 .tally
3462 .as_ref()
3463 .map(|t| t.met_quorum)
3464 .unwrap_or(false))
3465 }
3466
3467 async fn graft(&mut self) -> Result<()> {
3484 let Err(e) = self.graft_inner().await else {
3485 return Ok(());
3486 };
3487 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
3488 self.state.status = RunStatus::Blocked;
3489 self.state.event("graft", format!("blocked: {e:#}"));
3490 self.state.save()?;
3491 return Err(e);
3492 }
3493 if let Some(rec) = self.state.graft.clone()
3498 && !rec.outcome.settled()
3499 {
3500 if let Some(winner) = self.state.winner().cloned() {
3501 if let Err(re) = Self::graft_restore(&winner.worktree, &rec.pre).await {
3502 self.state.status = RunStatus::Blocked;
3503 self.state.event("graft", format!("blocked: {re:#}"));
3504 self.state.save()?;
3505 return Err(re);
3506 }
3507 }
3508 if let Some(g) = self.state.graft.as_mut() {
3509 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
3510 }
3511 }
3512 self.state.event("graft", format!("skipped: {e:#}"));
3513 if let Err(e) = self.state.save() {
3514 tracing::warn!("save after graft: {e:#}");
3515 }
3516 Ok(())
3517 }
3518
3519 async fn graft_restore(wt: &Path, pre: &str) -> Result<()> {
3522 let restored = git::restore_to(wt, pre).await;
3523 let tip = git::rev_parse(wt, "HEAD").await;
3524 match (restored, tip) {
3525 (Ok(()), Ok(tip)) if tip == pre => Ok(()),
3526 (Err(e), _) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
3527 (Ok(()), Ok(tip)) => {
3528 Err(GraftRestoreFailed(format!("the winner is at {tip}, not {pre}")).into())
3529 }
3530 (Ok(()), Err(e)) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
3531 }
3532 }
3533
3534 fn graft_skipped(&mut self, why: &str) {
3536 let message = format!("skipped: {why}");
3537 if !self
3538 .state
3539 .events
3540 .iter()
3541 .any(|e| e.node == "graft" && e.message == message)
3542 {
3543 self.state.event("graft", message);
3544 }
3545 }
3546
3547 async fn graft_inner(&mut self) -> Result<()> {
3548 if let Some(rec) = self.state.graft.clone()
3551 && !rec.outcome.settled()
3552 {
3553 if self.state.reviews.is_empty()
3557 && let Some(winner) = self.state.winner().cloned()
3558 {
3559 Self::graft_restore(&winner.worktree, &rec.pre).await?;
3560 }
3561 if let Some(g) = self.state.graft.as_mut() {
3562 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
3563 }
3564 self.state.event(
3565 "graft",
3566 "rolled back: the previous round was interrupted; the winner is untouched",
3567 );
3568 self.state.save()?;
3569 return Ok(());
3570 }
3571 let plan = match graft::plan(&self.state) {
3572 Ok(plan) => plan,
3573 Err(skip) => {
3574 if skip != graft::Skip::AlreadyDone {
3575 self.graft_skipped(skip.reason());
3576 self.state.save()?;
3577 }
3578 return Ok(());
3579 }
3580 };
3581 let Some(winner) = self.state.winner().cloned() else {
3582 return Ok(());
3583 };
3584 let wt = winner.worktree.clone();
3585 if !git::is_clean(&wt).await? {
3586 self.graft_skipped("the winner's worktree has uncommitted changes");
3587 self.state.save()?;
3588 return Ok(());
3589 }
3590 let pre = git::rev_parse(&wt, "HEAD").await?;
3591 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
3592 else {
3593 self.graft_skipped("the winner's diff could not be measured");
3594 self.state.save()?;
3595 return Ok(());
3596 };
3597 let limit = graft::limit(winner_lines);
3598 self.state.graft = Some(GraftRecord {
3601 pre: pre.clone(),
3602 post: None,
3603 lines: None,
3604 limit: Some(limit),
3605 agent: None,
3606 outcome: GraftOutcome::Running,
3607 });
3608 self.state.event(
3609 "graft",
3610 format!(
3611 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
3612 plan.strengths
3613 .iter()
3614 .map(|(l, _, _)| l.to_string())
3615 .collect::<Vec<_>>()
3616 .join(", "),
3617 plan.winner
3618 ),
3619 );
3620 self.state.save()?;
3621
3622 let fix_prompt = prompt::graft(
3623 &self.state.instruction,
3624 plan.winner,
3625 &plan.strengths,
3626 limit,
3627 &self.state.config.graph.language,
3628 );
3629 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
3630 let sessions = self.state.config.graph.sessions;
3631 let artifacts = agent::artifacts_dir(&self.state.dir());
3632 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
3633 let (_, seat, out) = self
3634 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
3635 SeatJob {
3636 prompt: fix_prompt.clone(),
3637 spec,
3638 seat,
3639 cwd: wt.clone(),
3640 timeout,
3641 allow_write: true,
3642 sessions,
3643 artifacts: artifacts.clone(),
3644 stem: "graft".to_owned(),
3645 handover: None,
3646 }
3647 })
3648 .await;
3649 let mut why: Option<String> = None;
3650 match &out {
3651 AgentOutcome::Ok(o) if agent::output_advances(o) => {
3652 why = Some("the fixer produced nothing usable".to_owned());
3653 }
3654 AgentOutcome::Ok(_) => {}
3655 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
3656 AgentOutcome::Quota(o) => {
3657 self.state.quota.push(QuotaLoss {
3658 seat: seat.key.clone(),
3659 node: "graft".to_owned(),
3660 at: Timestamp::now(),
3661 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3662 });
3663 why = Some("rate limited (quota)".to_owned());
3664 }
3665 AgentOutcome::Failed(e) => {
3666 why = Some(format!(
3667 "the fixer failed: {}",
3668 e.lines()
3669 .next()
3670 .unwrap_or("")
3671 .chars()
3672 .take(160)
3673 .collect::<String>()
3674 ));
3675 }
3676 }
3677 let agent_id = seat.agent.clone();
3678 self.state.seats.insert(seat.key.clone(), seat);
3679
3680 let mut post = None;
3681 let mut lines = None;
3682 if why.is_none() {
3683 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
3684 if let Ok(r) = &rescued {
3685 self.state.note_withheld("graft", &r.withheld);
3686 }
3687 let head = git::rev_parse(&wt, "HEAD").await?;
3688 let dirty = match &rescued {
3692 Ok(r) => git::status_porcelain(&wt)
3693 .await
3694 .map(|st| {
3695 st.lines().any(|l| {
3696 !r.withheld
3697 .iter()
3698 .any(|w| l.trim_end().ends_with(w.path.as_str()))
3699 })
3700 })
3701 .unwrap_or(true),
3702 Err(_) => true,
3703 };
3704 if dirty {
3705 why = Some("the fixer left changes that could not be committed".to_owned());
3706 } else if head == pre {
3707 why = Some("the fixer changed nothing".to_owned());
3708 } else {
3709 lines = git::changed_lines(&wt, &pre, &head).await;
3710 if graft::within_budget(lines, limit) {
3711 post = Some(head);
3712 } else {
3713 why = Some(format!(
3714 "{} changed lines against a limit of {limit}",
3715 lines.map_or_else(
3716 || "an unmeasurable number of".to_owned(),
3717 |l| l.to_string()
3718 )
3719 ));
3720 }
3721 }
3722 }
3723 let outcome = match why {
3724 None => {
3725 self.state.event(
3726 "graft",
3727 format!(
3728 "applied: {} changed lines (limit {limit})",
3729 lines.unwrap_or(0)
3730 ),
3731 );
3732 GraftOutcome::Applied
3733 }
3734 Some(why) => {
3735 Self::graft_restore(&wt, &pre).await?;
3738 self.state.event(
3739 "graft",
3740 format!("rolled back: {why}; the winner is untouched"),
3741 );
3742 GraftOutcome::RolledBack(why)
3743 }
3744 };
3745 self.state.graft = Some(GraftRecord {
3746 pre,
3747 post,
3748 lines,
3749 limit: Some(limit),
3750 agent: Some(agent_id),
3751 outcome,
3752 });
3753 self.state.save()?;
3754 Ok(())
3755 }
3756
3757 async fn fold_losers(&mut self) -> Result<()> {
3760 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3761 return Ok(());
3762 };
3763 let repo = self.state.repo.clone();
3764 let mut folded = Vec::new();
3765 for i in 0..self.state.candidates.len() {
3766 let c = &self.state.candidates[i];
3767 if c.label == winner || c.folded {
3768 continue;
3769 }
3770 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3771 git::worktree_remove(&repo, &wt).await.ok();
3772 git::branch_delete(&repo, &branch).await.ok();
3773 self.state.candidates[i].folded = true;
3774 folded.push(label.to_string());
3775 }
3776 let root = self.state.worktree_root();
3778 for j in 1..=self.roles.judges.len() {
3779 let wt = root.join(format!("judge-{j}"));
3780 if wt.exists() {
3781 git::worktree_remove(&repo, &wt).await.ok();
3782 }
3783 }
3784 if self.state.config.graph.advise {
3787 for k in 1..=self.state.config.graph.advisors {
3788 let wt = root.join(format!("advisor-{k}"));
3789 if wt.exists() {
3790 git::worktree_remove(&repo, &wt).await.ok();
3791 }
3792 }
3793 }
3794 if !folded.is_empty() {
3795 self.state
3796 .event("fold", format!("folded candidates {}", folded.join(", ")));
3797 self.state.save()?;
3798 }
3799 Ok(())
3800 }
3801
3802 async fn sync_to_base(&mut self) -> Result<()> {
3842 if self.state.status == RunStatus::AlreadyInBase {
3843 return Ok(());
3844 }
3845 let conflicted = self
3846 .state
3847 .base_sync
3848 .as_ref()
3849 .is_some_and(|s| s.conflict.is_some());
3850 let Some(winner) = self.state.winner().cloned() else {
3851 return Ok(());
3852 };
3853
3854 let repo = self.state.repo.clone();
3855 let remote = self.state.config.merge.remote.clone();
3856 let base_branch = self.state.base_branch.clone();
3857 let tracking = format!("{remote}/{base_branch}");
3858
3859 git::fetch(&repo, &remote, &base_branch).await.ok();
3860 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3865 return Ok(());
3866 };
3867
3868 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3869 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3870 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3871
3872 if (behind > 0 || conflicted || head == tip)
3881 && self
3882 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3883 .await?
3884 {
3885 return Ok(());
3886 }
3887 if conflicted {
3888 return Ok(());
3889 }
3890
3891 if behind == 0 {
3892 if let Some(from) = self
3901 .state
3902 .rebase_fixes
3903 .iter()
3904 .rev()
3905 .find_map(|r| r.from.clone())
3906 && from != head
3907 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3908 .await
3909 .is_ok_and(|o| o.ok())
3910 {
3911 git::sync_to_head(&winner.worktree).await?;
3912 }
3913 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3919 if let Some(why) = &conflict {
3920 self.state.status = RunStatus::Blocked;
3921 self.state.event("land", why.clone());
3922 }
3923 self.state.base_sync = Some(BaseSync {
3924 tip,
3925 behind: 0,
3926 attempts,
3927 conflict,
3928 already_in: None,
3929 });
3930 self.state.save()?;
3931 return Ok(());
3932 }
3933
3934 if attempts >= BASE_SYNC_ROUNDS {
3935 let why = format!(
3936 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3937 rebase(s); rebasing again would only race it",
3938 winner.branch
3939 );
3940 self.state.status = RunStatus::Blocked;
3941 self.state.base_sync = Some(BaseSync {
3942 tip,
3943 behind,
3944 attempts,
3945 conflict: Some(why.clone()),
3946 already_in: None,
3947 });
3948 self.state.event("land", why);
3949 self.state.save()?;
3950 return Ok(());
3951 }
3952
3953 self.state.event(
3954 "land",
3955 format!(
3956 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3957 winner.branch
3958 ),
3959 );
3960 self.state.save()?;
3961
3962 let branch_tracking = format!("{remote}/{}", winner.branch);
3968 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3969 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3970 git::rev_parse(&repo, &branch_tracking).await.ok()
3971 } else {
3972 None
3973 };
3974 if let Some(theirs) = &remote_tip
3978 && !git::is_ancestor(&repo, theirs, &head).await
3979 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3980 .await
3981 .is_ok_and(|missing| missing.is_empty())
3982 {
3983 let why = format!(
3984 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3985 them",
3986 short(theirs),
3987 winner.branch
3988 );
3989 self.state.status = RunStatus::Blocked;
3990 self.state.base_sync = Some(BaseSync {
3991 tip,
3992 behind,
3993 attempts,
3994 conflict: Some(why.clone()),
3995 already_in: None,
3996 });
3997 self.state.event("land", why);
3998 self.state.save()?;
3999 return Ok(());
4000 }
4001
4002 let scratch = self.state.dir().join("base-sync");
4003 let rebased = match crate::rebase::rebase_with_fixer(
4004 &mut self.state,
4005 &scratch,
4006 &winner.branch,
4007 &tracking,
4008 )
4009 .await
4010 {
4011 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4012 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4013 Err(e) => Err(e),
4014 };
4015 let attempts = attempts + 1;
4016 match rebased {
4017 Ok(None) => {
4018 git::sync_to_head(&winner.worktree).await?;
4022 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4023 let mut conflict = None;
4024 if let Some(pinned) = &remote_tip {
4025 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4026 match pushed {
4027 Ok(o) if o.ok() => self.state.event(
4028 "land",
4029 format!("pushed rebased {} to {remote}", winner.branch),
4030 ),
4031 Ok(o) => {
4032 conflict = Some(format!(
4033 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4034 winner.branch,
4035 short(pinned),
4036 o.stderr.chars().take(600).collect::<String>()
4037 ));
4038 }
4039 Err(e) => {
4040 conflict = Some(format!(
4041 "rebased {} locally but could not push it: {e:#}",
4042 winner.branch
4043 ));
4044 }
4045 }
4046 }
4047 if let Some(why) = &conflict {
4048 self.state.status = RunStatus::Blocked;
4049 self.state.event("land", why.clone());
4050 }
4051 self.state.base_sync = Some(BaseSync {
4052 tip: tip.clone(),
4053 behind: 0,
4054 attempts,
4055 conflict,
4056 already_in: None,
4057 });
4058 self.state
4059 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4060 }
4061 Ok(Some(conflict)) => {
4062 let why = format!(
4063 "{} conflicts with {tracking} and did not rebase: {}",
4064 winner.branch,
4065 conflict.chars().take(600).collect::<String>()
4066 );
4067 self.state.status = RunStatus::Blocked;
4068 self.state.base_sync = Some(BaseSync {
4069 tip,
4070 behind,
4071 attempts,
4072 conflict: Some(why.clone()),
4073 already_in: None,
4074 });
4075 self.state.event("land", why);
4076 }
4077 Err(e) => {
4078 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4079 self.state.status = RunStatus::Blocked;
4080 self.state.base_sync = Some(BaseSync {
4081 tip,
4082 behind,
4083 attempts,
4084 conflict: Some(why.clone()),
4085 already_in: None,
4086 });
4087 self.state.event("land", why);
4088 }
4089 }
4090 self.state.save()?;
4091 Ok(())
4092 }
4093
4094 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4100 let repo = self.state.repo.clone();
4101 let remote = self.state.config.merge.remote.clone();
4102 let fetched = git::fetch(&repo, &remote, branch).await;
4103 if !matches!(&fetched, Ok(o) if o.ok()) {
4104 return None;
4105 }
4106 let branch_tracking = format!("{remote}/{branch}");
4107 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4108 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4109 return None;
4110 }
4111 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4112 .await
4113 .is_ok_and(|missing| missing.is_empty())
4114 {
4115 return Some(format!(
4116 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4117 them",
4118 short(&theirs)
4119 ));
4120 }
4121 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4122 Ok(o) if o.ok() => {
4123 self.state
4124 .event("land", format!("pushed rebased {branch} to {remote}"));
4125 None
4126 }
4127 Ok(o) => Some(format!(
4128 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4129 someone may have pushed): {}",
4130 short(&theirs),
4131 o.stderr.chars().take(600).collect::<String>()
4132 )),
4133 Err(e) => Some(format!(
4134 "{branch} is rebased locally but could not be pushed: {e:#}"
4135 )),
4136 }
4137 }
4138
4139 async fn settle_already_in(
4156 &mut self,
4157 branch: &str,
4158 tip: &str,
4159 head: &str,
4160 attempts: usize,
4161 behind: usize,
4162 ) -> Result<bool> {
4163 let repo = self.state.repo.clone();
4164 let remote = self.state.config.merge.remote.clone();
4165 let start = self.state.base_commit.clone();
4166 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4167 Ok(Some(e)) => e,
4168 Ok(None) => return Ok(false),
4169 Err(e) => {
4170 tracing::warn!("already-in-base check for {branch}: {e:#}");
4171 return Ok(false);
4172 }
4173 };
4174 let mut verified = vec![head.to_owned()];
4175 let fetched = git::fetch(&repo, &remote, branch).await;
4176 if matches!(&fetched, Ok(o) if o.ok())
4177 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
4178 && theirs != head
4179 {
4180 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4181 Ok(Some(_)) => verified.push(theirs),
4182 _ => return Ok(false),
4183 }
4184 }
4185 let base_branch = self.state.base_branch.clone();
4186 let message = format!(
4187 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4188 land",
4189 evidence.names(),
4190 evidence.proof.as_str()
4191 );
4192 let closed =
4193 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
4194 match closed {
4195 Ok(Ok(url)) => self
4196 .state
4197 .event("land", format!("closed {url}: superseded on {base_branch}")),
4198 Ok(Err(why)) => self
4199 .state
4200 .event("land", format!("did not close a pull request: {why}")),
4201 Err(e) => {
4202 let why = format!(
4203 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4204 not be closed ({e:#}); resume to retry"
4205 );
4206 self.state.status = RunStatus::Blocked;
4207 self.state.base_sync = Some(BaseSync {
4208 tip: tip.to_owned(),
4209 behind,
4210 attempts,
4211 conflict: Some(why.clone()),
4212 already_in: None,
4213 });
4214 self.state.event("land", why);
4215 self.state.save()?;
4216 return Ok(true);
4217 }
4218 }
4219 self.state.status = RunStatus::AlreadyInBase;
4220 self.state.base_sync = Some(BaseSync {
4221 tip: tip.to_owned(),
4222 behind,
4223 attempts,
4224 conflict: None,
4225 already_in: Some(evidence),
4226 });
4227 self.state.event("land", message);
4228 self.state.save()?;
4229 self.settle_questions();
4230 Ok(true)
4231 }
4232
4233 fn landing_base(&self) -> String {
4243 self.state
4244 .base_sync
4245 .as_ref()
4246 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4247 }
4248
4249 pub async fn fix_selected(
4282 &mut self,
4283 ids: &[String],
4284 reason: &str,
4285 allow_stale: bool,
4286 ) -> Result<()> {
4287 let reason = reason.trim();
4288 if reason.is_empty() {
4289 bail!("a fix request needs a reason — that is the operator's own record of why");
4290 }
4291 if ids.is_empty() {
4292 bail!("no finding id given");
4293 }
4294 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4295 bail!(
4296 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4297 has already concluded — can be given a targeted fix. A run still \
4298 in progress should simply be resumed; a `merged` run's branch has \
4299 already landed, so its answer is a fresh `magi review <branch>`, \
4300 not reopening this run's own record",
4301 self.state.id,
4302 self.state.status.as_str()
4303 );
4304 }
4305 let Some(winner) = self.state.winner().cloned() else {
4306 bail!("run {} has no winning candidate to fix", self.state.id);
4307 };
4308 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4309 bail!(
4310 "branch `{}` no longer exists; this run cannot be extended",
4311 winner.branch
4312 );
4313 }
4314 let home = crate::run::home();
4315 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
4316 bail!(
4317 "run {} is currently being worked on by another magi process",
4318 self.state.id
4319 );
4320 }
4321 let _claim = FixClaim::acquire(&self.state.dir())?;
4327
4328 let mut seen = BTreeSet::new();
4332 let mut findings = Vec::new();
4333 let mut missing = Vec::new();
4334 for id in ids {
4335 if !seen.insert(id.clone()) {
4336 continue;
4337 }
4338 match self.state.finding(id) {
4339 Some((round, rec, f)) => findings.push(OperatorFixFinding {
4340 id: f.id.clone(),
4341 severity: f.severity,
4342 reviewer_vote: rec.vote,
4343 round: round.round,
4344 round_head: round.head.clone(),
4345 reviewer: rec.reviewer,
4346 agent: rec.agent.clone(),
4347 file: f.file.clone(),
4348 line: f.line,
4349 title: f.title.clone(),
4350 detail: f.detail.clone(),
4351 outcome: OperatorFixOutcome::Pending,
4352 }),
4353 None => missing.push(id.clone()),
4354 }
4355 }
4356 if !missing.is_empty() {
4357 bail!(
4358 "unknown finding id(s): {}; nothing was changed",
4359 missing.join(", ")
4360 );
4361 }
4362
4363 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
4364 let stale_details: Vec<(String, String)> = findings
4365 .iter()
4366 .filter(|f| f.round_head != head_at_request)
4367 .map(|f| (f.id.clone(), f.round_head.clone()))
4368 .collect();
4369 let stale = !stale_details.is_empty();
4370 if stale && !allow_stale {
4371 bail!(
4372 "the branch has moved since some finding(s) were raised — {} — now \
4373 at {}; pass --allow-stale to fix anyway, or re-run review first",
4374 stale_details
4375 .iter()
4376 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
4377 .collect::<Vec<_>>()
4378 .join(", "),
4379 short(&head_at_request)
4380 );
4381 }
4382
4383 let request = OperatorFixRequest {
4384 requested_at: Timestamp::now(),
4385 reason: reason.to_owned(),
4386 findings,
4387 head_at_request: head_at_request.clone(),
4388 allow_stale,
4389 stale,
4390 fix: None,
4391 result_head: None,
4392 follow_up_review_run: None,
4393 };
4394 self.state.event(
4395 "fix",
4396 format!(
4397 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4398 request.findings.len(),
4399 request
4400 .findings
4401 .iter()
4402 .map(|f| f.id.as_str())
4403 .collect::<Vec<_>>()
4404 .join(", "),
4405 ),
4406 );
4407 self.state.operator_fixes.push(request);
4414 self.state.save()?;
4415 let request_index = self.state.operator_fixes.len() - 1;
4416
4417 if winner.worktree.exists() {
4426 let dirty = git::git(
4429 &winner.worktree,
4430 &["status", "--porcelain", "--untracked-files=all"],
4431 )
4432 .await?;
4433 let only_withheld = dirty.lines().all(|l| {
4434 l.strip_prefix("?? ")
4435 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4436 });
4437 if !only_withheld {
4438 bail!(
4439 "`{}` has uncommitted changes; refusing to touch it — commit or \
4440 discard them first",
4441 winner.worktree.display()
4442 );
4443 }
4444 git::worktree_remove(&self.state.repo, &winner.worktree)
4445 .await
4446 .ok();
4447 }
4448 let fix_worktree = self.state.worktree_root().join("operator-fix");
4449 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
4450 git::git(
4451 &self.state.repo,
4452 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
4453 )
4454 .await
4455 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
4456 if !git::is_clean(&fix_worktree).await? {
4457 git::worktree_remove(&self.state.repo, &fix_worktree)
4458 .await
4459 .ok();
4460 bail!(
4461 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
4462 winner.branch
4463 );
4464 }
4465 let repo_for_setup = self.state.repo.clone();
4466 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
4467 Ok(()) => {
4468 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
4469 }
4470 Err(e) => Err(e),
4471 };
4472 if let Err(e) = setup {
4473 git::worktree_remove(&self.state.repo, &fix_worktree)
4474 .await
4475 .ok();
4476 return Err(e);
4477 }
4478
4479 let run_id = self.state.id.clone();
4480 let prompts = self.state.config.prompts.clone();
4481 let language = self.state.config.graph.language.clone();
4482 let sessions = self.state.config.graph.sessions;
4483 let artifacts = agent::artifacts_dir(&self.state.dir());
4484 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
4485 .findings
4486 .iter()
4487 .map(|f| Finding {
4488 id: f.id.clone(),
4489 severity: f.severity,
4490 file: f.file.clone(),
4491 line: f.line,
4492 title: f.title.clone(),
4493 detail: f.detail.clone(),
4494 })
4495 .collect();
4496 let fix_prompt = prompt::operator_fix(
4497 &self.state.instruction,
4498 &finding_list,
4499 reason,
4500 &stale_details,
4501 &head_at_request,
4502 &language,
4503 );
4504 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4505 let (job, seat, out) = self
4506 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
4507 prompt: fix_prompt.clone(),
4508 spec,
4509 seat,
4510 cwd: fix_worktree.clone(),
4511 timeout,
4512 allow_write: true,
4513 sessions,
4514 artifacts: artifacts.clone(),
4515 stem: "operator-fix".to_owned(),
4516 handover: None,
4517 })
4518 .await;
4519 let agent_id = seat.agent.clone();
4520
4521 let mut fix = FixRecord {
4522 agent: agent_id,
4523 addressed: Vec::new(),
4524 rejected: Vec::new(),
4525 notes: String::new(),
4526 committed: false,
4527 failed: None,
4528 duration_ms: 0,
4529 continuation: None,
4530 };
4531 let mut final_seat = seat.clone();
4532 match out {
4533 AgentOutcome::Ok(o) => {
4534 fix.duration_ms = o.duration_ms;
4535 let parsed = verdict::extract_json::<FixReport>(&o.text);
4536 let incomplete_reason = match &parsed {
4537 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4538 "the reply parsed, but it reported a command whose own CLI \
4539 never confirmed an exit status"
4540 .to_owned(),
4541 ),
4542 Ok(_) => None,
4543 Err(e) => Some(e.to_string()),
4544 };
4545 match incomplete_reason {
4546 None => {
4547 let report = parsed.expect("checked Ok above");
4548 fix.addressed = report.addressed;
4549 fix.rejected = report.rejected;
4550 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4551 }
4552 Some(reason) => {
4553 let (resumed_seat, resolved, failure, cont) = self
4554 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4555 .await;
4556 fix.duration_ms += cont.cumulative_wait_ms;
4557 fix.continuation = Some(cont);
4558 final_seat = resumed_seat;
4559 match resolved {
4560 Some(report) => {
4561 fix.addressed = report.addressed;
4562 fix.rejected = report.rejected;
4563 fix.notes =
4564 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4565 }
4566 None => fix.failed = failure,
4567 }
4568 }
4569 }
4570 }
4571 AgentOutcome::Dropped(o) => {
4572 fix.duration_ms = o.duration_ms;
4573 let why = o
4574 .dropped
4575 .as_ref()
4576 .map(|d| d.why.as_str())
4577 .unwrap_or("the CLI ended the stream without delivering its answer");
4578 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4579 }
4580 AgentOutcome::Quota(o) => {
4581 self.state.quota.push(QuotaLoss {
4582 seat: final_seat.key.clone(),
4583 node: "fix".to_owned(),
4584 at: Timestamp::now(),
4585 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4586 });
4587 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4588 }
4589 AgentOutcome::Failed(e) => fix.failed = Some(e),
4590 }
4591 if fix.continuation.is_none() {
4592 fix.continuation = Some(ContinuationRecord::not_needed());
4593 }
4594 self.state.seats.insert(final_seat.key.clone(), final_seat);
4595
4596 let rescue_message = format!(
4597 "magi: operator-selected fix ({}) (uncommitted work)",
4598 self.state.operator_fixes[request_index]
4599 .findings
4600 .iter()
4601 .map(|f| f.id.as_str())
4602 .collect::<Vec<_>>()
4603 .join(", ")
4604 );
4605 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4606 self.state.note_withheld("fix", &r.withheld);
4607 }
4608 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4609 fix.committed = after != head_at_request;
4610 git::worktree_remove(&self.state.repo, &fix_worktree)
4611 .await
4612 .ok();
4613
4614 self.state.event(
4615 "fix",
4616 match &fix.failed {
4617 Some(reason) => format!(
4618 "operator fix: adoption report was lost ({reason}); {}",
4619 if fix.committed {
4620 "committed"
4621 } else {
4622 "NO new commit"
4623 }
4624 ),
4625 None => format!(
4626 "operator fix: {} addressed, {} rejected, {}",
4627 fix.addressed.len(),
4628 fix.rejected.len(),
4629 if fix.committed {
4630 "committed"
4631 } else {
4632 "NO new commit"
4633 }
4634 ),
4635 },
4636 );
4637
4638 for f in &mut self.state.operator_fixes[request_index].findings {
4645 f.outcome = if fix.failed.is_some() {
4646 OperatorFixOutcome::Unreported
4647 } else if fix.addressed.contains(&f.id) {
4648 OperatorFixOutcome::Addressed
4649 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4650 OperatorFixOutcome::Rejected { why: r.why.clone() }
4651 } else {
4652 OperatorFixOutcome::Unreported
4653 };
4654 }
4655
4656 let committed = fix.committed;
4657 if committed {
4658 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4659 }
4660 self.state.operator_fixes[request_index].fix = Some(fix);
4661 self.state.save()?;
4664
4665 if committed {
4666 self.state.event(
4667 "fix",
4668 format!(
4669 "operator fix committed {}; opening a follow-up review-only run",
4670 short(&after)
4671 ),
4672 );
4673 let origin =
4676 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4677 match Self::review(
4678 &self.state.repo,
4679 &winner.branch,
4680 self.state.config.clone(),
4681 origin,
4682 )
4683 .await
4684 {
4685 Ok(mut follow_up) => {
4686 follow_up.state.event(
4687 "start",
4688 format!(
4689 "requested by an operator fix on run {} for finding(s) {}",
4690 self.state.id,
4691 self.state.operator_fixes[request_index]
4692 .findings
4693 .iter()
4694 .map(|f| f.id.as_str())
4695 .collect::<Vec<_>>()
4696 .join(", "),
4697 ),
4698 );
4699 follow_up.state.save()?;
4700 let follow_up_id = follow_up.state.id.clone();
4701 let adopted = match crate::direct::adopt(
4704 &follow_up.state,
4705 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
4706 ) {
4707 Ok(a) => a,
4708 Err(e) => {
4709 self.state.event(
4712 "fix",
4713 format!(
4714 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
4715 ),
4716 );
4717 self.state.save()?;
4718 return Ok(());
4719 }
4720 };
4721 let executed = follow_up.execute().await;
4722 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
4723 if let Some(a) = adopted {
4724 a.finish(&follow_up.state, executed);
4725 }
4726 if let Some(e) = failure {
4727 self.state.event(
4728 "fix",
4729 format!(
4730 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4731 ),
4732 );
4733 }
4734 self.state.operator_fixes[request_index].follow_up_review_run =
4735 Some(follow_up_id);
4736 }
4737 Err(e) => {
4738 self.state.event(
4739 "fix",
4740 format!("committed the fix but could not open a follow-up review: {e:#}"),
4741 );
4742 }
4743 }
4744 self.state.save()?;
4745 }
4746
4747 Ok(())
4748 }
4749
4750 async fn ask_fixer(
4770 &mut self,
4771 winner: &Candidate,
4772 node: &'static str,
4773 round: Option<usize>,
4774 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
4775 ) -> (SeatJob, SeatState, AgentOutcome) {
4776 let attempts = fixer::attempts(&self.state, &self.roles, winner);
4777 self.ask_fixer_among(attempts, node, round, None, build)
4778 .await
4779 }
4780
4781 async fn ask_fixer_among(
4786 &mut self,
4787 attempts: Vec<(AgentSpec, String)>,
4788 node: &'static str,
4789 round: Option<usize>,
4790 restore: Option<(&Path, &str)>,
4791 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
4792 ) -> (SeatJob, SeatState, AgentOutcome) {
4793 let run_id = self.state.id.clone();
4794 let prompts = self.state.config.prompts.clone();
4795 let cache = self.state.config.cache_dir();
4796 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
4797 let mut last = None;
4798 for (i, (spec, key)) in attempts.into_iter().enumerate() {
4799 let seat = self.seat(&key, &spec.id);
4800 let mut job = build(spec.clone(), seat);
4801 if i > 0 {
4802 job.stem = format!("{}-{}", job.stem, spec.id);
4803 }
4804 let ctx = WaveCtx {
4805 carry_seats: false,
4806 run: &run_id,
4807 node,
4808 prompts: &prompts,
4809 cache: cache.as_deref(),
4810 round,
4811 };
4812 let (seat, out) =
4813 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4814 let advances = match &out {
4815 AgentOutcome::Ok(o) => agent::output_advances(o),
4816 _ => true,
4817 };
4818 if let Some(next) = ids.get(i + 1)
4819 && advances
4820 {
4821 self.state.seats.insert(seat.key.clone(), seat.clone());
4822 let class =
4823 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
4824 let reason = match &out {
4825 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
4826 other => fail_reason(other),
4827 };
4828 record_handover(
4829 &mut self.state,
4830 node,
4831 &seat.key,
4832 &spec.id,
4833 next,
4834 &class,
4835 &reason,
4836 );
4837 if let Some((wt, rev)) = restore
4841 && let Err(e) = git::restore_to(wt, rev).await
4842 {
4843 let why =
4844 format!("could not restore the worktree before the next fixer: {e:#}");
4845 tracing::warn!("{why}");
4846 last = Some((job, seat, AgentOutcome::Failed(why)));
4847 break;
4848 }
4849 continue;
4850 }
4851 last = Some((job, seat, out));
4852 break;
4853 }
4854 last.expect("the fixer chain always has an entry")
4855 }
4856
4857 async fn review_loop(&mut self) -> Result<()> {
4858 if self
4863 .state
4864 .base_sync
4865 .as_ref()
4866 .is_some_and(|s| s.conflict.is_some())
4867 {
4868 return Ok(());
4869 }
4870 let run_id = self.state.id.clone();
4875 let prompts = self.state.config.prompts.clone();
4876 let Some(winner) = self.state.winner().cloned() else {
4877 return Ok(());
4878 };
4879 let max_rounds = self.state.config.graph.review_rounds;
4880 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4890 if status == RunStatus::Gating {
4893 self.record_contested_handoff();
4894 }
4895 self.state.status = status;
4896 self.state.save()?;
4897 return Ok(());
4898 }
4899 self.state.status = RunStatus::Reviewing;
4900 if self
4910 .state
4911 .reviews
4912 .last()
4913 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4914 {
4915 let shell = self.state.config.shell();
4916 return self
4917 .stop_reviewing(
4918 "the last round's own verification never resolved",
4919 &shell,
4920 &winner.worktree,
4921 )
4922 .await;
4923 }
4924
4925 let repo = self.state.repo.clone();
4926 let root = self.state.worktree_root();
4927 let language = self.state.config.graph.language.clone();
4928 let sessions = self.state.config.graph.sessions;
4929 let artifacts = agent::artifacts_dir(&self.state.dir());
4930 let base = self.landing_base();
4931 let base_short = short(&base);
4932 let reviewers = self.roles.reviewers.clone();
4933 let shell = self.state.config.shell();
4934
4935 for round in (self.state.reviews.len() + 1)..=max_rounds {
4936 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4937 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4938 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4939 let prev_verification = self
4948 .state
4949 .reviews
4950 .last()
4951 .and_then(|r| r.verification_summary(&head));
4952
4953 let mut jobs = Vec::new();
4957 for (r, spec) in reviewers.iter().cloned().enumerate() {
4958 let wt = root.join(format!("review-{}", r + 1));
4959 if wt.exists() {
4962 git::reset_detached(&wt, &head).await?;
4963 } else {
4964 git::worktree_add_detached(&repo, &wt, &head).await?;
4965 }
4966 self.setup_seat_worktree(&repo, &wt, false).await?;
4967 let seat_key = format!("review-{}", r + 1);
4968 let spec = pick_start_spec(
4971 &self.roles.reviewer_roster,
4972 spec,
4973 self.state.seat_history.get(&seat_key),
4974 );
4975 let seat = self.seat(&seat_key, &spec.id);
4976 jobs.push(SeatJob {
4977 prompt: prompt::review(&prompt::ReviewCtx {
4978 instruction: &self.state.instruction,
4979 branch: &winner.branch,
4980 base_short: &base_short,
4981 stat: &stat,
4982 patch: &patch,
4983 verification: prev_verification.as_ref(),
4984 reviewers: reviewers.len(),
4985 round,
4986 rounds: max_rounds,
4987 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4990 lens: Lens::for_seat(r),
4991 language: &language,
4992 }),
4993 spec,
4994 seat,
4995 cwd: wt,
4996 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4997 allow_write: false,
4998 sessions,
4999 artifacts: artifacts.clone(),
5000 stem: format!("review-{round}-{}", r + 1),
5001 handover: None,
5002 });
5003 }
5004
5005 self.state.event(
5006 "review",
5007 format!(
5008 "round {round}: {} reviewers on {}",
5009 jobs.len(),
5010 short(&head)
5011 ),
5012 );
5013 let mut quota_losses = Vec::new();
5014 let review_retries = self.state.config.graph.retries;
5015 let review_cache = self.state.config.cache_dir();
5016 let ctx = WaveCtx {
5017 carry_seats: true,
5018 run: &run_id,
5019 node: "review",
5020 prompts: &prompts,
5021 cache: review_cache.as_deref(),
5022 round: Some(round),
5023 };
5024 let results = ask_json_wave::<Review>(
5025 jobs,
5026 Arc::clone(&self.sem),
5027 review_retries,
5028 &self.roles.reviewer_roster,
5029 &ctx,
5030 &mut quota_losses,
5031 &mut self.state,
5032 &|_: &Review| Ok(()),
5033 )
5034 .await;
5035 let round_quota_missing = quota_losses.len();
5039 self.state.quota.extend(quota_losses);
5040
5041 let mut records = Vec::new();
5042 let mut all_findings = Vec::new();
5043 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5044 let agent_id = seat.agent.clone();
5045 self.state.seats.insert(seat.key.clone(), seat);
5046 let mut record = ReviewRecord {
5047 reviewer: r + 1,
5048 agent: agent_id,
5049 summary: String::new(),
5050 findings: Vec::new(),
5051 vote: None,
5052 failed: None,
5053 duration_ms: 0,
5054 attempts,
5060 };
5061 match res {
5062 Ok((review, out)) => {
5063 record.summary =
5071 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5072 record.vote = Some(review.vote);
5073 record.duration_ms = out.duration_ms;
5074 for (n, mut f) in review.findings.into_iter().enumerate() {
5075 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
5078 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
5079 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
5080 f.file = f
5086 .file
5087 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
5088 all_findings.push(f.clone());
5089 record.findings.push(f);
5090 }
5091 self.state.event(
5092 "review",
5093 format!(
5094 "round {round}: reviewer {} voted {} with {} finding(s)",
5095 r + 1,
5096 review.vote.label(),
5097 record.findings.len()
5098 ),
5099 );
5100 }
5101 Err(e) => {
5102 record.failed = Some(e.to_string());
5103 self.state.event(
5104 "review",
5105 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
5106 );
5107 }
5108 }
5109 records.push(record);
5110 }
5111
5112 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
5119 let vote_split =
5120 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
5121 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
5122 if vote_split {
5123 self.state.event(
5124 "review",
5125 format!(
5126 "round {round}: votes split ({}) — one round of reconsideration",
5127 initial_votes
5128 .iter()
5129 .map(|v| v.label())
5130 .collect::<Vec<_>>()
5131 .join(", ")
5132 ),
5133 );
5134 let panel: Vec<ReviewSeatReport<'_>> = records
5137 .iter()
5138 .filter_map(|r| {
5139 r.vote.map(|vote| ReviewSeatReport {
5140 reviewer: r.reviewer,
5141 vote,
5142 summary: &r.summary,
5143 findings: &r.findings,
5144 })
5145 })
5146 .collect();
5147
5148 let mut jobs = Vec::new();
5149 let mut seats_at = Vec::new();
5150 for (r, spec) in reviewers.iter().cloned().enumerate() {
5151 if records[r].vote.is_none() {
5155 continue;
5156 }
5157 let wt = root.join(format!("review-{}", r + 1));
5158 let seat_key = format!("review-{}", r + 1);
5159 let spec = self.occupant(&seat_key, spec);
5160 let seat = self.seat(&seat_key, &spec.id);
5161 let build = |with_patch: bool| {
5169 prompt::review_reconsider(&ReviewReconsiderCtx {
5170 instruction: &self.state.instruction,
5171 reviewer: r + 1,
5172 lens: Lens::for_seat(r),
5173 panel: &panel,
5174 patch: with_patch.then_some(ReviewPatch {
5175 branch: &winner.branch,
5176 base_short: &base_short,
5177 stat: &stat,
5178 patch: &patch,
5179 }),
5180 round,
5181 rounds: max_rounds,
5182 language: &language,
5183 })
5184 };
5185 let full = build(true);
5186 let prompt = if has_context(&spec, &seat, sessions) {
5187 build(false)
5188 } else {
5189 full.clone()
5190 };
5191 jobs.push(SeatJob {
5192 prompt,
5193 spec,
5194 seat,
5195 cwd: wt,
5196 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5197 allow_write: false,
5198 sessions,
5199 artifacts: artifacts.clone(),
5200 stem: format!("review-{round}-reconsider-{}", r + 1),
5201 handover: Some(full),
5202 });
5203 seats_at.push(r);
5204 }
5205
5206 let mut recon_quota_losses = Vec::new();
5207 let recon_cache = self.state.config.cache_dir();
5208 let recon_ctx = WaveCtx {
5209 carry_seats: true,
5210 run: &run_id,
5211 node: "review",
5212 prompts: &prompts,
5213 cache: recon_cache.as_deref(),
5214 round: Some(round),
5215 };
5216 let recon_results = ask_json_wave::<ReviewRevote>(
5217 jobs,
5218 Arc::clone(&self.sem),
5219 review_retries,
5220 &self.roles.reviewer_roster,
5221 &recon_ctx,
5222 &mut recon_quota_losses,
5223 &mut self.state,
5224 &|_: &ReviewRevote| Ok(()),
5225 )
5226 .await;
5227 self.state.quota.extend(recon_quota_losses);
5228
5229 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
5230 let agent_id = seat.agent.clone();
5231 self.state.seats.insert(seat.key.clone(), seat);
5232 let mut rec = ReviewRevoteRecord {
5233 reviewer: r + 1,
5234 agent: agent_id,
5235 vote: None,
5236 reason: String::new(),
5237 failed: None,
5238 };
5239 match res {
5240 Ok((rv, _)) => {
5241 rec.vote = Some(rv.vote);
5242 rec.reason =
5243 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
5244 self.state.event(
5245 "review",
5246 format!(
5247 "round {round}: reviewer {} revoted {}",
5248 r + 1,
5249 rv.vote.label()
5250 ),
5251 );
5252 }
5253 Err(e) => {
5254 rec.failed = Some(e.to_string());
5255 self.state.event(
5256 "review",
5257 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
5258 );
5259 }
5260 }
5261 reconsideration.push(rec);
5262 }
5263 } else if initial_votes.len() > 1 {
5264 self.state.event(
5265 "review",
5266 format!(
5267 "round {round}: votes agreed ({}) — no reconsideration",
5268 initial_votes[0].label()
5269 ),
5270 );
5271 }
5272
5273 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
5274 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5275 let defer_e2e =
5286 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
5287 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
5288 let reason =
5289 format!("{blocking} blocking finding(s) already required a fix this round");
5290 self.state.event(
5291 "verify",
5292 format!(
5293 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
5294 {}); it will run once a round has none left",
5295 short(&head)
5296 ),
5297 );
5298 (Vec::new(), false, true, Some(reason))
5299 } else {
5300 let e2e_commands = self.state.config.verify.e2e.clone();
5301 let cache_dir = self.state.config.cache_dir();
5302 let context = format!("round {round}");
5303 let (e2e, verify_retried) = with_cache_lease(
5304 &mut self.state,
5305 cache_dir.as_deref(),
5306 "e2e",
5307 "e2e",
5308 &winner.worktree,
5309 &head,
5310 verify_timeout,
5311 &context,
5312 |state, budget| {
5313 let shell = shell.clone();
5314 let e2e_commands = e2e_commands.clone();
5315 let worktree = winner.worktree.clone();
5316 let context = context.clone();
5317 async move {
5318 run_e2e_with_retry(
5319 state,
5320 &shell,
5321 &e2e_commands,
5322 &worktree,
5323 budget,
5324 &context,
5325 )
5326 .await
5327 }
5328 },
5329 )
5330 .await;
5331 (e2e, verify_retried, false, None)
5332 };
5333
5334 let expected = records.len();
5335 let answered = records.iter().filter(|r| r.failed.is_none()).count();
5336 let incomplete = answered < expected;
5337 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
5338 let policy = self.state.config.graph.incomplete_review;
5339 let clean = round_is_clean(
5340 blocking,
5341 e2e_ok,
5342 answered,
5343 expected,
5344 round_quota_missing,
5345 policy,
5346 );
5347
5348 let mut round_record = ReviewRound {
5349 round,
5350 head: head.clone(),
5351 verified_head: None,
5352 verified_at: None,
5353 reviews: records,
5354 e2e,
5355 verify_retried,
5356 e2e_deferred,
5357 e2e_defer_reason,
5358 fix: None,
5359 blocking,
5360 answered,
5361 expected,
5362 clean,
5363 progressed: false,
5364 vote_split,
5365 reconsideration,
5366 verdict: None,
5367 };
5368 round_record.verdict = ReviewVote::worst(
5372 round_record
5373 .final_votes()
5374 .into_iter()
5375 .map(|(_, _, vote)| vote),
5376 );
5377 if !matches!(
5388 round_record.e2e_status(),
5389 E2eStatus::Deferred | E2eStatus::NotConfigured
5390 ) {
5391 round_record.verified_head = Some(head.clone());
5392 round_record.verified_at = Some(Timestamp::now());
5393 }
5394 let this_round_verification = round_record.verification_summary(&head);
5395
5396 if incomplete {
5397 let missing: Vec<String> = round_record
5398 .reviews
5399 .iter()
5400 .filter(|r| r.failed.is_some())
5401 .map(|r| format!("review-{}", r.reviewer))
5402 .collect();
5403 self.state.event(
5404 "review",
5405 format!(
5406 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
5407 missing.join(", ")
5408 ),
5409 );
5410 }
5411
5412 if clean {
5413 self.state.event(
5414 "review",
5415 if incomplete && policy == IncompleteReviewPolicy::Warn {
5416 format!(
5417 "round {round}: clean (warn policy, incomplete panel) — no \
5418 blocking findings from the seats that answered, verification green"
5419 )
5420 } else if incomplete {
5421 format!(
5422 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
5423 quorum) — no blocking findings from the seats that answered, \
5424 verification green",
5425 expected - answered
5426 )
5427 } else {
5428 format!("round {round}: clean — no blocking findings, verification green")
5429 },
5430 );
5431 self.state.reviews.push(round_record);
5432 self.state.status = RunStatus::Gating;
5433 self.state.save()?;
5434 return Ok(());
5435 }
5436
5437 if incomplete && blocking == 0 && e2e_ok {
5445 self.state.reviews.push(round_record);
5446 self.state.save()?;
5447 if round == max_rounds {
5448 self.state.status = RunStatus::Blocked;
5449 self.state.event(
5450 "review",
5451 format!(
5452 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
5453 refusing to call it clean",
5454 expected - answered
5455 ),
5456 );
5457 return Ok(());
5458 }
5459 continue;
5460 }
5461
5462 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
5474 self.state.reviews.push(round_record);
5475 return self
5476 .stop_reviewing(
5477 "the round's own verification could not run",
5478 &shell,
5479 &winner.worktree,
5480 )
5481 .await;
5482 }
5483
5484 if round == max_rounds {
5485 self.state.reviews.push(round_record);
5486 return self
5487 .stop_reviewing(
5488 &format!(
5489 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
5490 ),
5491 &shell,
5492 &winner.worktree,
5493 )
5494 .await;
5495 }
5496
5497 let blocking_findings: Vec<_> = all_findings
5500 .iter()
5501 .filter(|f| f.severity.blocks())
5502 .cloned()
5503 .collect();
5504 let fix_prompt = prompt::fix(
5505 &self.state.instruction,
5506 &blocking_findings,
5507 this_round_verification.as_ref(),
5508 round,
5509 max_rounds,
5510 &language,
5511 );
5512 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5513 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5514 let (job, seat, out) = self
5515 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
5516 prompt: fix_prompt.clone(),
5517 spec,
5518 seat,
5519 cwd: winner.worktree.clone(),
5520 timeout,
5521 allow_write: true,
5522 sessions,
5523 artifacts: artifacts.clone(),
5524 stem: format!("fix-{round}"),
5525 handover: None,
5526 })
5527 .await;
5528 let agent_id = seat.agent.clone();
5529
5530 let mut fix = FixRecord {
5531 agent: agent_id,
5532 addressed: Vec::new(),
5533 rejected: Vec::new(),
5534 notes: String::new(),
5535 committed: false,
5536 failed: None,
5537 duration_ms: 0,
5538 continuation: None,
5539 };
5540 let mut continuation = ContinuationRecord::not_needed();
5541 let mut final_seat = seat.clone();
5542 match out {
5543 AgentOutcome::Ok(o) => {
5544 fix.duration_ms = o.duration_ms;
5545 let parsed = verdict::extract_json::<FixReport>(&o.text);
5546 let incomplete_reason = match &parsed {
5553 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5554 "the reply parsed, but it reported a command whose own CLI never \
5555 confirmed an exit status"
5556 .to_owned(),
5557 ),
5558 Ok(_) => None,
5559 Err(e) => Some(e.to_string()),
5560 };
5561 match incomplete_reason {
5562 None => {
5563 let report = parsed.expect("checked Ok above");
5564 fix.addressed = report.addressed;
5565 fix.rejected = report.rejected;
5566 fix.notes =
5567 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5568 }
5569 Some(reason) => {
5570 let (resumed_seat, resolved, failure, cont) = self
5571 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
5572 .await;
5573 fix.duration_ms += cont.cumulative_wait_ms;
5574 continuation = cont;
5575 final_seat = resumed_seat;
5576 match resolved {
5577 Some(report) => {
5578 fix.addressed = report.addressed;
5579 fix.rejected = report.rejected;
5580 fix.notes = blind::sanitize_prose(
5581 &report.notes,
5582 &self.state.config.blind,
5583 );
5584 }
5585 None => fix.failed = failure,
5586 }
5587 }
5588 }
5589 }
5590 AgentOutcome::Dropped(o) => {
5592 fix.duration_ms = o.duration_ms;
5593 let why = o
5594 .dropped
5595 .as_ref()
5596 .map(|d| d.why.as_str())
5597 .unwrap_or("the CLI ended the stream without delivering its answer");
5598 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5599 }
5600 AgentOutcome::Quota(o) => {
5601 self.state.quota.push(QuotaLoss {
5602 seat: final_seat.key.clone(),
5603 node: "fix".to_owned(),
5604 at: Timestamp::now(),
5605 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5606 });
5607 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5608 }
5609 AgentOutcome::Failed(e) => fix.failed = Some(e),
5610 }
5611 fix.continuation = Some(continuation);
5612 self.state.seats.insert(final_seat.key.clone(), final_seat);
5613 if let Ok(r) = git::rescue_commit(
5614 &winner.worktree,
5615 &format!("magi: review round {round} fixes (uncommitted work)"),
5616 )
5617 .await
5618 {
5619 self.state.note_withheld("fix", &r.withheld);
5620 }
5621 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5622 fix.committed = after != before;
5623 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5631 let progressed = diff_after != patch;
5632 let commit_note = if fix.committed {
5633 "committed"
5634 } else {
5635 "NO new commit"
5636 };
5637 let tree_note = if progressed {
5638 "changed vs base"
5639 } else {
5640 "unchanged vs base"
5641 };
5642 self.state.event(
5643 "fix",
5644 match &fix.failed {
5645 Some(reason) => {
5651 format!(
5652 "round {round}: fixer's adoption report was lost ({reason}); \
5653 {commit_note}, tree {tree_note}"
5654 )
5655 }
5656 None => format!(
5657 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5658 {tree_note}{}",
5659 fix.addressed.len(),
5660 fix.rejected.len(),
5661 if continuation.outcome == ContinuationOutcome::Resumed {
5662 format!(
5663 " (adoption report recovered after {} continuation(s))",
5664 continuation.attempts
5665 )
5666 } else {
5667 String::new()
5668 },
5669 ),
5670 },
5671 );
5672 round_record.fix = Some(fix);
5673 round_record.progressed = progressed;
5674 self.state.reviews.push(round_record);
5675 self.state.save()?;
5676
5677 if matches!(
5690 continuation.outcome,
5691 ContinuationOutcome::Exhausted
5692 | ContinuationOutcome::QuotaLost
5693 | ContinuationOutcome::NoSession
5694 ) {
5695 return self
5696 .stop_reviewing(
5697 "the fixer's adoption report never came back, even after resuming its \
5698 own seat; refusing to start another round against the same worktree \
5699 while that is unresolved",
5700 &shell,
5701 &winner.worktree,
5702 )
5703 .await;
5704 }
5705
5706 let streak = self
5707 .state
5708 .reviews
5709 .iter()
5710 .rev()
5711 .take_while(|r| !r.progressed)
5712 .count();
5713 if streak >= STAGNANT_LIMIT {
5714 return self
5715 .stop_reviewing(
5716 &format!(
5717 "the tree has not moved against base for {streak} round(s) in a row"
5718 ),
5719 &shell,
5720 &winner.worktree,
5721 )
5722 .await;
5723 }
5724 }
5725 Ok(())
5726 }
5727
5728 fn record_contested_handoff(&mut self) {
5762 if self.state.contested_handoff.is_some() {
5763 return;
5764 }
5765 let Some(contested) = self
5766 .state
5767 .reviews
5768 .last()
5769 .and_then(ReviewRound::contested_handoff)
5770 else {
5771 return;
5772 };
5773 self.state.event(
5774 "review",
5775 format!(
5776 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5777 wait for the owner's approval",
5778 contested.findings.len(),
5779 contested.rejecters.len()
5780 ),
5781 );
5782 self.state.contested_handoff = Some(contested);
5783 }
5784
5785 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5786 let round_idx = self.state.reviews.len() - 1;
5787 let needs_catchup_run = matches!(
5795 self.state.reviews[round_idx].e2e_status(),
5796 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5797 );
5798 if needs_catchup_run {
5799 let round = self.state.reviews[round_idx].round;
5800 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5801 let commands = self.state.config.verify.e2e.clone();
5802 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5803 let cache_dir = self.state.config.cache_dir();
5804 let context = format!(
5805 "round {round}: verification unresolved, catching up before the final decision"
5806 );
5807 let (outcomes, verify_retried) = with_cache_lease(
5808 &mut self.state,
5809 cache_dir.as_deref(),
5810 "e2e",
5811 "e2e",
5812 worktree,
5813 &attempted_head,
5814 timeout,
5815 &context,
5816 |state, budget| {
5817 let shell = shell.to_vec();
5818 let commands = commands.clone();
5819 let context = context.clone();
5820 async move {
5821 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5822 .await
5823 }
5824 },
5825 )
5826 .await;
5827 let last = &mut self.state.reviews[round_idx];
5828 last.e2e = outcomes;
5829 last.verify_retried = verify_retried;
5830 last.verified_head = Some(attempted_head);
5837 last.verified_at = Some(Timestamp::now());
5838 if verify_inconclusive(&last.e2e) {
5839 self.state.save()?;
5846 return Ok(());
5847 }
5848 last.e2e_deferred = false;
5849 }
5850 let last = &self.state.reviews[round_idx];
5851 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5852
5853 match last.e2e_status() {
5854 E2eStatus::Failed => {
5855 let red: Vec<String> = last
5856 .e2e
5857 .iter()
5858 .filter(|o| !o.ok())
5859 .map(|o| {
5860 format!(
5861 "`{}` -> {:?}\n{}",
5862 o.command,
5863 o.code,
5864 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5865 )
5866 })
5867 .collect();
5868 self.state
5869 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5870 self.state.status = RunStatus::Blocked;
5871 }
5872 E2eStatus::ResourceBlocked => {
5877 self.state.event(
5878 "review",
5879 format!(
5880 "{why}; e2e could not run (shared build cache unavailable); not \
5881 deciding yet"
5882 ),
5883 );
5884 }
5885 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5886 self.state.event(
5887 "review",
5888 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5889 );
5890 self.record_contested_handoff();
5891 self.state.status = RunStatus::Gating;
5892 }
5893 }
5894 self.state.save()?;
5895 Ok(())
5896 }
5897
5898 async fn gate(&mut self) -> Result<()> {
5901 if self.state.status == RunStatus::Failed
5913 || self
5914 .state
5915 .base_sync
5916 .as_ref()
5917 .is_some_and(|s| s.conflict.is_some())
5918 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5919 != Some(RunStatus::Gating)
5920 {
5921 return Ok(());
5922 }
5923 if self.state.gate_ran {
5924 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5935 self.state.status = RunStatus::Blocked;
5936 self.state.save()?;
5937 }
5938 return Ok(());
5939 }
5940 let Some(winner) = self.state.winner().cloned() else {
5941 return Ok(());
5942 };
5943 self.state.status = RunStatus::Gating;
5944 let mut outcomes = self.run_gate(&winner).await?;
5945 loop {
5946 if verify_inconclusive(&outcomes) {
5957 self.state.save()?;
5958 return Ok(());
5959 }
5960 if outcomes.iter().all(CommandOutcome::ok) {
5961 break;
5962 }
5963 match self.gate_fix_round(&winner, &outcomes).await? {
5964 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5965 GateFix::Stop => break,
5966 GateFix::Defer => {
5967 self.state.save()?;
5968 return Ok(());
5969 }
5970 }
5971 }
5972 let passed = outcomes.iter().all(CommandOutcome::ok);
5973 self.state.gate = outcomes;
5974 self.state.gate_ran = true;
5975 if !passed {
5976 self.state.status = RunStatus::Blocked;
5977 let spent = self.state.gate_fixes.len();
5978 self.state.event(
5979 "gate",
5980 if spent == 0 {
5981 "gate failed; not merging".to_owned()
5982 } else {
5983 format!("gate failed after {spent} gate-fix round(s); not merging")
5984 },
5985 );
5986 }
5987 self.state.save()?;
5988 Ok(())
5989 }
5990
5991 async fn run_pre_gate(&mut self, winner: &Candidate) {
6001 let commands = self.state.config.verify.pre_gate.clone();
6002 if commands.is_empty() {
6003 return;
6004 }
6005 let shell = self.state.config.shell();
6006 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6007 let (outcomes, _) = run_commands(
6008 &mut self.state,
6009 "pre_gate",
6010 "pre_gate",
6011 0,
6012 &shell,
6013 &commands,
6014 &winner.worktree,
6015 timeout,
6016 )
6017 .await;
6018 for o in &outcomes {
6019 if !o.ok() {
6020 tracing::warn!(
6021 "pre_gate `{}` failed ({:?}); the gate decides",
6022 o.command,
6023 o.code
6024 );
6025 }
6026 self.state.event(
6027 "pre_gate",
6028 format!(
6029 "`{}` -> {}",
6030 o.command,
6031 if o.ok() {
6032 "pass".to_owned()
6033 } else {
6034 format!(
6035 "FAIL ({:?})\n{}",
6036 o.code,
6037 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6038 )
6039 }
6040 ),
6041 );
6042 }
6043 self.state.pre_gate = outcomes;
6044 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6045 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6046 Ok(head) => {
6047 self.state
6048 .event("pre_gate", format!("committed mechanical fixes ({head})"));
6049 self.state.pre_gate_commit = Some(head);
6050 }
6051 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6052 },
6053 Ok(false) => {}
6054 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6055 }
6056 if let Err(e) = self.state.save() {
6057 tracing::warn!("could not persist the pre_gate record: {e:#}");
6058 }
6059 }
6060
6061 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
6064 self.run_pre_gate(winner).await;
6065 let shell = self.state.config.shell();
6066 let gate_commands = self.state.config.verify.gate.clone();
6067 let outcomes = if gate_commands.is_empty() {
6076 Vec::new()
6077 } else {
6078 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6079 let cache_dir = self.state.config.cache_dir();
6080 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6081 let (outcomes, _) = with_cache_lease(
6082 &mut self.state,
6083 cache_dir.as_deref(),
6084 "gate",
6085 "gate",
6086 &winner.worktree,
6087 &head,
6088 timeout,
6089 "final gate",
6090 |state, budget| {
6091 let shell = shell.clone();
6092 let gate_commands = gate_commands.clone();
6093 let worktree = winner.worktree.clone();
6094 async move {
6095 let (outcomes, timed_out_pids) = run_commands(
6096 state,
6097 "gate",
6098 "gate",
6099 0,
6100 &shell,
6101 &gate_commands,
6102 &worktree,
6103 budget,
6104 )
6105 .await;
6106 (outcomes, false, timed_out_pids)
6107 }
6108 },
6109 )
6110 .await;
6111 outcomes
6112 };
6113 if outcomes.is_empty() {
6114 self.state.event(
6119 "gate",
6120 "no gate commands configured; nothing to check, passing",
6121 );
6122 }
6123 for o in &outcomes {
6124 self.state.event(
6125 "gate",
6126 format!(
6127 "`{}` -> {}",
6128 o.command,
6129 if o.ok() {
6130 "pass".to_owned()
6131 } else {
6132 format!(
6133 "FAIL ({:?})\n{}",
6134 o.code,
6135 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6136 )
6137 }
6138 ),
6139 );
6140 }
6141 Ok(outcomes)
6142 }
6143
6144 async fn gate_fix_round(
6157 &mut self,
6158 winner: &Candidate,
6159 outcomes: &[CommandOutcome],
6160 ) -> Result<GateFix> {
6161 let cap = self.state.config.graph.gate_fix_rounds;
6162 let spent = self.state.gate_fixes.len();
6163 if spent >= cap {
6164 if cap > 0 {
6165 self.state.event(
6166 "gate",
6167 format!("{spent} gate-fix round(s) spent and the gate still fails"),
6168 );
6169 }
6170 return Ok(GateFix::Stop);
6171 }
6172 if !gate_fixable(outcomes) {
6173 self.state.event(
6174 "gate",
6175 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
6176 command or similar); not spending a fix round on it",
6177 );
6178 return Ok(GateFix::Stop);
6179 }
6180 let min_free = self.state.config.disk.min_free_bytes;
6181 if min_free > 0 {
6182 match crate::disk::free_bytes(&winner.worktree) {
6183 Ok(free) if crate::disk::enough_space(free, min_free) => {}
6184 Ok(free) => {
6185 self.state.event(
6186 "gate",
6187 format!(
6188 "only {free} bytes free ({min_free} required by `[disk] \
6189 min_free_bytes`); not spending a fix round on a failure the disk \
6190 may explain"
6191 ),
6192 );
6193 return Ok(GateFix::Stop);
6194 }
6195 Err(e) => {
6196 self.state.event(
6197 "gate",
6198 format!("free disk space could not be measured ({e:#}); no fix round"),
6199 );
6200 return Ok(GateFix::Stop);
6201 }
6202 }
6203 }
6204
6205 let attempt = spent + 1;
6206 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
6207 let base = self.landing_base();
6208 let fix_prompt = prompt::gate_fix(
6209 &self.state.instruction,
6210 &failed,
6211 attempt,
6212 cap,
6213 &self.state.config.graph.language,
6214 );
6215 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6216 let sessions = self.state.config.graph.sessions;
6217 let artifacts = agent::artifacts_dir(&self.state.dir());
6218 self.state.event(
6219 "gate",
6220 format!("gate failed; gate-fix round {attempt} of {cap}"),
6221 );
6222 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6223 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6224 let (_, seat, out) = self
6225 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
6226 prompt: fix_prompt.clone(),
6227 spec,
6228 seat,
6229 cwd: winner.worktree.clone(),
6230 timeout,
6231 allow_write: true,
6232 sessions,
6233 artifacts: artifacts.clone(),
6234 stem: format!("gate-fix-{attempt}"),
6235 handover: None,
6236 })
6237 .await;
6238 let mut record = GateFixRecord {
6239 agent: seat.agent.clone(),
6240 failed,
6241 notes: String::new(),
6242 committed: false,
6243 error: None,
6244 };
6245 match out {
6246 AgentOutcome::Ok(o) => {
6247 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
6250 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6251 }
6252 }
6253 AgentOutcome::Dropped(_) => {
6254 record.error = Some("the CLI dropped the stream".to_owned());
6255 }
6256 AgentOutcome::Quota(o) => {
6257 self.state.quota.push(QuotaLoss {
6258 seat: seat.key.clone(),
6259 node: "gate-fix".to_owned(),
6260 at: Timestamp::now(),
6261 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6262 });
6263 record.error = Some("rate limited (quota); fixer could not run".to_owned());
6264 }
6265 AgentOutcome::Failed(e) => record.error = Some(e),
6266 }
6267 self.state.seats.insert(seat.key.clone(), seat);
6268 if let Ok(r) = git::rescue_commit(
6269 &winner.worktree,
6270 &format!("magi: gate fix {attempt} (uncommitted work)"),
6271 )
6272 .await
6273 {
6274 self.state.note_withheld("gate-fix", &r.withheld);
6275 }
6276 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6277 record.committed = after != before;
6278 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
6279 let note = record.error.clone();
6280 self.state.gate_fixes.push(record);
6281 self.state.save()?;
6282 if !changed {
6283 self.state.event(
6284 "gate",
6285 match note {
6286 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
6287 None => format!("gate-fix round {attempt}: the tree did not change"),
6288 },
6289 );
6290 return Ok(GateFix::Stop);
6291 }
6292 self.state.event(
6293 "gate",
6294 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
6295 );
6296
6297 let commands = self.state.config.verify.e2e.clone();
6298 if !commands.is_empty() {
6299 let shell = self.state.config.shell();
6300 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6301 let cache_dir = self.state.config.cache_dir();
6302 let context = format!("gate-fix round {attempt}");
6303 let (e2e, _) = with_cache_lease(
6304 &mut self.state,
6305 cache_dir.as_deref(),
6306 "e2e",
6307 "e2e",
6308 &winner.worktree,
6309 &after,
6310 timeout,
6311 &context,
6312 |state, budget| {
6313 let shell = shell.clone();
6314 let commands = commands.clone();
6315 let context = context.clone();
6316 let worktree = winner.worktree.clone();
6317 async move {
6318 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
6319 .await
6320 }
6321 },
6322 )
6323 .await;
6324 if verify_inconclusive(&e2e) {
6325 return Ok(GateFix::Defer);
6326 }
6327 if e2e.iter().any(|o| !o.ok()) {
6328 self.state.event(
6329 "gate",
6330 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
6331 );
6332 return Ok(GateFix::Stop);
6333 }
6334 }
6335 Ok(GateFix::Retry)
6336 }
6337
6338 async fn guarded_pr_message(
6345 &mut self,
6346 winner: &Candidate,
6347 facts: Option<&BranchFacts>,
6348 posting: bool,
6349 ) -> Result<Option<PrMessage>> {
6350 let mut pr = pr_message_raw(&self.state, winner.label, facts);
6351 if !posting {
6352 let identity = crate::scrub::Identity::current();
6353 return Ok(Some(PrMessage {
6354 title: crate::scrub::scrub(&pr.title, &identity),
6355 body: crate::scrub::scrub(&pr.body, &identity),
6356 }));
6357 }
6358 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
6361 if let Some(known) = self
6362 .state
6363 .github_text
6364 .clone()
6365 .filter(|g| g.fingerprint == raw_fp)
6366 {
6367 return self.settle_withheld(winner, &pr, known).await;
6368 }
6369 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
6370 let mut violations = crate::github_text::check_with(&pr.title, &pr.body, decision);
6371 let identity = crate::scrub::Identity::current();
6372 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
6373 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
6374 && !violations.contains(&crate::github_text::Violation::SensitiveData)
6375 {
6376 violations.push(crate::github_text::Violation::SensitiveData);
6377 }
6378 let has_language = violations
6382 .iter()
6383 .any(|v| *v != crate::github_text::Violation::SensitiveData);
6384 if self.state.config.graph.github_text_guard && !violations.is_empty() {
6385 self.state.event(
6386 "github-text",
6387 if has_language {
6388 format!("description rejected: {violations:?}; requesting one rewrite")
6389 } else {
6390 format!("description rejected: {violations:?}; asking the owner")
6391 },
6392 );
6393 let mut rewritten = false;
6394 if has_language
6395 && let Some(spec) = self
6396 .state
6397 .config
6398 .agents
6399 .iter()
6400 .find(|a| a.id == winner.agent)
6401 .cloned()
6402 {
6403 let key = format!("impl-{}", winner.label);
6404 let seat = self.seat(&key, &spec.id);
6405 let prompt = format!(
6406 "Rewrite only the following pull request description. The posting gate reported {violations:?}. Write English prose and remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the complete Markdown body. Preserve the magi run marker.\n\nTITLE: {}\n{}",
6407 pr.title, pr.body
6408 );
6409 let job = SeatJob {
6410 spec,
6411 seat,
6412 cwd: winner.worktree.clone(),
6413 prompt,
6414 timeout: retry_budget(
6415 Duration::from_secs(self.state.config.graph.timeout_implement),
6416 true,
6417 ),
6418 allow_write: false,
6419 sessions: self.state.config.graph.sessions,
6420 artifacts: agent::artifacts_dir(&self.state.dir()),
6421 stem: "github-text-rewrite".to_owned(),
6422 handover: None,
6423 };
6424 let prompts = self.state.config.prompts.clone();
6425 let cache = self.state.config.cache_dir();
6426 let run = self.state.id.clone();
6427 let ctx = WaveCtx {
6428 run: &run,
6429 node: "github-text",
6430 prompts: &prompts,
6431 cache: cache.as_deref(),
6432 round: None,
6433 carry_seats: false,
6434 };
6435 let (seat, outcome) =
6436 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
6437 self.state.seats.insert(seat.key.clone(), seat);
6438 if let AgentOutcome::Ok(output) = outcome
6439 && let Some(title) = summary_title(&output.text)
6440 {
6441 let mut body = summary_without_title(&output.text);
6442 let rewrite = PrMessage {
6443 title: title.clone(),
6444 body: body.clone(),
6445 };
6446 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
6447 if !body.trim().is_empty()
6448 && crate::github_text::check_with(&title, &body, redecision).is_empty()
6449 && crate::github_text::shareable(&title)
6450 && crate::github_text::shareable(&body)
6451 {
6452 decision = redecision;
6453 let marker = format!("magi:run/{}", self.state.id);
6454 if !body.contains(&marker) {
6455 body.push_str(&format!("\n\n{marker}"));
6456 }
6457 pr = PrMessage { title, body };
6458 rewritten = true;
6459 self.state
6460 .event("github-text", "description rewrite passed");
6461 }
6462 }
6463 }
6464 if !rewritten {
6465 self.state.event(
6466 "github-text",
6467 "description rewrite unavailable or rejected; asking the owner",
6468 );
6469 if let Some(w) =
6470 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
6471 {
6472 self.retire_github_text_question("the withheld text changed");
6475 let record = crate::run::GithubTextAsk {
6476 fingerprint: raw_fp,
6477 title: w.title,
6478 body: w.body,
6479 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
6480 question: None,
6481 asks: 0,
6482 resolved: false,
6483 chosen_title: None,
6484 };
6485 return self.settle_withheld(winner, &pr, record).await;
6486 }
6487 }
6488 }
6489 let (title, body) =
6490 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
6491 Ok(Some(PrMessage { title, body }))
6492 }
6493
6494 fn retire_github_text_question(&mut self, why: &str) {
6496 let Some(id) = self
6497 .state
6498 .github_text
6499 .as_ref()
6500 .and_then(|g| g.question.clone())
6501 else {
6502 return;
6503 };
6504 let _ = ask::Questions::open().update(&id, |q| {
6505 q.abandon(why);
6506 Ok(())
6507 });
6508 }
6509
6510 async fn settle_withheld(
6514 &mut self,
6515 winner: &Candidate,
6516 pr: &PrMessage,
6517 mut g: crate::run::GithubTextAsk,
6518 ) -> Result<Option<PrMessage>> {
6519 use crate::github_text as gt;
6520 let store = ask::Questions::open();
6521 let timeout = self.state.config.graph.answer_timeout;
6522 if !g.resolved {
6523 let mut standing = g
6524 .question
6525 .as_ref()
6526 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
6527 if let Some(q) = standing.as_ref()
6528 && q.status.open()
6529 && gt::expired(q, timeout)
6530 {
6531 let why = format!("no answer within {}s of asking", timeout.max(1));
6532 standing = store
6533 .update(&q.id, |q| {
6534 q.abandon(&why);
6535 Ok(())
6536 })
6537 .ok()
6538 .map(|(q, ())| q);
6539 }
6540 let mut ask_again: Option<(Vec<String>, Option<String>)> = None;
6541 match standing {
6542 None if g.asks <= 1 => {
6545 g.asks = 0;
6546 ask_again = Some((g.categories.clone(), None));
6547 }
6548 None => {
6549 g.resolved = true;
6550 g.chosen_title = None;
6551 }
6552 Some(q) if q.status.open() => {
6553 self.park_on_github_text(&g)?;
6554 return Ok(None);
6555 }
6556 Some(q) => match gt::read_reply(&q) {
6557 gt::Reply::Fallback => {
6558 g.resolved = true;
6559 g.chosen_title = None;
6560 }
6561 gt::Reply::Title(text) => {
6562 let first = text
6563 .lines()
6564 .map(str::trim)
6565 .find(|l| !l.is_empty())
6566 .unwrap_or("");
6567 let decision = if first.is_empty() {
6568 None
6569 } else {
6570 let candidate = PrMessage {
6571 title: first.to_owned(),
6572 body: String::new(),
6573 };
6574 self.judge_pr_language(&winner.worktree, &candidate).await
6575 };
6576 match gt::vet_title(&text, decision) {
6577 Ok(title) => {
6578 g.resolved = true;
6579 g.chosen_title = Some(title);
6580 }
6581 Err(cats) if g.asks < 2 => {
6582 ask_again = Some((
6583 cats.into_iter().map(str::to_owned).collect(),
6584 Some(first.to_owned()),
6585 ))
6586 }
6587 Err(_) => {
6588 g.resolved = true;
6589 g.chosen_title = None;
6590 }
6591 }
6592 }
6593 },
6594 }
6595 if let Some((categories, replacement)) = ask_again {
6596 let retry = replacement.is_some();
6597 let (shown_title, shown_body) = match replacement.as_deref() {
6600 Some(t) => (t, ""),
6601 None => (pr.title.as_str(), pr.body.as_str()),
6602 };
6603 let w = gt::Withheld {
6604 title: g.title || retry,
6605 body: g.body && !retry,
6606 categories,
6607 };
6608 let artifact = match crate::run::write_artifact(
6611 &self.state,
6612 >::artifact_name(>::fingerprint(shown_title, shown_body)),
6613 >::artifact_text(shown_title, shown_body),
6614 ) {
6615 Ok(p) => Some(p),
6616 Err(e) => {
6617 tracing::warn!("could not write the withheld-text artifact: {e:#}");
6618 None
6619 }
6620 };
6621 let mut q = ask::Question::new(
6622 self.state.id.clone(),
6623 gt::ASK_NODE.to_owned(),
6624 gt::ASK_SEAT.to_owned(),
6625 gt::question_summary(&self.state.config.graph.language, &w),
6626 gt::question_detail(&w, shown_title, shown_body, retry, artifact.as_deref()),
6627 gt::question_choices(&w),
6628 );
6629 q.answer_timeout = timeout;
6630 g.question = Some(q.id.clone());
6633 g.asks += 1;
6634 self.state.github_text = Some(g.clone());
6635 self.state.event(
6636 "github-text",
6637 format!("asking the owner about the withheld text ({})", q.short()),
6638 );
6639 self.state.save()?;
6640 store
6641 .put(&mut q)
6642 .context("file the posting-gate question")?;
6643 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
6644 tracing::warn!(
6645 "could not notify about posting-gate question {}: {e:#}",
6646 q.short()
6647 );
6648 }
6649 self.park_on_github_text(&g)?;
6650 return Ok(None);
6651 }
6652 }
6653 let resolved = g.chosen_title.clone();
6654 self.state.event(
6655 "github-text",
6656 if resolved.is_some() {
6657 "posting the owner's replacement title"
6658 } else {
6659 "posting the neutral text for the withheld fields"
6660 },
6661 );
6662 self.state.github_text = Some(g.clone());
6663 self.state.save()?;
6664 let neutral = |cat: &str| g.categories.iter().any(|c| c == cat);
6667 let title = match (g.title, resolved) {
6668 (true, Some(t)) => t,
6669 (true, None) if neutral("title-language") => gt::NEUTRAL_TITLE.to_owned(),
6670 _ => pr.title.clone(),
6671 };
6672 let body = if g.body && neutral("body-language") {
6673 format!("{}\n\nmagi:run/{}", gt::NEUTRAL_BODY, self.state.id)
6674 } else {
6675 pr.body.clone()
6676 };
6677 let english = gt::LanguageDecision {
6679 title_english: true,
6680 body_english: true,
6681 body_complete: true,
6682 };
6683 let (title, body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
6684 Ok(Some(PrMessage { title, body }))
6685 }
6686
6687 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
6688 self.state.github_text = Some(g.clone());
6689 self.state.parked = true;
6690 self.state.event(
6691 "github-text",
6692 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
6693 );
6694 self.state.save()
6695 }
6696
6697 async fn judge_pr_language(
6700 &mut self,
6701 cwd: &Path,
6702 pr: &PrMessage,
6703 ) -> Option<crate::github_text::LanguageDecision> {
6704 self.state.config.roles.language_judge.as_ref()?;
6705 let decision =
6706 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
6707 self.state.event(
6708 "github-text",
6709 if decision.is_some() {
6710 "language decided by the language judge"
6711 } else {
6712 "language judge unavailable; using built-in heuristics"
6713 },
6714 );
6715 decision
6716 }
6717
6718 async fn merge(&mut self) -> Result<()> {
6719 if self
6734 .state
6735 .base_sync
6736 .as_ref()
6737 .is_some_and(|s| s.conflict.is_some())
6738 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6739 != Some(RunStatus::Gating)
6740 || !self.state.gate_status().ok()
6749 {
6750 return Ok(());
6751 }
6752 if self.state.merge.is_some() {
6761 return Ok(());
6762 }
6763 let Some(winner) = self.state.winner().cloned() else {
6764 return Ok(());
6765 };
6766 let repo = self.state.repo.clone();
6767 let base = self.state.base_branch.clone();
6768 let mode = self.state.config.merge.mode;
6769 let style = self.state.config.merge.style;
6770 let facts = if is_review_run(&self.state) {
6771 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
6772 let start = review_base(
6773 &repo,
6774 &self.state.config.merge.remote,
6775 &base,
6776 &self.state.base_commit,
6777 &winner.branch,
6778 )
6779 .await;
6780 branch_facts(&repo, &start, &winner.branch).await
6781 } else {
6782 None
6783 };
6784 let leaked = if is_review_run(&self.state) {
6787 leaked_subjects(&self.state, &winner.branch).await
6788 } else {
6789 Some(Vec::new())
6790 };
6791 let Some(pr) = self
6795 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
6796 .await?
6797 else {
6798 return Ok(());
6799 };
6800 let message = pr.commit_message();
6801
6802 let outcome = match mode {
6803 MergeMode::None => MergeOutcome {
6804 mode,
6805 ok: true,
6806 detail: manual_merge_command(style, &repo, &winner.branch, &message),
6807 empty: false,
6808 },
6809 MergeMode::Pr | MergeMode::Local
6810 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
6811 {
6812 MergeOutcome {
6813 mode,
6814 ok: false,
6815 detail: empty_candidate_detail(&self.state, &base),
6816 empty: true,
6817 }
6818 }
6819 MergeMode::Local => {
6820 let on = git::current_branch(&repo).await?;
6821 if on.as_deref() != Some(base.as_str()) {
6822 MergeOutcome {
6823 mode,
6824 ok: false,
6825 detail: format!(
6826 "{} has {} checked out, not the base branch {base}",
6827 repo.display(),
6828 on.unwrap_or_else(|| "a detached HEAD".to_owned())
6829 ),
6830 empty: false,
6831 }
6832 } else if !git::is_clean(&repo).await? {
6833 MergeOutcome {
6834 mode,
6835 ok: false,
6836 detail: format!("{} is dirty; refusing to merge", repo.display()),
6837 empty: false,
6838 }
6839 } else {
6840 let out = match style {
6841 MergeStyle::Merge => {
6842 git::merge_no_ff(&repo, &winner.branch, &message).await?
6843 }
6844 MergeStyle::Squash => {
6845 git::merge_squash(&repo, &winner.branch, &message).await?
6846 }
6847 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
6848 };
6849 MergeOutcome {
6850 mode,
6851 ok: out.ok(),
6852 detail: if out.ok() { out.stdout } else { out.stderr },
6853 empty: false,
6854 }
6855 }
6856 }
6857 MergeMode::Pr => {
6858 let remote = self.state.config.merge.remote.clone();
6859 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
6860 if !pushed.ok() {
6861 MergeOutcome {
6862 mode,
6863 ok: false,
6864 detail: pushed.stderr,
6865 empty: false,
6866 }
6867 } else {
6868 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
6874 let out = match pr_merge_plan(found) {
6875 PrPlan::Create => {
6876 gh_pr_create(
6877 &winner.worktree,
6878 &base,
6879 &winner.branch,
6880 &pr.title,
6881 &pr.body,
6882 )
6883 .await
6884 }
6885 PrPlan::Adopt { url, title } => {
6886 self.state
6887 .event("merge", format!("Pr: adopted open pull request {url}"));
6888 if title != pr.title
6889 && (!is_review_run(&self.state)
6890 || leaked
6891 .as_deref()
6892 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
6893 && let Err(e) = land::set_pr_title(
6894 &mut self.state,
6895 &winner.worktree,
6896 &url,
6897 &pr.title,
6898 )
6899 .await
6900 {
6901 tracing::warn!("could not refresh title of {url}: {e:#}");
6902 self.state
6903 .event("merge", format!("Pr: title refresh failed: {e:#}"));
6904 }
6905 Ok(url)
6906 }
6907 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
6908 };
6909 match out {
6910 Ok(url) => MergeOutcome {
6911 mode,
6912 ok: true,
6913 detail: url,
6914 empty: false,
6915 },
6916 Err(e) => MergeOutcome {
6917 mode,
6918 ok: false,
6919 detail: e.to_string(),
6920 empty: false,
6921 },
6922 }
6923 }
6924 }
6925 };
6926
6927 self.state.status = match (mode, outcome.ok) {
6928 (MergeMode::None, _) => RunStatus::Ready,
6929 (_, true) => RunStatus::Merged,
6930 (_, false) => RunStatus::Blocked,
6931 };
6932 self.state.event(
6933 "merge",
6934 format!(
6935 "{:?}: {}",
6936 mode,
6937 outcome.detail.lines().next().unwrap_or("")
6938 ),
6939 );
6940 self.state.merge = Some(outcome);
6941 self.state.save()?;
6942
6943 if self.state.config.graph.land
6949 && mode == MergeMode::Pr
6950 && self.state.status == RunStatus::Merged
6951 {
6952 self.run_land().await?;
6953 }
6954 self.settle_questions();
6959 Ok(())
6960 }
6961
6962 async fn run_land(&mut self) -> Result<()> {
6973 let url = self
6974 .state
6975 .merge
6976 .as_ref()
6977 .map(|m| m.detail.clone())
6978 .unwrap_or_default();
6979 let url = url.lines().next().unwrap_or("").trim().to_owned();
6980 if !url.starts_with("http") {
6981 return Ok(());
6982 }
6983 match land::land(&mut self.state, &url).await {
6986 Ok(pr) if self.state.parked => {
6987 let _ = pr;
6991 }
6992 Ok(pr) => {
6993 self.state.status = match pr.state {
6994 land::PrLifecycle::Merged => RunStatus::Merged,
6995 _ => RunStatus::Blocked,
6996 };
6997 if bump::should_release_bump(self.state.status)
7004 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
7005 {
7006 self.state
7012 .event("bump", format!("release bump skipped: {e:#}"));
7013 }
7014 if self.state.status == RunStatus::Merged {
7017 crate::followup::after_merge(&mut self.state, &pr.url).await;
7018 }
7019 self.state.save()?;
7020 }
7021 Err(e) => {
7022 self.state.status = RunStatus::Blocked;
7023 self.state.event("land", format!("gave up: {e}"));
7024 self.state.save()?;
7025 }
7026 }
7027 Ok(())
7028 }
7029
7030 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
7034 if let Some(existing) = self.state.seats.get(key)
7035 && existing.agent == agent
7036 {
7037 return existing.clone();
7038 }
7039 let fresh = if self.state.seats.contains_key(key) {
7042 handover_seat(key, agent, self.state.next_seat_seed())
7043 } else {
7044 SeatState::new(key, agent, self.state.seed)
7045 };
7046 self.state.seats.insert(key.to_owned(), fresh.clone());
7047 fresh
7048 }
7049
7050 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
7056 match self.state.seats.get(key) {
7057 Some(s) if s.agent != spec.id => {
7058 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
7059 }
7060 _ => spec,
7061 }
7062 }
7063
7064 fn view(&self, c: &Candidate) -> CandidateView {
7066 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
7067 .unwrap_or_default();
7068 let (patch, _) = blind::sanitize_patch(
7069 &format!("candidate {} patch", c.label),
7070 &raw,
7071 &self.state.config.blind,
7072 );
7073 CandidateView {
7074 label: c.label,
7075 branch: c.branch.clone(),
7076 summary: c.summary.clone(),
7077 stat: c.stat.clone(),
7078 patch,
7079 }
7080 }
7081
7082 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
7084 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
7085 prompt::judge(
7086 "(see above)",
7087 &views,
7088 self.roles.judges.len(),
7089 base_short,
7090 "en",
7091 )
7092 }
7093
7094 fn vote_prompt_full(
7100 &self,
7101 j: usize,
7102 labels: &[char],
7103 language: &str,
7104 candidates: &[Candidate],
7105 base_short: &str,
7106 ) -> String {
7107 let mut text = format!(
7108 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
7109 prompt::final_vote(labels, language),
7110 self.state.instruction,
7111 self.candidate_block(candidates, base_short)
7112 );
7113 if let Some(own) = self
7114 .state
7115 .judgements
7116 .get(j)
7117 .filter(|r| !r.ranking.is_empty())
7118 {
7119 let reasons = own
7120 .reasons
7121 .iter()
7122 .map(|(k, v)| format!("- {k}: {v}"))
7123 .collect::<Vec<_>>()
7124 .join("\n");
7125 text.push_str(&format!(
7126 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
7127 own.ranking.iter().collect::<String>(),
7128 if reasons.is_empty() {
7129 ""
7130 } else {
7131 ", because:\n"
7132 }
7133 ));
7134 }
7135 if !self.state.deliberation.is_empty() {
7136 text.push_str("\n# What was argued before this vote\n");
7137 for t in self.transcript(&[], j) {
7138 text.push_str(&format!(
7139 "\n## {}{}\n\n{}\n",
7140 t.who,
7141 if t.is_self { " (you)" } else { "" },
7142 t.body.trim()
7143 ));
7144 }
7145 }
7146 text
7147 }
7148
7149 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
7156 let mut turns = Vec::new();
7157 for j in &self.state.judgements {
7158 if j.ranking.is_empty() {
7159 continue;
7160 }
7161 let reasons = j
7162 .reasons
7163 .iter()
7164 .map(|(k, v)| format!("- {k}: {v}"))
7165 .collect::<Vec<_>>()
7166 .join("\n");
7167 turns.push(Turn {
7168 who: format!("Judge {} (opening ranking)", j.judge),
7169 is_self: j.judge == self_idx + 1,
7170 body: format!(
7171 "Ranked {}{}{reasons}",
7172 j.ranking.iter().collect::<String>(),
7173 if reasons.is_empty() {
7174 ""
7175 } else {
7176 ", because:\n"
7177 }
7178 ),
7179 });
7180 }
7181 for t in self
7182 .state
7183 .deliberation
7184 .iter()
7185 .flat_map(|r| r.turns.iter())
7186 .chain(current)
7187 {
7188 turns.push(Turn {
7189 who: format!("Judge {}", t.judge),
7190 is_self: t.judge == self_idx + 1,
7191 body: t.body.clone(),
7192 });
7193 }
7194 turns
7195 }
7196}
7197
7198fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
7200 agent::has_session(spec.kind, seat, sessions)
7201}
7202
7203fn next_untried_in_roster<'a>(
7224 roster: &'a [AgentSpec],
7225 start: usize,
7226 tried: &BTreeSet<String>,
7227) -> Option<&'a AgentSpec> {
7228 roster
7229 .get(start + 1..)?
7230 .iter()
7231 .find(|s| !tried.contains(&s.id))
7232}
7233
7234fn pick_successor<'a>(
7255 roster: &'a [AgentSpec],
7256 start: usize,
7257 tried: &BTreeSet<String>,
7258 carried: Option<&BTreeSet<String>>,
7259 others: &BTreeSet<String>,
7260) -> Option<&'a AgentSpec> {
7261 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
7262 roster
7263 .get(start + 1..)
7264 .and_then(|tail| {
7265 tail.iter()
7266 .filter(free)
7267 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
7268 })
7269 .or_else(|| {
7270 carried
7271 .is_some()
7272 .then(|| roster.iter().find(free))
7273 .flatten()
7274 })
7275 .or_else(|| match carried {
7276 Some(c) => next_for_seat(roster, start, tried, c),
7277 None => next_untried_in_roster(roster, start, tried),
7278 })
7279}
7280
7281fn next_for_seat<'a>(
7291 roster: &'a [AgentSpec],
7292 start: usize,
7293 round_tried: &BTreeSet<String>,
7294 carried_failed: &BTreeSet<String>,
7295) -> Option<&'a AgentSpec> {
7296 roster
7297 .get(start + 1..)?
7298 .iter()
7299 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
7300 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
7301}
7302
7303fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
7309 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
7310 return spec;
7311 };
7312 let ok = |id: &str| !h.failed.contains(id);
7313 if let Some(last) = h.last_ok.as_deref()
7314 && ok(last)
7315 && let Some(s) = roster.iter().find(|s| s.id == last)
7316 {
7317 return s.clone();
7318 }
7319 if ok(&spec.id) {
7320 return spec;
7321 }
7322 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
7323 roster
7324 .get(from + 1..)
7325 .into_iter()
7326 .flatten()
7327 .chain(roster.iter())
7328 .find(|s| ok(&s.id))
7329 .cloned()
7330 .unwrap_or(spec)
7331}
7332
7333pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
7337 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
7338}
7339
7340const TIMED_OUT: &str = "timed out";
7343
7344impl FailClass {
7345 fn of(out: &AgentOutcome) -> Option<Self> {
7347 match out {
7348 AgentOutcome::Ok(_) => None,
7349 AgentOutcome::Quota(_) => Some(Self::Quota),
7350 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
7351 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
7352 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
7353 }
7354 }
7355
7356 fn stem_word(&self) -> &'static str {
7358 match self {
7359 Self::Quota => "quota",
7360 _ => "handover",
7361 }
7362 }
7363}
7364
7365fn failure_signature(msg: &str) -> String {
7369 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
7370 let mut out = Vec::new();
7371 for word in line.split_whitespace() {
7372 if word.contains('/') || word.contains('\\') {
7373 out.push("<path>".to_owned());
7374 continue;
7375 }
7376 let mut w = String::new();
7377 let mut in_digits = false;
7378 for c in word.chars() {
7379 if c.is_ascii_digit() {
7380 if !in_digits {
7381 w.push('#');
7382 }
7383 in_digits = true;
7384 } else {
7385 in_digits = false;
7386 w.push(c);
7387 }
7388 }
7389 out.push(w);
7390 }
7391 out.join(" ").chars().take(120).collect()
7392}
7393
7394fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
7401 match cur {
7402 FailClass::Quota | FailClass::Timeout => true,
7403 FailClass::Other(_) => prev != Some(cur),
7404 }
7405}
7406
7407fn fail_reason(out: &AgentOutcome) -> String {
7409 match out {
7410 AgentOutcome::Ok(_) => String::new(),
7411 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
7412 AgentOutcome::Dropped(o) => format!(
7413 "the CLI dropped the stream ({})",
7414 o.dropped
7415 .as_ref()
7416 .map(|d| d.why.as_str())
7417 .unwrap_or("it ended without delivering its answer")
7418 ),
7419 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
7420 }
7421}
7422
7423pub(crate) fn record_handover(
7426 state: &mut RunState,
7427 node: &str,
7428 seat: &str,
7429 from: &str,
7430 to: &str,
7431 class: &FailClass,
7432 reason: &str,
7433) {
7434 let message = if *class == FailClass::Quota {
7435 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
7436 } else {
7437 format!("{seat}: handed over {from} -> {to} ({reason})")
7438 };
7439 state.event(node, message);
7440 state.handovers.push(Handover {
7441 at: Timestamp::now(),
7442 node: node.to_owned(),
7443 seat: seat.to_owned(),
7444 from: from.to_owned(),
7445 to: to.to_owned(),
7446 reason: reason.to_owned(),
7447 });
7448}
7449
7450fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
7464 commands.iter().any(|c| c.exit_code.is_none())
7465}
7466
7467fn verified_noop_claim(
7480 usable: bool,
7481 commands: &[agent::CommandEvidence],
7482 text: &str,
7483) -> Option<String> {
7484 (usable && !has_unconfirmed_command(commands))
7485 .then(|| verdict::verified_noop(text))
7486 .flatten()
7487}
7488
7489fn short(commit: &str) -> String {
7490 commit.chars().take(7).collect()
7491}
7492
7493fn make_executable(path: &Path) -> Result<()> {
7494 #[cfg(unix)]
7495 {
7496 use std::os::unix::fs::PermissionsExt as _;
7497 let mut perms = std::fs::metadata(path)?.permissions();
7498 perms.set_mode(0o755);
7499 std::fs::set_permissions(path, perms)?;
7500 }
7501 #[cfg(not(unix))]
7502 {
7503 let _ = path;
7504 }
7505 Ok(())
7506}
7507
7508struct WaveCtx<'a> {
7515 run: &'a str,
7518 node: &'a str,
7520 prompts: &'a Prompts,
7521 cache: Option<&'a Path>,
7523 round: Option<usize>,
7526 carry_seats: bool,
7530}
7531
7532async fn run_one(
7534 job: SeatJob,
7535 sem: Arc<Semaphore>,
7536 ctx: &WaveCtx<'_>,
7537 state: &mut RunState,
7538 attempt: usize,
7539) -> (SeatState, AgentOutcome) {
7540 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
7541 .await
7542 .pop()
7543 .expect("one job in, one result out");
7544 (seat, out)
7545}
7546
7547async fn wave(
7553 jobs: Vec<SeatJob>,
7554 sem: Arc<Semaphore>,
7555 ctx: &WaveCtx<'_>,
7556 state: &mut RunState,
7557 attempt: usize,
7558) -> Vec<(usize, SeatState, AgentOutcome)> {
7559 let WaveCtx {
7560 run,
7561 node,
7562 prompts,
7563 cache,
7564 round,
7565 carry_seats: _,
7566 } = *ctx;
7567 for job in &jobs {
7568 state.seat_started(node, &job.seat.key, job.timeout, attempt);
7569 }
7570 if let Err(e) = state.save() {
7571 tracing::warn!("could not persist in-progress seats: {e:#}");
7576 }
7577 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
7593 let wait_started = Instant::now();
7594 let cache_guard = if let Some(cache_dir) = cache {
7595 if jobs_had_a_writer {
7596 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
7597 let budget = jobs
7598 .iter()
7599 .map(|j| j.timeout)
7600 .max()
7601 .unwrap_or(Duration::from_secs(60));
7602 acquire_cache_lease(state, cache_dir, &owner, budget, node)
7603 .await
7604 .ok()
7605 } else {
7606 None
7607 }
7608 } else {
7609 None
7610 };
7611 let waited_for_lease = wait_started.elapsed();
7618 let mut set = tokio::task::JoinSet::new();
7619 let overlay = prompts.overlay(node);
7620 for (i, mut job) in jobs.into_iter().enumerate() {
7621 job.timeout = job.timeout.saturating_sub(waited_for_lease);
7622 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
7623 if cache.is_some() {
7624 job.prompt.push('\n');
7625 job.prompt
7626 .push_str(&prompt::build_cache_note(node, job.allow_write));
7627 }
7628 let sem = Arc::clone(&sem);
7629 let run = run.to_owned();
7630 let node = node.to_owned();
7631 let attachments = if node == "implement" {
7634 state.attachments.clone()
7635 } else {
7636 Vec::new()
7637 };
7638 let cache = cache
7649 .filter(|_| job.allow_write && cache_guard.is_some())
7650 .map(Path::to_path_buf);
7651 set.spawn(async move {
7652 let _permit = sem.acquire().await;
7653 let mut seat = job.seat;
7654 let out = agent::invoke(
7655 &job.spec,
7656 &mut seat,
7657 &Invocation {
7658 cwd: &job.cwd,
7659 prompt: &job.prompt,
7660 timeout: job.timeout,
7661 allow_write: job.allow_write,
7662 unsandboxed: false,
7663 sessions: job.sessions,
7664 artifacts: &job.artifacts,
7665 stem: &job.stem,
7666 run: &run,
7667 node: &node,
7668 cache_dir: cache.as_deref(),
7669 attachments: &attachments,
7670 writable: &[],
7671 },
7672 )
7673 .await;
7674 let out = match out {
7675 Ok(o) if o.usable() => AgentOutcome::Ok(o),
7676 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
7677 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
7685 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
7686 Ok(o) => AgentOutcome::Failed(format!(
7687 "exited with {:?} and no usable output",
7688 o.exit_code
7689 )),
7690 Err(e) => AgentOutcome::Failed(e.to_string()),
7691 };
7692 (i, seat, out)
7693 });
7694 }
7695 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
7696 while let Some(joined) = set.join_next().await {
7697 let (i, seat, out) = match joined {
7698 Ok(v) => v,
7699 Err(e) => {
7703 tracing::error!("agent task panicked: {e}");
7704 continue;
7705 }
7706 };
7707 state.seat_finished(&seat.key);
7708 record_jobs(state, node, round, &seat.key, &out);
7709 if let Err(e) = state.save() {
7710 tracing::warn!("could not persist a seat's completion: {e:#}");
7711 }
7712 if collected.len() <= i {
7713 collected.resize_with(i + 1, || None);
7714 }
7715 collected[i] = Some((i, seat, out));
7716 }
7717 if state
7723 .active
7724 .values()
7725 .any(|a| a.node == node && a.attempt == attempt)
7726 {
7727 state
7728 .active
7729 .retain(|_, a| !(a.node == node && a.attempt == attempt));
7730 if let Err(e) = state.save() {
7731 tracing::warn!("could not persist the end of a wave: {e:#}");
7732 }
7733 }
7734 if let Some(cache_dir) = cache
7741 && jobs_had_a_writer
7742 {
7743 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
7744 }
7745 if let Some(guard) = cache_guard {
7746 guard.release();
7747 }
7748 collected.into_iter().flatten().collect()
7749}
7750
7751fn record_jobs(
7762 state: &mut RunState,
7763 node: &str,
7764 round: Option<usize>,
7765 seat: &str,
7766 out: &AgentOutcome,
7767) {
7768 let commands: &[agent::CommandEvidence] = match out {
7769 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
7770 AgentOutcome::Failed(_) => &[],
7771 };
7772 let checked_at = Timestamp::now();
7773 for c in commands {
7774 state.jobs.push(JobRecord {
7775 node: node.to_owned(),
7776 round,
7777 seat: seat.to_owned(),
7778 id: c.id.clone(),
7779 description: c.description.clone(),
7780 checked_at,
7781 status: match c.exit_code {
7782 Some(0) => JobStatus::Completed,
7783 Some(_) => JobStatus::Failed,
7784 None => JobStatus::Unknown,
7785 },
7786 exit_code: c.exit_code,
7787 result_summary: c.result_summary.clone(),
7788 source: c.source.clone(),
7789 });
7790 }
7791}
7792
7793fn round_is_clean(
7814 blocking: usize,
7815 e2e_ok: bool,
7816 answered: usize,
7817 expected: usize,
7818 quota_missing: usize,
7819 policy: IncompleteReviewPolicy,
7820) -> bool {
7821 if blocking != 0 || !e2e_ok {
7822 return false;
7823 }
7824 if answered == expected || policy == IncompleteReviewPolicy::Warn {
7825 return true;
7826 }
7827 answered > 0 && expected - answered <= quota_missing
7828}
7829
7830fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
7854 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
7855 return Some(RunStatus::Gating);
7856 }
7857 let last = reviews.last()?;
7858 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
7859 if reviews.len() < max_rounds && !stagnant {
7860 return None;
7861 }
7862 if last.incomplete() && last.blocking == 0 {
7863 return Some(RunStatus::Blocked);
7864 }
7865 if last.e2e_status() == E2eStatus::ResourceBlocked {
7866 return None;
7867 }
7868 Some(if last.e2e.iter().all(CommandOutcome::ok) {
7869 RunStatus::Gating
7870 } else {
7871 RunStatus::Blocked
7872 })
7873}
7874
7875pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
7890 if nudged {
7891 (full / 4).max(Duration::from_secs(120)).min(full)
7892 } else {
7893 full
7894 }
7895}
7896
7897#[allow(clippy::too_many_arguments)]
7926async fn ask_json_wave<T>(
7927 jobs: Vec<SeatJob>,
7928 sem: Arc<Semaphore>,
7929 retries: usize,
7930 roster: &[AgentSpec],
7931 ctx: &WaveCtx<'_>,
7932 losses: &mut Vec<QuotaLoss>,
7933 state: &mut RunState,
7934 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
7935) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7936where
7937 T: serde::de::DeserializeOwned + Send + 'static,
7938{
7939 ask_wave_with(
7940 jobs,
7941 sem,
7942 retries,
7943 roster,
7944 ctx,
7945 losses,
7946 state,
7947 &|text: &str| {
7948 let v = verdict::extract_json::<T>(text)?;
7949 validate(&v)?;
7950 Ok(v)
7951 },
7952 )
7953 .await
7954}
7955
7956#[allow(clippy::too_many_arguments)]
7965async fn ask_wave_with<T>(
7966 jobs: Vec<SeatJob>,
7967 sem: Arc<Semaphore>,
7968 retries: usize,
7969 roster: &[AgentSpec],
7970 ctx: &WaveCtx<'_>,
7971 losses: &mut Vec<QuotaLoss>,
7972 state: &mut RunState,
7973 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
7974) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7975where
7976 T: Send + 'static,
7977{
7978 let n = jobs.len();
7979 let originals: Vec<SeatJob> = jobs;
7980 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
7981 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
7982 let mut nudges: Vec<usize> = vec![0; n];
7989 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
7993 let mut tried: Vec<BTreeSet<String>> = specs
7994 .iter()
7995 .map(|s| BTreeSet::from([s.id.clone()]))
7996 .collect();
7997 let starts: Vec<usize> = specs
7998 .iter()
7999 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
8000 .collect();
8001 let carry = ctx.carry_seats && !roster.is_empty();
8002 let carried: Vec<BTreeSet<String>> = originals
8005 .iter()
8006 .map(|j| {
8007 state
8008 .seat_history
8009 .get(&j.seat.key)
8010 .filter(|_| carry)
8011 .map(|h| h.failed.clone())
8012 .unwrap_or_default()
8013 })
8014 .collect();
8015 let mut prev: Vec<Option<FailClass>> = originals
8016 .iter()
8017 .map(|j| {
8018 state
8019 .seat_history
8020 .get(&j.seat.key)
8021 .filter(|_| carry)
8022 .and_then(|h| h.last_fail.clone())
8023 })
8024 .collect();
8025 let next_agent =
8026 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
8027 let others: BTreeSet<String> = specs
8028 .iter()
8029 .enumerate()
8030 .filter(|(j, _)| *j != i)
8031 .map(|(_, s)| s.id.clone())
8032 .collect();
8033 pick_successor(
8034 roster,
8035 starts[i],
8036 tried,
8037 carry.then(|| &carried[i]),
8038 &others,
8039 )
8040 .cloned()
8041 };
8042 let mut fresh: Vec<Option<String>> = vec![None; n];
8043 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
8044 let mut pending: Vec<usize> = (0..n).collect();
8045
8046 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
8051 for round in 0..max_rounds {
8052 if pending.is_empty() {
8053 break;
8054 }
8055 let mut batch = Vec::with_capacity(pending.len());
8056 let mut renudged: Vec<&str> = Vec::new();
8057 for &i in &pending {
8058 let src = &originals[i];
8059 let full: &str = match &src.handover {
8064 Some(h) if specs[i].id != src.spec.id => h,
8065 _ => &src.prompt,
8066 };
8067 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
8070 (
8071 full.to_owned(),
8072 src.timeout,
8073 format!("{}-{word}-{}", src.stem, specs[i].id),
8074 )
8075 } else if nudges[i] == 0 {
8076 (full.to_owned(), src.timeout, src.stem.clone())
8077 } else {
8078 renudged.push(src.seat.key.as_str());
8079 let why = done[i]
8080 .as_ref()
8081 .and_then(|r| r.as_ref().err().map(ToString::to_string))
8082 .unwrap_or_else(|| "no parsable answer".to_owned());
8083 let nudge = prompt::nudge(&why);
8084 let nudged = has_context(&specs[i], &seats[i], src.sessions);
8085 let prompt = if nudged {
8086 nudge
8087 } else {
8088 format!("{full}\n\n---\n\n{nudge}")
8089 };
8090 (
8091 prompt,
8092 retry_budget(src.timeout, nudged),
8093 format!("{}-retry{}", src.stem, nudges[i]),
8094 )
8095 };
8096 batch.push(SeatJob {
8097 spec: specs[i].clone(),
8098 seat: seats[i].clone(),
8099 cwd: src.cwd.clone(),
8100 prompt,
8101 timeout,
8102 allow_write: src.allow_write,
8103 sessions: src.sessions,
8104 artifacts: src.artifacts.clone(),
8105 stem,
8106 handover: None,
8107 });
8108 }
8109
8110 if !renudged.is_empty() {
8111 state.event(
8112 ctx.node,
8113 format!("retry {round}: re-asking {}", renudged.join(", ")),
8114 );
8115 }
8116 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
8117 let mut still = Vec::new();
8118 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
8119 seats[i] = seat;
8120 let class = FailClass::of(&out);
8121 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
8124 && nudges[i] < retries
8125 && !roster.is_empty();
8126 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
8127 let next = should_hand_over(prev[i].as_ref(), &cur)
8128 .then(|| next_agent(i, &tried[i], &specs))
8129 .flatten();
8130 if carry {
8131 let h = state
8132 .seat_history
8133 .entry(originals[i].seat.key.clone())
8134 .or_default();
8135 h.failed.insert(specs[i].id.clone());
8136 h.last_fail = Some(cur.clone());
8137 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
8138 h.last_ok = None;
8139 }
8140 if let Err(e) = state.save() {
8143 tracing::warn!("could not persist a seat's failure history: {e:#}");
8144 }
8145 }
8146 if let Some(next) = next {
8147 record_handover(
8148 state,
8149 ctx.node,
8150 &originals[i].seat.key,
8151 &specs[i].id,
8152 &next.id,
8153 &cur,
8154 &fail_reason(&out),
8155 );
8156 tried[i].insert(next.id.clone());
8157 prev[i] = Some(cur.clone());
8158 seats[i] =
8159 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
8160 specs[i] = next;
8161 fresh[i] = Some(cur.stem_word().to_owned());
8162 nudges[i] = 0;
8163 done[i] = Some(Err(anyhow::anyhow!(
8164 "handed over after: {}",
8165 fail_reason(&out)
8166 )));
8167 still.push(i);
8168 continue;
8169 }
8170 }
8171 if carry
8172 && !nudge_first
8173 && let Some(cur) = class.clone()
8174 {
8175 let h = state
8177 .seat_history
8178 .entry(originals[i].seat.key.clone())
8179 .or_default();
8180 h.failed.insert(specs[i].id.clone());
8181 h.last_fail = Some(cur);
8182 }
8183 let parsed = match out {
8184 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
8185 AgentOutcome::Quota(o) => {
8186 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
8187 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
8188 }
8189 AgentOutcome::Dropped(o) => {
8193 let why = o
8194 .dropped
8195 .as_ref()
8196 .map(|d| d.why.as_str())
8197 .unwrap_or("the CLI ended the stream without delivering its answer");
8198 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
8199 }
8200 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
8201 };
8202 let quota = class == Some(FailClass::Quota);
8203 let failed = parsed.is_err();
8204 done[i] = Some(parsed);
8205 if carry && !failed {
8206 let h = state
8207 .seat_history
8208 .entry(originals[i].seat.key.clone())
8209 .or_default();
8210 h.failed.remove(&specs[i].id);
8211 h.last_ok = Some(specs[i].id.clone());
8212 h.last_fail = None;
8213 }
8214 let agent_failure = class.is_some()
8222 && !nudge_first
8223 && !roster.is_empty()
8224 && next_agent(i, &tried[i], &specs).is_some();
8225 if failed && !quota && !agent_failure && nudges[i] < retries {
8226 nudges[i] += 1;
8227 still.push(i);
8228 }
8229 }
8230 pending = still;
8231 }
8232
8233 for (i, q) in last_quota.into_iter().enumerate() {
8236 if let Some(reset) = q {
8237 losses.push(QuotaLoss {
8238 seat: originals[i].seat.key.clone(),
8239 node: ctx.node.to_owned(),
8240 at: Timestamp::now(),
8241 reset,
8242 });
8243 }
8244 }
8245
8246 seats
8247 .into_iter()
8248 .zip(done)
8249 .zip(nudges)
8250 .map(|((seat, res), attempts)| {
8251 (
8252 seat,
8253 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
8254 attempts,
8255 )
8256 })
8257 .collect()
8258}
8259
8260async fn acquire_cache_lease(
8273 state: &mut RunState,
8274 cache_dir: &Path,
8275 owner: &crate::cache::Owner,
8276 budget: Duration,
8277 context: &str,
8278) -> Result<crate::cache::Guard> {
8279 let home = crate::run::home();
8280 let started = Instant::now();
8281 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
8282 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
8283 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
8284 Err(e) => {
8285 state.event(
8286 "verify",
8287 format!("{context}: could not check the shared build cache: {e:#}"),
8288 );
8289 if let Err(e2) = state.save() {
8290 tracing::warn!("could not persist a cache-check failure: {e2:#}");
8291 }
8292 return Err(e);
8293 }
8294 };
8295 state.event(
8296 "verify",
8297 format!(
8298 "{context}: waiting for the shared build cache at {} ({})",
8299 cache_dir.display(),
8300 busy.describe()
8301 ),
8302 );
8303 if let Err(e) = state.save() {
8304 tracing::warn!("could not persist a cache wait: {e:#}");
8305 }
8306 let remaining = budget.saturating_sub(started.elapsed());
8307 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
8308 Ok(g) => Ok(g),
8309 Err(e) => {
8310 state.event("verify", format!("{context}: {e:#}"));
8311 if let Err(e2) = state.save() {
8312 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
8313 }
8314 Err(e)
8315 }
8316 }
8317}
8318
8319#[allow(clippy::too_many_arguments)]
8340async fn with_cache_lease<'s, F, Fut>(
8341 state: &'s mut RunState,
8342 cache_dir: Option<&Path>,
8343 node: &str,
8344 seat: &str,
8345 worktree: &Path,
8346 head: &str,
8347 budget: Duration,
8348 context: &str,
8349 body: F,
8350) -> (Vec<CommandOutcome>, bool)
8351where
8352 F: FnOnce(&'s mut RunState, Duration) -> Fut,
8353 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
8354{
8355 let Some(cache_dir) = cache_dir else {
8356 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
8357 return (outcomes, retried);
8358 };
8359 let home = crate::run::home();
8360 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
8361 let started = Instant::now();
8362 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
8363 Ok(g) => g,
8364 Err(e) => {
8365 return (
8366 vec![CommandOutcome {
8367 command: "(waiting for the shared build cache)".to_owned(),
8368 code: None,
8369 output_tail: e.to_string(),
8370 duration_ms: started.elapsed().as_millis() as u64,
8371 resource_blocked: true,
8372 }],
8373 false,
8374 );
8375 }
8376 };
8377 let identity = crate::cache::Identity::new(worktree, head);
8378 if let Err(e) = crate::cache::ensure_fresh(
8379 &home,
8380 cache_dir,
8381 &identity,
8382 &state.config.verify.cargo_manifests(),
8383 ) {
8384 state.event(
8392 "verify",
8393 format!(
8394 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
8395 worktree.display(),
8396 short(head)
8397 ),
8398 );
8399 guard.release();
8400 return (
8401 vec![CommandOutcome {
8402 command: "(confirming the shared build cache is fresh)".to_owned(),
8403 code: None,
8404 output_tail: e.to_string(),
8405 duration_ms: started.elapsed().as_millis() as u64,
8406 resource_blocked: true,
8407 }],
8408 false,
8409 );
8410 }
8411 let remaining = budget.saturating_sub(started.elapsed());
8412 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
8413 if !timed_out_pids.is_empty() {
8418 wait_for_timed_out_children_to_die(&timed_out_pids).await;
8419 }
8420 guard.release();
8421 (outcomes, retried)
8422}
8423
8424async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
8436 wait_for_pids_with(
8437 pids,
8438 crate::proc::pid_alive,
8439 LEASE_RELEASE_POLL,
8440 LEASE_RELEASE_MAX_WAIT,
8441 )
8442 .await;
8443}
8444
8445async fn wait_for_pids_with<F: Fn(u32) -> bool>(
8451 pids: &[u32],
8452 alive: F,
8453 poll: Duration,
8454 max_wait: Duration,
8455) {
8456 let deadline = Instant::now() + max_wait;
8457 loop {
8458 if pids.iter().all(|&pid| !alive(pid)) {
8459 return;
8460 }
8461 if Instant::now() >= deadline {
8462 return;
8463 }
8464 tokio::time::sleep(poll).await;
8465 }
8466}
8467
8468fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
8475 outcomes.iter().any(|o| o.resource_blocked)
8476}
8477
8478enum GateFix {
8480 Retry,
8482 Stop,
8485 Defer,
8488}
8489
8490fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
8498 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
8499 red.peek().is_some()
8500 && red.all(|o| {
8501 !o.resource_blocked
8502 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
8503 && !o.output_tail.trim().is_empty()
8504 })
8505}
8506
8507fn e2e_outcome_label(o: &CommandOutcome) -> String {
8511 if o.ok() {
8512 return "pass".to_owned();
8513 }
8514 let reason = if o.build_failed() {
8515 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
8516 } else {
8517 format!("FAIL ({:?})", o.code)
8518 };
8519 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
8520}
8521
8522async fn run_e2e_with_retry(
8530 state: &mut RunState,
8531 shell: &[String],
8532 commands: &[String],
8533 worktree: &Path,
8534 timeout: Duration,
8535 context: &str,
8536) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
8537 let (mut e2e, mut timed_out_pids) = run_commands(
8538 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
8539 )
8540 .await;
8541 for o in &e2e {
8542 state.event(
8543 "verify",
8544 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
8545 );
8546 }
8547 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
8551 if verify_retried {
8552 state.event(
8553 "verify",
8554 format!(
8555 "{context}: verify could not build/link, not a test result — retrying once \
8556 before concluding"
8557 ),
8558 );
8559 let retried = run_commands(
8560 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
8561 )
8562 .await;
8563 e2e = retried.0;
8564 timed_out_pids.extend(retried.1);
8567 for o in &e2e {
8568 state.event(
8569 "verify",
8570 format!(
8571 "{context}: retry `{}` -> {}",
8572 o.command,
8573 e2e_outcome_label(o)
8574 ),
8575 );
8576 }
8577 }
8578 (e2e, verify_retried, timed_out_pids)
8579}
8580
8581#[allow(clippy::too_many_arguments)]
8597async fn run_commands(
8598 state: &mut RunState,
8599 node: &str,
8600 task: &str,
8601 attempt: usize,
8602 shell: &[String],
8603 commands: &[String],
8604 cwd: &Path,
8605 timeout: Duration,
8606) -> (Vec<CommandOutcome>, Vec<u32>) {
8607 if commands.is_empty() {
8608 return (Vec::new(), Vec::new());
8613 }
8614 let mut out = Vec::new();
8615 let mut timed_out_pids = Vec::new();
8616 let total = commands.len();
8617 for (idx, command) in commands.iter().enumerate() {
8618 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
8619 if let Err(e) = state.save() {
8620 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
8621 }
8622 let started = Instant::now();
8623 let mut cmd = tokio::process::Command::new(&shell[0]);
8624 cmd.quiet();
8625 cmd.args(&shell[1..])
8626 .arg(command)
8627 .current_dir(cwd)
8628 .stdin(std::process::Stdio::null())
8629 .stdout(std::process::Stdio::piped())
8630 .stderr(std::process::Stdio::piped())
8631 .kill_on_drop(true);
8632 let spawned = cmd.spawn();
8633 let (code, body) = match spawned {
8634 Ok(child) => {
8635 let pid = child.id();
8640 match tokio::time::timeout(timeout, child.wait_with_output()).await {
8641 Ok(Ok(o)) => {
8642 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
8643 body.push_str(&String::from_utf8_lossy(&o.stderr));
8644 (o.status.code(), body)
8645 }
8646 Ok(Err(e)) => (None, format!("failed to run: {e}")),
8647 Err(_) => {
8648 if let Some(pid) = pid {
8649 timed_out_pids.push(pid);
8650 }
8651 (None, format!("timed out after {}s", timeout.as_secs()))
8652 }
8653 }
8654 }
8655 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
8656 };
8657 out.push(CommandOutcome {
8658 command: command.clone(),
8659 code,
8660 output_tail: tail(&body, OUTPUT_TAIL),
8661 duration_ms: started.elapsed().as_millis() as u64,
8662 resource_blocked: false,
8663 });
8664 }
8665 state.task_finished(task);
8666 if let Err(e) = state.save() {
8667 tracing::warn!("could not persist the end of {task}: {e:#}");
8668 }
8669 (out, timed_out_pids)
8670}
8671
8672fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
8684 let repo = repo.display();
8685 match style {
8686 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
8687 MergeStyle::Squash => {
8688 let subject = message
8691 .lines()
8692 .next()
8693 .unwrap_or(branch)
8694 .replace(['\\', '"', '$', '`'], "");
8695 format!(
8696 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
8697 )
8698 }
8699 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
8700 }
8701}
8702
8703const PR_TITLE_MAX: usize = 240;
8714
8715fn english_title(text: &str, max: usize) -> Option<String> {
8726 let line = queue::first_line(text)?;
8727 let chars: Vec<char> = line.chars().collect();
8728 let (kept, mark) = if chars.len() <= max {
8729 (line.to_owned(), "")
8730 } else if let Some(end) = sentence_end(&chars, max) {
8731 (chars[..end].iter().collect::<String>(), "")
8732 } else {
8733 let room = max.saturating_sub(3);
8734 let cut = if chars[room].is_whitespace() {
8737 room
8738 } else {
8739 chars[..room]
8740 .iter()
8741 .rposition(|c| c.is_whitespace())
8742 .unwrap_or(room)
8743 };
8744 let head: String = chars[..cut].iter().collect();
8745 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
8746 (head.to_owned(), "...")
8747 };
8748 if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
8749 return None;
8750 }
8751 Some(format!("{kept}{mark}"))
8752}
8753
8754fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
8758 const MIN: usize = 20;
8759 for i in MIN..max.min(chars.len()) {
8760 if !matches!(chars[i], '.' | '!' | '?') {
8761 continue;
8762 }
8763 let Some(&next) = chars.get(i + 1) else {
8764 continue;
8765 };
8766 if !next.is_whitespace() {
8767 continue;
8768 }
8769 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
8770 if after.is_some_and(|c| c.is_ascii_lowercase()) {
8771 continue;
8772 }
8773 let word: String = chars[..i]
8774 .iter()
8775 .rev()
8776 .take_while(|c| !c.is_whitespace())
8777 .collect::<Vec<_>>()
8778 .into_iter()
8779 .rev()
8780 .collect();
8781 let word = word.to_ascii_lowercase();
8782 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
8783 continue;
8784 }
8785 let end = chars[..i]
8786 .iter()
8787 .rposition(|c| !c.is_whitespace())
8788 .map_or(i, |p| p + 1);
8789 return Some(end);
8790 }
8791 None
8792}
8793
8794struct PrMessage {
8799 title: String,
8800 body: String,
8801}
8802
8803impl PrMessage {
8804 fn commit_message(&self) -> String {
8808 format!("{}\n\n{}", self.title, self.body)
8809 }
8810}
8811
8812fn title_marker(line: &str) -> Option<&str> {
8814 let line = line.trim();
8815 let head = line.get(..6)?;
8816 head.eq_ignore_ascii_case("title:")
8817 .then(|| line[6..].trim())
8818}
8819
8820fn summary_title(summary: &str) -> Option<String> {
8825 let first = summary.lines().find(|l| !l.trim().is_empty())?;
8826 let raw = title_marker(first)?;
8827 if raw.is_empty() {
8828 return None;
8829 }
8830 let title = queue::title_from(raw, PR_TITLE_MAX);
8831 let lower = title.to_ascii_lowercase();
8832 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
8833 return None;
8834 }
8835 Some(title)
8836}
8837
8838const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
8840
8841const EXISTING_BRANCH: &str = "(existing branch)";
8843
8844fn is_review_run(state: &RunState) -> bool {
8848 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
8849}
8850
8851fn review_title(state: &RunState) -> Option<String> {
8856 english_subject(state.reviewed_commits.as_ref()?.first()?)
8857}
8858
8859fn english_subject(raw: &str) -> Option<String> {
8862 let raw = raw.trim();
8863 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
8864 return None;
8865 }
8866 let title = queue::title_from(raw, PR_TITLE_MAX);
8867 let lower = title.to_ascii_lowercase();
8868 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
8869 return None;
8870 }
8871 Some(title)
8872}
8873
8874#[derive(Debug, Clone, PartialEq, Eq)]
8877struct BranchFacts {
8878 commits: Vec<(String, String)>,
8880 stat: String,
8882}
8883
8884const STAT_FILE_LINES: usize = 25;
8886const COMMITS_MAX_CHARS: usize = 20_000;
8888
8889async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
8892 let commits = git::commit_log(repo, base, branch).await.ok()?;
8893 if commits.is_empty() {
8894 return None;
8895 }
8896 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
8897 let lines: Vec<&str> = stat.lines().collect();
8898 let stat = if lines.len() > STAT_FILE_LINES + 1 {
8899 let omitted = lines.len() - 1 - STAT_FILE_LINES;
8900 let more = format!(" ... {omitted} more file(s)");
8901 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
8902 kept.push(&more);
8903 kept.push(lines[lines.len() - 1]);
8904 kept.join("\n")
8905 } else {
8906 lines.join("\n")
8907 };
8908 Some(BranchFacts { commits, stat })
8909}
8910
8911fn markdown_safe(text: &str) -> String {
8914 text.replace("</details>", "</details>")
8915 .replace("\x60\x60\x60", "~~~")
8916}
8917
8918fn neutral_title(state: &RunState, winner: char) -> String {
8919 format!(
8920 "chore: land candidate {} of run {}",
8921 winner.to_ascii_uppercase(),
8922 state.id
8923 )
8924}
8925
8926pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
8932 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
8933 ""
8934 } else {
8935 pr_title
8936 }
8937}
8938
8939pub fn landing_subject_source(state: &RunState) -> String {
8943 if is_review_run(state) {
8944 let winner = state.candidates.first().map_or('A', |c| c.label);
8945 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
8946 }
8947 state.instruction.clone()
8948}
8949
8950fn summary_without_title(summary: &str) -> String {
8953 let mut lines = summary.trim().lines().peekable();
8954 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
8955 lines.next();
8956 }
8957 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
8958}
8959
8960#[cfg(test)]
8974fn pr_message(state: &RunState, winner: char) -> PrMessage {
8975 pr_message_with(state, winner, None)
8976}
8977
8978#[cfg(test)]
8981fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8982 let raw = pr_message_raw(state, winner, facts);
8983 let id = crate::scrub::Identity::current();
8984 PrMessage {
8985 title: crate::scrub::scrub(&raw.title, &id),
8986 body: crate::scrub::scrub(&raw.body, &id),
8987 }
8988}
8989
8990fn pr_message_raw(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8991 let summary = state
8992 .candidates
8993 .iter()
8994 .find(|c| c.label == winner)
8995 .map(|c| c.summary.as_str())
8996 .unwrap_or_default();
8997 let review = is_review_run(state);
9000 let title = if review {
9001 facts
9002 .and_then(|f| english_subject(&f.commits.first()?.0))
9003 .or_else(|| review_title(state))
9004 .or_else(|| {
9005 state
9006 .candidates
9007 .iter()
9008 .find(|c| c.label == winner)
9009 .filter(|c| !c.branch.starts_with("magi/"))
9010 .and_then(|c| english_subject(&c.branch))
9011 })
9012 .unwrap_or_else(|| neutral_title(state, winner))
9013 } else {
9014 summary_title(summary).unwrap_or_else(|| {
9015 english_title(&state.instruction, PR_TITLE_MAX)
9016 .unwrap_or_else(|| neutral_title(state, winner))
9017 })
9018 };
9019
9020 let mut body = String::new();
9021 let what = summary_without_title(summary);
9022 if !what.is_empty() {
9023 body.push_str("## Summary\n\n");
9024 body.push_str(&what);
9025 body.push_str("\n\n");
9026 }
9027
9028 let notes: Vec<(usize, &str)> = state
9031 .reviews
9032 .iter()
9033 .filter_map(|r| {
9034 let n = r.fix.as_ref()?.notes.trim();
9035 (!n.is_empty()).then_some((r.round, n))
9036 })
9037 .collect();
9038 if !notes.is_empty() {
9039 body.push_str("## Review fixes\n\n");
9040 if let [(_, only)] = notes.as_slice() {
9041 body.push_str(only);
9042 body.push_str("\n\n");
9043 } else {
9044 for (round, n) in ¬es {
9045 body.push_str(&format!("### Round {round}\n\n{n}\n\n"));
9046 }
9047 }
9048 }
9049 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
9050
9051 let open = state.open_findings();
9052 if !open.is_empty() {
9053 body.push_str("## Open review findings\n\n");
9054 for f in &open {
9055 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
9056 }
9057 body.push('\n');
9058 }
9059
9060 if let Some(fix) = fix
9061 && !fix.rejected.is_empty()
9062 {
9063 body.push_str("## Declined by the fixer\n\n");
9064 for r in &fix.rejected {
9065 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
9066 }
9067 body.push('\n');
9068 }
9069
9070 if review {
9071 body.push_str("## Commits under review\n\n");
9073 if let Some(facts) = facts {
9074 let mut left = COMMITS_MAX_CHARS;
9075 for (i, (subject, text)) in facts.commits.iter().enumerate() {
9076 let mut entry = format!("- {}\n", markdown_safe(subject));
9077 for l in markdown_safe(text).lines() {
9078 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
9079 }
9080 if left == 0 {
9081 body.push_str(&format!(
9082 "- ... {} more commit(s)\n",
9083 facts.commits.len() - i
9084 ));
9085 break;
9086 }
9087 if entry.len() > left {
9088 let mut end = left;
9091 while !entry.is_char_boundary(end) {
9092 end -= 1;
9093 }
9094 entry.truncate(end);
9095 entry.push_str("\n ... (truncated)\n");
9096 left = 0;
9097 } else {
9098 left -= entry.len();
9099 }
9100 body.push_str(&entry);
9101 }
9102 if !facts.stat.trim().is_empty() {
9103 body.push_str(&format!(
9104 "\n## Diff stat\n\n```\n{}\n```\n",
9105 markdown_safe(facts.stat.trim())
9106 ));
9107 }
9108 } else {
9109 match &state.reviewed_commits {
9110 Some(subjects) => {
9111 for s in subjects {
9112 body.push_str(&format!("- {}\n", s.trim()));
9113 }
9114 }
9115 None => {
9116 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
9119 body.push_str(rest.trim());
9120 body.push('\n');
9121 }
9122 }
9123 }
9124 } else {
9125 let task = state.instruction.trim();
9126 let task = if task.is_empty() {
9127 "(empty task)"
9128 } else {
9129 task
9130 };
9131 body.push_str(&format!(
9132 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
9133 task.replace("</details>", "</details>")
9134 ));
9135 }
9136
9137 body.push_str(&format!(
9138 "\n---\nmagi:run/{} magi:candidate-{}\n",
9139 state.id,
9140 winner.to_ascii_lowercase()
9141 ));
9142
9143 PrMessage { title, body }
9144}
9145
9146fn seeded_instruction(state: &RunState) -> String {
9150 match refs::describe(&state.seeds) {
9151 Some(facts) => format!(
9152 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
9153 Candidates start from the unmerged branch named above, when there \
9154 is one, and carry any unmerged commit named by sha as a \
9155 cherry-pick. Check that this is what the task meant before \
9156 building on it.",
9157 state.instruction
9158 ),
9159 None => state.instruction.clone(),
9160 }
9161}
9162
9163async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
9167 let base = &state.base_branch;
9168 let mut against = base.clone();
9169 if mode == MergeMode::Pr {
9170 let remote = &state.config.merge.remote;
9174 let tracking = format!("{remote}/{base}");
9175 let fetched = git::fetch(repo, remote, base).await;
9176 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
9177 against = tracking;
9178 } else {
9179 return false;
9180 }
9181 }
9182 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
9183}
9184
9185fn empty_candidate_detail(state: &RunState, base: &str) -> String {
9188 let mut detail = format!(
9189 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
9190 nothing to open a pull request for"
9191 );
9192 match refs::describe(&state.seeds) {
9193 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
9194 None => detail.push_str(
9195 "\nThe task names no existing branch or commit; if it means to land work \
9196 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
9197 ),
9198 }
9199 detail
9200}
9201
9202#[derive(Debug, PartialEq, Eq)]
9205enum PrPlan {
9206 Create,
9207 Adopt { url: String, title: String },
9208 Stop(String),
9209}
9210
9211fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
9214 match found {
9215 Ok(land::OpenPr::None) => PrPlan::Create,
9216 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
9217 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
9218 "several open pull requests exist for this branch, not picking one: {}",
9219 urls.join(" ")
9220 )),
9221 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
9222 }
9223}
9224
9225async fn gh_pr_create(
9227 cwd: &Path,
9228 base: &str,
9229 head: &str,
9230 title: &str,
9231 body: &str,
9232) -> Result<String> {
9233 let out = tokio::process::Command::new("gh")
9234 .args([
9235 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
9236 ])
9237 .current_dir(cwd)
9238 .quiet()
9239 .stdin(std::process::Stdio::null())
9240 .output()
9241 .await
9242 .context("spawn gh")?;
9243 if out.status.success() {
9244 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
9245 } else {
9246 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
9247 }
9248}
9249
9250pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
9259 let repo = state.repo.clone();
9260 let root = state.worktree_root();
9261 let winner = state.tally.as_ref().map(|t| t.winner);
9262 let mut removed = Vec::new();
9263
9264 for i in 0..state.candidates.len() {
9265 let c = state.candidates[i].clone();
9266 let is_winner = Some(c.label) == winner;
9267 if is_winner && !drop_winner {
9268 continue;
9269 }
9270 if c.worktree.exists() {
9271 git::worktree_remove(&repo, &c.worktree).await.ok();
9272 removed.push(c.worktree.to_string_lossy().into_owned());
9273 }
9274 let handed_over = state.released_branches.contains(&c.branch);
9277 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
9278 git::branch_delete(&repo, &c.branch).await.ok();
9279 removed.push(c.branch.clone());
9280 }
9281 state.candidates[i].folded = true;
9282 }
9283
9284 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
9285 let path = name.path();
9286 let keep = !drop_winner
9287 && winner.is_some_and(|w| {
9288 path.file_name()
9289 .is_some_and(|n| n == format!("cand-{w}").as_str())
9290 });
9291 if keep {
9292 continue;
9293 }
9294 git::worktree_remove(&repo, &path).await.ok();
9295 removed.push(path.to_string_lossy().into_owned());
9296 }
9297
9298 remove_if_empty(&root);
9307
9308 if state.enabled_worktree_config && drop_winner {
9309 git::release_worktree_config(&repo).await.ok();
9313 state.enabled_worktree_config = false;
9314 }
9315 state.save_under(home)?;
9316 Ok(removed)
9317}
9318
9319fn remove_if_empty(dir: &Path) {
9330 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
9331 std::fs::remove_dir(dir).ok();
9332 }
9333}
9334
9335pub fn worst_open(state: &RunState) -> Option<Severity> {
9337 state
9338 .reviews
9339 .last()?
9340 .reviews
9341 .iter()
9342 .flat_map(|r| r.findings.iter())
9343 .map(|f| f.severity)
9344 .max()
9345}
9346
9347async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
9351 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
9352 git::acquire_worktree_config(repo).await?;
9353 state.enabled_worktree_config = true;
9354 }
9355 Ok(())
9356}
9357
9358#[cfg(test)]
9359mod tests {
9360 #[test]
9361 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
9362 let leaked = vec!["chore(deps): update a crate".to_owned()];
9363 let own = "fix(daemon): apply a chosen action";
9364 assert!(should_retitle("", own, &leaked));
9365 assert!(should_retitle(
9366 &format!("{REVIEW_PROMPT_OPENING} `x`"),
9367 own,
9368 &leaked
9369 ));
9370 assert!(should_retitle(
9371 "chore: land candidate A of run 1",
9372 own,
9373 &leaked
9374 ));
9375 assert!(should_retitle(
9376 "magi: candidate A (uncommitted work)",
9377 own,
9378 &leaked
9379 ));
9380 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
9381 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
9382 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
9383 }
9384
9385 #[test]
9386 fn pr_merge_plan_creates_adopts_or_stops() {
9387 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
9388 assert_eq!(
9389 pr_merge_plan(Ok(land::OpenPr::One {
9390 url: "u".into(),
9391 title: "t".into()
9392 })),
9393 PrPlan::Adopt {
9394 url: "u".into(),
9395 title: "t".into()
9396 }
9397 );
9398 let PrPlan::Stop(many) =
9399 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
9400 else {
9401 panic!("many must stop");
9402 };
9403 assert!(many.contains('a') && many.contains('b'));
9404 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
9405 panic!("a failed lookup must stop");
9406 };
9407 assert!(err.contains("bad token"));
9408 }
9409
9410 use super::*;
9411 use crate::run::GateStatus;
9412 use std::collections::BTreeMap;
9413 use std::time::Duration;
9414
9415 fn conductor() -> AgentSpec {
9416 AgentSpec {
9417 id: "conductor".to_owned(),
9418 kind: crate::config::AgentKind::Command,
9419 model: None,
9420 command: vec!["true".to_owned()],
9421 extra_args: Vec::new(),
9422 env: BTreeMap::new(),
9423 prompt_delivery: None,
9424 }
9425 }
9426
9427 fn spec(id: &str) -> AgentSpec {
9428 AgentSpec {
9429 id: id.to_owned(),
9430 kind: crate::config::AgentKind::Command,
9431 model: None,
9432 command: vec!["true".to_owned()],
9433 extra_args: Vec::new(),
9434 env: BTreeMap::new(),
9435 prompt_delivery: None,
9436 }
9437 }
9438
9439 fn ids(xs: &[&str]) -> BTreeSet<String> {
9440 xs.iter().map(|s| (*s).to_owned()).collect()
9441 }
9442
9443 #[test]
9444 fn pick_successor_skips_an_agent_another_seat_holds() {
9445 let roster = [spec("a"), spec("b"), spec("c")];
9447 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
9448 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
9449 }
9450
9451 #[test]
9452 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
9453 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
9455 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
9456 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
9457 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
9458 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
9459 }
9460
9461 #[test]
9462 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
9463 let roster = [spec("a"), spec("b")];
9464 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
9465 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
9466 let carried = ids(&["b"]);
9467 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
9468 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
9469 }
9470
9471 #[test]
9472 fn pick_successor_returns_none_without_any_untried_successor() {
9473 let roster = [spec("a"), spec("b")];
9474 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
9475 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
9476 let carried = ids(&["a"]);
9477 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
9478 }
9479
9480 #[test]
9481 fn pick_successor_rescue_avoids_another_seats_occupant() {
9482 let roster = [spec("a"), spec("b"), spec("c")];
9484 let carried = ids(&["b", "c"]);
9485 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
9486 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
9487 }
9488
9489 #[test]
9490 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
9491 let roster = [spec("a"), spec("b"), spec("c")];
9492 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
9493 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
9494 }
9495
9496 #[test]
9497 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
9498 let roster = [spec("a"), spec("b"), spec("c")];
9499 let failed = ids(&["a", "b", "c"]);
9500 let mut tried = ids(&["b"]);
9502 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
9503 assert_eq!(first.id, "a");
9504 tried.insert(first.id.clone());
9505 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
9506 assert_eq!(second.id, "c");
9507 tried.insert(second.id.clone());
9508 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
9509 }
9510
9511 #[test]
9512 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
9513 let roster = [spec("a"), spec("b")];
9514 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
9515 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
9516 }
9517
9518 #[test]
9519 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
9520 let roster = [spec("a"), spec("b"), spec("c")];
9521 let h = SeatHistory {
9522 failed: ids(&["a"]),
9523 last_ok: Some("b".to_owned()),
9524 last_fail: None,
9525 };
9526 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
9527 let gone = SeatHistory {
9529 last_ok: Some("zzz".to_owned()),
9530 ..h.clone()
9531 };
9532 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
9533 let failed = SeatHistory {
9534 failed: ids(&["a", "b"]),
9535 last_ok: Some("b".to_owned()),
9536 last_fail: None,
9537 };
9538 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
9539 }
9540
9541 #[test]
9542 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
9543 let first = SeatState::new("review-1", "alpha", 7);
9544 let next = handover_seat("review-1", "gamma", 7);
9545 assert_ne!(first.claude_session, next.claude_session);
9546 }
9547
9548 #[test]
9549 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
9550 let mut state = state_with_summary("x", "y");
9551 let a = handover_seat("review-1", "beta", state.next_seat_seed());
9552 let b = handover_seat("review-1", "beta", state.next_seat_seed());
9553 assert_ne!(a.claude_session, b.claude_session);
9554 }
9555
9556 #[test]
9557 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
9558 let roster = [spec("a"), spec("b")];
9559 let h = SeatHistory {
9560 failed: ids(&["a", "b"]),
9561 ..SeatHistory::default()
9562 };
9563 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
9564 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
9565 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
9566 }
9567
9568 #[test]
9575 fn failure_signature_ignores_numbers_and_paths() {
9576 assert_eq!(
9577 failure_signature("exited with Some(2) and no usable output"),
9578 failure_signature("exited with Some(137) and no usable output")
9579 );
9580 assert_eq!(
9581 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
9582 failure_signature("cannot open /var/x.txt: denied")
9583 );
9584 assert_ne!(failure_signature("boom"), failure_signature("bang"));
9585 }
9586
9587 #[test]
9588 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
9589 let other = FailClass::Other("x".into());
9590 assert!(should_hand_over(None, &FailClass::Quota));
9591 assert!(should_hand_over(Some(&other), &FailClass::Quota));
9592 assert!(should_hand_over(
9593 Some(&FailClass::Timeout),
9594 &FailClass::Timeout
9595 ));
9596 assert!(should_hand_over(None, &other));
9597 assert!(!should_hand_over(Some(&other), &other));
9598 assert!(should_hand_over(
9599 Some(&other),
9600 &FailClass::Other("y".into())
9601 ));
9602 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
9604 assert!(should_hand_over(Some(&FailClass::Quota), &other));
9605 }
9606
9607 #[test]
9608 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
9609 let a = SeatState::new("judge-1", "alpha", 7);
9610 let b = handover_seat("judge-1", "beta", 7);
9611 assert_ne!(a.claude_session, b.claude_session);
9612 assert_eq!(b.turns, 0);
9613 }
9614
9615 #[test]
9616 fn a_timeout_is_classified_apart_from_other_failures() {
9617 assert_eq!(
9618 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
9619 Some(FailClass::Timeout)
9620 );
9621 assert!(matches!(
9622 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
9623 Some(FailClass::Other(_))
9624 ));
9625 }
9626
9627 #[test]
9628 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
9629 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
9630 let tried = BTreeSet::from(["beta".to_owned()]);
9631 let next = next_untried_in_roster(&roster, 1, &tried);
9634 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
9635 }
9636
9637 #[test]
9638 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
9639 let roster = vec![spec("alpha"), spec("beta")];
9640 let tried = BTreeSet::from(["beta".to_owned()]);
9641 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
9645 }
9646
9647 #[test]
9648 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
9649 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
9650 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
9651 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
9655 }
9656
9657 #[test]
9658 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
9659 let roster = vec![spec("a"), spec("a"), spec("b")];
9660 let tried = BTreeSet::from(["a".to_owned()]);
9661 let next = next_untried_in_roster(&roster, 0, &tried);
9662 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
9663 }
9664
9665 #[test]
9666 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
9667 let roster = vec![spec("a"), spec("b")];
9668 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
9669 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
9670 }
9671
9672 #[test]
9673 fn remove_if_empty_only_ever_takes_a_bare_directory() {
9674 let dir = tempfile::tempdir().unwrap();
9675 let bay = dir.path().join("ffff");
9676
9677 remove_if_empty(&bay);
9679 assert!(!bay.exists());
9680
9681 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
9684 remove_if_empty(&bay);
9685 assert!(bay.exists(), "non-empty directory must survive");
9686
9687 std::fs::remove_dir(bay.join("cand-A")).unwrap();
9689 remove_if_empty(&bay);
9690 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
9691 }
9692
9693 #[test]
9702 fn a_full_panel_that_found_nothing_is_clean() {
9703 assert!(round_is_clean(
9704 0,
9705 true,
9706 2,
9707 2,
9708 0,
9709 IncompleteReviewPolicy::Block
9710 ));
9711 }
9712
9713 #[test]
9714 fn a_missing_seat_is_never_clean_under_the_default_policy() {
9715 assert!(!round_is_clean(
9716 0,
9717 true,
9718 1,
9719 2,
9720 0,
9721 IncompleteReviewPolicy::Block
9722 ));
9723 }
9724
9725 #[test]
9726 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
9727 assert!(!round_is_clean(
9728 1,
9729 true,
9730 1,
9731 2,
9732 0,
9733 IncompleteReviewPolicy::Warn
9734 ));
9735 }
9736
9737 #[test]
9738 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
9739 assert!(round_is_clean(
9740 0,
9741 true,
9742 1,
9743 2,
9744 0,
9745 IncompleteReviewPolicy::Warn
9746 ));
9747 }
9748
9749 #[test]
9750 fn a_full_panel_with_an_open_finding_is_not_clean() {
9751 assert!(!round_is_clean(
9752 1,
9753 true,
9754 2,
9755 2,
9756 0,
9757 IncompleteReviewPolicy::Block
9758 ));
9759 }
9760
9761 #[test]
9762 fn a_full_panel_with_a_red_e2e_is_not_clean() {
9763 assert!(!round_is_clean(
9764 0,
9765 false,
9766 2,
9767 2,
9768 0,
9769 IncompleteReviewPolicy::Block
9770 ));
9771 }
9772
9773 #[test]
9780 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
9781 assert!(round_is_clean(
9784 0,
9785 true,
9786 1,
9787 2,
9788 1,
9789 IncompleteReviewPolicy::Block
9790 ));
9791 }
9792
9793 #[test]
9794 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
9795 assert!(!round_is_clean(
9798 0,
9799 true,
9800 1,
9801 2,
9802 0,
9803 IncompleteReviewPolicy::Block
9804 ));
9805 }
9806
9807 #[test]
9808 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
9809 assert!(!round_is_clean(
9810 1,
9811 true,
9812 1,
9813 2,
9814 1,
9815 IncompleteReviewPolicy::Block
9816 ));
9817 assert!(!round_is_clean(
9818 0,
9819 false,
9820 1,
9821 2,
9822 1,
9823 IncompleteReviewPolicy::Block
9824 ));
9825 }
9826
9827 #[test]
9828 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
9829 assert!(!round_is_clean(
9833 0,
9834 true,
9835 0,
9836 2,
9837 2,
9838 IncompleteReviewPolicy::Block
9839 ));
9840 }
9841
9842 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
9843 CommandOutcome {
9844 command: "test".to_owned(),
9845 code,
9846 output_tail: String::new(),
9847 duration_ms: 0,
9848 resource_blocked,
9849 }
9850 }
9851
9852 #[test]
9853 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
9854 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
9855 assert!(
9856 !verify_inconclusive(&[outcome(Some(1), false)]),
9857 "an ordinary failure is still evidence about the patch"
9858 );
9859 assert!(verify_inconclusive(&[outcome(None, true)]));
9860 assert!(
9861 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
9862 "one inconclusive outcome taints the whole batch"
9863 );
9864 assert!(!verify_inconclusive(&[]));
9865 }
9866
9867 #[tokio::test]
9868 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
9869 let calls = std::sync::atomic::AtomicUsize::new(0);
9873 let started = Instant::now();
9874 wait_for_pids_with(
9875 &[123],
9876 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
9877 Duration::from_millis(5),
9878 Duration::from_secs(5),
9879 )
9880 .await;
9881 assert!(
9882 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
9883 "must keep checking rather than deciding on the first answer"
9884 );
9885 assert!(
9886 started.elapsed() < Duration::from_secs(1),
9887 "must return the moment it is confirmed dead, not wait out the ceiling"
9888 );
9889 }
9890
9891 #[tokio::test]
9892 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
9893 let started = Instant::now();
9894 wait_for_pids_with(
9895 &[123],
9896 |_| true, Duration::from_millis(5),
9898 Duration::from_millis(30),
9899 )
9900 .await;
9901 let elapsed = started.elapsed();
9902 assert!(
9903 elapsed >= Duration::from_millis(30),
9904 "must not give up before its own ceiling: {elapsed:?}"
9905 );
9906 assert!(
9907 elapsed < Duration::from_secs(1),
9908 "must not wait past its own ceiling either: {elapsed:?}"
9909 );
9910 }
9911
9912 #[tokio::test]
9913 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
9914 let started = Instant::now();
9915 wait_for_pids_with(
9916 &[],
9917 |_| true,
9918 Duration::from_secs(5),
9919 Duration::from_secs(5),
9920 )
9921 .await;
9922 assert!(
9923 started.elapsed() < Duration::from_millis(200),
9924 "an empty pid list has nothing to confirm"
9925 );
9926 }
9927
9928 fn review_round(
9934 clean: bool,
9935 blocking: usize,
9936 answered: usize,
9937 expected: usize,
9938 progressed: bool,
9939 e2e_ok: bool,
9940 ) -> ReviewRound {
9941 ReviewRound {
9942 round: 1,
9943 head: "h".to_owned(),
9944 verified_head: None,
9945 verified_at: None,
9946 reviews: Vec::new(),
9947 e2e: vec![CommandOutcome {
9948 command: "test".to_owned(),
9949 code: Some(if e2e_ok { 0 } else { 1 }),
9950 output_tail: String::new(),
9951 duration_ms: 0,
9952 resource_blocked: false,
9953 }],
9954 verify_retried: false,
9955 e2e_deferred: false,
9956 e2e_defer_reason: None,
9957 fix: None,
9958 blocking,
9959 answered,
9960 expected,
9961 clean,
9962 progressed,
9963 vote_split: false,
9964 reconsideration: Vec::new(),
9965 verdict: None,
9966 }
9967 }
9968
9969 #[test]
9970 fn review_conclusion_is_none_when_nothing_has_run() {
9971 assert_eq!(review_conclusion(&[], 3), None);
9972 }
9973
9974 #[test]
9975 fn review_conclusion_is_none_while_rounds_remain() {
9976 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
9977 assert_eq!(review_conclusion(&rounds, 3), None);
9978 }
9979
9980 #[test]
9981 fn review_conclusion_is_gating_once_a_round_is_clean() {
9982 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
9983 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
9984 }
9985
9986 #[test]
9987 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
9988 let rounds = vec![
9989 review_round(false, 1, 2, 2, true, true),
9990 review_round(false, 1, 2, 2, true, true),
9991 ];
9992 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
9993 }
9994
9995 #[test]
9996 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
9997 let rounds = vec![
9998 review_round(false, 1, 2, 2, true, true),
9999 review_round(false, 1, 2, 2, true, false),
10000 ];
10001 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
10002 }
10003
10004 #[test]
10005 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
10006 let mut blocked = review_round(false, 1, 2, 2, true, false);
10013 blocked.e2e[0].resource_blocked = true;
10014 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
10015 assert_eq!(review_conclusion(&rounds, 2), None);
10016 }
10017
10018 #[test]
10019 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
10020 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
10022 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
10023 }
10024
10025 #[test]
10026 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
10027 let rounds = vec![
10028 review_round(false, 1, 2, 2, false, true),
10029 review_round(false, 1, 2, 2, false, true),
10030 ];
10031 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
10032 }
10033
10034 fn secs(n: u64) -> Duration {
10035 Duration::from_secs(n)
10036 }
10037
10038 fn init_repo(dir: &Path) {
10041 let run = |args: &[&str]| {
10042 let out = std::process::Command::new("git")
10043 .args(args)
10044 .current_dir(dir)
10045 .quiet()
10046 .output()
10047 .expect("spawn git");
10048 assert!(
10049 out.status.success(),
10050 "git {args:?} failed: {}",
10051 String::from_utf8_lossy(&out.stderr)
10052 );
10053 };
10054 run(&["init", "-b", "main"]);
10055 run(&["config", "user.name", "magi test"]);
10056 run(&["config", "user.email", "magi@example.com"]);
10057 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
10058 run(&["add", "-A"]);
10059 run(&["commit", "-m", "init"]);
10060 }
10061
10062 fn ask_test_home() {
10070 crate::run::pin_test_home();
10071 }
10072
10073 fn runner_at(status: RunStatus) -> Runner {
10076 let mut state = RunState::new(
10077 PathBuf::from("/nonexistent/repo"),
10078 "main".to_owned(),
10079 "deadbeef".to_owned(),
10080 "task".to_owned(),
10081 Config::default(),
10082 );
10083 state.status = status;
10084 Runner {
10085 state,
10086 roles: ResolvedRoles {
10087 implementers: Vec::new(),
10088 judges: Vec::new(),
10089 reviewers: Vec::new(),
10090 fixer: None,
10091 conductor: conductor(),
10092 implementer_roster: Vec::new(),
10093 judge_roster: Vec::new(),
10094 reviewer_roster: Vec::new(),
10095 },
10096 sem: Arc::new(Semaphore::new(1)),
10097 pause: Pause::new(),
10098 interrupt: Pause::new(),
10099 }
10100 }
10101
10102 #[test]
10106 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
10107 crate::run::pin_test_home();
10108 let mut runner = runner_at(RunStatus::Implementing);
10109 let interrupt = Pause::new();
10110 runner.watch_interrupt(interrupt.clone());
10111
10112 interrupt.park_because("task a1b2 asked to run first");
10113
10114 assert!(runner.park_here().expect("park_here"));
10115 assert!(runner.state.parked);
10116 let last = runner.state.events.last().expect("a park event");
10117 assert_eq!(last.node, "park");
10118 assert!(
10119 last.message.contains("task a1b2 asked to run first"),
10120 "expected the interrupt reason in {:?}",
10121 last.message
10122 );
10123 }
10124
10125 #[test]
10133 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
10134 crate::run::pin_test_home();
10135 let mut runner = runner_at(RunStatus::Implementing);
10136 let shutdown = Pause::new();
10137 runner.on_pause(shutdown.clone());
10138 let interrupt = Pause::new();
10139 runner.watch_interrupt(interrupt.clone());
10140
10141 assert!(!runner.park_here().expect("park_here"));
10143 assert!(!runner.state.parked);
10144
10145 interrupt.park_because("test");
10147 assert!(!shutdown.parked());
10148 assert!(runner.park_here().expect("park_here"));
10149 }
10150
10151 #[tokio::test]
10165 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
10166 crate::run::pin_test_home();
10167 let mut runner = runner_at(RunStatus::Implementing);
10168 let interrupt = Pause::new();
10169 runner.watch_interrupt(interrupt.clone());
10170
10171 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
10172 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
10173
10174 let node = async move {
10178 started_tx.send(()).expect("send started");
10179 finish_rx.await.expect("recv finish");
10180 "node finished"
10181 };
10182
10183 let interrupter = async move {
10184 started_rx.await.expect("recv started");
10185 interrupt.park_because("higher-priority task waiting");
10187 tokio::task::yield_now().await;
10191 finish_tx.send(()).expect("send finish");
10192 };
10193
10194 let (node_result, ()) = tokio::join!(node, interrupter);
10195 assert_eq!(
10196 node_result, "node finished",
10197 "the in-flight call ran to completion"
10198 );
10199
10200 assert!(runner.park_here().expect("park_here"));
10203 assert!(runner.state.parked);
10204 }
10205
10206 #[test]
10212 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
10213 crate::run::pin_test_home();
10214 let mut runner = runner_at(RunStatus::Judging);
10215 runner.state.config.agents = vec![conductor()];
10219 runner.state.candidates = vec![Candidate {
10220 index: 0,
10221 label: 'A',
10222 agent: "alpha".to_owned(),
10223 branch: "magi/x/A".to_owned(),
10224 worktree: PathBuf::from("/nonexistent/worktree"),
10225 summary: "did the thing".to_owned(),
10226 stat: "1 file changed".to_owned(),
10227 files: 1,
10228 commits: 1,
10229 empty: false,
10230 failed: None,
10231 verified_noop: None,
10232 duration_ms: 1234,
10233 folded: false,
10234 }];
10235 let run_id = runner.state.id.clone();
10236
10237 let interrupt = Pause::new();
10238 runner.watch_interrupt(interrupt.clone());
10239 interrupt.park_because("task c3d4 asked to run first");
10240 assert!(runner.park_here().expect("park_here"));
10241
10242 let resumed = Runner::resume(&run_id).expect("resume");
10243 assert_eq!(resumed.state.candidates.len(), 1);
10244 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
10245 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
10246 assert_eq!(resumed.state.status, runner.state.status);
10247 assert!(
10248 resumed.state.parked,
10249 "still parked until `execute` actually walks the graph again"
10250 );
10251 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
10252 }
10253
10254 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
10256 let mut q = ask::Question::new(
10257 run.to_owned(),
10258 "implement".to_owned(),
10259 "impl-A".to_owned(),
10260 "Which storage backend should the cache use?".to_owned(),
10261 String::new(),
10262 vec!["SQLite".to_owned(), "Redis".to_owned()],
10263 );
10264 store.put(&mut q).unwrap();
10265 q
10266 }
10267
10268 #[test]
10269 fn a_failed_runs_open_question_is_abandoned() {
10270 ask_test_home();
10271 let store = ask::Questions::open();
10272 let mut runner = runner_at(RunStatus::Failed);
10273 let run = runner.state.id.clone();
10274 let q = ask_open_question(&store, &run);
10275
10276 runner.settle_questions();
10277
10278 let back = store.get(&q.id).unwrap();
10279 assert!(
10280 !back.status.open(),
10281 "the seat that asked died with the run; nobody is left to read an answer"
10282 );
10283 assert!(
10284 back.detail.contains(&run) && back.detail.contains("failed"),
10285 "the reason names what the run became, not just that it is gone: {}",
10286 back.detail
10287 );
10288 }
10289
10290 #[test]
10291 fn a_merged_runs_open_question_is_abandoned_too() {
10292 ask_test_home();
10293 let store = ask::Questions::open();
10294 for status in [RunStatus::Merged, RunStatus::Ready] {
10297 let mut runner = runner_at(status);
10298 let run = runner.state.id.clone();
10299 let q = ask_open_question(&store, &run);
10300
10301 runner.settle_questions();
10302
10303 let back = store.get(&q.id).unwrap();
10304 assert!(
10305 !back.status.open(),
10306 "{status:?} run's question must not outlive the run"
10307 );
10308 }
10309 }
10310
10311 #[test]
10312 fn a_still_resumable_runs_open_question_is_left_alone() {
10313 ask_test_home();
10314 let store = ask::Questions::open();
10315 for status in [RunStatus::Blocked, RunStatus::Stalled] {
10321 let mut runner = runner_at(status);
10322 let run = runner.state.id.clone();
10323 let q = ask_open_question(&store, &run);
10324
10325 runner.settle_questions();
10326
10327 let back = store.get(&q.id).unwrap();
10328 assert!(
10329 back.status.open(),
10330 "{status:?} is still alive; the question must still be waiting"
10331 );
10332 }
10333 }
10334
10335 #[test]
10336 fn settle_questions_never_touches_an_already_answered_question() {
10337 ask_test_home();
10338 let store = ask::Questions::open();
10339 let mut runner = runner_at(RunStatus::Failed);
10340 let run = runner.state.id.clone();
10341 let mut q = ask_open_question(&store, &run);
10342 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
10343 .unwrap();
10344 store.put(&mut q).unwrap();
10345
10346 runner.settle_questions();
10351 runner.settle_questions();
10352
10353 let back = store.get(&q.id).unwrap();
10354 assert_eq!(
10355 back.status,
10356 ask::QuestionStatus::Answered,
10357 "a real answer is a decision on record, never overwritten by a sweep"
10358 );
10359 }
10360
10361 #[tokio::test]
10372 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
10373 crate::run::pin_test_home();
10374 let tmp = tempfile::tempdir().expect("tempdir");
10375 let repo = tmp.path().join("repo");
10376 std::fs::create_dir_all(&repo).unwrap();
10377 init_repo(&repo);
10378
10379 let mut config = Config::default();
10380 config.graph.worktree_root = Some(tmp.path().join("wt"));
10381
10382 let mut state = RunState::new(
10383 repo.clone(),
10384 "main".to_owned(),
10385 "deadbeef".to_owned(),
10386 "task".to_owned(),
10387 config,
10388 );
10389 let root = state.worktree_root();
10390 let wt_a = root.join("cand-A");
10391 let wt_b = root.join("cand-B");
10392 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
10393 .await
10394 .expect("worktree A");
10395 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
10396 .await
10397 .expect("worktree B");
10398
10399 state.candidates = vec![
10400 Candidate {
10401 index: 0,
10402 label: 'A',
10403 agent: "alpha".to_owned(),
10404 branch: "magi/x/A".to_owned(),
10405 worktree: wt_a.clone(),
10406 summary: String::new(),
10407 stat: String::new(),
10408 files: 0,
10409 commits: 0,
10410 empty: false,
10411 failed: None,
10412 verified_noop: None,
10413 duration_ms: 0,
10414 folded: false,
10415 },
10416 Candidate {
10417 index: 1,
10418 label: 'B',
10419 agent: "beta".to_owned(),
10420 branch: "magi/x/B".to_owned(),
10421 worktree: wt_b.clone(),
10422 summary: String::new(),
10423 stat: String::new(),
10424 files: 0,
10425 commits: 0,
10426 empty: false,
10427 failed: None,
10428 verified_noop: None,
10429 duration_ms: 0,
10430 folded: false,
10431 },
10432 ];
10433 state.tally = Some(Tally {
10434 first_choice: BTreeMap::from([('A', 1)]),
10435 borda: BTreeMap::new(),
10436 winner: 'A',
10437 rankings: 1,
10438 unanimous_initial: true,
10439 deliberated: false,
10440 changed_votes: 0,
10441 unanimous_final: true,
10442 tie_break: None,
10443 judges: 1,
10444 present: 1,
10445 quorum: 1,
10446 met_quorum: true,
10447 uncontested: None,
10448 });
10449 state.status = RunStatus::Ready;
10450
10451 fold_run(&mut state, false, &crate::run::home())
10452 .await
10453 .expect("fold_run");
10454
10455 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
10456 assert!(
10457 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
10458 "the unmerged winner's branch survives"
10459 );
10460 assert!(
10461 !state.candidates[0].folded,
10462 "the winner is not marked folded"
10463 );
10464
10465 assert!(!wt_b.exists(), "the loser's worktree is removed");
10466 assert!(
10467 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
10468 "the loser's branch is removed"
10469 );
10470 assert!(state.candidates[1].folded, "the loser is marked folded");
10471 }
10472
10473 #[tokio::test]
10476 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
10477 let tmp = tempfile::tempdir().expect("tempdir");
10478 let repo = tmp.path().join("repo");
10479 std::fs::create_dir_all(&repo).unwrap();
10480 init_repo(&repo);
10481 let home = tmp.path().join("home");
10482
10483 let mut config = Config::default();
10484 config.graph.worktree_root = Some(tmp.path().join("wt"));
10485 let mut state = RunState::new(
10486 repo.clone(),
10487 "main".to_owned(),
10488 "deadbeef".to_owned(),
10489 "task".to_owned(),
10490 config,
10491 );
10492 git::git(&repo, &["branch", "magi/x/A", "main"])
10494 .await
10495 .expect("branch");
10496 state.candidates = vec![Candidate {
10497 index: 0,
10498 label: 'A',
10499 agent: "alpha".to_owned(),
10500 branch: "magi/x/A".to_owned(),
10501 worktree: state.worktree_root().join("cand-A"),
10502 summary: String::new(),
10503 stat: String::new(),
10504 files: 0,
10505 commits: 0,
10506 empty: false,
10507 failed: None,
10508 verified_noop: None,
10509 duration_ms: 0,
10510 folded: true,
10511 }];
10512 state.released_to = Some("20260901-000000-new1".to_owned());
10513 state.released_branches = vec!["magi/x/A".to_owned()];
10514
10515 fold_run(&mut state, true, &home).await.expect("fold_run");
10516
10517 assert!(
10518 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
10519 "the handed-over branch survives a fold"
10520 );
10521 }
10522
10523 #[tokio::test]
10527 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
10528 let tmp = tempfile::tempdir().expect("tempdir");
10529 let repo = tmp.path().join("repo");
10530 std::fs::create_dir_all(&repo).unwrap();
10531 init_repo(&repo);
10532 let run = |args: &[&str]| {
10533 let out = std::process::Command::new("git")
10534 .quiet()
10535 .args(args)
10536 .current_dir(&repo)
10537 .output()
10538 .expect("spawn git");
10539 assert!(out.status.success(), "git {args:?}");
10540 };
10541 run(&["branch", "magi/x/A"]);
10542 run(&["checkout", "-q", "-b", "magi/x/B"]);
10543 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
10544 run(&["add", "-A"]);
10545 run(&["commit", "-q", "-m", "work"]);
10546 run(&["checkout", "-q", "main"]);
10547
10548 let probe = RunState::new(
10552 repo.clone(),
10553 "main".to_owned(),
10554 "deadbeef".to_owned(),
10555 "task".to_owned(),
10556 Config::default(),
10557 );
10558 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
10559 let bare = tmp.path().join("origin.git");
10560 let out = std::process::Command::new("git")
10561 .quiet()
10562 .args(["init", "-q", "--bare"])
10563 .arg(&bare)
10564 .output()
10565 .expect("spawn git");
10566 assert!(out.status.success(), "git init --bare");
10567 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
10568 run(&["push", "-q", "origin", "main"]);
10569
10570 let mut state = RunState::new(
10571 repo.clone(),
10572 "main".to_owned(),
10573 "deadbeef".to_owned(),
10574 "task".to_owned(),
10575 Config::default(),
10576 );
10577 state.seeds = vec![refs::Seed {
10578 token: "magi/27b2/A".to_owned(),
10579 kind: refs::SeedKind::Unresolved,
10580 sha: String::new(),
10581 branch: true,
10582 detail: "no branch or commit named magi/27b2/A".to_owned(),
10583 }];
10584
10585 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
10586 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
10587 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
10588 let detail = empty_candidate_detail(&state, "main");
10589 assert!(detail.starts_with("empty candidate"), "{detail}");
10590 assert!(detail.contains("magi/27b2/A"), "{detail}");
10591 }
10592
10593 #[tokio::test]
10602 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
10603 let tmp = tempfile::tempdir().expect("tempdir");
10604 let repo = tmp.path().join("repo");
10605 std::fs::create_dir_all(&repo).unwrap();
10606 init_repo(&repo);
10607
10608 let mut config = Config::default();
10609 config.merge.mode = MergeMode::Local;
10610
10611 let mut state = RunState::new(
10612 repo.clone(),
10613 "main".to_owned(),
10614 "deadbeef".to_owned(),
10615 "task".to_owned(),
10616 config,
10617 );
10618 state.candidates = vec![Candidate {
10619 index: 0,
10620 label: 'A',
10621 agent: "alpha".to_owned(),
10622 branch: "does-not-exist".to_owned(),
10623 worktree: repo.clone(),
10624 summary: String::new(),
10625 stat: String::new(),
10626 files: 0,
10627 commits: 0,
10628 empty: false,
10629 failed: None,
10630 verified_noop: None,
10631 duration_ms: 0,
10632 folded: false,
10633 }];
10634 state.tally = Some(Tally {
10635 first_choice: BTreeMap::from([('A', 1)]),
10636 borda: BTreeMap::new(),
10637 winner: 'A',
10638 rankings: 1,
10639 unanimous_initial: true,
10640 deliberated: false,
10641 changed_votes: 0,
10642 unanimous_final: true,
10643 tie_break: None,
10644 judges: 0,
10645 present: 0,
10646 quorum: 0,
10647 met_quorum: true,
10648 uncontested: Some("only candidate A produced a change".to_owned()),
10649 });
10650 state.reviews = vec![ReviewRound {
10651 round: 1,
10652 head: "deadbeef".to_owned(),
10653 verified_head: None,
10654 verified_at: None,
10655 reviews: Vec::new(),
10656 e2e: Vec::new(),
10657 fix: None,
10658 blocking: 0,
10659 answered: 0,
10660 expected: 0,
10661 clean: true,
10662 verify_retried: false,
10663 e2e_deferred: false,
10664 e2e_defer_reason: None,
10665 progressed: false,
10666 vote_split: false,
10667 reconsideration: Vec::new(),
10668 verdict: None,
10669 }];
10670 state.gate = vec![CommandOutcome {
10671 command: "test".to_owned(),
10672 code: Some(0),
10673 output_tail: String::new(),
10674 duration_ms: 0,
10675 resource_blocked: false,
10676 }];
10677 state.gate_ran = true;
10678 state.status = RunStatus::Ready;
10683 state.merge = Some(MergeOutcome {
10684 mode: MergeMode::Local,
10685 ok: false,
10686 detail: "already concluded".to_owned(),
10687 empty: false,
10688 });
10689
10690 let mut runner = Runner {
10691 state,
10692 roles: ResolvedRoles {
10693 implementers: Vec::new(),
10694 judges: Vec::new(),
10695 reviewers: Vec::new(),
10696 fixer: None,
10697 conductor: conductor(),
10698 implementer_roster: Vec::new(),
10699 judge_roster: Vec::new(),
10700 reviewer_roster: Vec::new(),
10701 },
10702 sem: Arc::new(Semaphore::new(1)),
10703 pause: Pause::new(),
10704 interrupt: Pause::new(),
10705 };
10706
10707 runner.merge().await.expect("merge");
10708
10709 assert_eq!(
10710 runner.state.status,
10711 RunStatus::Ready,
10712 "a concluded run's status must not change on reentry"
10713 );
10714 assert_eq!(
10715 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10716 Some("already concluded"),
10717 "merge must not run again once the node already recorded an outcome"
10718 );
10719 }
10720
10721 #[tokio::test]
10730 async fn merge_refuses_a_gate_that_has_not_actually_run() {
10731 let tmp = tempfile::tempdir().expect("tempdir");
10732 let repo = tmp.path().join("repo");
10733 std::fs::create_dir_all(&repo).unwrap();
10734 init_repo(&repo);
10735
10736 let mut config = Config::default();
10737 config.merge.mode = MergeMode::Local;
10738
10739 let mut state = RunState::new(
10740 repo.clone(),
10741 "main".to_owned(),
10742 "deadbeef".to_owned(),
10743 "task".to_owned(),
10744 config,
10745 );
10746 state.candidates = vec![Candidate {
10747 index: 0,
10748 label: 'A',
10749 agent: "alpha".to_owned(),
10750 branch: "does-not-exist".to_owned(),
10751 worktree: repo.clone(),
10752 summary: String::new(),
10753 stat: String::new(),
10754 files: 0,
10755 commits: 0,
10756 empty: false,
10757 failed: None,
10758 verified_noop: None,
10759 duration_ms: 0,
10760 folded: false,
10761 }];
10762 state.tally = Some(Tally {
10763 first_choice: BTreeMap::from([('A', 1)]),
10764 borda: BTreeMap::new(),
10765 winner: 'A',
10766 rankings: 1,
10767 unanimous_initial: true,
10768 deliberated: false,
10769 changed_votes: 0,
10770 unanimous_final: true,
10771 tie_break: None,
10772 judges: 0,
10773 present: 0,
10774 quorum: 0,
10775 met_quorum: true,
10776 uncontested: Some("only candidate A produced a change".to_owned()),
10777 });
10778 state.reviews = vec![ReviewRound {
10779 round: 1,
10780 head: "deadbeef".to_owned(),
10781 verified_head: None,
10782 verified_at: None,
10783 reviews: Vec::new(),
10784 e2e: Vec::new(),
10785 fix: None,
10786 blocking: 0,
10787 answered: 0,
10788 expected: 0,
10789 clean: true,
10790 verify_retried: false,
10791 e2e_deferred: false,
10792 e2e_defer_reason: None,
10793 progressed: false,
10794 vote_split: false,
10795 reconsideration: Vec::new(),
10796 verdict: None,
10797 }];
10798 state.gate = Vec::new();
10800 state.gate_ran = false;
10801 state.status = RunStatus::Gating;
10802
10803 let mut runner = Runner {
10804 state,
10805 roles: ResolvedRoles {
10806 implementers: Vec::new(),
10807 judges: Vec::new(),
10808 reviewers: Vec::new(),
10809 fixer: None,
10810 conductor: conductor(),
10811 implementer_roster: Vec::new(),
10812 judge_roster: Vec::new(),
10813 reviewer_roster: Vec::new(),
10814 },
10815 sem: Arc::new(Semaphore::new(1)),
10816 pause: Pause::new(),
10817 interrupt: Pause::new(),
10818 };
10819
10820 runner.merge().await.expect("merge");
10821
10822 assert!(
10823 runner.state.merge.is_none(),
10824 "an empty gate must never be read as a passing one: {:?}",
10825 runner.state.merge
10826 );
10827 }
10828
10829 #[tokio::test]
10836 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
10837 let tmp = tempfile::tempdir().expect("tempdir");
10838 let repo = tmp.path().join("repo");
10839 std::fs::create_dir_all(&repo).unwrap();
10840 init_repo(&repo);
10841
10842 let config = Config::default();
10844
10845 let mut state = RunState::new(
10846 repo.clone(),
10847 "main".to_owned(),
10848 "deadbeef".to_owned(),
10849 "task".to_owned(),
10850 config,
10851 );
10852 state.candidates = vec![Candidate {
10853 index: 0,
10854 label: 'A',
10855 agent: "alpha".to_owned(),
10856 branch: "does-not-exist".to_owned(),
10857 worktree: repo.clone(),
10858 summary: String::new(),
10859 stat: String::new(),
10860 files: 0,
10861 commits: 0,
10862 empty: false,
10863 failed: None,
10864 verified_noop: None,
10865 duration_ms: 0,
10866 folded: false,
10867 }];
10868 state.tally = Some(Tally {
10869 first_choice: BTreeMap::from([('A', 1)]),
10870 borda: BTreeMap::new(),
10871 winner: 'A',
10872 rankings: 1,
10873 unanimous_initial: true,
10874 deliberated: false,
10875 changed_votes: 0,
10876 unanimous_final: true,
10877 tie_break: None,
10878 judges: 0,
10879 present: 0,
10880 quorum: 0,
10881 met_quorum: true,
10882 uncontested: Some("only candidate A produced a change".to_owned()),
10883 });
10884 state.reviews = vec![ReviewRound {
10885 round: 1,
10886 head: "deadbeef".to_owned(),
10887 verified_head: None,
10888 verified_at: None,
10889 reviews: Vec::new(),
10890 e2e: Vec::new(),
10891 fix: None,
10892 blocking: 0,
10893 answered: 0,
10894 expected: 0,
10895 clean: true,
10896 verify_retried: false,
10897 e2e_deferred: false,
10898 e2e_defer_reason: None,
10899 progressed: false,
10900 vote_split: false,
10901 reconsideration: Vec::new(),
10902 verdict: None,
10903 }];
10904
10905 let mut runner = Runner {
10906 state,
10907 roles: ResolvedRoles {
10908 implementers: Vec::new(),
10909 judges: Vec::new(),
10910 reviewers: Vec::new(),
10911 fixer: None,
10912 conductor: conductor(),
10913 implementer_roster: Vec::new(),
10914 judge_roster: Vec::new(),
10915 reviewer_roster: Vec::new(),
10916 },
10917 sem: Arc::new(Semaphore::new(1)),
10918 pause: Pause::new(),
10919 interrupt: Pause::new(),
10920 };
10921
10922 runner.gate().await.expect("gate");
10923 assert!(
10924 runner.state.gate_ran,
10925 "zero configured commands is still a real attempt, not an unrun gate"
10926 );
10927 assert!(runner.state.gate.is_empty());
10928 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
10929 assert_ne!(
10930 runner.state.status,
10931 RunStatus::Blocked,
10932 "a gate with nothing to check must not read as failed"
10933 );
10934
10935 runner.merge().await.expect("merge");
10936 assert_eq!(
10937 runner.state.status,
10938 RunStatus::Ready,
10939 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
10940 );
10941 }
10942
10943 #[tokio::test]
10954 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
10955 crate::run::pin_test_home();
10956 let home = crate::run::home();
10957
10958 let tmp = tempfile::tempdir().expect("tempdir");
10959 let repo = tmp.path().join("repo");
10960 std::fs::create_dir_all(&repo).unwrap();
10961 init_repo(&repo);
10962 let cache_dir = tmp.path().join("target");
10965
10966 let mut config = Config::default();
10967 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
10968 config.graph.timeout_verify = Some(2);
10971
10972 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10973 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10974 .expect("no io error acquiring directly")
10975 {
10976 crate::cache::AcquireOutcome::Acquired(g) => g,
10977 crate::cache::AcquireOutcome::Busy(b) => {
10978 panic!("expected the direct acquire to win the lease first: {b:?}")
10979 }
10980 };
10981
10982 let mut state = RunState::new(
10983 repo.clone(),
10984 "main".to_owned(),
10985 "deadbeef".to_owned(),
10986 "task".to_owned(),
10987 config,
10988 );
10989 state.candidates = vec![Candidate {
10990 index: 0,
10991 label: 'A',
10992 agent: "alpha".to_owned(),
10993 branch: "does-not-exist".to_owned(),
10994 worktree: repo.clone(),
10995 summary: String::new(),
10996 stat: String::new(),
10997 files: 0,
10998 commits: 0,
10999 empty: false,
11000 failed: None,
11001 verified_noop: None,
11002 duration_ms: 0,
11003 folded: false,
11004 }];
11005 state.tally = Some(Tally {
11006 first_choice: BTreeMap::from([('A', 1)]),
11007 borda: BTreeMap::new(),
11008 winner: 'A',
11009 rankings: 1,
11010 unanimous_initial: true,
11011 deliberated: false,
11012 changed_votes: 0,
11013 unanimous_final: true,
11014 tie_break: None,
11015 judges: 0,
11016 present: 0,
11017 quorum: 0,
11018 met_quorum: true,
11019 uncontested: Some("only candidate A produced a change".to_owned()),
11020 });
11021 state.reviews = vec![ReviewRound {
11022 round: 1,
11023 head: "deadbeef".to_owned(),
11024 verified_head: None,
11025 verified_at: None,
11026 reviews: Vec::new(),
11027 e2e: Vec::new(),
11028 fix: None,
11029 blocking: 0,
11030 answered: 0,
11031 expected: 0,
11032 clean: true,
11033 verify_retried: false,
11034 e2e_deferred: false,
11035 e2e_defer_reason: None,
11036 progressed: false,
11037 vote_split: false,
11038 reconsideration: Vec::new(),
11039 verdict: None,
11040 }];
11041
11042 let mut runner = Runner {
11043 state,
11044 roles: ResolvedRoles {
11045 implementers: Vec::new(),
11046 judges: Vec::new(),
11047 reviewers: Vec::new(),
11048 fixer: None,
11049 conductor: conductor(),
11050 implementer_roster: Vec::new(),
11051 judge_roster: Vec::new(),
11052 reviewer_roster: Vec::new(),
11053 },
11054 sem: Arc::new(Semaphore::new(1)),
11055 pause: Pause::new(),
11056 interrupt: Pause::new(),
11057 };
11058
11059 let started = std::time::Instant::now();
11060 runner.gate().await.expect("gate");
11061 assert!(
11062 started.elapsed() < Duration::from_secs(1),
11063 "a gate with nothing to run must never wait on a lease it never needed"
11064 );
11065 assert!(
11066 runner.state.gate_ran,
11067 "zero commands is still a real, immediate attempt"
11068 );
11069 assert!(runner.state.gate.is_empty());
11070 assert_ne!(
11071 runner.state.status,
11072 RunStatus::Blocked,
11073 "must not read as resource-blocked on a lease it never asked for"
11074 );
11075 }
11076
11077 #[tokio::test]
11087 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
11088 crate::run::pin_test_home();
11089
11090 let tmp = tempfile::tempdir().expect("tempdir");
11091 let repo = tmp.path().join("repo");
11092 std::fs::create_dir_all(&repo).unwrap();
11093 init_repo(&repo);
11094
11095 let mut config = Config::default();
11096 config.verify.gate = vec![
11097 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
11098 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
11099 .to_owned(),
11100 ];
11101
11102 let mut state = RunState::new(
11103 repo.clone(),
11104 "main".to_owned(),
11105 "deadbeef".to_owned(),
11106 "task".to_owned(),
11107 config,
11108 );
11109 let run_id = state.id.clone();
11110 state.candidates = vec![Candidate {
11111 index: 0,
11112 label: 'A',
11113 agent: "alpha".to_owned(),
11114 branch: "does-not-exist".to_owned(),
11115 worktree: repo.clone(),
11116 summary: String::new(),
11117 stat: String::new(),
11118 files: 0,
11119 commits: 0,
11120 empty: false,
11121 failed: None,
11122 verified_noop: None,
11123 duration_ms: 0,
11124 folded: false,
11125 }];
11126 state.tally = Some(Tally {
11127 first_choice: BTreeMap::from([('A', 1)]),
11128 borda: BTreeMap::new(),
11129 winner: 'A',
11130 rankings: 1,
11131 unanimous_initial: true,
11132 deliberated: false,
11133 changed_votes: 0,
11134 unanimous_final: true,
11135 tie_break: None,
11136 judges: 0,
11137 present: 0,
11138 quorum: 0,
11139 met_quorum: true,
11140 uncontested: Some("only candidate A produced a change".to_owned()),
11141 });
11142 state.reviews = vec![ReviewRound {
11143 round: 1,
11144 head: "deadbeef".to_owned(),
11145 verified_head: None,
11146 verified_at: None,
11147 reviews: Vec::new(),
11148 e2e: Vec::new(),
11149 fix: None,
11150 blocking: 0,
11151 answered: 0,
11152 expected: 0,
11153 clean: true,
11154 verify_retried: false,
11155 e2e_deferred: false,
11156 e2e_defer_reason: None,
11157 progressed: false,
11158 vote_split: false,
11159 reconsideration: Vec::new(),
11160 verdict: None,
11161 }];
11162
11163 let mut runner = Runner {
11164 state,
11165 roles: ResolvedRoles {
11166 implementers: Vec::new(),
11167 judges: Vec::new(),
11168 reviewers: Vec::new(),
11169 fixer: None,
11170 conductor: conductor(),
11171 implementer_roster: Vec::new(),
11172 judge_roster: Vec::new(),
11173 reviewer_roster: Vec::new(),
11174 },
11175 sem: Arc::new(Semaphore::new(1)),
11176 pause: Pause::new(),
11177 interrupt: Pause::new(),
11178 };
11179
11180 let started_marker = repo.join("started.marker");
11181 let release_marker = repo.join("release.marker");
11182 let poller = tokio::spawn(async move {
11183 for _ in 0..100 {
11188 if started_marker.exists()
11189 && let Ok(s) = crate::run::RunState::load(&run_id)
11190 && let Some(a) = s.active.get("gate")
11191 {
11192 std::fs::write(&release_marker, b"go").expect("release marker");
11193 return Some(a.clone());
11194 }
11195 tokio::time::sleep(Duration::from_millis(50)).await;
11196 }
11197 None
11198 });
11199
11200 runner.gate().await.expect("gate");
11201 let captured = poller.await.expect("poller task");
11202 let captured = captured.expect(
11203 "the poller never saw a `gate` task entry in run.json while the command was \
11204 still blocked on its own release marker",
11205 );
11206
11207 assert_eq!(captured.task.as_deref(), Some("gate"));
11208 assert_eq!(captured.node, "gate");
11209 assert_eq!(captured.index, Some(1));
11210 assert_eq!(captured.total, Some(1));
11211 assert!(
11212 captured
11213 .command
11214 .as_deref()
11215 .is_some_and(|c| c.contains("started.marker")),
11216 "{captured:?}"
11217 );
11218
11219 assert!(
11220 runner.state.active.is_empty(),
11221 "the entry must be cleared once the command actually finished: {:?}",
11222 runner.state.active
11223 );
11224 assert!(runner.state.gate_ran);
11225 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
11226 }
11227
11228 #[tokio::test]
11231 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
11232 use crate::verdict::{Finding, ReviewVote, Severity};
11233 crate::run::pin_test_home();
11234 let tmp = tempfile::tempdir().expect("tempdir");
11235 let repo = tmp.path().join("repo");
11236 std::fs::create_dir_all(&repo).unwrap();
11237 init_repo(&repo);
11238
11239 for (severity, vote, expect) in [
11240 (Severity::Major, ReviewVote::Reject, true),
11241 (Severity::Minor, ReviewVote::Reject, false),
11242 (Severity::Major, ReviewVote::Approve, false),
11243 ] {
11244 let mut round = review_round(false, 1, 1, 1, false, true);
11245 round.reviews = vec![ReviewRecord {
11246 reviewer: 1,
11247 agent: "alpha".to_owned(),
11248 summary: String::new(),
11249 findings: vec![Finding {
11250 id: "R1-1-1".to_owned(),
11251 severity,
11252 file: None,
11253 line: None,
11254 title: "t".to_owned(),
11255 detail: String::new(),
11256 }],
11257 vote: Some(vote),
11258 failed: None,
11259 duration_ms: 0,
11260 attempts: 0,
11261 }];
11262 let mut state = RunState::new(
11263 repo.clone(),
11264 "main".to_owned(),
11265 "deadbeef".to_owned(),
11266 "task".to_owned(),
11267 Config::default(),
11268 );
11269 state.reviews = vec![round];
11270 let mut runner = Runner {
11271 state,
11272 roles: ResolvedRoles {
11273 implementers: Vec::new(),
11274 judges: Vec::new(),
11275 reviewers: Vec::new(),
11276 fixer: None,
11277 conductor: conductor(),
11278 implementer_roster: Vec::new(),
11279 judge_roster: Vec::new(),
11280 reviewer_roster: Vec::new(),
11281 },
11282 sem: Arc::new(Semaphore::new(1)),
11283 pause: Pause::new(),
11284 interrupt: Pause::new(),
11285 };
11286 let shell = runner.state.config.shell();
11287 runner
11288 .stop_reviewing("round budget spent", &shell, &repo)
11289 .await
11290 .expect("stop_reviewing");
11291 assert_eq!(runner.state.status, RunStatus::Gating);
11292 assert_eq!(
11293 runner.state.contested_handoff.is_some(),
11294 expect,
11295 "{severity:?} + {vote:?}"
11296 );
11297 }
11298 }
11299
11300 #[tokio::test]
11313 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
11314 crate::run::pin_test_home();
11315 let home = crate::run::home();
11316
11317 let tmp = tempfile::tempdir().expect("tempdir");
11318 let repo = tmp.path().join("repo");
11319 std::fs::create_dir_all(&repo).unwrap();
11320 init_repo(&repo);
11321 let head = crate::git::rev_parse(&repo, "HEAD")
11322 .await
11323 .expect("rev-parse");
11324 let cache_dir = tmp.path().join("target");
11327
11328 let mut config = Config::default();
11329 config.verify.e2e = vec![format!(
11330 "CARGO_TARGET_DIR='{}' test -f README.md",
11331 cache_dir.display()
11332 )];
11333 config.graph.review_rounds = 1;
11334 config.graph.timeout_verify = Some(2);
11337
11338 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
11339 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
11340 .expect("no io error acquiring directly")
11341 {
11342 crate::cache::AcquireOutcome::Acquired(g) => g,
11343 crate::cache::AcquireOutcome::Busy(b) => {
11344 panic!("expected the direct acquire to win the lease first: {b:?}")
11345 }
11346 };
11347
11348 let mut state = RunState::new(
11349 repo.clone(),
11350 "main".to_owned(),
11351 head.clone(),
11352 "task".to_owned(),
11353 config,
11354 );
11355 state.candidates = vec![Candidate {
11356 index: 0,
11357 label: 'A',
11358 agent: "alpha".to_owned(),
11359 branch: "does-not-exist".to_owned(),
11360 worktree: repo.clone(),
11361 summary: String::new(),
11362 stat: String::new(),
11363 files: 0,
11364 commits: 0,
11365 empty: false,
11366 failed: None,
11367 verified_noop: None,
11368 duration_ms: 0,
11369 folded: false,
11370 }];
11371 state.tally = Some(Tally {
11372 first_choice: BTreeMap::from([('A', 1)]),
11373 borda: BTreeMap::new(),
11374 winner: 'A',
11375 rankings: 1,
11376 unanimous_initial: true,
11377 deliberated: false,
11378 changed_votes: 0,
11379 unanimous_final: true,
11380 tie_break: None,
11381 judges: 0,
11382 present: 0,
11383 quorum: 0,
11384 met_quorum: true,
11385 uncontested: Some("only candidate A produced a change".to_owned()),
11386 });
11387 state.reviews = vec![ReviewRound {
11391 round: 1,
11392 head: head.clone(),
11393 verified_head: None,
11394 verified_at: None,
11395 reviews: Vec::new(),
11396 e2e: Vec::new(),
11397 fix: None,
11398 blocking: 1,
11399 answered: 1,
11400 expected: 1,
11401 clean: false,
11402 verify_retried: false,
11403 e2e_deferred: true,
11404 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
11405 progressed: false,
11406 vote_split: false,
11407 reconsideration: Vec::new(),
11408 verdict: None,
11409 }];
11410
11411 let mut runner = Runner {
11412 state,
11413 roles: ResolvedRoles {
11414 implementers: Vec::new(),
11415 judges: Vec::new(),
11416 reviewers: Vec::new(),
11417 fixer: None,
11418 conductor: conductor(),
11419 implementer_roster: Vec::new(),
11420 judge_roster: Vec::new(),
11421 reviewer_roster: Vec::new(),
11422 },
11423 sem: Arc::new(Semaphore::new(1)),
11424 pause: Pause::new(),
11425 interrupt: Pause::new(),
11426 };
11427
11428 let shell = runner.state.config.shell();
11429 runner
11430 .stop_reviewing("round budget spent", &shell, &repo)
11431 .await
11432 .expect("stop_reviewing");
11433
11434 let last = runner.state.reviews.last().expect("round record");
11435 assert_eq!(
11436 last.e2e_status(),
11437 E2eStatus::ResourceBlocked,
11438 "the shared cache is still held; the attempt must read as blocked, not deferred or \
11439 failed: {last:?}"
11440 );
11441 assert_eq!(
11442 last.verified_head.as_deref(),
11443 Some(head.as_str()),
11444 "which commit this attempt targeted is known even though nothing finished checking \
11445 it"
11446 );
11447 let first_attempt_at = last
11448 .verified_at
11449 .expect("when this attempt ran is known too");
11450 assert_ne!(
11451 runner.state.status,
11452 RunStatus::Blocked,
11453 "contention is evidence about the machine, not the patch — it must not settle the \
11454 run as blocked: {:?}",
11455 runner.state.status
11456 );
11457 assert!(
11458 !runner
11459 .state
11460 .events
11461 .iter()
11462 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
11463 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
11464 runner.state.events
11465 );
11466
11467 runner
11472 .stop_reviewing("round budget spent", &shell, &repo)
11473 .await
11474 .expect("stop_reviewing retry");
11475 assert_eq!(
11476 runner.state.reviews.len(),
11477 1,
11478 "no new round was started: {:?}",
11479 runner.state.reviews
11480 );
11481 let last = runner.state.reviews.last().expect("round record");
11482 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
11483 assert!(
11484 last.verified_at.expect("still known") > first_attempt_at,
11485 "a second reentry must be a fresh attempt, not a stale copy of the first"
11486 );
11487 assert_ne!(runner.state.status, RunStatus::Blocked);
11488
11489 held.release();
11490 }
11491
11492 #[tokio::test]
11504 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
11505 crate::run::pin_test_home();
11506 let home = crate::run::home();
11507
11508 let tmp = tempfile::tempdir().expect("tempdir");
11509 let repo = tmp.path().join("repo");
11510 std::fs::create_dir_all(&repo).unwrap();
11511 init_repo(&repo);
11512 let head = crate::git::rev_parse(&repo, "HEAD")
11513 .await
11514 .expect("rev-parse");
11515 let cache_dir = tmp.path().join("target");
11516
11517 let mut config = Config::default();
11518 config.verify.e2e = vec![format!(
11519 "CARGO_TARGET_DIR='{}' test -f README.md",
11520 cache_dir.display()
11521 )];
11522 config.graph.review_rounds = 1;
11523 config.graph.timeout_verify = Some(2);
11524
11525 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
11526 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
11527 .expect("no io error acquiring directly")
11528 {
11529 crate::cache::AcquireOutcome::Acquired(g) => g,
11530 crate::cache::AcquireOutcome::Busy(b) => {
11531 panic!("expected the direct acquire to win the lease first: {b:?}")
11532 }
11533 };
11534
11535 let mut state = RunState::new(
11536 repo.clone(),
11537 "main".to_owned(),
11538 head.clone(),
11539 "task".to_owned(),
11540 config,
11541 );
11542 state.candidates = vec![Candidate {
11543 index: 0,
11544 label: 'A',
11545 agent: "alpha".to_owned(),
11546 branch: "does-not-exist".to_owned(),
11547 worktree: repo.clone(),
11548 summary: String::new(),
11549 stat: String::new(),
11550 files: 0,
11551 commits: 0,
11552 empty: false,
11553 failed: None,
11554 verified_noop: None,
11555 duration_ms: 0,
11556 folded: false,
11557 }];
11558 state.tally = Some(Tally {
11559 first_choice: BTreeMap::from([('A', 1)]),
11560 borda: BTreeMap::new(),
11561 winner: 'A',
11562 rankings: 1,
11563 unanimous_initial: true,
11564 deliberated: false,
11565 changed_votes: 0,
11566 unanimous_final: true,
11567 tie_break: None,
11568 judges: 0,
11569 present: 0,
11570 quorum: 0,
11571 met_quorum: true,
11572 uncontested: Some("only candidate A produced a change".to_owned()),
11573 });
11574 state.reviews = vec![ReviewRound {
11578 round: 1,
11579 head: head.clone(),
11580 verified_head: Some(head.clone()),
11581 verified_at: Some(jiff::Timestamp::now()),
11582 reviews: Vec::new(),
11583 e2e: vec![CommandOutcome {
11584 command: format!(
11585 "CARGO_TARGET_DIR='{}' test -f README.md",
11586 cache_dir.display()
11587 ),
11588 code: None,
11589 output_tail: "waiting for the shared build cache".to_owned(),
11590 duration_ms: 0,
11591 resource_blocked: true,
11592 }],
11593 fix: None,
11594 blocking: 1,
11595 answered: 1,
11596 expected: 1,
11597 clean: false,
11598 verify_retried: false,
11599 e2e_deferred: false,
11600 e2e_defer_reason: None,
11601 progressed: false,
11602 vote_split: false,
11603 reconsideration: Vec::new(),
11604 verdict: None,
11605 }];
11606
11607 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
11608 let mut runner = Runner {
11609 state,
11610 roles: ResolvedRoles {
11611 implementers: Vec::new(),
11612 judges: Vec::new(),
11613 reviewers: Vec::new(),
11614 fixer: None,
11615 conductor: conductor(),
11616 implementer_roster: Vec::new(),
11617 judge_roster: Vec::new(),
11618 reviewer_roster: Vec::new(),
11619 },
11620 sem: Arc::new(Semaphore::new(1)),
11621 pause: Pause::new(),
11622 interrupt: Pause::new(),
11623 };
11624
11625 runner.review_loop().await.expect("review_loop");
11630
11631 assert_eq!(
11632 runner.state.reviews.len(),
11633 1,
11634 "no new round was started on top of the unresolved one: {:?}",
11635 runner.state.reviews
11636 );
11637 let last = &runner.state.reviews[0];
11638 assert_eq!(
11639 last.e2e_status(),
11640 E2eStatus::ResourceBlocked,
11641 "still contended: {last:?}"
11642 );
11643 assert!(
11644 last.verified_at.expect("still known") > first_attempt_at,
11645 "review_loop must have actually retried the check, not left it exactly as found"
11646 );
11647 assert_ne!(
11648 runner.state.status,
11649 RunStatus::Blocked,
11650 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
11651 runner.state.status
11652 );
11653
11654 held.release();
11655 }
11656
11657 #[tokio::test]
11658 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
11659 crate::run::pin_test_home();
11660 let tmp = tempfile::tempdir().expect("tempdir");
11661 let repo = tmp.path().join("repo");
11662 std::fs::create_dir_all(&repo).unwrap();
11663 init_repo(&repo);
11664
11665 let mut config = Config::default();
11666 config.merge.mode = MergeMode::Pr;
11667 config.graph.land = true;
11668 config.graph.land_approval = false;
11669
11670 let mut state = RunState::new(
11671 repo.clone(),
11672 "main".to_owned(),
11673 "deadbeef".to_owned(),
11674 "task".to_owned(),
11675 config,
11676 );
11677 state.candidates = vec![Candidate {
11678 index: 0,
11679 label: 'A',
11680 agent: "alpha".to_owned(),
11681 branch: "does-not-exist".to_owned(),
11682 worktree: repo.clone(),
11683 summary: String::new(),
11684 stat: String::new(),
11685 files: 0,
11686 commits: 0,
11687 empty: false,
11688 failed: None,
11689 verified_noop: None,
11690 duration_ms: 0,
11691 folded: false,
11692 }];
11693 state.tally = Some(Tally {
11694 first_choice: BTreeMap::from([('A', 1)]),
11695 borda: BTreeMap::new(),
11696 winner: 'A',
11697 rankings: 1,
11698 unanimous_initial: true,
11699 deliberated: false,
11700 changed_votes: 0,
11701 unanimous_final: true,
11702 tie_break: None,
11703 judges: 0,
11704 present: 0,
11705 quorum: 0,
11706 met_quorum: true,
11707 uncontested: Some("only candidate A produced a change".to_owned()),
11708 });
11709 state.reviews = vec![ReviewRound {
11710 round: 1,
11711 head: "deadbeef".to_owned(),
11712 verified_head: None,
11713 verified_at: None,
11714 reviews: Vec::new(),
11715 e2e: Vec::new(),
11716 fix: None,
11717 blocking: 0,
11718 answered: 0,
11719 expected: 0,
11720 clean: true,
11721 verify_retried: false,
11722 e2e_deferred: false,
11723 e2e_defer_reason: None,
11724 progressed: false,
11725 vote_split: false,
11726 reconsideration: Vec::new(),
11727 verdict: None,
11728 }];
11729 state.gate = vec![CommandOutcome {
11730 command: "test".to_owned(),
11731 code: Some(0),
11732 output_tail: String::new(),
11733 duration_ms: 0,
11734 resource_blocked: false,
11735 }];
11736 state.gate_ran = true;
11737 state.status = RunStatus::Landing;
11741 state.merge = Some(MergeOutcome {
11742 mode: MergeMode::Pr,
11743 ok: true,
11744 detail: "https://example.invalid/x/y/pull/1".to_owned(),
11745 empty: false,
11746 });
11747
11748 ask_test_home();
11752 let store = ask::Questions::open();
11753 let q = ask_open_question(&store, &state.id);
11754
11755 let mut runner = Runner {
11756 state,
11757 roles: ResolvedRoles {
11758 implementers: Vec::new(),
11759 judges: Vec::new(),
11760 reviewers: Vec::new(),
11761 fixer: None,
11762 conductor: conductor(),
11763 implementer_roster: Vec::new(),
11764 judge_roster: Vec::new(),
11765 reviewer_roster: Vec::new(),
11766 },
11767 sem: Arc::new(Semaphore::new(1)),
11768 pause: Pause::new(),
11769 interrupt: Pause::new(),
11770 };
11771
11772 runner.execute().await.expect("execute");
11777
11778 assert_eq!(
11779 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
11780 Some("https://example.invalid/x/y/pull/1"),
11781 "reentry must not push again or open a second pull request over the \
11782 one `land` is already watching"
11783 );
11784 assert_ne!(
11785 runner.state.status,
11786 RunStatus::Landing,
11787 "land could not actually reach the fake pull request, so it must \
11788 have given up rather than left the run silently parked forever"
11789 );
11790 assert_eq!(runner.state.status, RunStatus::Blocked);
11794 assert!(
11795 store.get(&q.id).unwrap().status.open(),
11796 "Blocked is still alive; settle_questions must have been a no-op here"
11797 );
11798 }
11799
11800 fn state_with_round(round: ReviewRound) -> RunState {
11801 let mut s = RunState::new(
11802 PathBuf::from("/repo"),
11803 "main".to_owned(),
11804 "abc1234".to_owned(),
11805 "add retries".to_owned(),
11806 Config::default(),
11807 );
11808 s.reviews = vec![round];
11809 s
11810 }
11811
11812 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
11813 crate::verdict::Finding {
11814 id: id.to_owned(),
11815 severity,
11816 file: None,
11817 line: None,
11818 title: title.to_owned(),
11819 detail: String::new(),
11820 }
11821 }
11822
11823 #[test]
11824 fn pr_body_names_open_findings_and_declined_ones() {
11825 let round = ReviewRound {
11826 round: 2,
11827 head: "deadbee".to_owned(),
11828 verified_head: None,
11829 verified_at: None,
11830 reviews: vec![ReviewRecord {
11831 attempts: 0,
11832 reviewer: 1,
11833 agent: "alpha".to_owned(),
11834 summary: String::new(),
11835 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
11836 vote: None,
11837 failed: None,
11838 duration_ms: 0,
11839 }],
11840 e2e: vec![CommandOutcome {
11841 command: "cargo test".to_owned(),
11842 code: Some(0),
11843 output_tail: String::new(),
11844 duration_ms: 0,
11845 resource_blocked: false,
11846 }],
11847 verify_retried: false,
11848 e2e_deferred: false,
11849 e2e_defer_reason: None,
11850 fix: Some(FixRecord {
11851 agent: "alpha".to_owned(),
11852 addressed: Vec::new(),
11853 rejected: vec![crate::verdict::Rejection {
11854 id: "R1-1-1".to_owned(),
11855 why: "not reachable from any caller".to_owned(),
11856 }],
11857 notes: String::new(),
11858 committed: true,
11859 failed: None,
11860 duration_ms: 0,
11861 continuation: None,
11862 }),
11863 blocking: 0,
11864 answered: 1,
11865 expected: 1,
11866 clean: false,
11867 progressed: true,
11868 vote_split: false,
11869 reconsideration: Vec::new(),
11870 verdict: None,
11871 };
11872 let state = state_with_round(round);
11873 let body = pr_message(&state, 'A').body;
11874
11875 assert!(body.contains("add retries"), "the task must still be there");
11876 assert!(body.contains("R2-1-1"), "{body}");
11877 assert!(body.contains("unused import"), "{body}");
11878 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
11879 assert!(
11880 body.contains("not reachable from any caller"),
11881 "the reason it was declined: {body}"
11882 );
11883 }
11884
11885 #[test]
11886 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
11887 let round = ReviewRound {
11888 round: 1,
11889 head: "deadbee".to_owned(),
11890 verified_head: None,
11891 verified_at: None,
11892 reviews: vec![ReviewRecord {
11893 attempts: 0,
11894 reviewer: 1,
11895 agent: "alpha".to_owned(),
11896 summary: String::new(),
11897 findings: Vec::new(),
11898 vote: None,
11899 failed: None,
11900 duration_ms: 0,
11901 }],
11902 e2e: Vec::new(),
11903 verify_retried: false,
11904 e2e_deferred: false,
11905 e2e_defer_reason: None,
11906 fix: None,
11907 blocking: 0,
11908 answered: 1,
11909 expected: 1,
11910 clean: true,
11911 progressed: false,
11912 vote_split: false,
11913 reconsideration: Vec::new(),
11914 verdict: None,
11915 };
11916 let state = state_with_round(round);
11917 let body = pr_message(&state, 'A').body;
11918 assert!(!body.contains("Open review findings"), "{body}");
11919 assert!(!body.contains("Declined"), "{body}");
11920 }
11921
11922 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
11923 let mut state = RunState::new(
11924 PathBuf::from("/repo"),
11925 "main".to_owned(),
11926 "abc1234".to_owned(),
11927 instruction.to_owned(),
11928 Config::default(),
11929 );
11930 state.candidates.push(Candidate {
11931 index: 0,
11932 label: 'A',
11933 agent: "alpha".to_owned(),
11934 branch: "magi/x/A".to_owned(),
11935 worktree: PathBuf::from("/wt"),
11936 summary: summary.to_owned(),
11937 stat: String::new(),
11938 files: 1,
11939 commits: 1,
11940 empty: false,
11941 failed: None,
11942 verified_noop: None,
11943 folded: false,
11944 duration_ms: 0,
11945 });
11946 state
11947 }
11948
11949 fn review_state(subjects: &[&str]) -> RunState {
11950 let mut state = state_with_summary(
11951 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
11952 "",
11953 );
11954 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
11955 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
11956 state
11957 }
11958
11959 #[tokio::test]
11962 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
11963 ask_test_home();
11964 let tmp = tempfile::tempdir().unwrap();
11965 let repo = tmp.path().join("repo");
11966 std::fs::create_dir_all(&repo).unwrap();
11967 init_repo(&repo);
11968 let origin = tmp.path().join("origin.git");
11969 let g = |dir: &Path, args: &[&str]| {
11970 let out = std::process::Command::new("git")
11971 .args(args)
11972 .current_dir(dir)
11973 .quiet()
11974 .output()
11975 .expect("spawn git");
11976 assert!(
11977 out.status.success(),
11978 "git {args:?}: {}",
11979 String::from_utf8_lossy(&out.stderr)
11980 );
11981 };
11982 g(
11983 tmp.path(),
11984 &[
11985 "clone",
11986 "--bare",
11987 "-q",
11988 repo.to_str().unwrap(),
11989 origin.to_str().unwrap(),
11990 ],
11991 );
11992 g(
11993 &repo,
11994 &["remote", "add", "origin", origin.to_str().unwrap()],
11995 );
11996 let c1 = git::rev_parse(&repo, "main").await.unwrap();
11997
11998 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
12000 g(&repo, &["add", "-A"]);
12001 g(
12002 &repo,
12003 &["commit", "-q", "-m", "chore(deps): update a crate"],
12004 );
12005 g(&repo, &["push", "-q", "origin", "main"]);
12006 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
12007 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
12008 g(&repo, &["add", "-A"]);
12009 g(
12010 &repo,
12011 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
12012 );
12013 g(&repo, &["checkout", "-q", "main"]);
12014
12015 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
12016 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
12017 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
12019 assert_eq!(fallback, c1);
12020
12021 let mut state = review_state(&[
12022 "chore(deps): update a crate",
12023 "fix(daemon): apply a chosen action",
12024 ]);
12025 state.repo = repo.clone();
12026 state.base_branch = "main".to_owned();
12027 state.base_commit = c1;
12028 refresh_reviewed_commits(&mut state, "feat/own").await;
12029 assert_eq!(
12030 state.reviewed_commits,
12031 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
12032 );
12033 assert_eq!(
12034 review_title(&state).as_deref(),
12035 Some("fix(daemon): apply a chosen action")
12036 );
12037 assert_eq!(
12038 leaked_subjects(&state, "feat/own").await,
12039 Some(vec!["chore(deps): update a crate".to_owned()])
12040 );
12041 state.config.merge.remote = "nowhere".to_owned();
12044 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
12045 }
12046
12047 #[test]
12048 fn pr_message_review_single_commit_uses_its_subject() {
12049 let state = review_state(&["feat(nats): per-role user"]);
12050 let m = pr_message(&state, 'A');
12051 assert_eq!(m.title, "feat(nats): per-role user");
12052 assert!(!m.body.contains("Review the work already"), "{}", m.body);
12053 assert!(m.body.contains("## Commits under review"), "{}", m.body);
12054 }
12055
12056 #[test]
12057 fn pr_message_review_multi_commit_takes_the_oldest() {
12058 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
12059 let m = pr_message(&state, 'A');
12060 assert_eq!(m.title, "feat: the change");
12061 for s in ["feat: the change", "fix: typo", "fix: again"] {
12062 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
12063 }
12064 }
12065
12066 #[test]
12067 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
12068 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
12069 let mut state = review_state(&[first, "fix: later fixup"]);
12070 state.candidates[0].branch = "機能/ログイン".to_owned();
12071 let m = pr_message(&state, 'A');
12072 assert!(
12073 m.title.starts_with("chore: land candidate A of run"),
12074 "{}",
12075 m.title
12076 );
12077 }
12078 }
12079
12080 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
12081 BranchFacts {
12082 commits: commits
12083 .iter()
12084 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
12085 .collect(),
12086 stat: stat.to_owned(),
12087 }
12088 }
12089
12090 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
12091 let mut r = review_round(true, 0, 1, 1, true, true);
12092 r.round = round;
12093 r.fix = notes.map(|n| FixRecord {
12094 agent: "fixer".to_owned(),
12095 addressed: Vec::new(),
12096 rejected: Vec::new(),
12097 notes: n.to_owned(),
12098 committed: true,
12099 failed: None,
12100 duration_ms: 0,
12101 continuation: None,
12102 });
12103 r
12104 }
12105
12106 #[test]
12107 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
12108 let state = review_state(&["ignored"]);
12109 let f = facts(
12110 &[
12111 (
12112 "fix(login): resolve PATH on macOS",
12113 "Login shells skip rc files.",
12114 ),
12115 ("fix: address review", ""),
12116 ],
12117 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
12118 );
12119 let m = pr_message_with(&state, 'A', Some(&f));
12120 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
12121 assert!(
12122 m.body
12123 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
12124 "{}",
12125 m.body
12126 );
12127 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
12128 assert!(m.body.contains("## Diff stat"), "{}", m.body);
12129 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
12130 for banned in [
12131 "Review the work already",
12132 "no task statement",
12133 "Original task",
12134 ] {
12135 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
12136 }
12137 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
12138 }
12139
12140 #[test]
12141 fn pr_message_review_truncates_a_huge_first_commit_body() {
12142 let state = review_state(&["ignored"]);
12143 let f = facts(
12144 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
12145 "s",
12146 );
12147 let m = pr_message_with(&state, 'A', Some(&f));
12148 assert!(m.body.len() < 30_000, "{}", m.body.len());
12149 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
12150 assert!(
12151 m.body.contains("- ... 1 more commit(s)"),
12152 "{}",
12153 m.body.len()
12154 );
12155 assert!(m.body.ends_with("magi:candidate-a\n"));
12156 }
12157
12158 #[test]
12159 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
12160 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
12161 assert_eq!(m.title, "feat: x");
12162 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
12163 assert!(!m.body.contains("Diff stat"), "{}", m.body);
12164 assert!(!m.body.contains("no task statement"), "{}", m.body);
12165 }
12166
12167 #[test]
12168 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
12169 let mut state = review_state(&["日本語の件名"]);
12170 state.candidates[0].branch = "fix/macos-login-path".to_owned();
12171 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
12172 state.candidates[0].branch = "機能/ログイン".to_owned();
12173 assert!(
12174 pr_message(&state, 'A')
12175 .title
12176 .starts_with("chore: land candidate A")
12177 );
12178 }
12179
12180 #[test]
12181 fn pr_message_review_fixes_survive_a_clean_final_round() {
12182 let mut state = review_state(&["feat: x"]);
12183 state.reviews = vec![
12184 round_with_notes(1, Some("handled the PATH case")),
12185 round_with_notes(2, None),
12186 ];
12187 let body = pr_message(&state, 'A').body;
12188 assert!(
12189 body.contains("## Review fixes\n\nhandled the PATH case\n"),
12190 "{body}"
12191 );
12192 assert!(!body.contains("### Round"), "{body}");
12193
12194 state.reviews = vec![
12195 round_with_notes(1, Some("first fix")),
12196 round_with_notes(2, Some("")),
12197 round_with_notes(3, Some("second fix")),
12198 round_with_notes(4, None),
12199 ];
12200 let body = pr_message(&state, 'A').body;
12201 assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
12202 assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
12203 assert!(!body.contains("### Round 2"), "{body}");
12204 }
12205
12206 #[test]
12207 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
12208 let state = state_with_summary(
12209 "add retries to the client",
12210 "TITLE: feat: retries\n\nDid it.",
12211 );
12212 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
12213 assert_eq!(m.title, "feat: retries");
12214 assert!(
12215 m.body.contains("<summary>Original task</summary>"),
12216 "{}",
12217 m.body
12218 );
12219 assert!(!m.body.contains("Commits under review"), "{}", m.body);
12220 assert!(
12221 m.body
12222 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
12223 "{}",
12224 m.body
12225 );
12226 }
12227
12228 #[test]
12229 fn pr_message_review_bounds_a_long_english_subject() {
12230 let long = format!("feat: {}", "word ".repeat(100));
12231 let m = pr_message(&review_state(&[&long]), 'A');
12232 assert!(m.title.starts_with("feat: word"), "{}", m.title);
12233 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
12234 }
12235
12236 #[test]
12237 fn pr_message_implementation_run_is_unchanged_by_review_support() {
12238 let state = state_with_summary("add retries\n\ndetails", "- did some things");
12239 let m = pr_message(&state, 'A');
12240 assert_eq!(m.title, "add retries");
12241 assert!(m.body.contains("<summary>Original task</summary>"));
12242 assert!(!m.body.contains("Commits under review"));
12243 assert_eq!(landing_subject_source(&state), state.instruction);
12244 }
12245
12246 #[test]
12247 fn review_run_squash_subject_is_the_change_not_the_prompt() {
12248 let state = review_state(&["feat: the change", "fix: typo"]);
12249 let source = landing_subject_source(&state);
12250 assert_eq!(land::merge_subject("", &source), "feat: the change");
12251 assert_eq!(
12252 land::merge_subject("magi: candidate A (uncommitted work)", &source),
12253 "feat: the change"
12254 );
12255 assert_eq!(
12257 land::merge_subject("feat: renamed by hand", &source),
12258 "feat: renamed by hand"
12259 );
12260 let blank = review_state(&["日本語"]);
12261 assert!(
12262 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
12263 );
12264 }
12265
12266 #[test]
12267 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
12268 let state = review_state(&["feat: the change"]);
12269 let source = landing_subject_source(&state);
12270 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
12271 assert_eq!(
12272 land::merge_subject(landing_title(&state, old), &source),
12273 "feat: the change"
12274 );
12275 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
12276 let task = state_with_summary("add retries", "");
12277 assert_eq!(landing_title(&task, old), old);
12278 }
12279
12280 #[test]
12281 fn pr_message_describes_the_change_not_the_task() {
12282 let state = state_with_summary(
12283 "今回やってほしいこと: results projector を直す",
12284 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
12285 );
12286 let m = pr_message(&state, 'A');
12287 assert_eq!(m.title, "fix(web): batch the runs list reads");
12288 assert!(
12289 m.body.starts_with("## Summary\n\n- reads run.json once"),
12290 "{}",
12291 m.body
12292 );
12293 assert!(!m.body.contains("TITLE:"), "{}", m.body);
12294 let task_at = m.body.find("今回やってほしいこと").unwrap();
12295 let details_at = m.body.find("<details>").unwrap();
12296 assert!(
12297 details_at < task_at,
12298 "the task lives inside <details>: {}",
12299 m.body
12300 );
12301 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
12302 assert!(m.body.contains("magi:candidate-a"));
12303 }
12304
12305 #[test]
12306 fn pr_message_falls_back_to_the_task_without_a_title_line() {
12307 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
12308 let m = pr_message(&state, 'A');
12309 assert_eq!(m.title, "add retries");
12310 assert!(
12311 m.body.contains("## Summary\n\n- did some things"),
12312 "{}",
12313 m.body
12314 );
12315
12316 let none = RunState::new(
12317 PathBuf::from("/repo"),
12318 "main".to_owned(),
12319 "abc1234".to_owned(),
12320 "add retries".to_owned(),
12321 Config::default(),
12322 );
12323 let m = pr_message(&none, 'A');
12324 assert_eq!(m.title, "add retries");
12325 assert!(!m.body.contains("## Summary"), "{}", m.body);
12326 }
12327
12328 #[test]
12329 fn pr_message_refuses_the_candidate_commit_subject() {
12330 for bad in [
12331 "TITLE: magi: candidate A (uncommitted work)",
12332 "TITLE: chore: stuff (uncommitted work)",
12333 "TITLE: ",
12334 ] {
12335 let state = state_with_summary("add retries", bad);
12336 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
12337 }
12338 }
12339
12340 #[test]
12341 fn pr_message_bounds_a_very_long_task_and_title() {
12342 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
12343 let state = state_with_summary(&long, "- nothing");
12344 let m = pr_message(&state, 'A');
12345 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
12346 assert!(!m.title.contains('\n'));
12347
12348 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
12349 let m = pr_message(&state, 'A');
12350 assert!(m.title.starts_with("feat: "));
12351 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
12352 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
12353 }
12354
12355 fn long_title_of(instruction: &str) -> String {
12356 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
12357 }
12358
12359 #[test]
12360 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
12361 let first = "Make the landing path keep a readable title for long tasks";
12362 let line = format!(
12363 "{first}. {}",
12364 "And then keep going with more words ".repeat(20)
12365 );
12366 let t = long_title_of(&line);
12367 assert_eq!(t, first);
12368 assert!(!t.starts_with("chore: land"));
12369 }
12370
12371 #[test]
12372 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
12373 let line = "word ".repeat(200);
12374 let t = long_title_of(&line);
12375 assert!(t.ends_with("word..."), "{t}");
12376 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
12377 }
12378
12379 #[test]
12380 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
12381 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
12382 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
12383 }
12384 }
12385
12386 #[test]
12387 fn pr_message_title_limit_is_exact() {
12388 let at = "a".repeat(PR_TITLE_MAX);
12389 assert_eq!(long_title_of(&at), at);
12390 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
12391 assert!(over.ends_with("..."), "{over}");
12392 assert_eq!(over.chars().count(), PR_TITLE_MAX);
12393 }
12394
12395 #[test]
12396 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
12397 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
12398 let t = long_title_of(&line);
12399 assert!(t.ends_with("..."), "{t}");
12400 assert!(t.is_ascii(), "{t}");
12401 }
12402
12403 #[test]
12404 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
12405 let line = format!(
12406 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
12407 "plus more filler words ".repeat(20)
12408 );
12409 let t = long_title_of(&line);
12410 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
12411 }
12412
12413 #[test]
12414 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
12415 let line = format!("\n\n# {}", "title words ".repeat(40));
12416 let state = state_with_summary(&line, "- nothing");
12417 let m = pr_message(&state, 'A');
12418 assert!(m.title.starts_with("title words"), "{}", m.title);
12419 assert_eq!(
12420 land::merge_subject(&m.title, &landing_subject_source(&state)),
12421 m.title
12422 );
12423 assert_eq!(
12425 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
12426 "feat: renamed by hand"
12427 );
12428 }
12429
12430 #[tokio::test]
12431 async fn github_text_rewrite_is_bounded_and_falls_back() {
12432 crate::run::pin_test_home();
12433 for (reply, accepted) in [
12434 (
12435 "TITLE: fix: retries\nAdd retries for failed requests.",
12436 true,
12437 ),
12438 (
12439 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
12440 false,
12441 ),
12442 ("TITLE: fix: retries\nUse token=secret", false),
12443 ] {
12444 let dir = tempfile::tempdir().unwrap();
12445 let mut runner = runner_at(RunStatus::Gating);
12446 runner.state = state_with_summary(
12447 "add retries",
12448 "TITLE: fix: retries\n日本語の説明を書きます。",
12449 );
12450 runner.state.repo = dir.path().to_owned();
12451 runner.state.candidates[0].worktree = dir.path().to_owned();
12452 let mut author = spec("alpha");
12453 author.command = vec![
12454 "sh".into(),
12455 "-c".into(),
12456 "printf '%s' \"$REWRITE_REPLY\"".into(),
12457 ];
12458 author.env.insert("REWRITE_REPLY".into(), reply.into());
12459 runner.state.config.agents = vec![author];
12460 let winner = runner.state.candidates[0].clone();
12461 let first = runner
12462 .guarded_pr_message(&winner, None, true)
12463 .await
12464 .unwrap();
12465 let message = if accepted {
12466 first.expect("a passing rewrite never asks")
12467 } else {
12468 assert!(first.is_none());
12470 assert!(runner.state.parked);
12471 let store = ask::Questions::open();
12472 let asked: Vec<_> = store
12473 .list()
12474 .into_iter()
12475 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
12476 .collect();
12477 assert_eq!(asked.len(), 1);
12478 assert!(!asked[0].detail.contains("secret"));
12479 store
12482 .update(&asked[0].id, |q| {
12483 q.abandon("test");
12484 Ok(())
12485 })
12486 .unwrap();
12487 let again = runner
12488 .guarded_pr_message(&winner, None, true)
12489 .await
12490 .unwrap()
12491 .expect("an abandoned question falls back");
12492 let n = store
12493 .list()
12494 .into_iter()
12495 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
12496 .count();
12497 assert_eq!(n, 1, "asked once per rejected text");
12498 again
12499 };
12500 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
12501 assert_eq!(
12502 message.body.contains("Add retries for failed requests."),
12503 accepted
12504 );
12505 assert_eq!(
12506 runner.state.seats["impl-A"].turns, 1,
12507 "only one rewrite invocation"
12508 );
12509 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
12510 if !accepted {
12511 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
12512 assert!(
12513 message
12514 .body
12515 .contains(&format!("magi:run/{}", runner.state.id))
12516 );
12517 }
12518 }
12519 }
12520
12521 #[tokio::test]
12522 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
12523 crate::run::pin_test_home();
12524 let dir = tempfile::tempdir().unwrap();
12525 let mut runner = runner_at(RunStatus::Gating);
12526 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
12527 runner.state.repo = dir.path().to_owned();
12528 runner.state.candidates[0].worktree = dir.path().to_owned();
12529 let winner = runner.state.candidates[0].clone();
12530 assert!(
12531 runner
12532 .guarded_pr_message(&winner, None, true)
12533 .await
12534 .unwrap()
12535 .is_none()
12536 );
12537 let store = ask::Questions::open();
12538 let find = |run: &str| {
12539 store
12540 .list()
12541 .into_iter()
12542 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
12543 .unwrap()
12544 };
12545 let run = runner.state.id.clone();
12546 let q = find(&run);
12547 let fp = runner
12548 .state
12549 .github_text
12550 .as_ref()
12551 .unwrap()
12552 .fingerprint
12553 .clone();
12554 let name = crate::github_text::artifact_name(&fp);
12555 let path = crate::run::artifact_path(&runner.state, &name);
12556 assert!(q.detail.contains(&path.display().to_string()));
12557 assert!(q.detail.contains("title-language: title"));
12558 let saved = std::fs::read_to_string(&path).expect("artifact written");
12559 assert!(saved.contains("日本語のタイトル"));
12560 assert!(q.cwd.is_none());
12561 assert_eq!(q.choices, ["use fallback", "use my text"]);
12562 let say = |id: &str, text: &str, choice: &str| {
12564 store
12565 .update(id, |q| {
12566 q.thread.push(ask::Turn {
12567 who: ask::Who::Operator,
12568 body: text.to_owned(),
12569 at: jiff::Timestamp::now(),
12570 note: None,
12571 });
12572 q.answer(ask::Answer::Choice(choice.to_owned()))
12573 })
12574 .unwrap();
12575 };
12576 say(&q.id, "まだ日本語", "use my text");
12577 assert!(
12578 runner
12579 .guarded_pr_message(&winner, None, true)
12580 .await
12581 .unwrap()
12582 .is_none()
12583 );
12584 let second = find(&run);
12585 assert_ne!(second.id, q.id);
12586 assert!(second.detail.contains("did not pass"));
12587 assert!(second.detail.contains("まだ日本語"));
12588 let retry_path = crate::run::artifact_path(
12589 &runner.state,
12590 &crate::github_text::artifact_name(&crate::github_text::fingerprint("まだ日本語", "")),
12591 );
12592 assert!(second.detail.contains(&retry_path.display().to_string()));
12593 assert!(
12594 std::fs::read_to_string(&retry_path)
12595 .expect("retry artifact written")
12596 .contains("まだ日本語")
12597 );
12598 say(&second.id, "fix: retry failed requests", "use my text");
12599 let message = runner
12600 .guarded_pr_message(&winner, None, true)
12601 .await
12602 .unwrap()
12603 .expect("a vetted replacement is posted");
12604 assert_eq!(message.title, "fix: retry failed requests");
12605 assert_eq!(
12606 runner
12607 .state
12608 .github_text
12609 .as_ref()
12610 .unwrap()
12611 .chosen_title
12612 .as_deref(),
12613 Some("fix: retry failed requests")
12614 );
12615 assert!(runner.state.github_text.as_ref().unwrap().resolved);
12616 let again = runner
12618 .guarded_pr_message(&winner, None, true)
12619 .await
12620 .unwrap()
12621 .unwrap();
12622 assert_eq!(again.title, message.title);
12623 assert_eq!(
12624 store
12625 .list()
12626 .into_iter()
12627 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
12628 .count(),
12629 2
12630 );
12631 }
12632
12633 #[test]
12634 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
12635 let mut state = state_with_summary(
12639 "add retries",
12640 "TITLE: fix(web): batch reads\n- reads run.json once",
12641 );
12642 state.config.graph.language = "ja".to_owned();
12643 let m = pr_message(&state, 'A');
12644 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
12645 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
12646
12647 let task = "今回やってほしいこと: results projector を直す";
12650 let mut state = state_with_summary(task, "- no title line");
12651 state.config.graph.language = "ja".to_owned();
12652 let m = pr_message(&state, 'A');
12653 assert_eq!(
12654 m.title,
12655 format!("chore: land candidate A of run {}", state.id)
12656 );
12657 assert!(
12658 m.body.contains(&format!(
12659 "<summary>Original task</summary>\n\n{task}\n\n</details>"
12660 )),
12661 "{}",
12662 m.body
12663 );
12664 }
12665
12666 #[test]
12667 fn pr_message_scrubs_home_paths_and_addresses() {
12668 let state = state_with_summary(
12669 "fix it in /Users/someone/src/x",
12670 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
12671 );
12672 let m = pr_message(&state, 'A');
12673 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
12674 assert!(!m.body.contains(leak), "{}", m.body);
12675 }
12676 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
12677 }
12678
12679 #[test]
12680 fn pr_message_survives_a_task_that_closes_details() {
12681 let state = state_with_summary("a </details> b", "TITLE: fix: x");
12682 let m = pr_message(&state, 'A');
12683 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
12684 }
12685
12686 #[test]
12687 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
12688 let cmd = manual_merge_command(
12689 MergeStyle::Squash,
12690 Path::new("/repo"),
12691 "b",
12692 "fix: \"quoted\" $(x) `y`\n\nbody",
12693 );
12694 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
12695 }
12696
12697 #[test]
12698 fn manual_merge_command_matches_the_configured_style() {
12699 let repo = Path::new("/repo");
12700 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
12701
12702 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
12703 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
12704
12705 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
12706 assert_eq!(
12707 squash,
12708 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
12709 \"Merge magi run 0832 (candidate A)\""
12710 );
12711
12712 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
12713 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
12714 }
12715
12716 #[test]
12717 fn a_nudge_gets_a_quarter_of_the_budget() {
12718 assert_eq!(retry_budget(secs(1200), true), secs(300));
12720 assert_eq!(retry_budget(secs(3600), true), secs(900));
12721 }
12722
12723 #[test]
12724 fn a_resent_prompt_keeps_the_whole_budget() {
12725 assert_eq!(retry_budget(secs(1200), false), secs(1200));
12728 assert_eq!(retry_budget(secs(60), false), secs(60));
12729 }
12730
12731 #[test]
12732 fn the_floor_never_exceeds_the_original_budget() {
12733 assert_eq!(retry_budget(secs(60), true), secs(60));
12737 assert_eq!(retry_budget(secs(480), true), secs(120));
12738 assert_eq!(retry_budget(secs(0), true), secs(0));
12739 }
12740
12741 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
12742 agent::CommandEvidence {
12743 id: "item1".to_owned(),
12744 description: "cargo test".to_owned(),
12745 exit_code,
12746 result_summary: String::new(),
12747 source: "codex".to_owned(),
12748 }
12749 }
12750
12751 #[test]
12752 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
12753 assert!(!has_unconfirmed_command(&[]));
12757 }
12758
12759 #[test]
12760 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
12761 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
12765 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
12766 assert!(!has_unconfirmed_command(&[
12767 evidence(Some(0)),
12768 evidence(Some(101))
12769 ]));
12770 }
12771
12772 #[test]
12773 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
12774 assert!(has_unconfirmed_command(&[
12775 evidence(Some(0)),
12776 evidence(None)
12777 ]));
12778 }
12779
12780 #[test]
12781 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
12782 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
12783 assert_eq!(
12784 verified_noop_claim(true, &[], text).as_deref(),
12785 Some("already fixed by b32cfc4, on main.")
12786 );
12787 }
12788
12789 #[test]
12790 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
12791 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
12794 assert!(verified_noop_claim(false, &[], text).is_none());
12795 }
12796
12797 #[test]
12798 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
12799 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
12800 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
12801 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
12803 }
12804
12805 #[test]
12806 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
12807 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
12808 }
12809
12810 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
12813 runner.state.candidates = shape
12814 .iter()
12815 .enumerate()
12816 .map(|(i, &(empty, verified))| Candidate {
12817 index: i,
12818 label: (b'A' + i as u8) as char,
12819 agent: "sonnet".to_owned(),
12820 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
12821 worktree: PathBuf::from(format!("/wt/{i}")),
12822 summary: String::new(),
12823 stat: String::new(),
12824 files: 0,
12825 commits: 0,
12826 empty,
12827 failed: None,
12828 verified_noop: verified.map(str::to_owned),
12829 duration_ms: 0,
12830 folded: false,
12831 })
12832 .collect();
12833 }
12834
12835 #[test]
12836 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
12837 ask_test_home();
12838 let mut runner = runner_at(RunStatus::Implementing);
12839 set_candidates(
12840 &mut runner,
12841 &[
12842 (true, Some("already on main at b32cfc4")),
12843 (true, Some("same fix, see the existing test")),
12844 ],
12845 );
12846
12847 runner
12848 .after_implement()
12849 .expect("a verified no-op is not an error");
12850
12851 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
12852 }
12853
12854 #[test]
12855 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
12856 ask_test_home();
12857 let mut runner = runner_at(RunStatus::Implementing);
12858 set_candidates(
12862 &mut runner,
12863 &[(true, Some("already on main at b32cfc4")), (true, None)],
12864 );
12865
12866 let err = runner
12867 .after_implement()
12868 .expect_err("an unverified empty candidate must still fail the run");
12869
12870 assert!(
12871 err.to_string().contains("no candidate produced a change"),
12872 "{err}"
12873 );
12874 assert_eq!(runner.state.status, RunStatus::Failed);
12875 }
12876
12877 #[test]
12878 fn after_implement_still_fails_an_ordinary_all_empty_run() {
12879 ask_test_home();
12880 let mut runner = runner_at(RunStatus::Implementing);
12881 set_candidates(&mut runner, &[(true, None), (true, None)]);
12882
12883 let err = runner
12884 .after_implement()
12885 .expect_err("no candidate declared anything; this is an ordinary failure");
12886
12887 assert!(
12888 err.to_string().contains("no candidate produced a change"),
12889 "{err}"
12890 );
12891 assert_eq!(runner.state.status, RunStatus::Failed);
12892 }
12893}
12894
12895#[derive(Debug)]
12898struct GraftRestoreFailed(String);
12899
12900impl std::fmt::Display for GraftRestoreFailed {
12901 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
12902 write!(
12903 f,
12904 "could not restore the winner to its pre-graft commit: {}",
12905 self.0
12906 )
12907 }
12908}
12909
12910impl std::error::Error for GraftRestoreFailed {}