1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::land;
41use crate::proc::Quiet as _;
42use crate::prompt::{
43 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
44};
45use crate::queue;
46use crate::refs;
47use crate::run::{
48 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
49 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord, Handover,
50 JobRecord, JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome,
51 OperatorFixRequest, Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState,
52 RunStatus, SeatHistory, Tally, VoteRecord, tail, write_artifact,
53};
54use crate::verdict::{
55 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
56 ReviewVote, Severity,
57};
58
59const OUTPUT_TAIL: usize = 8_000;
61
62const EVENT_OUTPUT_TAIL: usize = 2_000;
65
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
69
70const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
87
88pub(crate) const STAGNANT_LIMIT: usize = 2;
102
103const BASE_SYNC_ROUNDS: usize = 4;
116
117const MAX_FIX_CONTINUATIONS: usize = 2;
133
134#[derive(Clone)]
140struct SeatJob {
141 spec: AgentSpec,
142 seat: SeatState,
143 cwd: PathBuf,
144 prompt: String,
145 timeout: Duration,
146 allow_write: bool,
147 sessions: bool,
148 artifacts: PathBuf,
149 stem: String,
150 handover: Option<String>,
155}
156
157enum AgentOutcome {
169 Ok(AgentOutput),
171 Quota(AgentOutput),
173 Dropped(AgentOutput),
176 Failed(String),
178}
179
180#[derive(Debug, Clone, Default)]
207pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
208
209impl Pause {
210 #[must_use]
212 pub fn new() -> Self {
213 Self::default()
214 }
215
216 pub fn park(&self) {
218 self.0.store(true, Ordering::SeqCst);
219 }
220
221 pub fn park_because(&self, reason: impl Into<String>) {
227 let mut reason_guard = self
228 .1
229 .lock()
230 .unwrap_or_else(std::sync::PoisonError::into_inner);
231 if reason_guard.is_none() {
232 *reason_guard = Some(reason.into());
233 }
234 drop(reason_guard);
235 self.park();
236 }
237
238 #[must_use]
240 pub fn parked(&self) -> bool {
241 self.0.load(Ordering::SeqCst)
242 }
243
244 #[must_use]
246 pub fn reason(&self) -> Option<String> {
247 self.1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner)
250 .clone()
251 }
252}
253
254pub struct Runner {
256 pub state: RunState,
258 roles: ResolvedRoles,
259 sem: Arc<Semaphore>,
260 pause: Pause,
264 interrupt: Pause,
270}
271
272async fn review_base(
277 repo: &Path,
278 remote: &str,
279 base_branch: &str,
280 base_commit: &str,
281 branch: &str,
282) -> String {
283 review_base_checked(repo, remote, base_branch, base_commit, branch)
284 .await
285 .0
286}
287
288async fn review_base_checked(
293 repo: &Path,
294 remote: &str,
295 base_branch: &str,
296 base_commit: &str,
297 branch: &str,
298) -> (String, bool) {
299 let tracking = format!("{remote}/{base_branch}");
300 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
301 if git::rev_exists(repo, &tracking).await
302 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
303 && !mb.is_empty()
304 {
305 return (mb, fresh);
306 }
307 let mb = git::merge_base(repo, base_commit, branch)
308 .await
309 .ok()
310 .filter(|mb| !mb.is_empty())
311 .unwrap_or_else(|| base_commit.to_owned());
312 (mb, false)
313}
314
315pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
319 if !is_review_run(state) {
320 return;
321 }
322 let base = review_base(
323 &state.repo,
324 &state.config.merge.remote,
325 &state.base_branch,
326 &state.base_commit,
327 branch,
328 )
329 .await;
330 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
331 && !subjects.is_empty()
332 && state.reviewed_commits.as_ref() != Some(&subjects)
333 {
334 state.reviewed_commits = Some(subjects);
335 state.save().ok();
336 }
337}
338
339async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
343 let (base, trusted) = review_base_checked(
344 &state.repo,
345 &state.config.merge.remote,
346 &state.base_branch,
347 &state.base_commit,
348 branch,
349 )
350 .await;
351 if !trusted {
352 return None;
353 }
354 git::subjects(&state.repo, &state.base_commit, &base)
355 .await
356 .ok()
357}
358
359fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
363 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
364 if is_leak(computed) {
365 return false;
366 }
367 let cur = current.trim();
368 cur.is_empty()
369 || cur.starts_with(REVIEW_PROMPT_OPENING)
370 || cur.starts_with("chore: land candidate")
371 || cur.starts_with("magi: candidate")
372 || is_leak(cur)
373}
374
375async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
404 let tracking = format!("{remote}/{branch}");
405 let fetched = git::fetch(repo, remote, branch).await;
406 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
407 let local_exists = git::branch_exists(repo, branch).await?;
408 if !fresh {
409 if !local_exists {
410 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
411 }
412 tracing::warn!(
413 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
414 );
415 return Ok(());
416 }
417 let remote_sha = git::rev_parse(repo, &tracking).await?;
418 if !local_exists {
419 git::git(repo, &["branch", branch, &tracking]).await?;
420 return Ok(());
421 }
422 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
423 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
424 return Ok(());
425 }
426 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
427 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
433 .await?
434 {
435 crate::reconcile::Reconciliation::Pushed => {
436 tracing::warn!(
437 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
438 short(&local_sha),
439 short(&remote_sha)
440 );
441 return Ok(());
442 }
443 crate::reconcile::Reconciliation::Placeholder => {}
444 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
445 }
446 }
447 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
448 if !out.ok() {
449 bail!(
450 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
451 short(&local_sha),
452 short(&remote_sha),
453 out.stderr
454 );
455 }
456 tracing::warn!(
457 "local `{branch}` was stale: fast-forwarded {} -> {}",
458 short(&local_sha),
459 short(&remote_sha)
460 );
461 Ok(())
462}
463
464async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
465 let tracking = format!("{remote}/{base_branch}");
466 let fetched = git::fetch(repo, remote, base_branch).await;
467 if let Ok(out) = &fetched
468 && out.ok()
469 && git::rev_exists(repo, &tracking).await
470 {
471 return git::rev_parse(repo, &tracking).await;
472 }
473 let why = match &fetched {
474 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
475 Ok(_) => format!("{remote} has no {base_branch}"),
476 Err(e) => e.to_string(),
477 };
478 bail!(
481 "cannot read {tracking} ({why}); refusing to branch off the local \
482 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
483 base / remote in magi.toml"
484 )
485}
486
487struct FixClaim {
504 path: PathBuf,
505}
506
507impl FixClaim {
508 fn acquire(dir: &Path) -> Result<Self> {
509 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
510 let path = dir.join("fix.lock");
511 match Self::create(&path) {
512 Ok(claim) => Ok(claim),
513 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
514 if Self::reclaim_if_dead(&path) {
515 Self::create(&path).with_context(|| format!("lock {}", path.display()))
516 } else {
517 bail!(
518 "another `magi fix` is already running for this run ({} exists)",
519 path.display()
520 )
521 }
522 }
523 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
524 }
525 }
526
527 fn create(path: &Path) -> std::io::Result<Self> {
528 let mut f = std::fs::OpenOptions::new()
529 .write(true)
530 .create_new(true)
531 .open(path)?;
532 use std::io::Write as _;
533 writeln!(f, "{}", std::process::id())?;
535 Ok(Self {
536 path: path.to_owned(),
537 })
538 }
539
540 fn reclaim_if_dead(path: &Path) -> bool {
544 let dead = std::fs::read_to_string(path)
545 .ok()
546 .and_then(|body| body.trim().parse::<u32>().ok())
547 .is_some_and(|pid| !crate::proc::pid_alive(pid));
548 dead && std::fs::remove_file(path).is_ok()
549 }
550}
551
552impl Drop for FixClaim {
553 fn drop(&mut self) {
554 let _ = std::fs::remove_file(&self.path);
555 }
556}
557
558impl Runner {
559 pub async fn start(
561 repo: &Path,
562 instruction: String,
563 config: Config,
564 origin: Origin,
565 ) -> Result<Self> {
566 Self::start_naming(repo, instruction, "", config, origin).await
567 }
568
569 pub async fn start_naming(
573 repo: &Path,
574 instruction: String,
575 also_scan: &str,
576 config: Config,
577 origin: Origin,
578 ) -> Result<Self> {
579 let repo = git::toplevel(repo).await?;
580 let missing = agent::missing_programs(&config.agents);
581 if !missing.is_empty() {
582 bail!(
583 "these agent programs are not on PATH: {}. Fix the roster in \
584 magi.toml or install them.",
585 missing.join(", ")
586 );
587 }
588 let base_branch =
589 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
590 .await?;
591 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
592 let roles = config.resolve_roles()?;
593 let max_parallel = config.graph.max_parallel.max(1);
594 let seeds = refs::resolve(
597 &repo,
598 &base_commit,
599 &config.merge.remote,
600 &format!("{also_scan}\n{instruction}"),
601 )
602 .await;
603 refs::plan(&repo, &seeds).await?;
604 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
605 state.origin = Some(origin);
609 for seed in &seeds {
610 state.event(
611 "seed",
612 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
613 );
614 }
615 state.seeds = seeds;
616 state.event("start", format!("run {} created", state.id));
617 state.save()?;
618 Ok(Self {
619 state,
620 roles,
621 sem: Arc::new(Semaphore::new(max_parallel)),
622 pause: Pause::new(),
623 interrupt: Pause::new(),
624 })
625 }
626
627 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
641 Self::review_taking_over(repo, branch, config, None, origin).await
642 }
643
644 pub async fn review_taking_over(
651 repo: &Path,
652 branch: &str,
653 config: Config,
654 takeover: Option<crate::handover::Takeover>,
655 origin: Origin,
656 ) -> Result<Self> {
657 let repo = git::toplevel(repo).await?;
658 let missing = agent::missing_programs(&config.agents);
659 if !missing.is_empty() {
660 bail!(
661 "these agent programs are not on PATH: {}. Fix the roster in \
662 magi.toml or install them.",
663 missing.join(", ")
664 );
665 }
666 let base_branch =
667 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
668 .await?;
669 if base_branch == branch {
670 bail!("`{branch}` is the base branch; there is nothing to review against");
671 }
672 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
673
674 let roles = config.resolve_roles()?;
675 let max_parallel = config.graph.max_parallel.max(1);
676 let mut state = RunState::new(
677 repo.clone(),
678 base_branch,
679 base_commit.clone(),
680 String::new(),
681 config,
682 );
683 state.origin = Some(origin);
684
685 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
692 earlier: Vec::new(),
693 home: crate::run::home(),
694 choice: None,
695 });
696 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
697 if let Some(released) = &released {
698 state.event(
699 "release",
700 format!(
701 "took `{branch}` over from run {}: its worktree was released: {}",
702 crate::run::short_of(&released.old_id),
703 released.audit
704 ),
705 );
706 }
707 if let Some(choice) = takeover.choice.as_ref()
711 && let Err(e) =
712 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
713 .await
714 {
715 if let Some(released) = &released {
716 released.restore(&repo, branch).await;
717 }
718 return Err(e.context("applying the owner's answer about the diverged branch"));
719 }
720 let opened =
721 Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
722 if opened.is_err()
723 && let Some(released) = &released
724 {
725 released.restore(&repo, branch).await;
726 }
727 opened
728 }
729
730 async fn open_review(
733 repo: &Path,
734 branch: &str,
735 mut state: RunState,
736 roles: ResolvedRoles,
737 max_parallel: usize,
738 base_commit: String,
739 ) -> Result<Self> {
740 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
741 let start = review_base(
744 repo,
745 &state.config.merge.remote,
746 &state.base_branch,
747 &base_commit,
748 branch,
749 )
750 .await;
751 let log = git::log_oneline(repo, &start, branch)
752 .await
753 .unwrap_or_default();
754 let instruction = format!(
755 "Review the work already on branch `{branch}`. There is no task \
756 statement: what the change claims to do is whatever its commits \
757 say.\n\n{}",
758 if log.trim().is_empty() {
759 "(no commit messages)"
760 } else {
761 log.trim()
762 }
763 );
764 state.instruction = instruction;
765 state.reviewed_commits = Some(
766 git::subjects(repo, &start, branch)
767 .await
768 .unwrap_or_default(),
769 );
770
771 let worktree = state.worktree_root().join("under-review");
774 if let Some(parent) = worktree.parent() {
775 tokio::fs::create_dir_all(parent).await.ok();
776 }
777 let path = worktree.to_string_lossy().to_string();
778 git::git(repo, &["worktree", "add", &path, branch])
779 .await
780 .with_context(|| {
781 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
782 })?;
783
784 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
785 .await
786 .unwrap_or(0);
787 if commits == 0 {
788 git::worktree_remove(repo, &worktree).await.ok();
789 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
790 }
791 let files = git::changed_files(&worktree, &base_commit, "HEAD")
792 .await
793 .map(|f| f.len())
794 .unwrap_or(0);
795 if files == 0
796 && let (Ok(head_tree), Ok(base_tree)) = (
797 git::tree_of(&worktree, "HEAD").await,
798 git::tree_of(&worktree, &base_commit).await,
799 )
800 && head_tree == base_tree
801 {
802 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
803 git::worktree_remove(repo, &worktree).await.ok();
804 bail!(
805 "`{branch}` at {} has a tree identical to base {}; this usually means \
806 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
807 against `{}/{branch}`) rather than an empty change",
808 short(&head),
809 short(&base_commit),
810 state.config.merge.remote
811 );
812 }
813 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
814 .await
815 .unwrap_or_default();
816
817 state.candidates.push(Candidate {
818 index: 0,
819 label: 'A',
820 agent: EXISTING_BRANCH.to_owned(),
823 branch: branch.to_owned(),
824 worktree,
825 summary: String::new(),
826 stat,
827 files,
828 commits,
829 empty: false,
830 failed: None,
831 verified_noop: None,
832 duration_ms: 0,
833 folded: false,
834 });
835 state.tally = Some(Tally {
836 first_choice: BTreeMap::from([('A', 0)]),
837 borda: BTreeMap::new(),
838 winner: 'A',
839 rankings: 0,
840 unanimous_initial: false,
841 deliberated: false,
842 changed_votes: 0,
843 unanimous_final: false,
844 tie_break: None,
845 judges: 0,
849 present: 0,
850 quorum: 0,
851 met_quorum: true,
852 uncontested: Some("review-only run: nothing competed".to_owned()),
853 });
854 state.status = RunStatus::Reviewing;
855 state.event(
856 "start",
857 format!(
858 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
859 state.id
860 ),
861 );
862 state.save()?;
863 Ok(Self {
864 state,
865 roles,
866 sem: Arc::new(Semaphore::new(max_parallel)),
867 pause: Pause::new(),
868 interrupt: Pause::new(),
869 })
870 }
871
872 pub fn resume(id: &str) -> Result<Self> {
874 let state = RunState::load(id)?;
875 if let Some(to) = &state.released_to {
876 bail!(
877 "run {} cannot be resumed: its worktree was released to run {}",
878 state.short(),
879 crate::run::short_of(to)
880 );
881 }
882 let roles = state.config.resolve_roles()?;
883 let max_parallel = state.config.graph.max_parallel.max(1);
884 Ok(Self {
885 state,
886 roles,
887 sem: Arc::new(Semaphore::new(max_parallel)),
888 pause: Pause::new(),
889 interrupt: Pause::new(),
890 })
891 }
892
893 pub async fn execute(&mut self) -> Result<()> {
900 let result = self.execute_graph().await;
901 self.mark_driver_exited();
902 let ended = if result.is_err() {
903 Some(crate::notices::run_stopped(&self.state.id, &self.state))
904 } else {
905 crate::notices::run_ended(&self.state)
906 };
907 if let Some(notice) = ended {
908 crate::notices::raise(notice);
909 }
910 result
911 }
912
913 fn mark_driver_exited(&mut self) {
922 self.state.driver_exited = true;
923 let pid = std::process::id();
924 let Ok(mut disk) = RunState::load(&self.state.id) else {
925 return;
926 };
927 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
928 return;
929 }
930 disk.driver_exited = true;
931 if let Err(e) = disk.save() {
932 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
933 }
934 }
935
936 async fn execute_graph(&mut self) -> Result<()> {
937 self.state.parked = false;
942 self.state.clear_active();
949 if let Ok(disk) = RunState::load(&self.state.id)
967 && let Some(to) = &disk.released_to
968 {
969 bail!(
970 "run {} cannot continue: its worktree was released to run {}",
971 self.state.short(),
972 crate::run::short_of(to)
973 );
974 }
975 let pid = std::process::id();
976 self.state.driver_pid = Some(pid);
977 self.state.driver_started_at = crate::proc::process_started_at(pid);
978 self.state.driver_exited = false;
979 self.state.save()?;
980 if self.state.status == RunStatus::Stalled {
993 if self.recover_stall().await? {
994 self.finish_after_tally().await?;
995 } else {
996 self.state.save()?;
998 }
999 return Ok(());
1000 }
1001 if self.state.status == RunStatus::Landing {
1011 self.run_land().await?;
1012 self.settle_questions();
1017 return Ok(());
1018 }
1019 self.prep().await?;
1020 if self.park_here()? {
1021 return Ok(());
1022 }
1023 self.advise().await?;
1024 if self.park_here()? {
1025 return Ok(());
1026 }
1027 self.implement().await?;
1028 if self.park_here()? {
1029 return Ok(());
1030 }
1031 if self.state.status == RunStatus::VerifiedNoop {
1035 return Ok(());
1036 }
1037 self.judge().await?;
1038 if self.park_here()? {
1039 return Ok(());
1040 }
1041 self.deliberate().await?;
1042 if self.park_here()? {
1043 return Ok(());
1044 }
1045 self.vote().await?;
1046 if self.park_here()? {
1047 return Ok(());
1048 }
1049 self.tally()?;
1050 if self.state.status == RunStatus::Stalled {
1055 self.state.save()?;
1059 return Ok(());
1060 }
1061 self.finish_after_tally().await?;
1062 Ok(())
1063 }
1064
1065 fn park_here(&mut self) -> Result<bool> {
1072 if !self.pause.parked() && !self.interrupt.parked() {
1077 return Ok(false);
1078 }
1079 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
1080 Some(reason) => format!(
1081 "parked after `{}` ({reason}) — resume to carry on from here",
1082 self.state.status.as_str()
1083 ),
1084 None => format!(
1085 "parked after `{}` — resume to carry on from here",
1086 self.state.status.as_str()
1087 ),
1088 };
1089 self.state.event("park", why);
1090 self.state.parked = true;
1091 self.state.save()?;
1092 Ok(true)
1093 }
1094
1095 pub fn on_pause(&mut self, pause: Pause) {
1097 self.pause = pause;
1098 }
1099
1100 pub fn watch_interrupt(&mut self, pause: Pause) {
1106 self.interrupt = pause;
1107 }
1108
1109 fn settle_questions(&mut self) {
1129 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1130 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1131 }
1132 }
1133
1134 async fn finish_after_tally(&mut self) -> Result<()> {
1137 self.fold_losers().await?;
1138 self.sync_to_base().await?;
1143 if self.state.status == RunStatus::AlreadyInBase {
1144 return Ok(());
1145 }
1146 self.review_loop().await?;
1147 self.sync_to_base().await?;
1148 if self.state.status == RunStatus::AlreadyInBase {
1149 return Ok(());
1150 }
1151 self.gate().await?;
1152 self.merge().await?;
1153 self.state.save()?;
1154 Ok(())
1155 }
1156
1157 async fn prep(&mut self) -> Result<()> {
1160 if !self.state.candidates.is_empty() {
1161 return Ok(());
1162 }
1163 self.state.status = RunStatus::Prep;
1164 let repo = self.state.repo.clone();
1165 let base = self.state.base_commit.clone();
1166 let plan = refs::plan(&repo, &self.state.seeds).await?;
1167 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1168 let root = self.state.worktree_root();
1169 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1170
1171 let hooks_dir = self.state.dir().join("hooks");
1174 if self.state.config.blind.commit_msg_hook {
1175 std::fs::create_dir_all(&hooks_dir)
1176 .with_context(|| format!("create {}", hooks_dir.display()))?;
1177 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1178 let path = hooks_dir.join("commit-msg");
1179 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1180 make_executable(&path)?;
1181 git::acquire_worktree_config(&repo).await?;
1189 self.state.enabled_worktree_config = true;
1190 }
1191
1192 for (index, (spec, label)) in self
1193 .roles
1194 .implementers
1195 .clone()
1196 .into_iter()
1197 .zip(labels)
1198 .enumerate()
1199 {
1200 let branch = self.state.branch_for(label);
1201 let worktree = root.join(format!("cand-{label}"));
1202 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1203 if self.state.config.blind.commit_msg_hook {
1204 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1205 }
1206 git::local_exclude(&worktree, "/.magi/").await?;
1207 for pick in &plan.picks {
1208 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1209 self.state.status = RunStatus::Blocked;
1210 self.state
1211 .event("prep", format!("cannot apply referenced commit: {e}"));
1212 self.state.save()?;
1213 return Err(e);
1214 }
1215 }
1216 self.state.candidates.push(Candidate {
1217 index,
1218 label,
1219 agent: spec.id.clone(),
1220 branch,
1221 worktree,
1222 summary: String::new(),
1223 stat: String::new(),
1224 files: 0,
1225 commits: 0,
1226 empty: false,
1227 failed: None,
1228 verified_noop: None,
1229 duration_ms: 0,
1230 folded: false,
1231 });
1232 }
1233
1234 for j in 1..=self.roles.judges.len() {
1235 let wt = root.join(format!("judge-{j}"));
1236 if !wt.exists() {
1237 git::worktree_add_detached(&repo, &wt, &base).await?;
1238 }
1239 }
1240
1241 if self.state.config.graph.advise {
1249 for k in 1..=self.state.config.graph.advisors {
1250 let wt = root.join(format!("advisor-{k}"));
1251 if !wt.exists() {
1252 git::worktree_add_detached(&repo, &wt, &base).await?;
1253 }
1254 }
1255 }
1256
1257 let authors: Vec<&str> = self
1262 .roles
1263 .implementers
1264 .iter()
1265 .map(|a| a.id.as_str())
1266 .collect();
1267 let overlap: Vec<String> = self
1268 .roles
1269 .judges
1270 .iter()
1271 .enumerate()
1272 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1273 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1274 .collect();
1275 if !overlap.is_empty() {
1276 let note = format!(
1277 "{} also authored a candidate; blind, but the panel is less \
1278 independent than {} distinct agents would be",
1279 overlap.join(", "),
1280 self.roles.judges.len()
1281 );
1282 self.state.event("prep", note);
1283 }
1284
1285 self.state.event(
1286 "prep",
1287 format!(
1288 "{} candidates, {} judges, base {} ({})",
1289 self.state.candidates.len(),
1290 self.roles.judges.len(),
1291 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1292 self.state.base_branch
1293 ),
1294 );
1295 self.state.status = RunStatus::Implementing;
1296 self.state.save()?;
1297 Ok(())
1298 }
1299
1300 async fn advise(&mut self) -> Result<()> {
1333 let implement_untouched = self
1334 .state
1335 .candidates
1336 .iter()
1337 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1338 if !self.state.config.graph.advise || self.state.advise_attempted {
1339 return Ok(());
1340 }
1341 if !implement_untouched {
1342 self.state.event(
1343 "advise",
1344 "skipping the design-deliberation stage: at least one \
1345 candidate already shows implementation progress, so this \
1346 run is past the point the stage exists to run before"
1347 .to_owned(),
1348 );
1349 self.state.advise_attempted = true;
1350 self.state.save()?;
1351 return Ok(());
1352 }
1353 let run_id = self.state.id.clone();
1354 let prompts = self.state.config.prompts.clone();
1355 let instruction = self.state.instruction.clone();
1356 let language = self.state.config.graph.language.clone();
1357 let root = self.state.worktree_root();
1358 let n = self.state.config.graph.advisors;
1359 let where_recorded = self.state.dir().join("run.json");
1360
1361 let seats = match self.state.config.advisors() {
1362 Ok(seats) if !seats.is_empty() => seats,
1363 Ok(_) => {
1364 self.state.event(
1365 "advise",
1366 format!(
1367 "[graph] advisors is 0; skipping the design-deliberation \
1368 stage and continuing without a synthesis brief (see {})",
1369 where_recorded.display()
1370 ),
1371 );
1372 self.state.advise_attempted = true;
1373 self.state.save()?;
1374 return Ok(());
1375 }
1376 Err(e) => {
1377 self.state.event(
1378 "advise",
1379 format!(
1380 "could not resolve advisor seats ({e:#}); continuing \
1381 without a design-deliberation brief (see {})",
1382 where_recorded.display()
1383 ),
1384 );
1385 self.state.advise_attempted = true;
1386 self.state.save()?;
1387 return Ok(());
1388 }
1389 };
1390
1391 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1392 let artifacts = agent::artifacts_dir(&self.state.dir());
1393 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1394
1395 let mut jobs = Vec::new();
1396 for (i, spec) in seats.iter().cloned().enumerate() {
1397 let seat_key = format!("advisor-{}", i + 1);
1398 let seat = self.seat(&seat_key, &spec.id);
1399 jobs.push(SeatJob {
1400 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1401 spec,
1402 seat,
1403 cwd: worktrees[i % worktrees.len()].clone(),
1404 timeout,
1405 allow_write: false,
1406 sessions: false,
1407 artifacts: artifacts.clone(),
1408 stem: seat_key,
1409 handover: None,
1410 });
1411 }
1412
1413 self.state.event(
1414 "advise",
1415 format!(
1416 "{} advisor seat(s) sketching a design in parallel",
1417 jobs.len()
1418 ),
1419 );
1420 let mut quota_losses = Vec::new();
1421 let cache = self.state.config.cache_dir();
1422 let ctx = WaveCtx {
1423 carry_seats: false,
1424 run: &run_id,
1425 node: "advise",
1426 prompts: &prompts,
1427 cache: cache.as_deref(),
1428 round: None,
1429 };
1430 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1431 let results = ask_json_wave::<Proposal>(
1432 jobs,
1433 Arc::clone(&self.sem),
1434 self.state.config.graph.retries,
1435 &advisor_roster,
1436 &ctx,
1437 &mut quota_losses,
1438 &mut self.state,
1439 &|p: &Proposal| p.validate(),
1440 )
1441 .await;
1442 self.state.quota.extend(quota_losses);
1443
1444 let mut records = Vec::with_capacity(results.len());
1445 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1446 let agent_id = seat.agent.clone();
1447 self.state.seats.insert(seat.key.clone(), seat);
1448 match res {
1449 Ok((proposal, out)) => {
1450 self.state
1451 .event("advise", format!("advisor-{} proposed a design", i + 1));
1452 records.push(advise::AdvisorRecord::proposed(
1453 i + 1,
1454 agent_id,
1455 proposal,
1456 out.duration_ms,
1457 ));
1458 }
1459 Err(e) => {
1460 self.state.event(
1461 "advise",
1462 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1463 );
1464 records.push(advise::AdvisorRecord::failed(
1465 i + 1,
1466 agent_id,
1467 e.to_string(),
1468 ));
1469 }
1470 }
1471 }
1472
1473 let mut advice = advise::Advice {
1474 records,
1475 synthesis: None,
1476 };
1477 if advice.proposals().is_empty() {
1478 self.state.event(
1479 "advise",
1480 "no advisor produced a usable proposal; continuing without a \
1481 synthesis brief"
1482 .to_owned(),
1483 );
1484 } else {
1485 match self
1486 .synthesize_brief(
1487 &advice,
1488 &instruction,
1489 &language,
1490 &worktrees[0],
1491 &artifacts,
1492 &run_id,
1493 &prompts,
1494 cache.as_deref(),
1495 )
1496 .await
1497 {
1498 Ok(Some(text)) => {
1499 self.state.event(
1500 "advise",
1501 "synthesized a design brief for the implementer".to_owned(),
1502 );
1503 advice.synthesis = Some(text);
1504 }
1505 Ok(None) => {
1506 self.state.event(
1507 "advise",
1508 "the synthesis seat produced nothing usable; continuing \
1509 without a design brief"
1510 .to_owned(),
1511 );
1512 }
1513 Err(e) => {
1514 self.state.event(
1515 "advise",
1516 format!("could not synthesize a design brief: {e:#}"),
1517 );
1518 }
1519 }
1520 }
1521 advise::apply_reflection(&mut advice);
1522
1523 self.state.advice = Some(advice);
1524 self.state.advise_attempted = true;
1525 self.state.save()?;
1526 Ok(())
1527 }
1528
1529 #[allow(clippy::too_many_arguments)]
1541 async fn synthesize_brief(
1542 &mut self,
1543 advice: &advise::Advice,
1544 instruction: &str,
1545 language: &str,
1546 cwd: &Path,
1547 artifacts: &Path,
1548 run_id: &str,
1549 prompts: &Prompts,
1550 cache: Option<&Path>,
1551 ) -> Result<Option<String>> {
1552 let chain = agent::pick_chain(
1553 &self.state.config.agents,
1554 self.state.config.roles.synthesizer.as_ref(),
1555 &agent::installed,
1556 "synthesizer",
1557 )?;
1558 let proposals = advice.proposals();
1559 let mut prompt = prompt::with_overlay(
1560 prompt::synthesize_brief(instruction, &proposals, language),
1561 prompts.overlay("advise"),
1562 );
1563 if cache.is_some() {
1564 prompt.push('\n');
1569 prompt.push_str(&prompt::build_cache_note("advise", false));
1570 }
1571 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1572 let mut last = None;
1577 for (n, spec) in chain.iter().enumerate() {
1578 if n > 0 {
1579 self.state
1580 .event("advise", format!("synthesis falling back to {}", spec.id));
1581 }
1582 let mut seat = self.seat("advise-synthesis", &spec.id);
1583 let outcome = agent::invoke(
1584 spec,
1585 &mut seat,
1586 &Invocation {
1587 cwd,
1588 prompt: &prompt,
1589 timeout,
1590 allow_write: false,
1591 sessions: false,
1592 artifacts,
1593 stem: &if n == 0 {
1594 "advise-synthesis".to_owned()
1595 } else {
1596 format!("advise-synthesis-{}", spec.id)
1597 },
1598 run: run_id,
1599 node: "advise",
1600 cache_dir: None,
1601 attachments: &[],
1602 writable: &[],
1603 },
1604 )
1605 .await;
1606 if outcome.is_ok() {
1607 self.state.seats.insert(seat.key.clone(), seat);
1608 }
1609 let advance = agent::chain_advances(&outcome);
1610 last = Some(outcome);
1611 if !advance {
1612 break;
1613 }
1614 }
1615 let out = last.expect("a chain holds at least one agent")?;
1618 if !out.usable() {
1619 return Ok(None);
1620 }
1621 let text =
1622 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1623 Ok((!text.trim().is_empty()).then_some(text))
1624 }
1625
1626 async fn implement(&mut self) -> Result<()> {
1629 let run_id = self.state.id.clone();
1634 let prompts = self.state.config.prompts.clone();
1635 let todo: Vec<usize> = self
1636 .state
1637 .candidates
1638 .iter()
1639 .enumerate()
1640 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1641 .map(|(i, _)| i)
1642 .collect();
1643 if todo.is_empty() {
1644 return self.after_implement();
1645 }
1646 self.state.status = RunStatus::Implementing;
1647
1648 let language = self.state.config.graph.language.clone();
1649 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1650 let sessions = self.state.config.graph.sessions;
1651 let artifacts = agent::artifacts_dir(&self.state.dir());
1652 let brief = self
1656 .state
1657 .advice
1658 .as_ref()
1659 .and_then(|a| a.synthesis.as_deref())
1660 .map(str::to_owned);
1661 let attachments = self.state.attachments.clone();
1662
1663 let mut jobs = Vec::new();
1664 for &i in &todo {
1665 let (index, label, worktree) = {
1666 let c = &self.state.candidates[i];
1667 (c.index, c.label, c.worktree.clone())
1668 };
1669 let spec = self.roles.implementers[index].clone();
1670 let seat_key = format!("impl-{label}");
1671 let seat = self.seat(&seat_key, &spec.id);
1672 let instruction = seeded_instruction(&self.state);
1673 jobs.push(SeatJob {
1674 spec,
1675 seat,
1676 prompt: prompt::implement(
1677 &instruction,
1678 &worktree.to_string_lossy(),
1679 &language,
1680 brief.as_deref(),
1681 &attachments,
1682 ),
1683 cwd: worktree,
1684 timeout,
1685 allow_write: true,
1686 sessions,
1687 artifacts: artifacts.clone(),
1688 stem: format!("impl-{label}"),
1689 handover: None,
1690 });
1691 }
1692
1693 self.state.event(
1694 "implement",
1695 format!("{} candidates in parallel", jobs.len()),
1696 );
1697 let mut sent = jobs.clone();
1703 let cache = self.state.config.cache_dir();
1704 let ctx = WaveCtx {
1705 carry_seats: false,
1706 run: &run_id,
1707 node: "implement",
1708 prompts: &prompts,
1709 cache: cache.as_deref(),
1710 round: None,
1711 };
1712 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1713 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1714 .await;
1715 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1716 .await;
1717 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1718 .await;
1719
1720 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1721 let seat_key = seat.key.clone();
1722 let agent = seat.agent.clone();
1731 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1732 self.state.seats.insert(seat.key.clone(), seat);
1733 let label = self.state.candidates[i].label;
1734 let worktree = self.state.candidates[i].worktree.clone();
1735 let base = self.state.base_commit.clone();
1736
1737 let (summary, duration, failed, verified_claim) = match out {
1738 AgentOutcome::Ok(o) => {
1739 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1740 let failed = (!o.usable()).then(|| {
1741 if o.timed_out {
1742 "agent timed out".to_owned()
1743 } else {
1744 format!("agent exited with {:?}", o.exit_code)
1745 }
1746 });
1747 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1748 (text, o.duration_ms, failed, verified_claim)
1749 }
1750 AgentOutcome::Dropped(o) => {
1756 let why = o
1757 .dropped
1758 .as_ref()
1759 .map(|d| d.why.as_str())
1760 .unwrap_or("the CLI ended the stream without delivering its answer");
1761 (
1762 String::new(),
1763 o.duration_ms,
1764 Some(format!("the CLI dropped the stream ({why})")),
1765 None,
1766 )
1767 }
1768 AgentOutcome::Quota(o) => {
1769 self.state.quota.push(QuotaLoss {
1770 seat: seat_key,
1771 node: "implement".to_owned(),
1772 at: Timestamp::now(),
1773 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1774 });
1775 (
1776 String::new(),
1777 o.duration_ms,
1778 Some("rate limited (quota); produced no change".to_owned()),
1779 None,
1780 )
1781 }
1782 AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1783 };
1784
1785 let rescued = match git::rescue_commit(
1788 &worktree,
1789 &format!("magi: candidate {label} (uncommitted work)"),
1790 )
1791 .await
1792 {
1793 Ok(r) => {
1794 self.state.note_withheld("implement", &r.withheld);
1795 r.committed
1796 }
1797 Err(_) => false,
1798 };
1799 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1800 .await
1801 .unwrap_or(0);
1802 let patch = git::diff(&worktree, &base, "HEAD")
1803 .await
1804 .unwrap_or_default();
1805 let stat = git::diff_stat(&worktree, &base, "HEAD")
1806 .await
1807 .unwrap_or_default();
1808 let files = git::changed_files(&worktree, &base, "HEAD")
1809 .await
1810 .map(|f| f.len())
1811 .unwrap_or(0);
1812 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1813
1814 let c = &mut self.state.candidates[i];
1815 if !exhausted_the_fallback_chain {
1816 c.agent = agent;
1817 }
1818 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1819 c.stat = stat;
1820 c.files = files;
1821 c.commits = commits;
1822 c.duration_ms = duration;
1823 c.empty = commits == 0 || patch.trim().is_empty();
1824 c.failed = match failed {
1827 Some(_) if c.empty => failed,
1828 _ => None,
1829 };
1830 c.verified_noop = if c.empty { verified_claim } else { None };
1835 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1836 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1837 (None, true, Some(_), _) => {
1838 format!("candidate {label}: no change produced (agent-verified no-op)")
1839 }
1840 (None, true, None, _) => format!("candidate {label}: no change produced"),
1841 (None, false, _, true) => {
1842 format!(
1843 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1844 )
1845 }
1846 (None, false, _, false) => {
1847 format!("candidate {label}: {files} files, {commits} commits")
1848 }
1849 };
1850 self.state.event("implement", note);
1851 self.state.save()?;
1852 }
1853
1854 self.after_implement()
1855 }
1856
1857 async fn resume_undelivered(
1885 &mut self,
1886 results: &mut [(usize, SeatState, AgentOutcome)],
1887 sent: &[SeatJob],
1888 prompts: &Prompts,
1889 run_id: &str,
1890 ) {
1891 for (wi, seat, out) in results.iter_mut() {
1892 let Some(dropped) = (match &*out {
1893 AgentOutcome::Dropped(o) => o.dropped.clone(),
1894 _ => None,
1895 }) else {
1896 continue;
1897 };
1898 let Some(job) = sent.get(*wi) else { continue };
1899 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1901 self.state.event(
1902 "implement",
1903 format!(
1904 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1905 work is in the tree",
1906 seat.key, dropped.output_tokens, dropped.why
1907 ),
1908 );
1909 continue;
1910 }
1911 if !has_context(&job.spec, seat, job.sessions) {
1919 self.state.event(
1920 "implement",
1921 format!(
1922 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1923 is no session left to resume",
1924 seat.key, dropped.output_tokens, dropped.why
1925 ),
1926 );
1927 continue;
1928 }
1929 self.state.event(
1930 "implement",
1931 format!(
1932 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1933 conversation",
1934 seat.key, dropped.output_tokens, dropped.why
1935 ),
1936 );
1937 let mut retry = job.clone();
1938 retry.seat = seat.clone();
1939 retry.prompt = prompt::resume_after_drop(&dropped.why);
1940 retry.timeout = retry_budget(job.timeout, true);
1941 retry.stem = format!("{}-resume", job.stem);
1942 let cache = self.state.config.cache_dir();
1943 let ctx = WaveCtx {
1944 carry_seats: false,
1945 run: run_id,
1946 node: "implement",
1947 prompts,
1948 cache: cache.as_deref(),
1949 round: None,
1950 };
1951 let (resumed_seat, resumed) =
1952 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1953 *seat = resumed_seat;
1954 *out = resumed;
1955 }
1956 }
1957
1958 async fn resume_seat_handovers(
2023 &mut self,
2024 results: &mut [(usize, SeatState, AgentOutcome)],
2025 sent: &mut [SeatJob],
2026 prompts: &Prompts,
2027 run_id: &str,
2028 ) {
2029 let instruction = seeded_instruction(&self.state);
2030 let language = self.state.config.graph.language.clone();
2031 let brief = self
2032 .state
2033 .advice
2034 .as_ref()
2035 .and_then(|a| a.synthesis.as_deref())
2036 .map(str::to_owned);
2037 let attachments = self.state.attachments.clone();
2038 for (wi, seat, out) in results.iter_mut() {
2039 let others: BTreeSet<String> = sent
2042 .iter()
2043 .enumerate()
2044 .filter(|(j, _)| j != wi)
2045 .map(|(_, j)| j.spec.id.clone())
2046 .collect();
2047 let Some(job) = sent.get_mut(*wi) else {
2048 continue;
2049 };
2050 let start = self
2055 .roles
2056 .implementer_roster
2057 .iter()
2058 .position(|s| s.id == job.spec.id)
2059 .unwrap_or(0);
2060 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2061 let mut fallback_attempt = 0usize;
2062 let mut prev: Option<FailClass> = None;
2063 while let Some(cur) = FailClass::of(&*out) {
2064 if !should_hand_over(prev.as_ref(), &cur) {
2065 break;
2066 }
2067 let Some(next) =
2068 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2069 .cloned()
2070 else {
2071 break;
2072 };
2073 tried.insert(next.id.clone());
2074 fallback_attempt += 1;
2075
2076 if let Ok(r) = git::rescue_commit(
2077 &job.cwd,
2078 &format!(
2079 "magi: candidate {} (uncommitted work before {} fallback)",
2080 seat.key,
2081 if cur == FailClass::Quota {
2082 "quota"
2083 } else {
2084 "handover"
2085 }
2086 ),
2087 )
2088 .await
2089 {
2090 self.state.note_withheld("implement", &r.withheld);
2091 }
2092
2093 record_handover(
2094 &mut self.state,
2095 "implement",
2096 &seat.key,
2097 &seat.agent,
2098 &next.id,
2099 &cur,
2100 &fail_reason(&*out),
2101 );
2102 prev = Some(cur.clone());
2103
2104 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2105 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2106 job.spec = next.clone();
2114 let mut retry = job.clone();
2115 retry.seat = new_seat;
2116 retry.prompt = prompt::implement(
2117 &instruction,
2118 &job.cwd.to_string_lossy(),
2119 &language,
2120 brief.as_deref(),
2121 &attachments,
2122 );
2123 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2124 let cache = self.state.config.cache_dir();
2125 let ctx = WaveCtx {
2126 carry_seats: false,
2127 run: run_id,
2128 node: "implement",
2129 prompts,
2130 cache: cache.as_deref(),
2131 round: None,
2132 };
2133 let (fallback_seat, fallback_out) = run_one(
2134 retry,
2135 Arc::clone(&self.sem),
2136 &ctx,
2137 &mut self.state,
2138 fallback_attempt,
2139 )
2140 .await;
2141 *seat = fallback_seat;
2142 *out = fallback_out;
2143 }
2144 }
2145 }
2146
2147 async fn resume_unconfirmed_commands(
2171 &mut self,
2172 results: &mut [(usize, SeatState, AgentOutcome)],
2173 sent: &[SeatJob],
2174 prompts: &Prompts,
2175 run_id: &str,
2176 ) {
2177 for (wi, seat, out) in results.iter_mut() {
2178 let AgentOutcome::Ok(o) = &*out else {
2179 continue;
2180 };
2181 if !has_unconfirmed_command(&o.commands) {
2182 continue;
2183 }
2184 let Some(job) = sent.get(*wi) else { continue };
2185 if !has_context(&job.spec, seat, job.sessions) {
2186 self.state.event(
2187 "implement",
2188 format!(
2189 "{}: the reply named a command whose own CLI never confirmed the exit \
2190 status of, but there is no session left to resume",
2191 seat.key
2192 ),
2193 );
2194 continue;
2195 }
2196 self.state.event(
2197 "implement",
2198 format!(
2199 "{}: the reply named a command whose own CLI never confirmed the exit \
2200 status of; resuming the conversation",
2201 seat.key
2202 ),
2203 );
2204 let mut retry = job.clone();
2205 retry.seat = seat.clone();
2206 retry.prompt = prompt::resume_incomplete(
2207 "a command in your last reply had no confirmed exit status",
2208 );
2209 retry.timeout = retry_budget(job.timeout, true);
2210 retry.stem = format!("{}-confirm", job.stem);
2211 let cache = self.state.config.cache_dir();
2212 let ctx = WaveCtx {
2213 carry_seats: false,
2214 run: run_id,
2215 node: "implement",
2216 prompts,
2217 cache: cache.as_deref(),
2218 round: None,
2219 };
2220 let (resumed_seat, resumed) =
2221 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2222 *seat = resumed_seat;
2223 *out = resumed;
2224 }
2225 }
2226
2227 async fn continue_fix_report(
2248 &mut self,
2249 mut seat: SeatState,
2250 parse_err: String,
2251 job: &SeatJob,
2252 prompts: &Prompts,
2253 run_id: &str,
2254 round: usize,
2255 ) -> (
2256 SeatState,
2257 Option<FixReport>,
2258 Option<String>,
2259 ContinuationRecord,
2260 ) {
2261 let mut last_err = parse_err;
2262 let mut cumulative_wait_ms = 0u64;
2263 let mut attempts = 0usize;
2264 loop {
2265 if !has_context(&job.spec, &seat, job.sessions) {
2266 self.state.event(
2267 "fix",
2268 format!(
2269 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2270 session left to resume into"
2271 ),
2272 );
2273 let outcome = if attempts == 0 {
2274 ContinuationOutcome::NoSession
2275 } else {
2276 ContinuationOutcome::Exhausted
2277 };
2278 return (
2279 seat,
2280 None,
2281 Some(format!("unparsable fix report: {last_err}")),
2282 ContinuationRecord {
2283 attempts,
2284 cumulative_wait_ms,
2285 outcome,
2286 },
2287 );
2288 }
2289 if attempts >= MAX_FIX_CONTINUATIONS {
2290 self.state.event(
2291 "fix",
2292 format!(
2293 "round {round}: fixer's reply still had no adoption report after \
2294 {attempts} continuation(s) ({last_err}); giving up"
2295 ),
2296 );
2297 return (
2298 seat,
2299 None,
2300 Some(format!(
2301 "unparsable fix report after {attempts} continuation(s): {last_err}"
2302 )),
2303 ContinuationRecord {
2304 attempts,
2305 cumulative_wait_ms,
2306 outcome: ContinuationOutcome::Exhausted,
2307 },
2308 );
2309 }
2310 attempts += 1;
2311 self.state.event(
2312 "fix",
2313 format!(
2314 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2315 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2316 ),
2317 );
2318 let mut retry = job.clone();
2319 retry.seat = seat.clone();
2320 retry.prompt = prompt::resume_incomplete(&last_err);
2321 retry.timeout = retry_budget(job.timeout, true);
2322 retry.stem = format!("{}-continue{attempts}", job.stem);
2323 let cache = self.state.config.cache_dir();
2324 let ctx = WaveCtx {
2325 carry_seats: false,
2326 run: run_id,
2327 node: "fix",
2328 prompts,
2329 cache: cache.as_deref(),
2330 round: Some(round),
2331 };
2332 let (resumed_seat, resumed_out) = run_one(
2333 retry,
2334 Arc::clone(&self.sem),
2335 &ctx,
2336 &mut self.state,
2337 attempts,
2338 )
2339 .await;
2340 seat = resumed_seat;
2341 match resumed_out {
2342 AgentOutcome::Ok(o) => {
2343 cumulative_wait_ms += o.duration_ms;
2344 match verdict::extract_json::<FixReport>(&o.text) {
2345 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2346 self.state.event(
2347 "fix",
2348 format!(
2349 "round {round}: fixer's adoption report recovered after \
2350 {attempts} continuation(s)"
2351 ),
2352 );
2353 return (
2354 seat,
2355 Some(report),
2356 None,
2357 ContinuationRecord {
2358 attempts,
2359 cumulative_wait_ms,
2360 outcome: ContinuationOutcome::Resumed,
2361 },
2362 );
2363 }
2364 Ok(_) => {
2372 last_err = "the reply parsed, but it reported a command whose own CLI \
2373 never confirmed an exit status"
2374 .to_owned();
2375 }
2376 Err(e) => last_err = e.to_string(),
2377 }
2378 }
2379 AgentOutcome::Quota(o) => {
2380 cumulative_wait_ms += o.duration_ms;
2381 self.state.quota.push(QuotaLoss {
2382 seat: seat.key.clone(),
2383 node: "fix".to_owned(),
2384 at: Timestamp::now(),
2385 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2386 });
2387 self.state.event(
2388 "fix",
2389 format!(
2390 "round {round}: continuation rate limited (quota); not retrying now"
2391 ),
2392 );
2393 return (
2394 seat,
2395 None,
2396 Some("rate limited (quota) while recovering the fix report".to_owned()),
2397 ContinuationRecord {
2398 attempts,
2399 cumulative_wait_ms,
2400 outcome: ContinuationOutcome::QuotaLost,
2401 },
2402 );
2403 }
2404 AgentOutcome::Dropped(o) => {
2405 cumulative_wait_ms += o.duration_ms;
2406 let why = o
2407 .dropped
2408 .as_ref()
2409 .map(|d| d.why.as_str())
2410 .unwrap_or("the CLI ended the stream without delivering its answer");
2411 last_err = format!("the CLI dropped the stream ({why})");
2412 }
2413 AgentOutcome::Failed(e) => last_err = e,
2414 }
2415 }
2416 }
2417
2418 fn after_implement(&mut self) -> Result<()> {
2419 if self.state.leaks.is_empty() {
2421 let cfg = self.state.config.blind.clone();
2422 let mut leaks = Vec::new();
2423 for c in &self.state.candidates {
2424 let Some(patch) =
2425 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2426 else {
2427 continue;
2428 };
2429 leaks.extend(blind::scan(
2430 &format!("candidate {} patch", c.label),
2431 &patch,
2432 &cfg.vendor_tokens,
2433 ));
2434 }
2435 if !leaks.is_empty() {
2436 let summary = leaks
2437 .iter()
2438 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2439 .collect::<Vec<_>>()
2440 .join(", ");
2441 match cfg.on_leak {
2442 LeakPolicy::Fail => {
2443 self.state.status = RunStatus::Failed;
2444 self.state
2445 .event("blind", format!("vendor text in a patch: {summary}"));
2446 self.state.leaks = leaks;
2447 self.state.save()?;
2448 self.settle_questions();
2449 bail!(
2450 "blind.on_leak = \"fail\" and vendor text reached a \
2451 judged patch: {summary}"
2452 );
2453 }
2454 LeakPolicy::Redact => self.state.event(
2455 "blind",
2456 format!("redacting vendor text for judging: {summary}"),
2457 ),
2458 LeakPolicy::Warn => self.state.event(
2459 "blind",
2460 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2461 ),
2462 }
2463 self.state.leaks = leaks;
2464 }
2465 }
2466
2467 if self.state.viable().is_empty() {
2468 if self.state.all_candidates_verified_noop() {
2469 self.state.status = RunStatus::VerifiedNoop;
2480 self.state.save()?;
2481 self.settle_questions();
2482 return Ok(());
2483 }
2484 self.state.status = RunStatus::Failed;
2485 self.state.save()?;
2486 self.settle_questions();
2487 bail!("no candidate produced a change; nothing to judge");
2488 }
2489 self.state.status = RunStatus::Judging;
2490 self.state.save()?;
2491 Ok(())
2492 }
2493
2494 async fn judge(&mut self) -> Result<()> {
2497 let run_id = self.state.id.clone();
2502 let prompts = self.state.config.prompts.clone();
2503 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2504 return Ok(());
2505 }
2506 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2507 if viable.len() == 1 {
2508 self.state.judge_skipped = true;
2515 self.state.event(
2516 "judge",
2517 format!(
2518 "only candidate {} produced a change; judging skipped",
2519 viable[0].label
2520 ),
2521 );
2522 self.state.save()?;
2523 return Ok(());
2524 }
2525 self.state.status = RunStatus::Judging;
2526
2527 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2528 let language = self.state.config.graph.language.clone();
2529 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2530 let sessions = self.state.config.graph.sessions;
2531 let artifacts = agent::artifacts_dir(&self.state.dir());
2532 let root = self.state.worktree_root();
2533 let base_short = short(&self.state.base_commit);
2534
2535 let mut jobs = Vec::new();
2536 let mut orders = Vec::new();
2537 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2538 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2539 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2540 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2541 let seat_key = format!("judge-{}", j + 1);
2542 let seat = self.seat(&seat_key, &spec.id);
2543 jobs.push(SeatJob {
2544 prompt: prompt::judge(
2545 &self.state.instruction,
2546 &views,
2547 self.roles.judges.len(),
2548 &base_short,
2549 &language,
2550 ),
2551 spec,
2552 seat,
2553 cwd: root.join(format!("judge-{}", j + 1)),
2554 timeout,
2555 allow_write: false,
2556 sessions,
2557 artifacts: artifacts.clone(),
2558 stem: format!("judge-{}", j + 1),
2559 handover: None,
2560 });
2561 }
2562
2563 self.state.event(
2564 "judge",
2565 format!(
2566 "{} judges ranking {} candidates blind",
2567 jobs.len(),
2568 viable.len()
2569 ),
2570 );
2571 let labels_for_check = labels.clone();
2572 let mut quota_losses = Vec::new();
2573 let cache = self.state.config.cache_dir();
2574 let ctx = WaveCtx {
2575 carry_seats: false,
2576 run: &run_id,
2577 node: "judge",
2578 prompts: &prompts,
2579 cache: cache.as_deref(),
2580 round: None,
2581 };
2582 let results = ask_json_wave::<Ranking>(
2583 jobs,
2584 Arc::clone(&self.sem),
2585 self.state.config.graph.retries,
2586 &self.roles.judge_roster,
2587 &ctx,
2588 &mut quota_losses,
2589 &mut self.state,
2590 &move |r: &Ranking| r.validate(&labels_for_check),
2591 )
2592 .await;
2593 self.state.quota.extend(quota_losses);
2594
2595 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2596 let agent_id = seat.agent.clone();
2597 self.state.seats.insert(seat.key.clone(), seat);
2598 let mut record = Judgement {
2599 judge: j + 1,
2600 seat: format!("judge-{}", j + 1),
2601 agent: agent_id,
2602 ranking: Vec::new(),
2603 reasons: BTreeMap::new(),
2604 confidence: None,
2605 order: orders[j].clone(),
2606 failed: None,
2607 duration_ms: 0,
2608 };
2609 match res {
2610 Ok((ranking, out)) => {
2611 record.ranking = ranking.normalized();
2612 record.reasons = ranking.reasons;
2613 record.confidence = ranking.confidence;
2614 record.duration_ms = out.duration_ms;
2615 self.state.event(
2616 "judge",
2617 format!(
2618 "judge {} ranked {}",
2619 j + 1,
2620 record.ranking.iter().collect::<String>()
2621 ),
2622 );
2623 }
2624 Err(e) => {
2625 record.failed = Some(e.to_string());
2626 self.state
2627 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2628 }
2629 }
2630 self.state.judgements.push(record);
2631 self.state.save()?;
2632 }
2633 Ok(())
2634 }
2635
2636 async fn deliberate(&mut self) -> Result<()> {
2639 let run_id = self.state.id.clone();
2644 let prompts = self.state.config.prompts.clone();
2645 if !self.state.deliberation.is_empty() {
2646 return Ok(());
2647 }
2648 let tops: Vec<char> = self
2649 .state
2650 .judgements
2651 .iter()
2652 .filter_map(|j| j.ranking.first().copied())
2653 .collect();
2654 let rounds = self.state.config.graph.deliberate_rounds;
2655 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2656 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2657 self.state.event(
2658 "deliberate",
2659 format!("judges agreed on {} outright; no deliberation", tops[0]),
2660 );
2661 }
2662 self.state.status = RunStatus::Voting;
2663 self.state.save()?;
2664 return Ok(());
2665 }
2666
2667 self.state.status = RunStatus::Deliberating;
2668 self.state.event(
2669 "deliberate",
2670 format!(
2671 "split: first choices were {} — opening {rounds} round(s)",
2672 tops.iter().collect::<String>()
2673 ),
2674 );
2675
2676 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2677 let language = self.state.config.graph.language.clone();
2678 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2679 let sessions = self.state.config.graph.sessions;
2680 let artifacts = agent::artifacts_dir(&self.state.dir());
2681 let root = self.state.worktree_root();
2682 let base_short = short(&self.state.base_commit);
2683
2684 for round in 1..=rounds {
2688 let mut turns: Vec<DeliberationTurn> = Vec::new();
2689 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2690 if self.state.judgements[j].failed.is_some() {
2691 continue;
2692 }
2693 let seat_key = format!("judge-{}", j + 1);
2694 let spec = self.occupant(&seat_key, spec);
2695 let mut seat = self.seat(&seat_key, &spec.id);
2696 let transcript = self.transcript(&turns, j);
2697 let build = |context: Option<&str>| {
2698 prompt::deliberate(
2699 &self.state.instruction,
2700 context,
2701 &transcript,
2702 round,
2703 rounds,
2704 &language,
2705 )
2706 };
2707 let block = self.candidate_block(&viable, &base_short);
2708 let full = build(Some(&block));
2709 let text = if has_context(&spec, &seat, sessions) {
2710 build(None)
2711 } else {
2712 full.clone()
2713 };
2714 let job = SeatJob {
2715 spec,
2716 seat: seat.clone(),
2717 prompt: text,
2718 cwd: root.join(format!("judge-{}", j + 1)),
2719 timeout,
2720 allow_write: false,
2721 sessions,
2722 artifacts: artifacts.clone(),
2723 stem: format!("delib-{round}-judge-{}", j + 1),
2724 handover: Some(full),
2725 };
2726 let cache = self.state.config.cache_dir();
2727 let ctx = WaveCtx {
2728 carry_seats: false,
2729 run: &run_id,
2730 node: "deliberate",
2731 prompts: &prompts,
2732 cache: cache.as_deref(),
2733 round: None,
2734 };
2735 let mut losses = Vec::new();
2739 let mut results = ask_wave_with::<String>(
2740 vec![job],
2741 Arc::clone(&self.sem),
2742 0,
2743 &self.roles.judge_roster,
2744 &ctx,
2745 &mut losses,
2746 &mut self.state,
2747 &|text: &str| {
2748 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
2749 },
2750 )
2751 .await;
2752 self.state.quota.extend(losses);
2753 let (updated, res, _) = results.pop().expect("one job in, one result out");
2754 seat = updated;
2755 let agent_id = seat.agent.clone();
2756 self.state.seats.insert(seat.key.clone(), seat);
2757 let body = match res {
2758 Ok((body, _)) => body,
2759 Err(e) => {
2762 self.state
2763 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2764 continue;
2765 }
2766 };
2767 let tentative = verdict::extract_json::<Position>(&body)
2768 .ok()
2769 .and_then(|p| p.tentative)
2770 .and_then(|s| s.trim().chars().next())
2771 .map(|c| c.to_ascii_uppercase());
2772 self.state.event(
2773 "deliberate",
2774 format!(
2775 "round {round}: judge {} now favours {}",
2776 j + 1,
2777 tentative.map_or("—".to_owned(), |c| c.to_string())
2778 ),
2779 );
2780 turns.push(DeliberationTurn {
2781 judge: j + 1,
2782 agent: agent_id,
2783 body: blind::sanitize_prose(&body, &self.state.config.blind),
2784 tentative,
2785 });
2786 }
2787 self.state
2788 .deliberation
2789 .push(DeliberationRound { round, turns });
2790 self.state.save()?;
2791 }
2792
2793 self.state.status = RunStatus::Voting;
2794 self.state.save()?;
2795 Ok(())
2796 }
2797
2798 async fn vote(&mut self) -> Result<()> {
2801 let run_id = self.state.id.clone();
2806 let prompts = self.state.config.prompts.clone();
2807 if !self.state.votes.is_empty() {
2808 return Ok(());
2809 }
2810 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2811 if viable.len() == 1 {
2812 return Ok(());
2813 }
2814 self.state.status = RunStatus::Voting;
2815
2816 let language = self.state.config.graph.language.clone();
2817 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2818 let sessions = self.state.config.graph.sessions;
2819 let artifacts = agent::artifacts_dir(&self.state.dir());
2820 let root = self.state.worktree_root();
2821 let base_short = short(&self.state.base_commit);
2822 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2823
2824 let mut jobs = Vec::new();
2825 let mut seats_at = Vec::new();
2826 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2827 if self
2828 .state
2829 .judgements
2830 .get(j)
2831 .is_some_and(|r| r.failed.is_some())
2832 {
2833 continue;
2834 }
2835 let seat_key = format!("judge-{}", j + 1);
2836 let spec = self.occupant(&seat_key, spec);
2837 let seat = self.seat(&seat_key, &spec.id);
2838 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
2839 let text = if has_context(&spec, &seat, sessions) {
2840 prompt::final_vote(&viable, &language)
2841 } else {
2842 full.clone()
2843 };
2844 jobs.push(SeatJob {
2845 spec,
2846 seat,
2847 prompt: text,
2848 cwd: root.join(format!("judge-{}", j + 1)),
2849 timeout,
2850 allow_write: false,
2851 sessions,
2852 artifacts: artifacts.clone(),
2853 stem: format!("vote-judge-{}", j + 1),
2854 handover: Some(full),
2855 });
2856 seats_at.push(j);
2857 }
2858
2859 self.state.event(
2860 "vote",
2861 format!(
2862 "collecting {} final votes one by one, privately",
2863 jobs.len()
2864 ),
2865 );
2866 let allowed = viable.clone();
2867 let mut quota_losses = Vec::new();
2868 let cache = self.state.config.cache_dir();
2869 let ctx = WaveCtx {
2870 carry_seats: false,
2871 run: &run_id,
2872 node: "vote",
2873 prompts: &prompts,
2874 cache: cache.as_deref(),
2875 round: None,
2876 };
2877 let results = ask_json_wave::<FinalVote>(
2878 jobs,
2879 Arc::clone(&self.sem),
2880 self.state.config.graph.retries,
2881 &self.roles.judge_roster,
2882 &ctx,
2883 &mut quota_losses,
2884 &mut self.state,
2885 &move |v: &FinalVote| match v.label() {
2886 Some(c) if allowed.contains(&c) => Ok(()),
2887 other => bail!("vote {other:?} is not one of {allowed:?}"),
2888 },
2889 )
2890 .await;
2891 self.state.quota.extend(quota_losses);
2892
2893 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2894 let agent_id = seat.agent.clone();
2895 self.state.seats.insert(seat.key.clone(), seat);
2896 let initial = self
2897 .state
2898 .judgements
2899 .get(j)
2900 .and_then(|r| r.ranking.first().copied());
2901 let mut record = VoteRecord {
2902 judge: j + 1,
2903 agent: agent_id,
2904 vote: None,
2905 reason: String::new(),
2906 changed: false,
2907 };
2908 match res {
2909 Ok((v, _)) => {
2910 record.vote = v.label();
2911 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2912 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2913 self.state.event(
2914 "vote",
2915 format!(
2916 "judge {} voted {}{}",
2917 j + 1,
2918 record.vote.unwrap_or('?'),
2919 if record.changed { " (changed)" } else { "" }
2920 ),
2921 );
2922 }
2923 Err(e) => {
2924 self.state
2925 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2926 }
2927 }
2928 self.state.votes.push(record);
2929 self.state.save()?;
2930 }
2931 Ok(())
2932 }
2933
2934 fn tally(&mut self) -> Result<()> {
2937 if self.state.tally.is_some() {
2938 return Ok(());
2939 }
2940 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2941 let tops: Vec<char> = self
2942 .state
2943 .judgements
2944 .iter()
2945 .filter_map(|j| j.ranking.first().copied())
2946 .collect();
2947 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2948
2949 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2952 let mut cast: Vec<char> = Vec::new();
2953 for (i, j) in self.state.judgements.iter().enumerate() {
2954 let vote = self
2955 .state
2956 .votes
2957 .iter()
2958 .find(|v| v.judge == i + 1)
2959 .and_then(|v| v.vote)
2960 .or_else(|| j.ranking.first().copied());
2961 if let Some(v) = vote {
2962 *first_choice.entry(v).or_insert(0) += 1;
2963 cast.push(v);
2964 }
2965 }
2966
2967 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2968 for j in &self.state.judgements {
2969 let n = j.ranking.len();
2970 for (pos, label) in j.ranking.iter().enumerate() {
2971 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2972 }
2973 }
2974
2975 let best = first_choice.values().copied().max().unwrap_or(0);
2976 let mut leaders: Vec<char> = first_choice
2977 .iter()
2978 .filter(|(_, v)| **v == best)
2979 .map(|(k, _)| *k)
2980 .collect();
2981 let mut tie_break = None;
2982 if leaders.len() > 1 {
2983 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2984 let borda_leaders: Vec<char> = leaders
2985 .iter()
2986 .copied()
2987 .filter(|l| borda[l] == top_borda)
2988 .collect();
2989 tie_break = Some(if borda_leaders.len() == 1 {
2990 format!(
2991 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2992 leaders.len()
2993 )
2994 } else {
2995 format!(
2996 "{} way tie on both first-choice votes and Borda points, broken by label order",
2997 leaders.len()
2998 )
2999 });
3000 leaders = borda_leaders;
3001 leaders.sort_unstable();
3002 }
3003 let winner = *leaders
3004 .first()
3005 .or(viable.first())
3006 .context("no candidate to declare a winner from")?;
3007
3008 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3009 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3010 let deliberated = !self.state.deliberation.is_empty();
3011
3012 let quota_seats: std::collections::BTreeSet<&str> =
3016 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3017 let mut present = 0usize;
3018 for (i, j) in self.state.judgements.iter().enumerate() {
3019 if quota_seats.contains(j.seat.as_str()) {
3020 continue;
3021 }
3022 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3023 let voted = self
3024 .state
3025 .votes
3026 .iter()
3027 .any(|v| v.judge == i + 1 && v.vote.is_some());
3028 if ranked || voted {
3029 present += 1;
3030 }
3031 }
3032 let needs_quorum = viable.len() > 1;
3038 let judges_total = if needs_quorum {
3039 self.roles.judges.len()
3040 } else {
3041 0
3042 };
3043 let quorum = if needs_quorum {
3044 judges_total / 2 + 1
3045 } else {
3046 0
3047 };
3048 let met_quorum = !needs_quorum || present >= quorum;
3049 let uncontested = (!needs_quorum).then(|| {
3050 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3051 });
3052
3053 self.state.event(
3054 "tally",
3055 match &uncontested {
3056 Some(reason) => format!("winner {winner} — {reason}"),
3057 None => format!(
3058 "winner {winner} — votes {} | initial {} | {} changed | \
3059 {present}/{judges_total} judges{}",
3060 first_choice
3061 .iter()
3062 .map(|(k, v)| format!("{k}:{v}"))
3063 .collect::<Vec<_>>()
3064 .join(" "),
3065 if unanimous_initial {
3066 "unanimous"
3067 } else {
3068 "split"
3069 },
3070 changed_votes,
3071 if met_quorum {
3072 String::new()
3073 } else {
3074 format!(" — below quorum ({quorum} required)")
3075 },
3076 ),
3077 },
3078 );
3079 if !met_quorum {
3080 self.state.event(
3081 "stall",
3082 format!(
3083 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3084 the run stops here, resumable"
3085 ),
3086 );
3087 }
3088 self.state.tally = Some(Tally {
3089 first_choice,
3090 borda,
3091 winner,
3092 rankings: tops.len(),
3093 unanimous_initial,
3094 deliberated,
3095 changed_votes,
3096 unanimous_final,
3097 tie_break,
3098 judges: judges_total,
3099 present,
3100 quorum,
3101 met_quorum,
3102 uncontested,
3103 });
3104 self.state.status = if met_quorum {
3105 RunStatus::Reviewing
3106 } else {
3107 RunStatus::Stalled
3108 };
3109 self.state.save()?;
3110 Ok(())
3111 }
3112
3113 #[allow(clippy::too_many_lines)]
3134 async fn recover_stall(&mut self) -> Result<bool> {
3135 let run_id = self.state.id.clone();
3140 let prompts = self.state.config.prompts.clone();
3141 let quota_seats: BTreeSet<&str> =
3146 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3147 let absent: Vec<String> = self
3148 .state
3149 .judgements
3150 .iter()
3151 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3152 .map(|j| j.seat.clone())
3153 .collect();
3154 if absent.is_empty() {
3155 return Ok(false);
3156 }
3157 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3158 if viable.len() <= 1 {
3159 return Ok(false);
3160 }
3161 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3162 let language = self.state.config.graph.language.clone();
3163 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3164 let sessions = self.state.config.graph.sessions;
3165 let artifacts = agent::artifacts_dir(&self.state.dir());
3166 let root = self.state.worktree_root();
3167 let base_short = short(&self.state.base_commit);
3168 let candidates: Vec<Candidate> = viable.clone();
3169
3170 let mut positions: Vec<usize> = absent
3172 .iter()
3173 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3174 .collect();
3175 if positions.is_empty() {
3176 return Ok(false);
3177 }
3178 positions.sort_unstable();
3179 positions.dedup();
3180
3181 let mut judge_jobs = Vec::new();
3183 for &j in &positions {
3184 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3185 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3186 let seat_key = format!("judge-{}", j + 1);
3187 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3188 let seat = self.seat(&seat_key, &spec.id);
3189 judge_jobs.push(SeatJob {
3190 spec,
3191 seat,
3192 prompt: prompt::judge(
3193 &self.state.instruction,
3194 &views,
3195 self.roles.judges.len(),
3196 &base_short,
3197 &language,
3198 ),
3199 cwd: root.join(seat_key),
3200 timeout,
3201 allow_write: false,
3202 sessions,
3203 artifacts: artifacts.clone(),
3204 stem: format!("judge-{}-recover", j + 1),
3205 handover: None,
3206 });
3207 }
3208
3209 let labels_for_check = labels.clone();
3210 let mut judge_losses = Vec::new();
3211 let retries = self.state.config.graph.retries;
3212 let cache = self.state.config.cache_dir();
3213 let ctx = WaveCtx {
3214 carry_seats: false,
3215 run: &run_id,
3216 node: "judge",
3217 prompts: &prompts,
3218 cache: cache.as_deref(),
3219 round: None,
3220 };
3221 let results = ask_json_wave::<Ranking>(
3222 judge_jobs,
3223 Arc::clone(&self.sem),
3224 retries,
3225 &self.roles.judge_roster,
3226 &ctx,
3227 &mut judge_losses,
3228 &mut self.state,
3229 &move |r: &Ranking| r.validate(&labels_for_check),
3230 )
3231 .await;
3232
3233 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3235 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3236 let agent_id = seat.agent.clone();
3237 self.state.seats.insert(seat.key.clone(), seat);
3238 let record = &mut self.state.judgements[j];
3239 match res {
3240 Ok((ranking, out)) => {
3241 record.agent = agent_id;
3242 record.ranking = ranking.normalized();
3243 record.reasons = ranking.reasons;
3244 record.confidence = ranking.confidence;
3245 record.failed = None;
3246 record.duration_ms = out.duration_ms;
3247 recovered.insert(j);
3248 self.state.event(
3249 "recover",
3250 format!("judge {} ranked again after the limit", j + 1),
3251 );
3252 }
3253 Err(e) => {
3254 self.state
3255 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3256 }
3257 }
3258 }
3259
3260 let mut vote_jobs = Vec::new();
3262 let mut vote_pos: Vec<usize> = Vec::new();
3263 for &j in &recovered {
3264 let seat_key = format!("judge-{}", j + 1);
3265 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3266 let seat = self.seat(&seat_key, &spec.id);
3267 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3268 let text = if has_context(&spec, &seat, sessions) {
3269 prompt::final_vote(&labels, &language)
3270 } else {
3271 full.clone()
3272 };
3273 vote_jobs.push(SeatJob {
3274 spec,
3275 seat,
3276 prompt: text,
3277 cwd: root.join(seat_key),
3278 timeout,
3279 allow_write: false,
3280 sessions,
3281 artifacts: artifacts.clone(),
3282 stem: format!("vote-judge-{}-recover", j + 1),
3283 handover: Some(full),
3284 });
3285 vote_pos.push(j);
3286 }
3287 let allowed = labels.clone();
3288 let mut vote_losses = Vec::new();
3289 let vote_retries = self.state.config.graph.retries;
3290 let vote_cache = self.state.config.cache_dir();
3291 let ctx = WaveCtx {
3292 carry_seats: false,
3293 run: &run_id,
3294 node: "vote",
3295 prompts: &prompts,
3296 cache: vote_cache.as_deref(),
3297 round: None,
3298 };
3299 let votes = ask_json_wave::<FinalVote>(
3300 vote_jobs,
3301 Arc::clone(&self.sem),
3302 vote_retries,
3303 &self.roles.judge_roster,
3304 &ctx,
3305 &mut vote_losses,
3306 &mut self.state,
3307 &move |v: &FinalVote| match v.label() {
3308 Some(c) if allowed.contains(&c) => Ok(()),
3309 other => bail!("vote {other:?} is not one of {allowed:?}"),
3310 },
3311 )
3312 .await;
3313 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3314 let agent_id = seat.agent.clone();
3315 self.state.seats.insert(seat.key.clone(), seat);
3316 match res {
3317 Ok((v, _)) => {
3318 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3319 rec.vote = v.label();
3320 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3321 } else {
3322 self.state.votes.push(VoteRecord {
3323 judge: j + 1,
3324 agent: agent_id,
3325 vote: v.label(),
3326 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3327 changed: false,
3328 });
3329 }
3330 self.state.event(
3331 "recover",
3332 format!("judge {} voted again after the limit", j + 1),
3333 );
3334 }
3335 Err(e) => {
3336 self.state
3337 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3338 }
3339 }
3340 }
3341
3342 let recovered_keys: BTreeSet<String> = recovered
3346 .iter()
3347 .map(|&j| format!("judge-{}", j + 1))
3348 .collect();
3349 self.state
3350 .quota
3351 .retain(|q| !recovered_keys.contains(&q.seat));
3352 for loss in judge_losses.into_iter().chain(vote_losses) {
3356 if recovered_keys.contains(&loss.seat) {
3357 continue;
3358 }
3359 self.state.quota.retain(|q| q.seat != loss.seat);
3360 self.state.quota.push(loss);
3361 }
3362
3363 self.state.tally = None;
3365 self.tally()?;
3366 Ok(self
3367 .state
3368 .tally
3369 .as_ref()
3370 .map(|t| t.met_quorum)
3371 .unwrap_or(false))
3372 }
3373
3374 async fn fold_losers(&mut self) -> Result<()> {
3377 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3378 return Ok(());
3379 };
3380 let repo = self.state.repo.clone();
3381 let mut folded = Vec::new();
3382 for i in 0..self.state.candidates.len() {
3383 let c = &self.state.candidates[i];
3384 if c.label == winner || c.folded {
3385 continue;
3386 }
3387 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3388 git::worktree_remove(&repo, &wt).await.ok();
3389 git::branch_delete(&repo, &branch).await.ok();
3390 self.state.candidates[i].folded = true;
3391 folded.push(label.to_string());
3392 }
3393 let root = self.state.worktree_root();
3395 for j in 1..=self.roles.judges.len() {
3396 let wt = root.join(format!("judge-{j}"));
3397 if wt.exists() {
3398 git::worktree_remove(&repo, &wt).await.ok();
3399 }
3400 }
3401 if self.state.config.graph.advise {
3404 for k in 1..=self.state.config.graph.advisors {
3405 let wt = root.join(format!("advisor-{k}"));
3406 if wt.exists() {
3407 git::worktree_remove(&repo, &wt).await.ok();
3408 }
3409 }
3410 }
3411 if !folded.is_empty() {
3412 self.state
3413 .event("fold", format!("folded candidates {}", folded.join(", ")));
3414 self.state.save()?;
3415 }
3416 Ok(())
3417 }
3418
3419 async fn sync_to_base(&mut self) -> Result<()> {
3459 if self.state.status == RunStatus::AlreadyInBase {
3460 return Ok(());
3461 }
3462 let conflicted = self
3463 .state
3464 .base_sync
3465 .as_ref()
3466 .is_some_and(|s| s.conflict.is_some());
3467 let Some(winner) = self.state.winner().cloned() else {
3468 return Ok(());
3469 };
3470
3471 let repo = self.state.repo.clone();
3472 let remote = self.state.config.merge.remote.clone();
3473 let base_branch = self.state.base_branch.clone();
3474 let tracking = format!("{remote}/{base_branch}");
3475
3476 git::fetch(&repo, &remote, &base_branch).await.ok();
3477 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3482 return Ok(());
3483 };
3484
3485 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3486 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3487 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3488
3489 if (behind > 0 || conflicted || head == tip)
3498 && self
3499 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3500 .await?
3501 {
3502 return Ok(());
3503 }
3504 if conflicted {
3505 return Ok(());
3506 }
3507
3508 if behind == 0 {
3509 if let Some(from) = self
3518 .state
3519 .rebase_fixes
3520 .iter()
3521 .rev()
3522 .find_map(|r| r.from.clone())
3523 && from != head
3524 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3525 .await
3526 .is_ok_and(|o| o.ok())
3527 {
3528 git::sync_to_head(&winner.worktree).await?;
3529 }
3530 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3536 if let Some(why) = &conflict {
3537 self.state.status = RunStatus::Blocked;
3538 self.state.event("land", why.clone());
3539 }
3540 self.state.base_sync = Some(BaseSync {
3541 tip,
3542 behind: 0,
3543 attempts,
3544 conflict,
3545 already_in: None,
3546 });
3547 self.state.save()?;
3548 return Ok(());
3549 }
3550
3551 if attempts >= BASE_SYNC_ROUNDS {
3552 let why = format!(
3553 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3554 rebase(s); rebasing again would only race it",
3555 winner.branch
3556 );
3557 self.state.status = RunStatus::Blocked;
3558 self.state.base_sync = Some(BaseSync {
3559 tip,
3560 behind,
3561 attempts,
3562 conflict: Some(why.clone()),
3563 already_in: None,
3564 });
3565 self.state.event("land", why);
3566 self.state.save()?;
3567 return Ok(());
3568 }
3569
3570 self.state.event(
3571 "land",
3572 format!(
3573 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3574 winner.branch
3575 ),
3576 );
3577 self.state.save()?;
3578
3579 let branch_tracking = format!("{remote}/{}", winner.branch);
3585 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3586 let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3587 git::rev_parse(&repo, &branch_tracking).await.ok()
3588 } else {
3589 None
3590 };
3591 if let Some(theirs) = &remote_tip
3595 && !git::is_ancestor(&repo, theirs, &head).await
3596 && !crate::reconcile::origin_missing(&repo, &head, theirs)
3597 .await
3598 .is_ok_and(|missing| missing.is_empty())
3599 {
3600 let why = format!(
3601 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3602 them",
3603 short(theirs),
3604 winner.branch
3605 );
3606 self.state.status = RunStatus::Blocked;
3607 self.state.base_sync = Some(BaseSync {
3608 tip,
3609 behind,
3610 attempts,
3611 conflict: Some(why.clone()),
3612 already_in: None,
3613 });
3614 self.state.event("land", why);
3615 self.state.save()?;
3616 return Ok(());
3617 }
3618
3619 let scratch = self.state.dir().join("base-sync");
3620 let rebased = match crate::rebase::rebase_with_fixer(
3621 &mut self.state,
3622 &scratch,
3623 &winner.branch,
3624 &tracking,
3625 )
3626 .await
3627 {
3628 Ok(crate::rebase::Rebased::Applied) => Ok(None),
3629 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3630 Err(e) => Err(e),
3631 };
3632 let attempts = attempts + 1;
3633 match rebased {
3634 Ok(None) => {
3635 git::sync_to_head(&winner.worktree).await?;
3639 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
3640 let mut conflict = None;
3641 if let Some(pinned) = &remote_tip {
3642 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3643 match pushed {
3644 Ok(o) if o.ok() => self.state.event(
3645 "land",
3646 format!("pushed rebased {} to {remote}", winner.branch),
3647 ),
3648 Ok(o) => {
3649 conflict = Some(format!(
3650 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
3651 winner.branch,
3652 short(pinned),
3653 o.stderr.chars().take(600).collect::<String>()
3654 ));
3655 }
3656 Err(e) => {
3657 conflict = Some(format!(
3658 "rebased {} locally but could not push it: {e:#}",
3659 winner.branch
3660 ));
3661 }
3662 }
3663 }
3664 if let Some(why) = &conflict {
3665 self.state.status = RunStatus::Blocked;
3666 self.state.event("land", why.clone());
3667 }
3668 self.state.base_sync = Some(BaseSync {
3669 tip: tip.clone(),
3670 behind: 0,
3671 attempts,
3672 conflict,
3673 already_in: None,
3674 });
3675 self.state
3676 .event("land", format!("rebased {} onto {tracking}", winner.branch));
3677 }
3678 Ok(Some(conflict)) => {
3679 let why = format!(
3680 "{} conflicts with {tracking} and did not rebase: {}",
3681 winner.branch,
3682 conflict.chars().take(600).collect::<String>()
3683 );
3684 self.state.status = RunStatus::Blocked;
3685 self.state.base_sync = Some(BaseSync {
3686 tip,
3687 behind,
3688 attempts,
3689 conflict: Some(why.clone()),
3690 already_in: None,
3691 });
3692 self.state.event("land", why);
3693 }
3694 Err(e) => {
3695 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3696 self.state.status = RunStatus::Blocked;
3697 self.state.base_sync = Some(BaseSync {
3698 tip,
3699 behind,
3700 attempts,
3701 conflict: Some(why.clone()),
3702 already_in: None,
3703 });
3704 self.state.event("land", why);
3705 }
3706 }
3707 self.state.save()?;
3708 Ok(())
3709 }
3710
3711 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
3717 let repo = self.state.repo.clone();
3718 let remote = self.state.config.merge.remote.clone();
3719 let fetched = git::fetch(&repo, &remote, branch).await;
3720 if !matches!(&fetched, Ok(o) if o.ok()) {
3721 return None;
3722 }
3723 let branch_tracking = format!("{remote}/{branch}");
3724 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
3725 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
3726 return None;
3727 }
3728 if !crate::reconcile::origin_missing(&repo, head, &theirs)
3729 .await
3730 .is_ok_and(|missing| missing.is_empty())
3731 {
3732 return Some(format!(
3733 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
3734 them",
3735 short(&theirs)
3736 ));
3737 }
3738 match git::push_pinned(&repo, &remote, branch, &theirs).await {
3739 Ok(o) if o.ok() => {
3740 self.state
3741 .event("land", format!("pushed rebased {branch} to {remote}"));
3742 None
3743 }
3744 Ok(o) => Some(format!(
3745 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
3746 someone may have pushed): {}",
3747 short(&theirs),
3748 o.stderr.chars().take(600).collect::<String>()
3749 )),
3750 Err(e) => Some(format!(
3751 "{branch} is rebased locally but could not be pushed: {e:#}"
3752 )),
3753 }
3754 }
3755
3756 async fn settle_already_in(
3773 &mut self,
3774 branch: &str,
3775 tip: &str,
3776 head: &str,
3777 attempts: usize,
3778 behind: usize,
3779 ) -> Result<bool> {
3780 let repo = self.state.repo.clone();
3781 let remote = self.state.config.merge.remote.clone();
3782 let start = self.state.base_commit.clone();
3783 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3784 Ok(Some(e)) => e,
3785 Ok(None) => return Ok(false),
3786 Err(e) => {
3787 tracing::warn!("already-in-base check for {branch}: {e:#}");
3788 return Ok(false);
3789 }
3790 };
3791 let mut verified = vec![head.to_owned()];
3792 let fetched = git::fetch(&repo, &remote, branch).await;
3793 if matches!(&fetched, Ok(o) if o.ok())
3794 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3795 && theirs != head
3796 {
3797 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3798 Ok(Some(_)) => verified.push(theirs),
3799 _ => return Ok(false),
3800 }
3801 }
3802 let base_branch = self.state.base_branch.clone();
3803 let message = format!(
3804 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3805 land",
3806 evidence.names(),
3807 evidence.proof.as_str()
3808 );
3809 let closed =
3810 crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3811 match closed {
3812 Ok(Ok(url)) => self
3813 .state
3814 .event("land", format!("closed {url}: superseded on {base_branch}")),
3815 Ok(Err(why)) => self
3816 .state
3817 .event("land", format!("did not close a pull request: {why}")),
3818 Err(e) => {
3819 let why = format!(
3820 "{branch} is already in {remote}/{base_branch}, but its pull request could \
3821 not be closed ({e:#}); resume to retry"
3822 );
3823 self.state.status = RunStatus::Blocked;
3824 self.state.base_sync = Some(BaseSync {
3825 tip: tip.to_owned(),
3826 behind,
3827 attempts,
3828 conflict: Some(why.clone()),
3829 already_in: None,
3830 });
3831 self.state.event("land", why);
3832 self.state.save()?;
3833 return Ok(true);
3834 }
3835 }
3836 self.state.status = RunStatus::AlreadyInBase;
3837 self.state.base_sync = Some(BaseSync {
3838 tip: tip.to_owned(),
3839 behind,
3840 attempts,
3841 conflict: None,
3842 already_in: Some(evidence),
3843 });
3844 self.state.event("land", message);
3845 self.state.save()?;
3846 self.settle_questions();
3847 Ok(true)
3848 }
3849
3850 fn landing_base(&self) -> String {
3860 self.state
3861 .base_sync
3862 .as_ref()
3863 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3864 }
3865
3866 pub async fn fix_selected(
3899 &mut self,
3900 ids: &[String],
3901 reason: &str,
3902 allow_stale: bool,
3903 ) -> Result<()> {
3904 let reason = reason.trim();
3905 if reason.is_empty() {
3906 bail!("a fix request needs a reason — that is the operator's own record of why");
3907 }
3908 if ids.is_empty() {
3909 bail!("no finding id given");
3910 }
3911 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3912 bail!(
3913 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3914 has already concluded — can be given a targeted fix. A run still \
3915 in progress should simply be resumed; a `merged` run's branch has \
3916 already landed, so its answer is a fresh `magi review <branch>`, \
3917 not reopening this run's own record",
3918 self.state.id,
3919 self.state.status.as_str()
3920 );
3921 }
3922 let Some(winner) = self.state.winner().cloned() else {
3923 bail!("run {} has no winning candidate to fix", self.state.id);
3924 };
3925 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3926 bail!(
3927 "branch `{}` no longer exists; this run cannot be extended",
3928 winner.branch
3929 );
3930 }
3931 let home = crate::run::home();
3932 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3933 bail!(
3934 "run {} is currently being worked on by another magi process",
3935 self.state.id
3936 );
3937 }
3938 let _claim = FixClaim::acquire(&self.state.dir())?;
3944
3945 let mut seen = BTreeSet::new();
3949 let mut findings = Vec::new();
3950 let mut missing = Vec::new();
3951 for id in ids {
3952 if !seen.insert(id.clone()) {
3953 continue;
3954 }
3955 match self.state.finding(id) {
3956 Some((round, rec, f)) => findings.push(OperatorFixFinding {
3957 id: f.id.clone(),
3958 severity: f.severity,
3959 reviewer_vote: rec.vote,
3960 round: round.round,
3961 round_head: round.head.clone(),
3962 reviewer: rec.reviewer,
3963 agent: rec.agent.clone(),
3964 file: f.file.clone(),
3965 line: f.line,
3966 title: f.title.clone(),
3967 detail: f.detail.clone(),
3968 outcome: OperatorFixOutcome::Pending,
3969 }),
3970 None => missing.push(id.clone()),
3971 }
3972 }
3973 if !missing.is_empty() {
3974 bail!(
3975 "unknown finding id(s): {}; nothing was changed",
3976 missing.join(", ")
3977 );
3978 }
3979
3980 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3981 let stale_details: Vec<(String, String)> = findings
3982 .iter()
3983 .filter(|f| f.round_head != head_at_request)
3984 .map(|f| (f.id.clone(), f.round_head.clone()))
3985 .collect();
3986 let stale = !stale_details.is_empty();
3987 if stale && !allow_stale {
3988 bail!(
3989 "the branch has moved since some finding(s) were raised — {} — now \
3990 at {}; pass --allow-stale to fix anyway, or re-run review first",
3991 stale_details
3992 .iter()
3993 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3994 .collect::<Vec<_>>()
3995 .join(", "),
3996 short(&head_at_request)
3997 );
3998 }
3999
4000 let request = OperatorFixRequest {
4001 requested_at: Timestamp::now(),
4002 reason: reason.to_owned(),
4003 findings,
4004 head_at_request: head_at_request.clone(),
4005 allow_stale,
4006 stale,
4007 fix: None,
4008 result_head: None,
4009 follow_up_review_run: None,
4010 };
4011 self.state.event(
4012 "fix",
4013 format!(
4014 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4015 request.findings.len(),
4016 request
4017 .findings
4018 .iter()
4019 .map(|f| f.id.as_str())
4020 .collect::<Vec<_>>()
4021 .join(", "),
4022 ),
4023 );
4024 self.state.operator_fixes.push(request);
4031 self.state.save()?;
4032 let request_index = self.state.operator_fixes.len() - 1;
4033
4034 if winner.worktree.exists() {
4043 let dirty = git::git(
4046 &winner.worktree,
4047 &["status", "--porcelain", "--untracked-files=all"],
4048 )
4049 .await?;
4050 let only_withheld = dirty.lines().all(|l| {
4051 l.strip_prefix("?? ")
4052 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4053 });
4054 if !only_withheld {
4055 bail!(
4056 "`{}` has uncommitted changes; refusing to touch it — commit or \
4057 discard them first",
4058 winner.worktree.display()
4059 );
4060 }
4061 git::worktree_remove(&self.state.repo, &winner.worktree)
4062 .await
4063 .ok();
4064 }
4065 let fix_worktree = self.state.worktree_root().join("operator-fix");
4066 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
4067 git::git(
4068 &self.state.repo,
4069 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
4070 )
4071 .await
4072 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
4073 if !git::is_clean(&fix_worktree).await? {
4074 git::worktree_remove(&self.state.repo, &fix_worktree)
4075 .await
4076 .ok();
4077 bail!(
4078 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
4079 winner.branch
4080 );
4081 }
4082
4083 let run_id = self.state.id.clone();
4084 let prompts = self.state.config.prompts.clone();
4085 let language = self.state.config.graph.language.clone();
4086 let sessions = self.state.config.graph.sessions;
4087 let artifacts = agent::artifacts_dir(&self.state.dir());
4088 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
4089 .findings
4090 .iter()
4091 .map(|f| Finding {
4092 id: f.id.clone(),
4093 severity: f.severity,
4094 file: f.file.clone(),
4095 line: f.line,
4096 title: f.title.clone(),
4097 detail: f.detail.clone(),
4098 })
4099 .collect();
4100 let fix_prompt = prompt::operator_fix(
4101 &self.state.instruction,
4102 &finding_list,
4103 reason,
4104 &stale_details,
4105 &head_at_request,
4106 &language,
4107 );
4108 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4109 let (job, seat, out) = self
4110 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
4111 prompt: fix_prompt.clone(),
4112 spec,
4113 seat,
4114 cwd: fix_worktree.clone(),
4115 timeout,
4116 allow_write: true,
4117 sessions,
4118 artifacts: artifacts.clone(),
4119 stem: "operator-fix".to_owned(),
4120 handover: None,
4121 })
4122 .await;
4123 let agent_id = seat.agent.clone();
4124
4125 let mut fix = FixRecord {
4126 agent: agent_id,
4127 addressed: Vec::new(),
4128 rejected: Vec::new(),
4129 notes: String::new(),
4130 committed: false,
4131 failed: None,
4132 duration_ms: 0,
4133 continuation: None,
4134 };
4135 let mut final_seat = seat.clone();
4136 match out {
4137 AgentOutcome::Ok(o) => {
4138 fix.duration_ms = o.duration_ms;
4139 let parsed = verdict::extract_json::<FixReport>(&o.text);
4140 let incomplete_reason = match &parsed {
4141 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4142 "the reply parsed, but it reported a command whose own CLI \
4143 never confirmed an exit status"
4144 .to_owned(),
4145 ),
4146 Ok(_) => None,
4147 Err(e) => Some(e.to_string()),
4148 };
4149 match incomplete_reason {
4150 None => {
4151 let report = parsed.expect("checked Ok above");
4152 fix.addressed = report.addressed;
4153 fix.rejected = report.rejected;
4154 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4155 }
4156 Some(reason) => {
4157 let (resumed_seat, resolved, failure, cont) = self
4158 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4159 .await;
4160 fix.duration_ms += cont.cumulative_wait_ms;
4161 fix.continuation = Some(cont);
4162 final_seat = resumed_seat;
4163 match resolved {
4164 Some(report) => {
4165 fix.addressed = report.addressed;
4166 fix.rejected = report.rejected;
4167 fix.notes =
4168 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4169 }
4170 None => fix.failed = failure,
4171 }
4172 }
4173 }
4174 }
4175 AgentOutcome::Dropped(o) => {
4176 fix.duration_ms = o.duration_ms;
4177 let why = o
4178 .dropped
4179 .as_ref()
4180 .map(|d| d.why.as_str())
4181 .unwrap_or("the CLI ended the stream without delivering its answer");
4182 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4183 }
4184 AgentOutcome::Quota(o) => {
4185 self.state.quota.push(QuotaLoss {
4186 seat: final_seat.key.clone(),
4187 node: "fix".to_owned(),
4188 at: Timestamp::now(),
4189 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4190 });
4191 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4192 }
4193 AgentOutcome::Failed(e) => fix.failed = Some(e),
4194 }
4195 if fix.continuation.is_none() {
4196 fix.continuation = Some(ContinuationRecord::not_needed());
4197 }
4198 self.state.seats.insert(final_seat.key.clone(), final_seat);
4199
4200 let rescue_message = format!(
4201 "magi: operator-selected fix ({}) (uncommitted work)",
4202 self.state.operator_fixes[request_index]
4203 .findings
4204 .iter()
4205 .map(|f| f.id.as_str())
4206 .collect::<Vec<_>>()
4207 .join(", ")
4208 );
4209 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4210 self.state.note_withheld("fix", &r.withheld);
4211 }
4212 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4213 fix.committed = after != head_at_request;
4214 git::worktree_remove(&self.state.repo, &fix_worktree)
4215 .await
4216 .ok();
4217
4218 self.state.event(
4219 "fix",
4220 match &fix.failed {
4221 Some(reason) => format!(
4222 "operator fix: adoption report was lost ({reason}); {}",
4223 if fix.committed {
4224 "committed"
4225 } else {
4226 "NO new commit"
4227 }
4228 ),
4229 None => format!(
4230 "operator fix: {} addressed, {} rejected, {}",
4231 fix.addressed.len(),
4232 fix.rejected.len(),
4233 if fix.committed {
4234 "committed"
4235 } else {
4236 "NO new commit"
4237 }
4238 ),
4239 },
4240 );
4241
4242 for f in &mut self.state.operator_fixes[request_index].findings {
4249 f.outcome = if fix.failed.is_some() {
4250 OperatorFixOutcome::Unreported
4251 } else if fix.addressed.contains(&f.id) {
4252 OperatorFixOutcome::Addressed
4253 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4254 OperatorFixOutcome::Rejected { why: r.why.clone() }
4255 } else {
4256 OperatorFixOutcome::Unreported
4257 };
4258 }
4259
4260 let committed = fix.committed;
4261 if committed {
4262 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4263 }
4264 self.state.operator_fixes[request_index].fix = Some(fix);
4265 self.state.save()?;
4268
4269 if committed {
4270 self.state.event(
4271 "fix",
4272 format!(
4273 "operator fix committed {}; opening a follow-up review-only run",
4274 short(&after)
4275 ),
4276 );
4277 let origin =
4280 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4281 match Self::review(
4282 &self.state.repo,
4283 &winner.branch,
4284 self.state.config.clone(),
4285 origin,
4286 )
4287 .await
4288 {
4289 Ok(mut follow_up) => {
4290 follow_up.state.event(
4291 "start",
4292 format!(
4293 "requested by an operator fix on run {} for finding(s) {}",
4294 self.state.id,
4295 self.state.operator_fixes[request_index]
4296 .findings
4297 .iter()
4298 .map(|f| f.id.as_str())
4299 .collect::<Vec<_>>()
4300 .join(", "),
4301 ),
4302 );
4303 follow_up.state.save()?;
4304 let follow_up_id = follow_up.state.id.clone();
4305 let adopted = match crate::direct::adopt(
4308 &follow_up.state,
4309 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
4310 ) {
4311 Ok(a) => a,
4312 Err(e) => {
4313 self.state.event(
4316 "fix",
4317 format!(
4318 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
4319 ),
4320 );
4321 self.state.save()?;
4322 return Ok(());
4323 }
4324 };
4325 let executed = follow_up.execute().await;
4326 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
4327 if let Some(a) = adopted {
4328 a.finish(&follow_up.state, executed);
4329 }
4330 if let Some(e) = failure {
4331 self.state.event(
4332 "fix",
4333 format!(
4334 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4335 ),
4336 );
4337 }
4338 self.state.operator_fixes[request_index].follow_up_review_run =
4339 Some(follow_up_id);
4340 }
4341 Err(e) => {
4342 self.state.event(
4343 "fix",
4344 format!("committed the fix but could not open a follow-up review: {e:#}"),
4345 );
4346 }
4347 }
4348 self.state.save()?;
4349 }
4350
4351 Ok(())
4352 }
4353
4354 async fn ask_fixer(
4374 &mut self,
4375 winner: &Candidate,
4376 node: &'static str,
4377 round: Option<usize>,
4378 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
4379 ) -> (SeatJob, SeatState, AgentOutcome) {
4380 let run_id = self.state.id.clone();
4381 let prompts = self.state.config.prompts.clone();
4382 let cache = self.state.config.cache_dir();
4383 let attempts = fixer::attempts(&self.state, &self.roles, winner);
4384 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
4385 let mut last = None;
4386 for (i, (spec, key)) in attempts.into_iter().enumerate() {
4387 let seat = self.seat(&key, &spec.id);
4388 let mut job = build(spec.clone(), seat);
4389 if i > 0 {
4390 job.stem = format!("{}-{}", job.stem, spec.id);
4391 }
4392 let ctx = WaveCtx {
4393 carry_seats: false,
4394 run: &run_id,
4395 node,
4396 prompts: &prompts,
4397 cache: cache.as_deref(),
4398 round,
4399 };
4400 let (seat, out) =
4401 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4402 let advances = match &out {
4403 AgentOutcome::Ok(o) => agent::output_advances(o),
4404 _ => true,
4405 };
4406 if let Some(next) = ids.get(i + 1)
4407 && advances
4408 {
4409 self.state.seats.insert(seat.key.clone(), seat.clone());
4410 let class =
4411 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
4412 let reason = match &out {
4413 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
4414 other => fail_reason(other),
4415 };
4416 record_handover(
4417 &mut self.state,
4418 node,
4419 &seat.key,
4420 &spec.id,
4421 next,
4422 &class,
4423 &reason,
4424 );
4425 continue;
4426 }
4427 last = Some((job, seat, out));
4428 break;
4429 }
4430 last.expect("the fixer chain always has an entry")
4431 }
4432
4433 async fn review_loop(&mut self) -> Result<()> {
4434 if self
4439 .state
4440 .base_sync
4441 .as_ref()
4442 .is_some_and(|s| s.conflict.is_some())
4443 {
4444 return Ok(());
4445 }
4446 let run_id = self.state.id.clone();
4451 let prompts = self.state.config.prompts.clone();
4452 let Some(winner) = self.state.winner().cloned() else {
4453 return Ok(());
4454 };
4455 let max_rounds = self.state.config.graph.review_rounds;
4456 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4466 if status == RunStatus::Gating {
4469 self.record_contested_handoff();
4470 }
4471 self.state.status = status;
4472 self.state.save()?;
4473 return Ok(());
4474 }
4475 self.state.status = RunStatus::Reviewing;
4476 if self
4486 .state
4487 .reviews
4488 .last()
4489 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4490 {
4491 let shell = self.state.config.shell();
4492 return self
4493 .stop_reviewing(
4494 "the last round's own verification never resolved",
4495 &shell,
4496 &winner.worktree,
4497 )
4498 .await;
4499 }
4500
4501 let repo = self.state.repo.clone();
4502 let root = self.state.worktree_root();
4503 let language = self.state.config.graph.language.clone();
4504 let sessions = self.state.config.graph.sessions;
4505 let artifacts = agent::artifacts_dir(&self.state.dir());
4506 let base = self.landing_base();
4507 let base_short = short(&base);
4508 let reviewers = self.roles.reviewers.clone();
4509 let shell = self.state.config.shell();
4510
4511 for round in (self.state.reviews.len() + 1)..=max_rounds {
4512 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4513 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4514 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4515 let prev_verification = self
4524 .state
4525 .reviews
4526 .last()
4527 .and_then(|r| r.verification_summary(&head));
4528
4529 let mut jobs = Vec::new();
4533 for (r, spec) in reviewers.iter().cloned().enumerate() {
4534 let wt = root.join(format!("review-{}", r + 1));
4535 if wt.exists() {
4536 git::reset_detached(&wt, &head).await?;
4537 } else {
4538 git::worktree_add_detached(&repo, &wt, &head).await?;
4539 }
4540 let seat_key = format!("review-{}", r + 1);
4541 let spec = pick_start_spec(
4544 &self.roles.reviewer_roster,
4545 spec,
4546 self.state.seat_history.get(&seat_key),
4547 );
4548 let seat = self.seat(&seat_key, &spec.id);
4549 jobs.push(SeatJob {
4550 prompt: prompt::review(&prompt::ReviewCtx {
4551 instruction: &self.state.instruction,
4552 branch: &winner.branch,
4553 base_short: &base_short,
4554 stat: &stat,
4555 patch: &patch,
4556 verification: prev_verification.as_ref(),
4557 reviewers: reviewers.len(),
4558 round,
4559 rounds: max_rounds,
4560 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4563 lens: Lens::for_seat(r),
4564 language: &language,
4565 }),
4566 spec,
4567 seat,
4568 cwd: wt,
4569 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4570 allow_write: false,
4571 sessions,
4572 artifacts: artifacts.clone(),
4573 stem: format!("review-{round}-{}", r + 1),
4574 handover: None,
4575 });
4576 }
4577
4578 self.state.event(
4579 "review",
4580 format!(
4581 "round {round}: {} reviewers on {}",
4582 jobs.len(),
4583 short(&head)
4584 ),
4585 );
4586 let mut quota_losses = Vec::new();
4587 let review_retries = self.state.config.graph.retries;
4588 let review_cache = self.state.config.cache_dir();
4589 let ctx = WaveCtx {
4590 carry_seats: true,
4591 run: &run_id,
4592 node: "review",
4593 prompts: &prompts,
4594 cache: review_cache.as_deref(),
4595 round: Some(round),
4596 };
4597 let results = ask_json_wave::<Review>(
4598 jobs,
4599 Arc::clone(&self.sem),
4600 review_retries,
4601 &self.roles.reviewer_roster,
4602 &ctx,
4603 &mut quota_losses,
4604 &mut self.state,
4605 &|_: &Review| Ok(()),
4606 )
4607 .await;
4608 let round_quota_missing = quota_losses.len();
4612 self.state.quota.extend(quota_losses);
4613
4614 let mut records = Vec::new();
4615 let mut all_findings = Vec::new();
4616 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4617 let agent_id = seat.agent.clone();
4618 self.state.seats.insert(seat.key.clone(), seat);
4619 let mut record = ReviewRecord {
4620 reviewer: r + 1,
4621 agent: agent_id,
4622 summary: String::new(),
4623 findings: Vec::new(),
4624 vote: None,
4625 failed: None,
4626 duration_ms: 0,
4627 attempts,
4633 };
4634 match res {
4635 Ok((review, out)) => {
4636 record.summary =
4644 blind::sanitize_prose(&review.summary, &self.state.config.blind);
4645 record.vote = Some(review.vote);
4646 record.duration_ms = out.duration_ms;
4647 for (n, mut f) in review.findings.into_iter().enumerate() {
4648 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4651 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4652 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4653 f.file = f
4659 .file
4660 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4661 all_findings.push(f.clone());
4662 record.findings.push(f);
4663 }
4664 self.state.event(
4665 "review",
4666 format!(
4667 "round {round}: reviewer {} voted {} with {} finding(s)",
4668 r + 1,
4669 review.vote.label(),
4670 record.findings.len()
4671 ),
4672 );
4673 }
4674 Err(e) => {
4675 record.failed = Some(e.to_string());
4676 self.state.event(
4677 "review",
4678 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4679 );
4680 }
4681 }
4682 records.push(record);
4683 }
4684
4685 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4692 let vote_split =
4693 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4694 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4695 if vote_split {
4696 self.state.event(
4697 "review",
4698 format!(
4699 "round {round}: votes split ({}) — one round of reconsideration",
4700 initial_votes
4701 .iter()
4702 .map(|v| v.label())
4703 .collect::<Vec<_>>()
4704 .join(", ")
4705 ),
4706 );
4707 let panel: Vec<ReviewSeatReport<'_>> = records
4710 .iter()
4711 .filter_map(|r| {
4712 r.vote.map(|vote| ReviewSeatReport {
4713 reviewer: r.reviewer,
4714 vote,
4715 summary: &r.summary,
4716 findings: &r.findings,
4717 })
4718 })
4719 .collect();
4720
4721 let mut jobs = Vec::new();
4722 let mut seats_at = Vec::new();
4723 for (r, spec) in reviewers.iter().cloned().enumerate() {
4724 if records[r].vote.is_none() {
4728 continue;
4729 }
4730 let wt = root.join(format!("review-{}", r + 1));
4731 let seat_key = format!("review-{}", r + 1);
4732 let spec = self.occupant(&seat_key, spec);
4733 let seat = self.seat(&seat_key, &spec.id);
4734 let build = |with_patch: bool| {
4742 prompt::review_reconsider(&ReviewReconsiderCtx {
4743 instruction: &self.state.instruction,
4744 reviewer: r + 1,
4745 lens: Lens::for_seat(r),
4746 panel: &panel,
4747 patch: with_patch.then_some(ReviewPatch {
4748 branch: &winner.branch,
4749 base_short: &base_short,
4750 stat: &stat,
4751 patch: &patch,
4752 }),
4753 round,
4754 rounds: max_rounds,
4755 language: &language,
4756 })
4757 };
4758 let full = build(true);
4759 let prompt = if has_context(&spec, &seat, sessions) {
4760 build(false)
4761 } else {
4762 full.clone()
4763 };
4764 jobs.push(SeatJob {
4765 prompt,
4766 spec,
4767 seat,
4768 cwd: wt,
4769 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4770 allow_write: false,
4771 sessions,
4772 artifacts: artifacts.clone(),
4773 stem: format!("review-{round}-reconsider-{}", r + 1),
4774 handover: Some(full),
4775 });
4776 seats_at.push(r);
4777 }
4778
4779 let mut recon_quota_losses = Vec::new();
4780 let recon_cache = self.state.config.cache_dir();
4781 let recon_ctx = WaveCtx {
4782 carry_seats: true,
4783 run: &run_id,
4784 node: "review",
4785 prompts: &prompts,
4786 cache: recon_cache.as_deref(),
4787 round: Some(round),
4788 };
4789 let recon_results = ask_json_wave::<ReviewRevote>(
4790 jobs,
4791 Arc::clone(&self.sem),
4792 review_retries,
4793 &self.roles.reviewer_roster,
4794 &recon_ctx,
4795 &mut recon_quota_losses,
4796 &mut self.state,
4797 &|_: &ReviewRevote| Ok(()),
4798 )
4799 .await;
4800 self.state.quota.extend(recon_quota_losses);
4801
4802 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4803 let agent_id = seat.agent.clone();
4804 self.state.seats.insert(seat.key.clone(), seat);
4805 let mut rec = ReviewRevoteRecord {
4806 reviewer: r + 1,
4807 agent: agent_id,
4808 vote: None,
4809 reason: String::new(),
4810 failed: None,
4811 };
4812 match res {
4813 Ok((rv, _)) => {
4814 rec.vote = Some(rv.vote);
4815 rec.reason =
4816 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4817 self.state.event(
4818 "review",
4819 format!(
4820 "round {round}: reviewer {} revoted {}",
4821 r + 1,
4822 rv.vote.label()
4823 ),
4824 );
4825 }
4826 Err(e) => {
4827 rec.failed = Some(e.to_string());
4828 self.state.event(
4829 "review",
4830 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4831 );
4832 }
4833 }
4834 reconsideration.push(rec);
4835 }
4836 } else if initial_votes.len() > 1 {
4837 self.state.event(
4838 "review",
4839 format!(
4840 "round {round}: votes agreed ({}) — no reconsideration",
4841 initial_votes[0].label()
4842 ),
4843 );
4844 }
4845
4846 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4847 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4848 let defer_e2e =
4859 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4860 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4861 let reason =
4862 format!("{blocking} blocking finding(s) already required a fix this round");
4863 self.state.event(
4864 "verify",
4865 format!(
4866 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4867 {}); it will run once a round has none left",
4868 short(&head)
4869 ),
4870 );
4871 (Vec::new(), false, true, Some(reason))
4872 } else {
4873 let e2e_commands = self.state.config.verify.e2e.clone();
4874 let cache_dir = self.state.config.cache_dir();
4875 let context = format!("round {round}");
4876 let (e2e, verify_retried) = with_cache_lease(
4877 &mut self.state,
4878 cache_dir.as_deref(),
4879 "e2e",
4880 "e2e",
4881 &winner.worktree,
4882 &head,
4883 verify_timeout,
4884 &context,
4885 |state, budget| {
4886 let shell = shell.clone();
4887 let e2e_commands = e2e_commands.clone();
4888 let worktree = winner.worktree.clone();
4889 let context = context.clone();
4890 async move {
4891 run_e2e_with_retry(
4892 state,
4893 &shell,
4894 &e2e_commands,
4895 &worktree,
4896 budget,
4897 &context,
4898 )
4899 .await
4900 }
4901 },
4902 )
4903 .await;
4904 (e2e, verify_retried, false, None)
4905 };
4906
4907 let expected = records.len();
4908 let answered = records.iter().filter(|r| r.failed.is_none()).count();
4909 let incomplete = answered < expected;
4910 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4911 let policy = self.state.config.graph.incomplete_review;
4912 let clean = round_is_clean(
4913 blocking,
4914 e2e_ok,
4915 answered,
4916 expected,
4917 round_quota_missing,
4918 policy,
4919 );
4920
4921 let mut round_record = ReviewRound {
4922 round,
4923 head: head.clone(),
4924 verified_head: None,
4925 verified_at: None,
4926 reviews: records,
4927 e2e,
4928 verify_retried,
4929 e2e_deferred,
4930 e2e_defer_reason,
4931 fix: None,
4932 blocking,
4933 answered,
4934 expected,
4935 clean,
4936 progressed: false,
4937 vote_split,
4938 reconsideration,
4939 verdict: None,
4940 };
4941 round_record.verdict = ReviewVote::worst(
4945 round_record
4946 .final_votes()
4947 .into_iter()
4948 .map(|(_, _, vote)| vote),
4949 );
4950 if !matches!(
4961 round_record.e2e_status(),
4962 E2eStatus::Deferred | E2eStatus::NotConfigured
4963 ) {
4964 round_record.verified_head = Some(head.clone());
4965 round_record.verified_at = Some(Timestamp::now());
4966 }
4967 let this_round_verification = round_record.verification_summary(&head);
4968
4969 if incomplete {
4970 let missing: Vec<String> = round_record
4971 .reviews
4972 .iter()
4973 .filter(|r| r.failed.is_some())
4974 .map(|r| format!("review-{}", r.reviewer))
4975 .collect();
4976 self.state.event(
4977 "review",
4978 format!(
4979 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4980 missing.join(", ")
4981 ),
4982 );
4983 }
4984
4985 if clean {
4986 self.state.event(
4987 "review",
4988 if incomplete && policy == IncompleteReviewPolicy::Warn {
4989 format!(
4990 "round {round}: clean (warn policy, incomplete panel) — no \
4991 blocking findings from the seats that answered, verification green"
4992 )
4993 } else if incomplete {
4994 format!(
4995 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4996 quorum) — no blocking findings from the seats that answered, \
4997 verification green",
4998 expected - answered
4999 )
5000 } else {
5001 format!("round {round}: clean — no blocking findings, verification green")
5002 },
5003 );
5004 self.state.reviews.push(round_record);
5005 self.state.status = RunStatus::Gating;
5006 self.state.save()?;
5007 return Ok(());
5008 }
5009
5010 if incomplete && blocking == 0 && e2e_ok {
5018 self.state.reviews.push(round_record);
5019 self.state.save()?;
5020 if round == max_rounds {
5021 self.state.status = RunStatus::Blocked;
5022 self.state.event(
5023 "review",
5024 format!(
5025 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
5026 refusing to call it clean",
5027 expected - answered
5028 ),
5029 );
5030 return Ok(());
5031 }
5032 continue;
5033 }
5034
5035 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
5047 self.state.reviews.push(round_record);
5048 return self
5049 .stop_reviewing(
5050 "the round's own verification could not run",
5051 &shell,
5052 &winner.worktree,
5053 )
5054 .await;
5055 }
5056
5057 if round == max_rounds {
5058 self.state.reviews.push(round_record);
5059 return self
5060 .stop_reviewing(
5061 &format!(
5062 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
5063 ),
5064 &shell,
5065 &winner.worktree,
5066 )
5067 .await;
5068 }
5069
5070 let blocking_findings: Vec<_> = all_findings
5073 .iter()
5074 .filter(|f| f.severity.blocks())
5075 .cloned()
5076 .collect();
5077 let fix_prompt = prompt::fix(
5078 &self.state.instruction,
5079 &blocking_findings,
5080 this_round_verification.as_ref(),
5081 round,
5082 max_rounds,
5083 &language,
5084 );
5085 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5086 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5087 let (job, seat, out) = self
5088 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
5089 prompt: fix_prompt.clone(),
5090 spec,
5091 seat,
5092 cwd: winner.worktree.clone(),
5093 timeout,
5094 allow_write: true,
5095 sessions,
5096 artifacts: artifacts.clone(),
5097 stem: format!("fix-{round}"),
5098 handover: None,
5099 })
5100 .await;
5101 let agent_id = seat.agent.clone();
5102
5103 let mut fix = FixRecord {
5104 agent: agent_id,
5105 addressed: Vec::new(),
5106 rejected: Vec::new(),
5107 notes: String::new(),
5108 committed: false,
5109 failed: None,
5110 duration_ms: 0,
5111 continuation: None,
5112 };
5113 let mut continuation = ContinuationRecord::not_needed();
5114 let mut final_seat = seat.clone();
5115 match out {
5116 AgentOutcome::Ok(o) => {
5117 fix.duration_ms = o.duration_ms;
5118 let parsed = verdict::extract_json::<FixReport>(&o.text);
5119 let incomplete_reason = match &parsed {
5126 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5127 "the reply parsed, but it reported a command whose own CLI never \
5128 confirmed an exit status"
5129 .to_owned(),
5130 ),
5131 Ok(_) => None,
5132 Err(e) => Some(e.to_string()),
5133 };
5134 match incomplete_reason {
5135 None => {
5136 let report = parsed.expect("checked Ok above");
5137 fix.addressed = report.addressed;
5138 fix.rejected = report.rejected;
5139 fix.notes =
5140 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5141 }
5142 Some(reason) => {
5143 let (resumed_seat, resolved, failure, cont) = self
5144 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
5145 .await;
5146 fix.duration_ms += cont.cumulative_wait_ms;
5147 continuation = cont;
5148 final_seat = resumed_seat;
5149 match resolved {
5150 Some(report) => {
5151 fix.addressed = report.addressed;
5152 fix.rejected = report.rejected;
5153 fix.notes = blind::sanitize_prose(
5154 &report.notes,
5155 &self.state.config.blind,
5156 );
5157 }
5158 None => fix.failed = failure,
5159 }
5160 }
5161 }
5162 }
5163 AgentOutcome::Dropped(o) => {
5165 fix.duration_ms = o.duration_ms;
5166 let why = o
5167 .dropped
5168 .as_ref()
5169 .map(|d| d.why.as_str())
5170 .unwrap_or("the CLI ended the stream without delivering its answer");
5171 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5172 }
5173 AgentOutcome::Quota(o) => {
5174 self.state.quota.push(QuotaLoss {
5175 seat: final_seat.key.clone(),
5176 node: "fix".to_owned(),
5177 at: Timestamp::now(),
5178 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5179 });
5180 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5181 }
5182 AgentOutcome::Failed(e) => fix.failed = Some(e),
5183 }
5184 fix.continuation = Some(continuation);
5185 self.state.seats.insert(final_seat.key.clone(), final_seat);
5186 if let Ok(r) = git::rescue_commit(
5187 &winner.worktree,
5188 &format!("magi: review round {round} fixes (uncommitted work)"),
5189 )
5190 .await
5191 {
5192 self.state.note_withheld("fix", &r.withheld);
5193 }
5194 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5195 fix.committed = after != before;
5196 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5204 let progressed = diff_after != patch;
5205 let commit_note = if fix.committed {
5206 "committed"
5207 } else {
5208 "NO new commit"
5209 };
5210 let tree_note = if progressed {
5211 "changed vs base"
5212 } else {
5213 "unchanged vs base"
5214 };
5215 self.state.event(
5216 "fix",
5217 match &fix.failed {
5218 Some(reason) => {
5224 format!(
5225 "round {round}: fixer's adoption report was lost ({reason}); \
5226 {commit_note}, tree {tree_note}"
5227 )
5228 }
5229 None => format!(
5230 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5231 {tree_note}{}",
5232 fix.addressed.len(),
5233 fix.rejected.len(),
5234 if continuation.outcome == ContinuationOutcome::Resumed {
5235 format!(
5236 " (adoption report recovered after {} continuation(s))",
5237 continuation.attempts
5238 )
5239 } else {
5240 String::new()
5241 },
5242 ),
5243 },
5244 );
5245 round_record.fix = Some(fix);
5246 round_record.progressed = progressed;
5247 self.state.reviews.push(round_record);
5248 self.state.save()?;
5249
5250 if matches!(
5263 continuation.outcome,
5264 ContinuationOutcome::Exhausted
5265 | ContinuationOutcome::QuotaLost
5266 | ContinuationOutcome::NoSession
5267 ) {
5268 return self
5269 .stop_reviewing(
5270 "the fixer's adoption report never came back, even after resuming its \
5271 own seat; refusing to start another round against the same worktree \
5272 while that is unresolved",
5273 &shell,
5274 &winner.worktree,
5275 )
5276 .await;
5277 }
5278
5279 let streak = self
5280 .state
5281 .reviews
5282 .iter()
5283 .rev()
5284 .take_while(|r| !r.progressed)
5285 .count();
5286 if streak >= STAGNANT_LIMIT {
5287 return self
5288 .stop_reviewing(
5289 &format!(
5290 "the tree has not moved against base for {streak} round(s) in a row"
5291 ),
5292 &shell,
5293 &winner.worktree,
5294 )
5295 .await;
5296 }
5297 }
5298 Ok(())
5299 }
5300
5301 fn record_contested_handoff(&mut self) {
5335 if self.state.contested_handoff.is_some() {
5336 return;
5337 }
5338 let Some(contested) = self
5339 .state
5340 .reviews
5341 .last()
5342 .and_then(ReviewRound::contested_handoff)
5343 else {
5344 return;
5345 };
5346 self.state.event(
5347 "review",
5348 format!(
5349 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5350 wait for the owner's approval",
5351 contested.findings.len(),
5352 contested.rejecters.len()
5353 ),
5354 );
5355 self.state.contested_handoff = Some(contested);
5356 }
5357
5358 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5359 let round_idx = self.state.reviews.len() - 1;
5360 let needs_catchup_run = matches!(
5368 self.state.reviews[round_idx].e2e_status(),
5369 E2eStatus::Deferred | E2eStatus::ResourceBlocked
5370 );
5371 if needs_catchup_run {
5372 let round = self.state.reviews[round_idx].round;
5373 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5374 let commands = self.state.config.verify.e2e.clone();
5375 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5376 let cache_dir = self.state.config.cache_dir();
5377 let context = format!(
5378 "round {round}: verification unresolved, catching up before the final decision"
5379 );
5380 let (outcomes, verify_retried) = with_cache_lease(
5381 &mut self.state,
5382 cache_dir.as_deref(),
5383 "e2e",
5384 "e2e",
5385 worktree,
5386 &attempted_head,
5387 timeout,
5388 &context,
5389 |state, budget| {
5390 let shell = shell.to_vec();
5391 let commands = commands.clone();
5392 let context = context.clone();
5393 async move {
5394 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5395 .await
5396 }
5397 },
5398 )
5399 .await;
5400 let last = &mut self.state.reviews[round_idx];
5401 last.e2e = outcomes;
5402 last.verify_retried = verify_retried;
5403 last.verified_head = Some(attempted_head);
5410 last.verified_at = Some(Timestamp::now());
5411 if verify_inconclusive(&last.e2e) {
5412 self.state.save()?;
5419 return Ok(());
5420 }
5421 last.e2e_deferred = false;
5422 }
5423 let last = &self.state.reviews[round_idx];
5424 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5425
5426 match last.e2e_status() {
5427 E2eStatus::Failed => {
5428 let red: Vec<String> = last
5429 .e2e
5430 .iter()
5431 .filter(|o| !o.ok())
5432 .map(|o| {
5433 format!(
5434 "`{}` -> {:?}\n{}",
5435 o.command,
5436 o.code,
5437 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5438 )
5439 })
5440 .collect();
5441 self.state
5442 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5443 self.state.status = RunStatus::Blocked;
5444 }
5445 E2eStatus::ResourceBlocked => {
5450 self.state.event(
5451 "review",
5452 format!(
5453 "{why}; e2e could not run (shared build cache unavailable); not \
5454 deciding yet"
5455 ),
5456 );
5457 }
5458 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5459 self.state.event(
5460 "review",
5461 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5462 );
5463 self.record_contested_handoff();
5464 self.state.status = RunStatus::Gating;
5465 }
5466 }
5467 self.state.save()?;
5468 Ok(())
5469 }
5470
5471 async fn gate(&mut self) -> Result<()> {
5474 if self.state.status == RunStatus::Failed
5486 || self
5487 .state
5488 .base_sync
5489 .as_ref()
5490 .is_some_and(|s| s.conflict.is_some())
5491 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5492 != Some(RunStatus::Gating)
5493 {
5494 return Ok(());
5495 }
5496 if self.state.gate_ran {
5497 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5508 self.state.status = RunStatus::Blocked;
5509 self.state.save()?;
5510 }
5511 return Ok(());
5512 }
5513 let Some(winner) = self.state.winner().cloned() else {
5514 return Ok(());
5515 };
5516 self.state.status = RunStatus::Gating;
5517 let mut outcomes = self.run_gate(&winner).await?;
5518 loop {
5519 if verify_inconclusive(&outcomes) {
5530 self.state.save()?;
5531 return Ok(());
5532 }
5533 if outcomes.iter().all(CommandOutcome::ok) {
5534 break;
5535 }
5536 match self.gate_fix_round(&winner, &outcomes).await? {
5537 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5538 GateFix::Stop => break,
5539 GateFix::Defer => {
5540 self.state.save()?;
5541 return Ok(());
5542 }
5543 }
5544 }
5545 let passed = outcomes.iter().all(CommandOutcome::ok);
5546 self.state.gate = outcomes;
5547 self.state.gate_ran = true;
5548 if !passed {
5549 self.state.status = RunStatus::Blocked;
5550 let spent = self.state.gate_fixes.len();
5551 self.state.event(
5552 "gate",
5553 if spent == 0 {
5554 "gate failed; not merging".to_owned()
5555 } else {
5556 format!("gate failed after {spent} gate-fix round(s); not merging")
5557 },
5558 );
5559 }
5560 self.state.save()?;
5561 Ok(())
5562 }
5563
5564 async fn run_pre_gate(&mut self, winner: &Candidate) {
5574 let commands = self.state.config.verify.pre_gate.clone();
5575 if commands.is_empty() {
5576 return;
5577 }
5578 let shell = self.state.config.shell();
5579 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5580 let (outcomes, _) = run_commands(
5581 &mut self.state,
5582 "pre_gate",
5583 "pre_gate",
5584 0,
5585 &shell,
5586 &commands,
5587 &winner.worktree,
5588 timeout,
5589 )
5590 .await;
5591 for o in &outcomes {
5592 if !o.ok() {
5593 tracing::warn!(
5594 "pre_gate `{}` failed ({:?}); the gate decides",
5595 o.command,
5596 o.code
5597 );
5598 }
5599 self.state.event(
5600 "pre_gate",
5601 format!(
5602 "`{}` -> {}",
5603 o.command,
5604 if o.ok() {
5605 "pass".to_owned()
5606 } else {
5607 format!(
5608 "FAIL ({:?})\n{}",
5609 o.code,
5610 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5611 )
5612 }
5613 ),
5614 );
5615 }
5616 self.state.pre_gate = outcomes;
5617 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5618 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5619 Ok(head) => {
5620 self.state
5621 .event("pre_gate", format!("committed mechanical fixes ({head})"));
5622 self.state.pre_gate_commit = Some(head);
5623 }
5624 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5625 },
5626 Ok(false) => {}
5627 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5628 }
5629 if let Err(e) = self.state.save() {
5630 tracing::warn!("could not persist the pre_gate record: {e:#}");
5631 }
5632 }
5633
5634 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5637 self.run_pre_gate(winner).await;
5638 let shell = self.state.config.shell();
5639 let gate_commands = self.state.config.verify.gate.clone();
5640 let outcomes = if gate_commands.is_empty() {
5649 Vec::new()
5650 } else {
5651 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5652 let cache_dir = self.state.config.cache_dir();
5653 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5654 let (outcomes, _) = with_cache_lease(
5655 &mut self.state,
5656 cache_dir.as_deref(),
5657 "gate",
5658 "gate",
5659 &winner.worktree,
5660 &head,
5661 timeout,
5662 "final gate",
5663 |state, budget| {
5664 let shell = shell.clone();
5665 let gate_commands = gate_commands.clone();
5666 let worktree = winner.worktree.clone();
5667 async move {
5668 let (outcomes, timed_out_pids) = run_commands(
5669 state,
5670 "gate",
5671 "gate",
5672 0,
5673 &shell,
5674 &gate_commands,
5675 &worktree,
5676 budget,
5677 )
5678 .await;
5679 (outcomes, false, timed_out_pids)
5680 }
5681 },
5682 )
5683 .await;
5684 outcomes
5685 };
5686 if outcomes.is_empty() {
5687 self.state.event(
5692 "gate",
5693 "no gate commands configured; nothing to check, passing",
5694 );
5695 }
5696 for o in &outcomes {
5697 self.state.event(
5698 "gate",
5699 format!(
5700 "`{}` -> {}",
5701 o.command,
5702 if o.ok() {
5703 "pass".to_owned()
5704 } else {
5705 format!(
5706 "FAIL ({:?})\n{}",
5707 o.code,
5708 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5709 )
5710 }
5711 ),
5712 );
5713 }
5714 Ok(outcomes)
5715 }
5716
5717 async fn gate_fix_round(
5730 &mut self,
5731 winner: &Candidate,
5732 outcomes: &[CommandOutcome],
5733 ) -> Result<GateFix> {
5734 let cap = self.state.config.graph.gate_fix_rounds;
5735 let spent = self.state.gate_fixes.len();
5736 if spent >= cap {
5737 if cap > 0 {
5738 self.state.event(
5739 "gate",
5740 format!("{spent} gate-fix round(s) spent and the gate still fails"),
5741 );
5742 }
5743 return Ok(GateFix::Stop);
5744 }
5745 if !gate_fixable(outcomes) {
5746 self.state.event(
5747 "gate",
5748 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5749 command or similar); not spending a fix round on it",
5750 );
5751 return Ok(GateFix::Stop);
5752 }
5753 let min_free = self.state.config.disk.min_free_bytes;
5754 if min_free > 0 {
5755 match crate::disk::free_bytes(&winner.worktree) {
5756 Ok(free) if crate::disk::enough_space(free, min_free) => {}
5757 Ok(free) => {
5758 self.state.event(
5759 "gate",
5760 format!(
5761 "only {free} bytes free ({min_free} required by `[disk] \
5762 min_free_bytes`); not spending a fix round on a failure the disk \
5763 may explain"
5764 ),
5765 );
5766 return Ok(GateFix::Stop);
5767 }
5768 Err(e) => {
5769 self.state.event(
5770 "gate",
5771 format!("free disk space could not be measured ({e:#}); no fix round"),
5772 );
5773 return Ok(GateFix::Stop);
5774 }
5775 }
5776 }
5777
5778 let attempt = spent + 1;
5779 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5780 let base = self.landing_base();
5781 let fix_prompt = prompt::gate_fix(
5782 &self.state.instruction,
5783 &failed,
5784 attempt,
5785 cap,
5786 &self.state.config.graph.language,
5787 );
5788 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5789 let sessions = self.state.config.graph.sessions;
5790 let artifacts = agent::artifacts_dir(&self.state.dir());
5791 self.state.event(
5792 "gate",
5793 format!("gate failed; gate-fix round {attempt} of {cap}"),
5794 );
5795 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5796 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5797 let (_, seat, out) = self
5798 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
5799 prompt: fix_prompt.clone(),
5800 spec,
5801 seat,
5802 cwd: winner.worktree.clone(),
5803 timeout,
5804 allow_write: true,
5805 sessions,
5806 artifacts: artifacts.clone(),
5807 stem: format!("gate-fix-{attempt}"),
5808 handover: None,
5809 })
5810 .await;
5811 let mut record = GateFixRecord {
5812 agent: seat.agent.clone(),
5813 failed,
5814 notes: String::new(),
5815 committed: false,
5816 error: None,
5817 };
5818 match out {
5819 AgentOutcome::Ok(o) => {
5820 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5823 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5824 }
5825 }
5826 AgentOutcome::Dropped(_) => {
5827 record.error = Some("the CLI dropped the stream".to_owned());
5828 }
5829 AgentOutcome::Quota(o) => {
5830 self.state.quota.push(QuotaLoss {
5831 seat: seat.key.clone(),
5832 node: "gate-fix".to_owned(),
5833 at: Timestamp::now(),
5834 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5835 });
5836 record.error = Some("rate limited (quota); fixer could not run".to_owned());
5837 }
5838 AgentOutcome::Failed(e) => record.error = Some(e),
5839 }
5840 self.state.seats.insert(seat.key.clone(), seat);
5841 if let Ok(r) = git::rescue_commit(
5842 &winner.worktree,
5843 &format!("magi: gate fix {attempt} (uncommitted work)"),
5844 )
5845 .await
5846 {
5847 self.state.note_withheld("gate-fix", &r.withheld);
5848 }
5849 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5850 record.committed = after != before;
5851 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5852 let note = record.error.clone();
5853 self.state.gate_fixes.push(record);
5854 self.state.save()?;
5855 if !changed {
5856 self.state.event(
5857 "gate",
5858 match note {
5859 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5860 None => format!("gate-fix round {attempt}: the tree did not change"),
5861 },
5862 );
5863 return Ok(GateFix::Stop);
5864 }
5865 self.state.event(
5866 "gate",
5867 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5868 );
5869
5870 let commands = self.state.config.verify.e2e.clone();
5871 if !commands.is_empty() {
5872 let shell = self.state.config.shell();
5873 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5874 let cache_dir = self.state.config.cache_dir();
5875 let context = format!("gate-fix round {attempt}");
5876 let (e2e, _) = with_cache_lease(
5877 &mut self.state,
5878 cache_dir.as_deref(),
5879 "e2e",
5880 "e2e",
5881 &winner.worktree,
5882 &after,
5883 timeout,
5884 &context,
5885 |state, budget| {
5886 let shell = shell.clone();
5887 let commands = commands.clone();
5888 let context = context.clone();
5889 let worktree = winner.worktree.clone();
5890 async move {
5891 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5892 .await
5893 }
5894 },
5895 )
5896 .await;
5897 if verify_inconclusive(&e2e) {
5898 return Ok(GateFix::Defer);
5899 }
5900 if e2e.iter().any(|o| !o.ok()) {
5901 self.state.event(
5902 "gate",
5903 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5904 );
5905 return Ok(GateFix::Stop);
5906 }
5907 }
5908 Ok(GateFix::Retry)
5909 }
5910
5911 async fn guarded_pr_message(
5915 &mut self,
5916 winner: &Candidate,
5917 facts: Option<&BranchFacts>,
5918 posting: bool,
5919 ) -> PrMessage {
5920 let mut pr = pr_message_raw(&self.state, winner.label, facts);
5921 if !posting {
5922 let identity = crate::scrub::Identity::current();
5923 return PrMessage {
5924 title: crate::scrub::scrub(&pr.title, &identity),
5925 body: crate::scrub::scrub(&pr.body, &identity),
5926 };
5927 }
5928 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
5929 let mut violations = crate::github_text::check_with(&pr.title, &pr.body, decision);
5930 let identity = crate::scrub::Identity::current();
5931 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
5932 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
5933 && !violations.contains(&crate::github_text::Violation::SensitiveData)
5934 {
5935 violations.push(crate::github_text::Violation::SensitiveData);
5936 }
5937 violations.retain(|v| *v != crate::github_text::Violation::SensitiveData);
5941 if self.state.config.graph.github_text_guard && !violations.is_empty() {
5942 self.state.event(
5943 "github-text",
5944 format!("description rejected: {violations:?}; requesting one rewrite"),
5945 );
5946 let mut rewritten = false;
5947 if let Some(spec) = self
5948 .state
5949 .config
5950 .agents
5951 .iter()
5952 .find(|a| a.id == winner.agent)
5953 .cloned()
5954 {
5955 let key = format!("impl-{}", winner.label);
5956 let seat = self.seat(&key, &spec.id);
5957 let prompt = format!(
5958 "Rewrite only the following pull request description. The posting gate reported {violations:?}. Write English prose and remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the complete Markdown body. Preserve the magi run marker.\n\nTITLE: {}\n{}",
5959 pr.title, pr.body
5960 );
5961 let job = SeatJob {
5962 spec,
5963 seat,
5964 cwd: winner.worktree.clone(),
5965 prompt,
5966 timeout: retry_budget(
5967 Duration::from_secs(self.state.config.graph.timeout_implement),
5968 true,
5969 ),
5970 allow_write: false,
5971 sessions: self.state.config.graph.sessions,
5972 artifacts: agent::artifacts_dir(&self.state.dir()),
5973 stem: "github-text-rewrite".to_owned(),
5974 handover: None,
5975 };
5976 let prompts = self.state.config.prompts.clone();
5977 let cache = self.state.config.cache_dir();
5978 let run = self.state.id.clone();
5979 let ctx = WaveCtx {
5980 run: &run,
5981 node: "github-text",
5982 prompts: &prompts,
5983 cache: cache.as_deref(),
5984 round: None,
5985 carry_seats: false,
5986 };
5987 let (seat, outcome) =
5988 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
5989 self.state.seats.insert(seat.key.clone(), seat);
5990 if let AgentOutcome::Ok(output) = outcome
5991 && let Some(title) = summary_title(&output.text)
5992 {
5993 let mut body = summary_without_title(&output.text);
5994 let rewrite = PrMessage {
5995 title: title.clone(),
5996 body: body.clone(),
5997 };
5998 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
5999 if !body.trim().is_empty()
6000 && crate::github_text::check_with(&title, &body, redecision).is_empty()
6001 {
6002 decision = redecision;
6003 let marker = format!("magi:run/{}", self.state.id);
6004 if !body.contains(&marker) {
6005 body.push_str(&format!("\n\n{marker}"));
6006 }
6007 pr = PrMessage { title, body };
6008 rewritten = true;
6009 self.state
6010 .event("github-text", "description rewrite passed");
6011 }
6012 }
6013 }
6014 if !rewritten {
6015 self.state.event(
6016 "github-text",
6017 "description rewrite unavailable or rejected; using neutral body",
6018 );
6019 pr = PrMessage {
6020 title: pr.title,
6021 body: format!(
6022 "{}\n\nmagi:run/{}",
6023 crate::github_text::NEUTRAL_BODY,
6024 self.state.id
6025 ),
6026 };
6027 decision = decision.map(|d| crate::github_text::LanguageDecision {
6030 body_english: true,
6031 ..d
6032 });
6033 }
6034 }
6035 let (title, body) =
6036 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
6037 PrMessage { title, body }
6038 }
6039
6040 async fn judge_pr_language(
6043 &mut self,
6044 cwd: &Path,
6045 pr: &PrMessage,
6046 ) -> Option<crate::github_text::LanguageDecision> {
6047 self.state.config.roles.language_judge.as_ref()?;
6048 let decision =
6049 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
6050 self.state.event(
6051 "github-text",
6052 if decision.is_some() {
6053 "language decided by the language judge"
6054 } else {
6055 "language judge unavailable; using built-in heuristics"
6056 },
6057 );
6058 decision
6059 }
6060
6061 async fn merge(&mut self) -> Result<()> {
6062 if self
6077 .state
6078 .base_sync
6079 .as_ref()
6080 .is_some_and(|s| s.conflict.is_some())
6081 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6082 != Some(RunStatus::Gating)
6083 || !self.state.gate_status().ok()
6092 {
6093 return Ok(());
6094 }
6095 if self.state.merge.is_some() {
6104 return Ok(());
6105 }
6106 let Some(winner) = self.state.winner().cloned() else {
6107 return Ok(());
6108 };
6109 let repo = self.state.repo.clone();
6110 let base = self.state.base_branch.clone();
6111 let mode = self.state.config.merge.mode;
6112 let style = self.state.config.merge.style;
6113 let facts = if is_review_run(&self.state) {
6114 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
6115 let start = review_base(
6116 &repo,
6117 &self.state.config.merge.remote,
6118 &base,
6119 &self.state.base_commit,
6120 &winner.branch,
6121 )
6122 .await;
6123 branch_facts(&repo, &start, &winner.branch).await
6124 } else {
6125 None
6126 };
6127 let leaked = if is_review_run(&self.state) {
6130 leaked_subjects(&self.state, &winner.branch).await
6131 } else {
6132 Some(Vec::new())
6133 };
6134 let pr = self
6135 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
6136 .await;
6137 let message = pr.commit_message();
6138
6139 let outcome = match mode {
6140 MergeMode::None => MergeOutcome {
6141 mode,
6142 ok: true,
6143 detail: manual_merge_command(style, &repo, &winner.branch, &message),
6144 empty: false,
6145 },
6146 MergeMode::Pr | MergeMode::Local
6147 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
6148 {
6149 MergeOutcome {
6150 mode,
6151 ok: false,
6152 detail: empty_candidate_detail(&self.state, &base),
6153 empty: true,
6154 }
6155 }
6156 MergeMode::Local => {
6157 let on = git::current_branch(&repo).await?;
6158 if on.as_deref() != Some(base.as_str()) {
6159 MergeOutcome {
6160 mode,
6161 ok: false,
6162 detail: format!(
6163 "{} has {} checked out, not the base branch {base}",
6164 repo.display(),
6165 on.unwrap_or_else(|| "a detached HEAD".to_owned())
6166 ),
6167 empty: false,
6168 }
6169 } else if !git::is_clean(&repo).await? {
6170 MergeOutcome {
6171 mode,
6172 ok: false,
6173 detail: format!("{} is dirty; refusing to merge", repo.display()),
6174 empty: false,
6175 }
6176 } else {
6177 let out = match style {
6178 MergeStyle::Merge => {
6179 git::merge_no_ff(&repo, &winner.branch, &message).await?
6180 }
6181 MergeStyle::Squash => {
6182 git::merge_squash(&repo, &winner.branch, &message).await?
6183 }
6184 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
6185 };
6186 MergeOutcome {
6187 mode,
6188 ok: out.ok(),
6189 detail: if out.ok() { out.stdout } else { out.stderr },
6190 empty: false,
6191 }
6192 }
6193 }
6194 MergeMode::Pr => {
6195 let remote = self.state.config.merge.remote.clone();
6196 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
6197 if !pushed.ok() {
6198 MergeOutcome {
6199 mode,
6200 ok: false,
6201 detail: pushed.stderr,
6202 empty: false,
6203 }
6204 } else {
6205 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
6211 let out = match pr_merge_plan(found) {
6212 PrPlan::Create => {
6213 gh_pr_create(
6214 &winner.worktree,
6215 &base,
6216 &winner.branch,
6217 &pr.title,
6218 &pr.body,
6219 )
6220 .await
6221 }
6222 PrPlan::Adopt { url, title } => {
6223 self.state
6224 .event("merge", format!("Pr: adopted open pull request {url}"));
6225 if title != pr.title
6226 && (!is_review_run(&self.state)
6227 || leaked
6228 .as_deref()
6229 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
6230 && let Err(e) = land::set_pr_title(
6231 &mut self.state,
6232 &winner.worktree,
6233 &url,
6234 &pr.title,
6235 )
6236 .await
6237 {
6238 tracing::warn!("could not refresh title of {url}: {e:#}");
6239 self.state
6240 .event("merge", format!("Pr: title refresh failed: {e:#}"));
6241 }
6242 Ok(url)
6243 }
6244 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
6245 };
6246 match out {
6247 Ok(url) => MergeOutcome {
6248 mode,
6249 ok: true,
6250 detail: url,
6251 empty: false,
6252 },
6253 Err(e) => MergeOutcome {
6254 mode,
6255 ok: false,
6256 detail: e.to_string(),
6257 empty: false,
6258 },
6259 }
6260 }
6261 }
6262 };
6263
6264 self.state.status = match (mode, outcome.ok) {
6265 (MergeMode::None, _) => RunStatus::Ready,
6266 (_, true) => RunStatus::Merged,
6267 (_, false) => RunStatus::Blocked,
6268 };
6269 self.state.event(
6270 "merge",
6271 format!(
6272 "{:?}: {}",
6273 mode,
6274 outcome.detail.lines().next().unwrap_or("")
6275 ),
6276 );
6277 self.state.merge = Some(outcome);
6278 self.state.save()?;
6279
6280 if self.state.config.graph.land
6286 && mode == MergeMode::Pr
6287 && self.state.status == RunStatus::Merged
6288 {
6289 self.run_land().await?;
6290 }
6291 self.settle_questions();
6296 Ok(())
6297 }
6298
6299 async fn run_land(&mut self) -> Result<()> {
6310 let url = self
6311 .state
6312 .merge
6313 .as_ref()
6314 .map(|m| m.detail.clone())
6315 .unwrap_or_default();
6316 let url = url.lines().next().unwrap_or("").trim().to_owned();
6317 if !url.starts_with("http") {
6318 return Ok(());
6319 }
6320 match land::land(&mut self.state, &url).await {
6323 Ok(pr) if self.state.parked => {
6324 let _ = pr;
6328 }
6329 Ok(pr) => {
6330 self.state.status = match pr.state {
6331 land::PrLifecycle::Merged => RunStatus::Merged,
6332 _ => RunStatus::Blocked,
6333 };
6334 if bump::should_release_bump(self.state.status)
6341 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
6342 {
6343 self.state
6349 .event("bump", format!("release bump skipped: {e:#}"));
6350 }
6351 if self.state.status == RunStatus::Merged {
6354 crate::followup::after_merge(&mut self.state, &pr.url).await;
6355 }
6356 self.state.save()?;
6357 }
6358 Err(e) => {
6359 self.state.status = RunStatus::Blocked;
6360 self.state.event("land", format!("gave up: {e}"));
6361 self.state.save()?;
6362 }
6363 }
6364 Ok(())
6365 }
6366
6367 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
6371 if let Some(existing) = self.state.seats.get(key)
6372 && existing.agent == agent
6373 {
6374 return existing.clone();
6375 }
6376 let fresh = if self.state.seats.contains_key(key) {
6379 handover_seat(key, agent, self.state.next_seat_seed())
6380 } else {
6381 SeatState::new(key, agent, self.state.seed)
6382 };
6383 self.state.seats.insert(key.to_owned(), fresh.clone());
6384 fresh
6385 }
6386
6387 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
6393 match self.state.seats.get(key) {
6394 Some(s) if s.agent != spec.id => {
6395 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
6396 }
6397 _ => spec,
6398 }
6399 }
6400
6401 fn view(&self, c: &Candidate) -> CandidateView {
6403 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
6404 .unwrap_or_default();
6405 let (patch, _) = blind::sanitize_patch(
6406 &format!("candidate {} patch", c.label),
6407 &raw,
6408 &self.state.config.blind,
6409 );
6410 CandidateView {
6411 label: c.label,
6412 branch: c.branch.clone(),
6413 summary: c.summary.clone(),
6414 stat: c.stat.clone(),
6415 patch,
6416 }
6417 }
6418
6419 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
6421 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
6422 prompt::judge(
6423 "(see above)",
6424 &views,
6425 self.roles.judges.len(),
6426 base_short,
6427 "en",
6428 )
6429 }
6430
6431 fn vote_prompt_full(
6437 &self,
6438 j: usize,
6439 labels: &[char],
6440 language: &str,
6441 candidates: &[Candidate],
6442 base_short: &str,
6443 ) -> String {
6444 let mut text = format!(
6445 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
6446 prompt::final_vote(labels, language),
6447 self.state.instruction,
6448 self.candidate_block(candidates, base_short)
6449 );
6450 if let Some(own) = self
6451 .state
6452 .judgements
6453 .get(j)
6454 .filter(|r| !r.ranking.is_empty())
6455 {
6456 let reasons = own
6457 .reasons
6458 .iter()
6459 .map(|(k, v)| format!("- {k}: {v}"))
6460 .collect::<Vec<_>>()
6461 .join("\n");
6462 text.push_str(&format!(
6463 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
6464 own.ranking.iter().collect::<String>(),
6465 if reasons.is_empty() {
6466 ""
6467 } else {
6468 ", because:\n"
6469 }
6470 ));
6471 }
6472 if !self.state.deliberation.is_empty() {
6473 text.push_str("\n# What was argued before this vote\n");
6474 for t in self.transcript(&[], j) {
6475 text.push_str(&format!(
6476 "\n## {}{}\n\n{}\n",
6477 t.who,
6478 if t.is_self { " (you)" } else { "" },
6479 t.body.trim()
6480 ));
6481 }
6482 }
6483 text
6484 }
6485
6486 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6493 let mut turns = Vec::new();
6494 for j in &self.state.judgements {
6495 if j.ranking.is_empty() {
6496 continue;
6497 }
6498 let reasons = j
6499 .reasons
6500 .iter()
6501 .map(|(k, v)| format!("- {k}: {v}"))
6502 .collect::<Vec<_>>()
6503 .join("\n");
6504 turns.push(Turn {
6505 who: format!("Judge {} (opening ranking)", j.judge),
6506 is_self: j.judge == self_idx + 1,
6507 body: format!(
6508 "Ranked {}{}{reasons}",
6509 j.ranking.iter().collect::<String>(),
6510 if reasons.is_empty() {
6511 ""
6512 } else {
6513 ", because:\n"
6514 }
6515 ),
6516 });
6517 }
6518 for t in self
6519 .state
6520 .deliberation
6521 .iter()
6522 .flat_map(|r| r.turns.iter())
6523 .chain(current)
6524 {
6525 turns.push(Turn {
6526 who: format!("Judge {}", t.judge),
6527 is_self: t.judge == self_idx + 1,
6528 body: t.body.clone(),
6529 });
6530 }
6531 turns
6532 }
6533}
6534
6535fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6537 agent::has_session(spec.kind, seat, sessions)
6538}
6539
6540fn next_untried_in_roster<'a>(
6561 roster: &'a [AgentSpec],
6562 start: usize,
6563 tried: &BTreeSet<String>,
6564) -> Option<&'a AgentSpec> {
6565 roster
6566 .get(start + 1..)?
6567 .iter()
6568 .find(|s| !tried.contains(&s.id))
6569}
6570
6571fn pick_successor<'a>(
6592 roster: &'a [AgentSpec],
6593 start: usize,
6594 tried: &BTreeSet<String>,
6595 carried: Option<&BTreeSet<String>>,
6596 others: &BTreeSet<String>,
6597) -> Option<&'a AgentSpec> {
6598 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
6599 roster
6600 .get(start + 1..)
6601 .and_then(|tail| {
6602 tail.iter()
6603 .filter(free)
6604 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
6605 })
6606 .or_else(|| {
6607 carried
6608 .is_some()
6609 .then(|| roster.iter().find(free))
6610 .flatten()
6611 })
6612 .or_else(|| match carried {
6613 Some(c) => next_for_seat(roster, start, tried, c),
6614 None => next_untried_in_roster(roster, start, tried),
6615 })
6616}
6617
6618fn next_for_seat<'a>(
6628 roster: &'a [AgentSpec],
6629 start: usize,
6630 round_tried: &BTreeSet<String>,
6631 carried_failed: &BTreeSet<String>,
6632) -> Option<&'a AgentSpec> {
6633 roster
6634 .get(start + 1..)?
6635 .iter()
6636 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
6637 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
6638}
6639
6640fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
6646 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
6647 return spec;
6648 };
6649 let ok = |id: &str| !h.failed.contains(id);
6650 if let Some(last) = h.last_ok.as_deref()
6651 && ok(last)
6652 && let Some(s) = roster.iter().find(|s| s.id == last)
6653 {
6654 return s.clone();
6655 }
6656 if ok(&spec.id) {
6657 return spec;
6658 }
6659 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
6660 roster
6661 .get(from + 1..)
6662 .into_iter()
6663 .flatten()
6664 .chain(roster.iter())
6665 .find(|s| ok(&s.id))
6666 .cloned()
6667 .unwrap_or(spec)
6668}
6669
6670pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6674 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6675}
6676
6677const TIMED_OUT: &str = "timed out";
6680
6681impl FailClass {
6682 fn of(out: &AgentOutcome) -> Option<Self> {
6684 match out {
6685 AgentOutcome::Ok(_) => None,
6686 AgentOutcome::Quota(_) => Some(Self::Quota),
6687 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6688 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6689 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6690 }
6691 }
6692
6693 fn stem_word(&self) -> &'static str {
6695 match self {
6696 Self::Quota => "quota",
6697 _ => "handover",
6698 }
6699 }
6700}
6701
6702fn failure_signature(msg: &str) -> String {
6706 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6707 let mut out = Vec::new();
6708 for word in line.split_whitespace() {
6709 if word.contains('/') || word.contains('\\') {
6710 out.push("<path>".to_owned());
6711 continue;
6712 }
6713 let mut w = String::new();
6714 let mut in_digits = false;
6715 for c in word.chars() {
6716 if c.is_ascii_digit() {
6717 if !in_digits {
6718 w.push('#');
6719 }
6720 in_digits = true;
6721 } else {
6722 in_digits = false;
6723 w.push(c);
6724 }
6725 }
6726 out.push(w);
6727 }
6728 out.join(" ").chars().take(120).collect()
6729}
6730
6731fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6738 match cur {
6739 FailClass::Quota | FailClass::Timeout => true,
6740 FailClass::Other(_) => prev != Some(cur),
6741 }
6742}
6743
6744fn fail_reason(out: &AgentOutcome) -> String {
6746 match out {
6747 AgentOutcome::Ok(_) => String::new(),
6748 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6749 AgentOutcome::Dropped(o) => format!(
6750 "the CLI dropped the stream ({})",
6751 o.dropped
6752 .as_ref()
6753 .map(|d| d.why.as_str())
6754 .unwrap_or("it ended without delivering its answer")
6755 ),
6756 AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6757 }
6758}
6759
6760pub(crate) fn record_handover(
6763 state: &mut RunState,
6764 node: &str,
6765 seat: &str,
6766 from: &str,
6767 to: &str,
6768 class: &FailClass,
6769 reason: &str,
6770) {
6771 let message = if *class == FailClass::Quota {
6772 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6773 } else {
6774 format!("{seat}: handed over {from} -> {to} ({reason})")
6775 };
6776 state.event(node, message);
6777 state.handovers.push(Handover {
6778 at: Timestamp::now(),
6779 node: node.to_owned(),
6780 seat: seat.to_owned(),
6781 from: from.to_owned(),
6782 to: to.to_owned(),
6783 reason: reason.to_owned(),
6784 });
6785}
6786
6787fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6801 commands.iter().any(|c| c.exit_code.is_none())
6802}
6803
6804fn verified_noop_claim(
6817 usable: bool,
6818 commands: &[agent::CommandEvidence],
6819 text: &str,
6820) -> Option<String> {
6821 (usable && !has_unconfirmed_command(commands))
6822 .then(|| verdict::verified_noop(text))
6823 .flatten()
6824}
6825
6826fn short(commit: &str) -> String {
6827 commit.chars().take(7).collect()
6828}
6829
6830fn make_executable(path: &Path) -> Result<()> {
6831 #[cfg(unix)]
6832 {
6833 use std::os::unix::fs::PermissionsExt as _;
6834 let mut perms = std::fs::metadata(path)?.permissions();
6835 perms.set_mode(0o755);
6836 std::fs::set_permissions(path, perms)?;
6837 }
6838 #[cfg(not(unix))]
6839 {
6840 let _ = path;
6841 }
6842 Ok(())
6843}
6844
6845struct WaveCtx<'a> {
6852 run: &'a str,
6855 node: &'a str,
6857 prompts: &'a Prompts,
6858 cache: Option<&'a Path>,
6860 round: Option<usize>,
6863 carry_seats: bool,
6867}
6868
6869async fn run_one(
6871 job: SeatJob,
6872 sem: Arc<Semaphore>,
6873 ctx: &WaveCtx<'_>,
6874 state: &mut RunState,
6875 attempt: usize,
6876) -> (SeatState, AgentOutcome) {
6877 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6878 .await
6879 .pop()
6880 .expect("one job in, one result out");
6881 (seat, out)
6882}
6883
6884async fn wave(
6890 jobs: Vec<SeatJob>,
6891 sem: Arc<Semaphore>,
6892 ctx: &WaveCtx<'_>,
6893 state: &mut RunState,
6894 attempt: usize,
6895) -> Vec<(usize, SeatState, AgentOutcome)> {
6896 let WaveCtx {
6897 run,
6898 node,
6899 prompts,
6900 cache,
6901 round,
6902 carry_seats: _,
6903 } = *ctx;
6904 for job in &jobs {
6905 state.seat_started(node, &job.seat.key, job.timeout, attempt);
6906 }
6907 if let Err(e) = state.save() {
6908 tracing::warn!("could not persist in-progress seats: {e:#}");
6913 }
6914 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6930 let wait_started = Instant::now();
6931 let cache_guard = if let Some(cache_dir) = cache {
6932 if jobs_had_a_writer {
6933 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6934 let budget = jobs
6935 .iter()
6936 .map(|j| j.timeout)
6937 .max()
6938 .unwrap_or(Duration::from_secs(60));
6939 acquire_cache_lease(state, cache_dir, &owner, budget, node)
6940 .await
6941 .ok()
6942 } else {
6943 None
6944 }
6945 } else {
6946 None
6947 };
6948 let waited_for_lease = wait_started.elapsed();
6955 let mut set = tokio::task::JoinSet::new();
6956 let overlay = prompts.overlay(node);
6957 for (i, mut job) in jobs.into_iter().enumerate() {
6958 job.timeout = job.timeout.saturating_sub(waited_for_lease);
6959 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6960 if cache.is_some() {
6961 job.prompt.push('\n');
6962 job.prompt
6963 .push_str(&prompt::build_cache_note(node, job.allow_write));
6964 }
6965 let sem = Arc::clone(&sem);
6966 let run = run.to_owned();
6967 let node = node.to_owned();
6968 let attachments = if node == "implement" {
6971 state.attachments.clone()
6972 } else {
6973 Vec::new()
6974 };
6975 let cache = cache
6986 .filter(|_| job.allow_write && cache_guard.is_some())
6987 .map(Path::to_path_buf);
6988 set.spawn(async move {
6989 let _permit = sem.acquire().await;
6990 let mut seat = job.seat;
6991 let out = agent::invoke(
6992 &job.spec,
6993 &mut seat,
6994 &Invocation {
6995 cwd: &job.cwd,
6996 prompt: &job.prompt,
6997 timeout: job.timeout,
6998 allow_write: job.allow_write,
6999 sessions: job.sessions,
7000 artifacts: &job.artifacts,
7001 stem: &job.stem,
7002 run: &run,
7003 node: &node,
7004 cache_dir: cache.as_deref(),
7005 attachments: &attachments,
7006 writable: &[],
7007 },
7008 )
7009 .await;
7010 let out = match out {
7011 Ok(o) if o.usable() => AgentOutcome::Ok(o),
7012 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
7013 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
7021 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
7022 Ok(o) => AgentOutcome::Failed(format!(
7023 "exited with {:?} and no usable output",
7024 o.exit_code
7025 )),
7026 Err(e) => AgentOutcome::Failed(e.to_string()),
7027 };
7028 (i, seat, out)
7029 });
7030 }
7031 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
7032 while let Some(joined) = set.join_next().await {
7033 let (i, seat, out) = match joined {
7034 Ok(v) => v,
7035 Err(e) => {
7039 tracing::error!("agent task panicked: {e}");
7040 continue;
7041 }
7042 };
7043 state.seat_finished(&seat.key);
7044 record_jobs(state, node, round, &seat.key, &out);
7045 if let Err(e) = state.save() {
7046 tracing::warn!("could not persist a seat's completion: {e:#}");
7047 }
7048 if collected.len() <= i {
7049 collected.resize_with(i + 1, || None);
7050 }
7051 collected[i] = Some((i, seat, out));
7052 }
7053 if state
7059 .active
7060 .values()
7061 .any(|a| a.node == node && a.attempt == attempt)
7062 {
7063 state
7064 .active
7065 .retain(|_, a| !(a.node == node && a.attempt == attempt));
7066 if let Err(e) = state.save() {
7067 tracing::warn!("could not persist the end of a wave: {e:#}");
7068 }
7069 }
7070 if let Some(cache_dir) = cache
7077 && jobs_had_a_writer
7078 {
7079 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
7080 }
7081 if let Some(guard) = cache_guard {
7082 guard.release();
7083 }
7084 collected.into_iter().flatten().collect()
7085}
7086
7087fn record_jobs(
7098 state: &mut RunState,
7099 node: &str,
7100 round: Option<usize>,
7101 seat: &str,
7102 out: &AgentOutcome,
7103) {
7104 let commands: &[agent::CommandEvidence] = match out {
7105 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
7106 AgentOutcome::Failed(_) => &[],
7107 };
7108 let checked_at = Timestamp::now();
7109 for c in commands {
7110 state.jobs.push(JobRecord {
7111 node: node.to_owned(),
7112 round,
7113 seat: seat.to_owned(),
7114 id: c.id.clone(),
7115 description: c.description.clone(),
7116 checked_at,
7117 status: match c.exit_code {
7118 Some(0) => JobStatus::Completed,
7119 Some(_) => JobStatus::Failed,
7120 None => JobStatus::Unknown,
7121 },
7122 exit_code: c.exit_code,
7123 result_summary: c.result_summary.clone(),
7124 source: c.source.clone(),
7125 });
7126 }
7127}
7128
7129fn round_is_clean(
7150 blocking: usize,
7151 e2e_ok: bool,
7152 answered: usize,
7153 expected: usize,
7154 quota_missing: usize,
7155 policy: IncompleteReviewPolicy,
7156) -> bool {
7157 if blocking != 0 || !e2e_ok {
7158 return false;
7159 }
7160 if answered == expected || policy == IncompleteReviewPolicy::Warn {
7161 return true;
7162 }
7163 answered > 0 && expected - answered <= quota_missing
7164}
7165
7166fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
7190 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
7191 return Some(RunStatus::Gating);
7192 }
7193 let last = reviews.last()?;
7194 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
7195 if reviews.len() < max_rounds && !stagnant {
7196 return None;
7197 }
7198 if last.incomplete() && last.blocking == 0 {
7199 return Some(RunStatus::Blocked);
7200 }
7201 if last.e2e_status() == E2eStatus::ResourceBlocked {
7202 return None;
7203 }
7204 Some(if last.e2e.iter().all(CommandOutcome::ok) {
7205 RunStatus::Gating
7206 } else {
7207 RunStatus::Blocked
7208 })
7209}
7210
7211pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
7226 if nudged {
7227 (full / 4).max(Duration::from_secs(120)).min(full)
7228 } else {
7229 full
7230 }
7231}
7232
7233#[allow(clippy::too_many_arguments)]
7262async fn ask_json_wave<T>(
7263 jobs: Vec<SeatJob>,
7264 sem: Arc<Semaphore>,
7265 retries: usize,
7266 roster: &[AgentSpec],
7267 ctx: &WaveCtx<'_>,
7268 losses: &mut Vec<QuotaLoss>,
7269 state: &mut RunState,
7270 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
7271) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7272where
7273 T: serde::de::DeserializeOwned + Send + 'static,
7274{
7275 ask_wave_with(
7276 jobs,
7277 sem,
7278 retries,
7279 roster,
7280 ctx,
7281 losses,
7282 state,
7283 &|text: &str| {
7284 let v = verdict::extract_json::<T>(text)?;
7285 validate(&v)?;
7286 Ok(v)
7287 },
7288 )
7289 .await
7290}
7291
7292#[allow(clippy::too_many_arguments)]
7301async fn ask_wave_with<T>(
7302 jobs: Vec<SeatJob>,
7303 sem: Arc<Semaphore>,
7304 retries: usize,
7305 roster: &[AgentSpec],
7306 ctx: &WaveCtx<'_>,
7307 losses: &mut Vec<QuotaLoss>,
7308 state: &mut RunState,
7309 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
7310) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
7311where
7312 T: Send + 'static,
7313{
7314 let n = jobs.len();
7315 let originals: Vec<SeatJob> = jobs;
7316 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
7317 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
7318 let mut nudges: Vec<usize> = vec![0; n];
7325 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
7329 let mut tried: Vec<BTreeSet<String>> = specs
7330 .iter()
7331 .map(|s| BTreeSet::from([s.id.clone()]))
7332 .collect();
7333 let starts: Vec<usize> = specs
7334 .iter()
7335 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
7336 .collect();
7337 let carry = ctx.carry_seats && !roster.is_empty();
7338 let carried: Vec<BTreeSet<String>> = originals
7341 .iter()
7342 .map(|j| {
7343 state
7344 .seat_history
7345 .get(&j.seat.key)
7346 .filter(|_| carry)
7347 .map(|h| h.failed.clone())
7348 .unwrap_or_default()
7349 })
7350 .collect();
7351 let mut prev: Vec<Option<FailClass>> = originals
7352 .iter()
7353 .map(|j| {
7354 state
7355 .seat_history
7356 .get(&j.seat.key)
7357 .filter(|_| carry)
7358 .and_then(|h| h.last_fail.clone())
7359 })
7360 .collect();
7361 let next_agent =
7362 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
7363 let others: BTreeSet<String> = specs
7364 .iter()
7365 .enumerate()
7366 .filter(|(j, _)| *j != i)
7367 .map(|(_, s)| s.id.clone())
7368 .collect();
7369 pick_successor(
7370 roster,
7371 starts[i],
7372 tried,
7373 carry.then(|| &carried[i]),
7374 &others,
7375 )
7376 .cloned()
7377 };
7378 let mut fresh: Vec<Option<String>> = vec![None; n];
7379 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
7380 let mut pending: Vec<usize> = (0..n).collect();
7381
7382 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
7387 for round in 0..max_rounds {
7388 if pending.is_empty() {
7389 break;
7390 }
7391 let mut batch = Vec::with_capacity(pending.len());
7392 let mut renudged: Vec<&str> = Vec::new();
7393 for &i in &pending {
7394 let src = &originals[i];
7395 let full: &str = match &src.handover {
7400 Some(h) if specs[i].id != src.spec.id => h,
7401 _ => &src.prompt,
7402 };
7403 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
7406 (
7407 full.to_owned(),
7408 src.timeout,
7409 format!("{}-{word}-{}", src.stem, specs[i].id),
7410 )
7411 } else if nudges[i] == 0 {
7412 (full.to_owned(), src.timeout, src.stem.clone())
7413 } else {
7414 renudged.push(src.seat.key.as_str());
7415 let why = done[i]
7416 .as_ref()
7417 .and_then(|r| r.as_ref().err().map(ToString::to_string))
7418 .unwrap_or_else(|| "no parsable answer".to_owned());
7419 let nudge = prompt::nudge(&why);
7420 let nudged = has_context(&specs[i], &seats[i], src.sessions);
7421 let prompt = if nudged {
7422 nudge
7423 } else {
7424 format!("{full}\n\n---\n\n{nudge}")
7425 };
7426 (
7427 prompt,
7428 retry_budget(src.timeout, nudged),
7429 format!("{}-retry{}", src.stem, nudges[i]),
7430 )
7431 };
7432 batch.push(SeatJob {
7433 spec: specs[i].clone(),
7434 seat: seats[i].clone(),
7435 cwd: src.cwd.clone(),
7436 prompt,
7437 timeout,
7438 allow_write: src.allow_write,
7439 sessions: src.sessions,
7440 artifacts: src.artifacts.clone(),
7441 stem,
7442 handover: None,
7443 });
7444 }
7445
7446 if !renudged.is_empty() {
7447 state.event(
7448 ctx.node,
7449 format!("retry {round}: re-asking {}", renudged.join(", ")),
7450 );
7451 }
7452 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
7453 let mut still = Vec::new();
7454 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
7455 seats[i] = seat;
7456 let class = FailClass::of(&out);
7457 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
7460 && nudges[i] < retries
7461 && !roster.is_empty();
7462 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
7463 let next = should_hand_over(prev[i].as_ref(), &cur)
7464 .then(|| next_agent(i, &tried[i], &specs))
7465 .flatten();
7466 if carry {
7467 let h = state
7468 .seat_history
7469 .entry(originals[i].seat.key.clone())
7470 .or_default();
7471 h.failed.insert(specs[i].id.clone());
7472 h.last_fail = Some(cur.clone());
7473 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
7474 h.last_ok = None;
7475 }
7476 if let Err(e) = state.save() {
7479 tracing::warn!("could not persist a seat's failure history: {e:#}");
7480 }
7481 }
7482 if let Some(next) = next {
7483 record_handover(
7484 state,
7485 ctx.node,
7486 &originals[i].seat.key,
7487 &specs[i].id,
7488 &next.id,
7489 &cur,
7490 &fail_reason(&out),
7491 );
7492 tried[i].insert(next.id.clone());
7493 prev[i] = Some(cur.clone());
7494 seats[i] =
7495 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
7496 specs[i] = next;
7497 fresh[i] = Some(cur.stem_word().to_owned());
7498 nudges[i] = 0;
7499 done[i] = Some(Err(anyhow::anyhow!(
7500 "handed over after: {}",
7501 fail_reason(&out)
7502 )));
7503 still.push(i);
7504 continue;
7505 }
7506 }
7507 if carry
7508 && !nudge_first
7509 && let Some(cur) = class.clone()
7510 {
7511 let h = state
7513 .seat_history
7514 .entry(originals[i].seat.key.clone())
7515 .or_default();
7516 h.failed.insert(specs[i].id.clone());
7517 h.last_fail = Some(cur);
7518 }
7519 let parsed = match out {
7520 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
7521 AgentOutcome::Quota(o) => {
7522 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
7523 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
7524 }
7525 AgentOutcome::Dropped(o) => {
7529 let why = o
7530 .dropped
7531 .as_ref()
7532 .map(|d| d.why.as_str())
7533 .unwrap_or("the CLI ended the stream without delivering its answer");
7534 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
7535 }
7536 AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
7537 };
7538 let quota = class == Some(FailClass::Quota);
7539 let failed = parsed.is_err();
7540 done[i] = Some(parsed);
7541 if carry && !failed {
7542 let h = state
7543 .seat_history
7544 .entry(originals[i].seat.key.clone())
7545 .or_default();
7546 h.failed.remove(&specs[i].id);
7547 h.last_ok = Some(specs[i].id.clone());
7548 h.last_fail = None;
7549 }
7550 let agent_failure = class.is_some()
7558 && !nudge_first
7559 && !roster.is_empty()
7560 && next_agent(i, &tried[i], &specs).is_some();
7561 if failed && !quota && !agent_failure && nudges[i] < retries {
7562 nudges[i] += 1;
7563 still.push(i);
7564 }
7565 }
7566 pending = still;
7567 }
7568
7569 for (i, q) in last_quota.into_iter().enumerate() {
7572 if let Some(reset) = q {
7573 losses.push(QuotaLoss {
7574 seat: originals[i].seat.key.clone(),
7575 node: ctx.node.to_owned(),
7576 at: Timestamp::now(),
7577 reset,
7578 });
7579 }
7580 }
7581
7582 seats
7583 .into_iter()
7584 .zip(done)
7585 .zip(nudges)
7586 .map(|((seat, res), attempts)| {
7587 (
7588 seat,
7589 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
7590 attempts,
7591 )
7592 })
7593 .collect()
7594}
7595
7596async fn acquire_cache_lease(
7609 state: &mut RunState,
7610 cache_dir: &Path,
7611 owner: &crate::cache::Owner,
7612 budget: Duration,
7613 context: &str,
7614) -> Result<crate::cache::Guard> {
7615 let home = crate::run::home();
7616 let started = Instant::now();
7617 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
7618 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
7619 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
7620 Err(e) => {
7621 state.event(
7622 "verify",
7623 format!("{context}: could not check the shared build cache: {e:#}"),
7624 );
7625 if let Err(e2) = state.save() {
7626 tracing::warn!("could not persist a cache-check failure: {e2:#}");
7627 }
7628 return Err(e);
7629 }
7630 };
7631 state.event(
7632 "verify",
7633 format!(
7634 "{context}: waiting for the shared build cache at {} ({})",
7635 cache_dir.display(),
7636 busy.describe()
7637 ),
7638 );
7639 if let Err(e) = state.save() {
7640 tracing::warn!("could not persist a cache wait: {e:#}");
7641 }
7642 let remaining = budget.saturating_sub(started.elapsed());
7643 match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
7644 Ok(g) => Ok(g),
7645 Err(e) => {
7646 state.event("verify", format!("{context}: {e:#}"));
7647 if let Err(e2) = state.save() {
7648 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
7649 }
7650 Err(e)
7651 }
7652 }
7653}
7654
7655#[allow(clippy::too_many_arguments)]
7676async fn with_cache_lease<'s, F, Fut>(
7677 state: &'s mut RunState,
7678 cache_dir: Option<&Path>,
7679 node: &str,
7680 seat: &str,
7681 worktree: &Path,
7682 head: &str,
7683 budget: Duration,
7684 context: &str,
7685 body: F,
7686) -> (Vec<CommandOutcome>, bool)
7687where
7688 F: FnOnce(&'s mut RunState, Duration) -> Fut,
7689 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
7690{
7691 let Some(cache_dir) = cache_dir else {
7692 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
7693 return (outcomes, retried);
7694 };
7695 let home = crate::run::home();
7696 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
7697 let started = Instant::now();
7698 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
7699 Ok(g) => g,
7700 Err(e) => {
7701 return (
7702 vec![CommandOutcome {
7703 command: "(waiting for the shared build cache)".to_owned(),
7704 code: None,
7705 output_tail: e.to_string(),
7706 duration_ms: started.elapsed().as_millis() as u64,
7707 resource_blocked: true,
7708 }],
7709 false,
7710 );
7711 }
7712 };
7713 let identity = crate::cache::Identity::new(worktree, head);
7714 if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7715 state.event(
7723 "verify",
7724 format!(
7725 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7726 worktree.display(),
7727 short(head)
7728 ),
7729 );
7730 guard.release();
7731 return (
7732 vec![CommandOutcome {
7733 command: "(confirming the shared build cache is fresh)".to_owned(),
7734 code: None,
7735 output_tail: e.to_string(),
7736 duration_ms: started.elapsed().as_millis() as u64,
7737 resource_blocked: true,
7738 }],
7739 false,
7740 );
7741 }
7742 let remaining = budget.saturating_sub(started.elapsed());
7743 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7744 if !timed_out_pids.is_empty() {
7749 wait_for_timed_out_children_to_die(&timed_out_pids).await;
7750 }
7751 guard.release();
7752 (outcomes, retried)
7753}
7754
7755async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7767 wait_for_pids_with(
7768 pids,
7769 crate::proc::pid_alive,
7770 LEASE_RELEASE_POLL,
7771 LEASE_RELEASE_MAX_WAIT,
7772 )
7773 .await;
7774}
7775
7776async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7782 pids: &[u32],
7783 alive: F,
7784 poll: Duration,
7785 max_wait: Duration,
7786) {
7787 let deadline = Instant::now() + max_wait;
7788 loop {
7789 if pids.iter().all(|&pid| !alive(pid)) {
7790 return;
7791 }
7792 if Instant::now() >= deadline {
7793 return;
7794 }
7795 tokio::time::sleep(poll).await;
7796 }
7797}
7798
7799fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7806 outcomes.iter().any(|o| o.resource_blocked)
7807}
7808
7809enum GateFix {
7811 Retry,
7813 Stop,
7816 Defer,
7819}
7820
7821fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7829 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7830 red.peek().is_some()
7831 && red.all(|o| {
7832 !o.resource_blocked
7833 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7834 && !o.output_tail.trim().is_empty()
7835 })
7836}
7837
7838fn e2e_outcome_label(o: &CommandOutcome) -> String {
7842 if o.ok() {
7843 return "pass".to_owned();
7844 }
7845 let reason = if o.build_failed() {
7846 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7847 } else {
7848 format!("FAIL ({:?})", o.code)
7849 };
7850 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7851}
7852
7853async fn run_e2e_with_retry(
7861 state: &mut RunState,
7862 shell: &[String],
7863 commands: &[String],
7864 worktree: &Path,
7865 timeout: Duration,
7866 context: &str,
7867) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7868 let (mut e2e, mut timed_out_pids) = run_commands(
7869 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7870 )
7871 .await;
7872 for o in &e2e {
7873 state.event(
7874 "verify",
7875 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7876 );
7877 }
7878 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7882 if verify_retried {
7883 state.event(
7884 "verify",
7885 format!(
7886 "{context}: verify could not build/link, not a test result — retrying once \
7887 before concluding"
7888 ),
7889 );
7890 let retried = run_commands(
7891 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7892 )
7893 .await;
7894 e2e = retried.0;
7895 timed_out_pids.extend(retried.1);
7898 for o in &e2e {
7899 state.event(
7900 "verify",
7901 format!(
7902 "{context}: retry `{}` -> {}",
7903 o.command,
7904 e2e_outcome_label(o)
7905 ),
7906 );
7907 }
7908 }
7909 (e2e, verify_retried, timed_out_pids)
7910}
7911
7912#[allow(clippy::too_many_arguments)]
7928async fn run_commands(
7929 state: &mut RunState,
7930 node: &str,
7931 task: &str,
7932 attempt: usize,
7933 shell: &[String],
7934 commands: &[String],
7935 cwd: &Path,
7936 timeout: Duration,
7937) -> (Vec<CommandOutcome>, Vec<u32>) {
7938 if commands.is_empty() {
7939 return (Vec::new(), Vec::new());
7944 }
7945 let mut out = Vec::new();
7946 let mut timed_out_pids = Vec::new();
7947 let total = commands.len();
7948 for (idx, command) in commands.iter().enumerate() {
7949 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7950 if let Err(e) = state.save() {
7951 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7952 }
7953 let started = Instant::now();
7954 let mut cmd = tokio::process::Command::new(&shell[0]);
7955 cmd.quiet();
7956 cmd.args(&shell[1..])
7957 .arg(command)
7958 .current_dir(cwd)
7959 .stdin(std::process::Stdio::null())
7960 .stdout(std::process::Stdio::piped())
7961 .stderr(std::process::Stdio::piped())
7962 .kill_on_drop(true);
7963 let spawned = cmd.spawn();
7964 let (code, body) = match spawned {
7965 Ok(child) => {
7966 let pid = child.id();
7971 match tokio::time::timeout(timeout, child.wait_with_output()).await {
7972 Ok(Ok(o)) => {
7973 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7974 body.push_str(&String::from_utf8_lossy(&o.stderr));
7975 (o.status.code(), body)
7976 }
7977 Ok(Err(e)) => (None, format!("failed to run: {e}")),
7978 Err(_) => {
7979 if let Some(pid) = pid {
7980 timed_out_pids.push(pid);
7981 }
7982 (None, format!("timed out after {}s", timeout.as_secs()))
7983 }
7984 }
7985 }
7986 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7987 };
7988 out.push(CommandOutcome {
7989 command: command.clone(),
7990 code,
7991 output_tail: tail(&body, OUTPUT_TAIL),
7992 duration_ms: started.elapsed().as_millis() as u64,
7993 resource_blocked: false,
7994 });
7995 }
7996 state.task_finished(task);
7997 if let Err(e) = state.save() {
7998 tracing::warn!("could not persist the end of {task}: {e:#}");
7999 }
8000 (out, timed_out_pids)
8001}
8002
8003fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
8015 let repo = repo.display();
8016 match style {
8017 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
8018 MergeStyle::Squash => {
8019 let subject = message
8022 .lines()
8023 .next()
8024 .unwrap_or(branch)
8025 .replace(['\\', '"', '$', '`'], "");
8026 format!(
8027 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
8028 )
8029 }
8030 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
8031 }
8032}
8033
8034const PR_TITLE_MAX: usize = 240;
8045
8046fn english_title(text: &str, max: usize) -> Option<String> {
8057 let line = queue::first_line(text)?;
8058 let chars: Vec<char> = line.chars().collect();
8059 let (kept, mark) = if chars.len() <= max {
8060 (line.to_owned(), "")
8061 } else if let Some(end) = sentence_end(&chars, max) {
8062 (chars[..end].iter().collect::<String>(), "")
8063 } else {
8064 let room = max.saturating_sub(3);
8065 let cut = if chars[room].is_whitespace() {
8068 room
8069 } else {
8070 chars[..room]
8071 .iter()
8072 .rposition(|c| c.is_whitespace())
8073 .unwrap_or(room)
8074 };
8075 let head: String = chars[..cut].iter().collect();
8076 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
8077 (head.to_owned(), "...")
8078 };
8079 if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
8080 return None;
8081 }
8082 Some(format!("{kept}{mark}"))
8083}
8084
8085fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
8089 const MIN: usize = 20;
8090 for i in MIN..max.min(chars.len()) {
8091 if !matches!(chars[i], '.' | '!' | '?') {
8092 continue;
8093 }
8094 let Some(&next) = chars.get(i + 1) else {
8095 continue;
8096 };
8097 if !next.is_whitespace() {
8098 continue;
8099 }
8100 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
8101 if after.is_some_and(|c| c.is_ascii_lowercase()) {
8102 continue;
8103 }
8104 let word: String = chars[..i]
8105 .iter()
8106 .rev()
8107 .take_while(|c| !c.is_whitespace())
8108 .collect::<Vec<_>>()
8109 .into_iter()
8110 .rev()
8111 .collect();
8112 let word = word.to_ascii_lowercase();
8113 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
8114 continue;
8115 }
8116 let end = chars[..i]
8117 .iter()
8118 .rposition(|c| !c.is_whitespace())
8119 .map_or(i, |p| p + 1);
8120 return Some(end);
8121 }
8122 None
8123}
8124
8125struct PrMessage {
8130 title: String,
8131 body: String,
8132}
8133
8134impl PrMessage {
8135 fn commit_message(&self) -> String {
8139 format!("{}\n\n{}", self.title, self.body)
8140 }
8141}
8142
8143fn title_marker(line: &str) -> Option<&str> {
8145 let line = line.trim();
8146 let head = line.get(..6)?;
8147 head.eq_ignore_ascii_case("title:")
8148 .then(|| line[6..].trim())
8149}
8150
8151fn summary_title(summary: &str) -> Option<String> {
8156 let first = summary.lines().find(|l| !l.trim().is_empty())?;
8157 let raw = title_marker(first)?;
8158 if raw.is_empty() {
8159 return None;
8160 }
8161 let title = queue::title_from(raw, PR_TITLE_MAX);
8162 let lower = title.to_ascii_lowercase();
8163 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
8164 return None;
8165 }
8166 Some(title)
8167}
8168
8169const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
8171
8172const EXISTING_BRANCH: &str = "(existing branch)";
8174
8175fn is_review_run(state: &RunState) -> bool {
8179 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
8180}
8181
8182fn review_title(state: &RunState) -> Option<String> {
8187 english_subject(state.reviewed_commits.as_ref()?.first()?)
8188}
8189
8190fn english_subject(raw: &str) -> Option<String> {
8193 let raw = raw.trim();
8194 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
8195 return None;
8196 }
8197 let title = queue::title_from(raw, PR_TITLE_MAX);
8198 let lower = title.to_ascii_lowercase();
8199 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
8200 return None;
8201 }
8202 Some(title)
8203}
8204
8205#[derive(Debug, Clone, PartialEq, Eq)]
8208struct BranchFacts {
8209 commits: Vec<(String, String)>,
8211 stat: String,
8213}
8214
8215const STAT_FILE_LINES: usize = 25;
8217const COMMITS_MAX_CHARS: usize = 20_000;
8219
8220async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
8223 let commits = git::commit_log(repo, base, branch).await.ok()?;
8224 if commits.is_empty() {
8225 return None;
8226 }
8227 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
8228 let lines: Vec<&str> = stat.lines().collect();
8229 let stat = if lines.len() > STAT_FILE_LINES + 1 {
8230 let omitted = lines.len() - 1 - STAT_FILE_LINES;
8231 let more = format!(" ... {omitted} more file(s)");
8232 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
8233 kept.push(&more);
8234 kept.push(lines[lines.len() - 1]);
8235 kept.join("\n")
8236 } else {
8237 lines.join("\n")
8238 };
8239 Some(BranchFacts { commits, stat })
8240}
8241
8242fn markdown_safe(text: &str) -> String {
8245 text.replace("</details>", "</details>")
8246 .replace("\x60\x60\x60", "~~~")
8247}
8248
8249fn neutral_title(state: &RunState, winner: char) -> String {
8250 format!(
8251 "chore: land candidate {} of run {}",
8252 winner.to_ascii_uppercase(),
8253 state.id
8254 )
8255}
8256
8257pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
8263 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
8264 ""
8265 } else {
8266 pr_title
8267 }
8268}
8269
8270pub fn landing_subject_source(state: &RunState) -> String {
8274 if is_review_run(state) {
8275 let winner = state.candidates.first().map_or('A', |c| c.label);
8276 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
8277 }
8278 state.instruction.clone()
8279}
8280
8281fn summary_without_title(summary: &str) -> String {
8284 let mut lines = summary.trim().lines().peekable();
8285 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
8286 lines.next();
8287 }
8288 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
8289}
8290
8291#[cfg(test)]
8305fn pr_message(state: &RunState, winner: char) -> PrMessage {
8306 pr_message_with(state, winner, None)
8307}
8308
8309#[cfg(test)]
8312fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8313 let raw = pr_message_raw(state, winner, facts);
8314 let id = crate::scrub::Identity::current();
8315 PrMessage {
8316 title: crate::scrub::scrub(&raw.title, &id),
8317 body: crate::scrub::scrub(&raw.body, &id),
8318 }
8319}
8320
8321fn pr_message_raw(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
8322 let summary = state
8323 .candidates
8324 .iter()
8325 .find(|c| c.label == winner)
8326 .map(|c| c.summary.as_str())
8327 .unwrap_or_default();
8328 let review = is_review_run(state);
8331 let title = if review {
8332 facts
8333 .and_then(|f| english_subject(&f.commits.first()?.0))
8334 .or_else(|| review_title(state))
8335 .or_else(|| {
8336 state
8337 .candidates
8338 .iter()
8339 .find(|c| c.label == winner)
8340 .filter(|c| !c.branch.starts_with("magi/"))
8341 .and_then(|c| english_subject(&c.branch))
8342 })
8343 .unwrap_or_else(|| neutral_title(state, winner))
8344 } else {
8345 summary_title(summary).unwrap_or_else(|| {
8346 english_title(&state.instruction, PR_TITLE_MAX)
8347 .unwrap_or_else(|| neutral_title(state, winner))
8348 })
8349 };
8350
8351 let mut body = String::new();
8352 let what = summary_without_title(summary);
8353 if !what.is_empty() {
8354 body.push_str("## Summary\n\n");
8355 body.push_str(&what);
8356 body.push_str("\n\n");
8357 }
8358
8359 let notes: Vec<(usize, &str)> = state
8362 .reviews
8363 .iter()
8364 .filter_map(|r| {
8365 let n = r.fix.as_ref()?.notes.trim();
8366 (!n.is_empty()).then_some((r.round, n))
8367 })
8368 .collect();
8369 if !notes.is_empty() {
8370 body.push_str("## Review fixes\n\n");
8371 if let [(_, only)] = notes.as_slice() {
8372 body.push_str(only);
8373 body.push_str("\n\n");
8374 } else {
8375 for (round, n) in ¬es {
8376 body.push_str(&format!("### Round {round}\n\n{n}\n\n"));
8377 }
8378 }
8379 }
8380 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
8381
8382 let open = state.open_findings();
8383 if !open.is_empty() {
8384 body.push_str("## Open review findings\n\n");
8385 for f in &open {
8386 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
8387 }
8388 body.push('\n');
8389 }
8390
8391 if let Some(fix) = fix
8392 && !fix.rejected.is_empty()
8393 {
8394 body.push_str("## Declined by the fixer\n\n");
8395 for r in &fix.rejected {
8396 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
8397 }
8398 body.push('\n');
8399 }
8400
8401 if review {
8402 body.push_str("## Commits under review\n\n");
8404 if let Some(facts) = facts {
8405 let mut left = COMMITS_MAX_CHARS;
8406 for (i, (subject, text)) in facts.commits.iter().enumerate() {
8407 let mut entry = format!("- {}\n", markdown_safe(subject));
8408 for l in markdown_safe(text).lines() {
8409 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
8410 }
8411 if left == 0 {
8412 body.push_str(&format!(
8413 "- ... {} more commit(s)\n",
8414 facts.commits.len() - i
8415 ));
8416 break;
8417 }
8418 if entry.len() > left {
8419 let mut end = left;
8422 while !entry.is_char_boundary(end) {
8423 end -= 1;
8424 }
8425 entry.truncate(end);
8426 entry.push_str("\n ... (truncated)\n");
8427 left = 0;
8428 } else {
8429 left -= entry.len();
8430 }
8431 body.push_str(&entry);
8432 }
8433 if !facts.stat.trim().is_empty() {
8434 body.push_str(&format!(
8435 "\n## Diff stat\n\n```\n{}\n```\n",
8436 markdown_safe(facts.stat.trim())
8437 ));
8438 }
8439 } else {
8440 match &state.reviewed_commits {
8441 Some(subjects) => {
8442 for s in subjects {
8443 body.push_str(&format!("- {}\n", s.trim()));
8444 }
8445 }
8446 None => {
8447 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
8450 body.push_str(rest.trim());
8451 body.push('\n');
8452 }
8453 }
8454 }
8455 } else {
8456 let task = state.instruction.trim();
8457 let task = if task.is_empty() {
8458 "(empty task)"
8459 } else {
8460 task
8461 };
8462 body.push_str(&format!(
8463 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
8464 task.replace("</details>", "</details>")
8465 ));
8466 }
8467
8468 body.push_str(&format!(
8469 "\n---\nmagi:run/{} magi:candidate-{}\n",
8470 state.id,
8471 winner.to_ascii_lowercase()
8472 ));
8473
8474 PrMessage { title, body }
8475}
8476
8477fn seeded_instruction(state: &RunState) -> String {
8481 match refs::describe(&state.seeds) {
8482 Some(facts) => format!(
8483 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
8484 Candidates start from the unmerged branch named above, when there \
8485 is one, and carry any unmerged commit named by sha as a \
8486 cherry-pick. Check that this is what the task meant before \
8487 building on it.",
8488 state.instruction
8489 ),
8490 None => state.instruction.clone(),
8491 }
8492}
8493
8494async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
8498 let base = &state.base_branch;
8499 let mut against = base.clone();
8500 if mode == MergeMode::Pr {
8501 let remote = &state.config.merge.remote;
8505 let tracking = format!("{remote}/{base}");
8506 let fetched = git::fetch(repo, remote, base).await;
8507 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
8508 against = tracking;
8509 } else {
8510 return false;
8511 }
8512 }
8513 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
8514}
8515
8516fn empty_candidate_detail(state: &RunState, base: &str) -> String {
8519 let mut detail = format!(
8520 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
8521 nothing to open a pull request for"
8522 );
8523 match refs::describe(&state.seeds) {
8524 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
8525 None => detail.push_str(
8526 "\nThe task names no existing branch or commit; if it means to land work \
8527 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
8528 ),
8529 }
8530 detail
8531}
8532
8533#[derive(Debug, PartialEq, Eq)]
8536enum PrPlan {
8537 Create,
8538 Adopt { url: String, title: String },
8539 Stop(String),
8540}
8541
8542fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
8545 match found {
8546 Ok(land::OpenPr::None) => PrPlan::Create,
8547 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
8548 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
8549 "several open pull requests exist for this branch, not picking one: {}",
8550 urls.join(" ")
8551 )),
8552 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
8553 }
8554}
8555
8556async fn gh_pr_create(
8558 cwd: &Path,
8559 base: &str,
8560 head: &str,
8561 title: &str,
8562 body: &str,
8563) -> Result<String> {
8564 let out = tokio::process::Command::new("gh")
8565 .args([
8566 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
8567 ])
8568 .current_dir(cwd)
8569 .quiet()
8570 .stdin(std::process::Stdio::null())
8571 .output()
8572 .await
8573 .context("spawn gh")?;
8574 if out.status.success() {
8575 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
8576 } else {
8577 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
8578 }
8579}
8580
8581pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
8590 let repo = state.repo.clone();
8591 let root = state.worktree_root();
8592 let winner = state.tally.as_ref().map(|t| t.winner);
8593 let mut removed = Vec::new();
8594
8595 for i in 0..state.candidates.len() {
8596 let c = state.candidates[i].clone();
8597 let is_winner = Some(c.label) == winner;
8598 if is_winner && !drop_winner {
8599 continue;
8600 }
8601 if c.worktree.exists() {
8602 git::worktree_remove(&repo, &c.worktree).await.ok();
8603 removed.push(c.worktree.to_string_lossy().into_owned());
8604 }
8605 let handed_over = state.released_branches.contains(&c.branch);
8608 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
8609 git::branch_delete(&repo, &c.branch).await.ok();
8610 removed.push(c.branch.clone());
8611 }
8612 state.candidates[i].folded = true;
8613 }
8614
8615 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
8616 let path = name.path();
8617 let keep = !drop_winner
8618 && winner.is_some_and(|w| {
8619 path.file_name()
8620 .is_some_and(|n| n == format!("cand-{w}").as_str())
8621 });
8622 if keep {
8623 continue;
8624 }
8625 git::worktree_remove(&repo, &path).await.ok();
8626 removed.push(path.to_string_lossy().into_owned());
8627 }
8628
8629 remove_if_empty(&root);
8638
8639 if state.enabled_worktree_config && drop_winner {
8640 git::release_worktree_config(&repo).await.ok();
8644 state.enabled_worktree_config = false;
8645 }
8646 state.save_under(home)?;
8647 Ok(removed)
8648}
8649
8650fn remove_if_empty(dir: &Path) {
8661 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
8662 std::fs::remove_dir(dir).ok();
8663 }
8664}
8665
8666pub fn worst_open(state: &RunState) -> Option<Severity> {
8668 state
8669 .reviews
8670 .last()?
8671 .reviews
8672 .iter()
8673 .flat_map(|r| r.findings.iter())
8674 .map(|f| f.severity)
8675 .max()
8676}
8677
8678#[cfg(test)]
8679mod tests {
8680 #[test]
8681 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
8682 let leaked = vec!["chore(deps): update a crate".to_owned()];
8683 let own = "fix(daemon): apply a chosen action";
8684 assert!(should_retitle("", own, &leaked));
8685 assert!(should_retitle(
8686 &format!("{REVIEW_PROMPT_OPENING} `x`"),
8687 own,
8688 &leaked
8689 ));
8690 assert!(should_retitle(
8691 "chore: land candidate A of run 1",
8692 own,
8693 &leaked
8694 ));
8695 assert!(should_retitle(
8696 "magi: candidate A (uncommitted work)",
8697 own,
8698 &leaked
8699 ));
8700 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
8701 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
8702 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
8703 }
8704
8705 #[test]
8706 fn pr_merge_plan_creates_adopts_or_stops() {
8707 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
8708 assert_eq!(
8709 pr_merge_plan(Ok(land::OpenPr::One {
8710 url: "u".into(),
8711 title: "t".into()
8712 })),
8713 PrPlan::Adopt {
8714 url: "u".into(),
8715 title: "t".into()
8716 }
8717 );
8718 let PrPlan::Stop(many) =
8719 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
8720 else {
8721 panic!("many must stop");
8722 };
8723 assert!(many.contains('a') && many.contains('b'));
8724 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
8725 panic!("a failed lookup must stop");
8726 };
8727 assert!(err.contains("bad token"));
8728 }
8729
8730 use super::*;
8731 use crate::run::GateStatus;
8732 use std::collections::BTreeMap;
8733 use std::time::Duration;
8734
8735 fn conductor() -> AgentSpec {
8736 AgentSpec {
8737 id: "conductor".to_owned(),
8738 kind: crate::config::AgentKind::Command,
8739 model: None,
8740 command: vec!["true".to_owned()],
8741 extra_args: Vec::new(),
8742 env: BTreeMap::new(),
8743 prompt_delivery: None,
8744 }
8745 }
8746
8747 fn spec(id: &str) -> AgentSpec {
8748 AgentSpec {
8749 id: id.to_owned(),
8750 kind: crate::config::AgentKind::Command,
8751 model: None,
8752 command: vec!["true".to_owned()],
8753 extra_args: Vec::new(),
8754 env: BTreeMap::new(),
8755 prompt_delivery: None,
8756 }
8757 }
8758
8759 fn ids(xs: &[&str]) -> BTreeSet<String> {
8760 xs.iter().map(|s| (*s).to_owned()).collect()
8761 }
8762
8763 #[test]
8764 fn pick_successor_skips_an_agent_another_seat_holds() {
8765 let roster = [spec("a"), spec("b"), spec("c")];
8767 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
8768 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8769 }
8770
8771 #[test]
8772 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
8773 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
8775 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
8776 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8777 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
8778 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
8779 }
8780
8781 #[test]
8782 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
8783 let roster = [spec("a"), spec("b")];
8784 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
8785 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8786 let carried = ids(&["b"]);
8787 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
8788 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8789 }
8790
8791 #[test]
8792 fn pick_successor_returns_none_without_any_untried_successor() {
8793 let roster = [spec("a"), spec("b")];
8794 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
8795 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
8796 let carried = ids(&["a"]);
8797 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
8798 }
8799
8800 #[test]
8801 fn pick_successor_rescue_avoids_another_seats_occupant() {
8802 let roster = [spec("a"), spec("b"), spec("c")];
8804 let carried = ids(&["b", "c"]);
8805 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
8806 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8807 }
8808
8809 #[test]
8810 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
8811 let roster = [spec("a"), spec("b"), spec("c")];
8812 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
8813 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8814 }
8815
8816 #[test]
8817 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
8818 let roster = [spec("a"), spec("b"), spec("c")];
8819 let failed = ids(&["a", "b", "c"]);
8820 let mut tried = ids(&["b"]);
8822 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8823 assert_eq!(first.id, "a");
8824 tried.insert(first.id.clone());
8825 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8826 assert_eq!(second.id, "c");
8827 tried.insert(second.id.clone());
8828 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
8829 }
8830
8831 #[test]
8832 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
8833 let roster = [spec("a"), spec("b")];
8834 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
8835 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8836 }
8837
8838 #[test]
8839 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
8840 let roster = [spec("a"), spec("b"), spec("c")];
8841 let h = SeatHistory {
8842 failed: ids(&["a"]),
8843 last_ok: Some("b".to_owned()),
8844 last_fail: None,
8845 };
8846 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
8847 let gone = SeatHistory {
8849 last_ok: Some("zzz".to_owned()),
8850 ..h.clone()
8851 };
8852 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
8853 let failed = SeatHistory {
8854 failed: ids(&["a", "b"]),
8855 last_ok: Some("b".to_owned()),
8856 last_fail: None,
8857 };
8858 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
8859 }
8860
8861 #[test]
8862 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
8863 let first = SeatState::new("review-1", "alpha", 7);
8864 let next = handover_seat("review-1", "gamma", 7);
8865 assert_ne!(first.claude_session, next.claude_session);
8866 }
8867
8868 #[test]
8869 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
8870 let mut state = state_with_summary("x", "y");
8871 let a = handover_seat("review-1", "beta", state.next_seat_seed());
8872 let b = handover_seat("review-1", "beta", state.next_seat_seed());
8873 assert_ne!(a.claude_session, b.claude_session);
8874 }
8875
8876 #[test]
8877 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
8878 let roster = [spec("a"), spec("b")];
8879 let h = SeatHistory {
8880 failed: ids(&["a", "b"]),
8881 ..SeatHistory::default()
8882 };
8883 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
8884 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
8885 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
8886 }
8887
8888 #[test]
8895 fn failure_signature_ignores_numbers_and_paths() {
8896 assert_eq!(
8897 failure_signature("exited with Some(2) and no usable output"),
8898 failure_signature("exited with Some(137) and no usable output")
8899 );
8900 assert_eq!(
8901 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
8902 failure_signature("cannot open /var/x.txt: denied")
8903 );
8904 assert_ne!(failure_signature("boom"), failure_signature("bang"));
8905 }
8906
8907 #[test]
8908 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
8909 let other = FailClass::Other("x".into());
8910 assert!(should_hand_over(None, &FailClass::Quota));
8911 assert!(should_hand_over(Some(&other), &FailClass::Quota));
8912 assert!(should_hand_over(
8913 Some(&FailClass::Timeout),
8914 &FailClass::Timeout
8915 ));
8916 assert!(should_hand_over(None, &other));
8917 assert!(!should_hand_over(Some(&other), &other));
8918 assert!(should_hand_over(
8919 Some(&other),
8920 &FailClass::Other("y".into())
8921 ));
8922 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
8924 assert!(should_hand_over(Some(&FailClass::Quota), &other));
8925 }
8926
8927 #[test]
8928 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
8929 let a = SeatState::new("judge-1", "alpha", 7);
8930 let b = handover_seat("judge-1", "beta", 7);
8931 assert_ne!(a.claude_session, b.claude_session);
8932 assert_eq!(b.turns, 0);
8933 }
8934
8935 #[test]
8936 fn a_timeout_is_classified_apart_from_other_failures() {
8937 assert_eq!(
8938 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
8939 Some(FailClass::Timeout)
8940 );
8941 assert!(matches!(
8942 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
8943 Some(FailClass::Other(_))
8944 ));
8945 }
8946
8947 #[test]
8948 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
8949 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8950 let tried = BTreeSet::from(["beta".to_owned()]);
8951 let next = next_untried_in_roster(&roster, 1, &tried);
8954 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
8955 }
8956
8957 #[test]
8958 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
8959 let roster = vec![spec("alpha"), spec("beta")];
8960 let tried = BTreeSet::from(["beta".to_owned()]);
8961 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8965 }
8966
8967 #[test]
8968 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
8969 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8970 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
8971 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8975 }
8976
8977 #[test]
8978 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
8979 let roster = vec![spec("a"), spec("a"), spec("b")];
8980 let tried = BTreeSet::from(["a".to_owned()]);
8981 let next = next_untried_in_roster(&roster, 0, &tried);
8982 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8983 }
8984
8985 #[test]
8986 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
8987 let roster = vec![spec("a"), spec("b")];
8988 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
8989 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
8990 }
8991
8992 #[test]
8993 fn remove_if_empty_only_ever_takes_a_bare_directory() {
8994 let dir = tempfile::tempdir().unwrap();
8995 let bay = dir.path().join("ffff");
8996
8997 remove_if_empty(&bay);
8999 assert!(!bay.exists());
9000
9001 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
9004 remove_if_empty(&bay);
9005 assert!(bay.exists(), "non-empty directory must survive");
9006
9007 std::fs::remove_dir(bay.join("cand-A")).unwrap();
9009 remove_if_empty(&bay);
9010 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
9011 }
9012
9013 #[test]
9022 fn a_full_panel_that_found_nothing_is_clean() {
9023 assert!(round_is_clean(
9024 0,
9025 true,
9026 2,
9027 2,
9028 0,
9029 IncompleteReviewPolicy::Block
9030 ));
9031 }
9032
9033 #[test]
9034 fn a_missing_seat_is_never_clean_under_the_default_policy() {
9035 assert!(!round_is_clean(
9036 0,
9037 true,
9038 1,
9039 2,
9040 0,
9041 IncompleteReviewPolicy::Block
9042 ));
9043 }
9044
9045 #[test]
9046 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
9047 assert!(!round_is_clean(
9048 1,
9049 true,
9050 1,
9051 2,
9052 0,
9053 IncompleteReviewPolicy::Warn
9054 ));
9055 }
9056
9057 #[test]
9058 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
9059 assert!(round_is_clean(
9060 0,
9061 true,
9062 1,
9063 2,
9064 0,
9065 IncompleteReviewPolicy::Warn
9066 ));
9067 }
9068
9069 #[test]
9070 fn a_full_panel_with_an_open_finding_is_not_clean() {
9071 assert!(!round_is_clean(
9072 1,
9073 true,
9074 2,
9075 2,
9076 0,
9077 IncompleteReviewPolicy::Block
9078 ));
9079 }
9080
9081 #[test]
9082 fn a_full_panel_with_a_red_e2e_is_not_clean() {
9083 assert!(!round_is_clean(
9084 0,
9085 false,
9086 2,
9087 2,
9088 0,
9089 IncompleteReviewPolicy::Block
9090 ));
9091 }
9092
9093 #[test]
9100 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
9101 assert!(round_is_clean(
9104 0,
9105 true,
9106 1,
9107 2,
9108 1,
9109 IncompleteReviewPolicy::Block
9110 ));
9111 }
9112
9113 #[test]
9114 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
9115 assert!(!round_is_clean(
9118 0,
9119 true,
9120 1,
9121 2,
9122 0,
9123 IncompleteReviewPolicy::Block
9124 ));
9125 }
9126
9127 #[test]
9128 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
9129 assert!(!round_is_clean(
9130 1,
9131 true,
9132 1,
9133 2,
9134 1,
9135 IncompleteReviewPolicy::Block
9136 ));
9137 assert!(!round_is_clean(
9138 0,
9139 false,
9140 1,
9141 2,
9142 1,
9143 IncompleteReviewPolicy::Block
9144 ));
9145 }
9146
9147 #[test]
9148 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
9149 assert!(!round_is_clean(
9153 0,
9154 true,
9155 0,
9156 2,
9157 2,
9158 IncompleteReviewPolicy::Block
9159 ));
9160 }
9161
9162 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
9163 CommandOutcome {
9164 command: "test".to_owned(),
9165 code,
9166 output_tail: String::new(),
9167 duration_ms: 0,
9168 resource_blocked,
9169 }
9170 }
9171
9172 #[test]
9173 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
9174 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
9175 assert!(
9176 !verify_inconclusive(&[outcome(Some(1), false)]),
9177 "an ordinary failure is still evidence about the patch"
9178 );
9179 assert!(verify_inconclusive(&[outcome(None, true)]));
9180 assert!(
9181 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
9182 "one inconclusive outcome taints the whole batch"
9183 );
9184 assert!(!verify_inconclusive(&[]));
9185 }
9186
9187 #[tokio::test]
9188 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
9189 let calls = std::sync::atomic::AtomicUsize::new(0);
9193 let started = Instant::now();
9194 wait_for_pids_with(
9195 &[123],
9196 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
9197 Duration::from_millis(5),
9198 Duration::from_secs(5),
9199 )
9200 .await;
9201 assert!(
9202 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
9203 "must keep checking rather than deciding on the first answer"
9204 );
9205 assert!(
9206 started.elapsed() < Duration::from_secs(1),
9207 "must return the moment it is confirmed dead, not wait out the ceiling"
9208 );
9209 }
9210
9211 #[tokio::test]
9212 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
9213 let started = Instant::now();
9214 wait_for_pids_with(
9215 &[123],
9216 |_| true, Duration::from_millis(5),
9218 Duration::from_millis(30),
9219 )
9220 .await;
9221 let elapsed = started.elapsed();
9222 assert!(
9223 elapsed >= Duration::from_millis(30),
9224 "must not give up before its own ceiling: {elapsed:?}"
9225 );
9226 assert!(
9227 elapsed < Duration::from_secs(1),
9228 "must not wait past its own ceiling either: {elapsed:?}"
9229 );
9230 }
9231
9232 #[tokio::test]
9233 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
9234 let started = Instant::now();
9235 wait_for_pids_with(
9236 &[],
9237 |_| true,
9238 Duration::from_secs(5),
9239 Duration::from_secs(5),
9240 )
9241 .await;
9242 assert!(
9243 started.elapsed() < Duration::from_millis(200),
9244 "an empty pid list has nothing to confirm"
9245 );
9246 }
9247
9248 fn review_round(
9254 clean: bool,
9255 blocking: usize,
9256 answered: usize,
9257 expected: usize,
9258 progressed: bool,
9259 e2e_ok: bool,
9260 ) -> ReviewRound {
9261 ReviewRound {
9262 round: 1,
9263 head: "h".to_owned(),
9264 verified_head: None,
9265 verified_at: None,
9266 reviews: Vec::new(),
9267 e2e: vec![CommandOutcome {
9268 command: "test".to_owned(),
9269 code: Some(if e2e_ok { 0 } else { 1 }),
9270 output_tail: String::new(),
9271 duration_ms: 0,
9272 resource_blocked: false,
9273 }],
9274 verify_retried: false,
9275 e2e_deferred: false,
9276 e2e_defer_reason: None,
9277 fix: None,
9278 blocking,
9279 answered,
9280 expected,
9281 clean,
9282 progressed,
9283 vote_split: false,
9284 reconsideration: Vec::new(),
9285 verdict: None,
9286 }
9287 }
9288
9289 #[test]
9290 fn review_conclusion_is_none_when_nothing_has_run() {
9291 assert_eq!(review_conclusion(&[], 3), None);
9292 }
9293
9294 #[test]
9295 fn review_conclusion_is_none_while_rounds_remain() {
9296 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
9297 assert_eq!(review_conclusion(&rounds, 3), None);
9298 }
9299
9300 #[test]
9301 fn review_conclusion_is_gating_once_a_round_is_clean() {
9302 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
9303 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
9304 }
9305
9306 #[test]
9307 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
9308 let rounds = vec![
9309 review_round(false, 1, 2, 2, true, true),
9310 review_round(false, 1, 2, 2, true, true),
9311 ];
9312 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
9313 }
9314
9315 #[test]
9316 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
9317 let rounds = vec![
9318 review_round(false, 1, 2, 2, true, true),
9319 review_round(false, 1, 2, 2, true, false),
9320 ];
9321 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
9322 }
9323
9324 #[test]
9325 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
9326 let mut blocked = review_round(false, 1, 2, 2, true, false);
9333 blocked.e2e[0].resource_blocked = true;
9334 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
9335 assert_eq!(review_conclusion(&rounds, 2), None);
9336 }
9337
9338 #[test]
9339 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
9340 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
9342 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
9343 }
9344
9345 #[test]
9346 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
9347 let rounds = vec![
9348 review_round(false, 1, 2, 2, false, true),
9349 review_round(false, 1, 2, 2, false, true),
9350 ];
9351 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
9352 }
9353
9354 fn secs(n: u64) -> Duration {
9355 Duration::from_secs(n)
9356 }
9357
9358 fn init_repo(dir: &Path) {
9361 let run = |args: &[&str]| {
9362 let out = std::process::Command::new("git")
9363 .args(args)
9364 .current_dir(dir)
9365 .quiet()
9366 .output()
9367 .expect("spawn git");
9368 assert!(
9369 out.status.success(),
9370 "git {args:?} failed: {}",
9371 String::from_utf8_lossy(&out.stderr)
9372 );
9373 };
9374 run(&["init", "-b", "main"]);
9375 run(&["config", "user.name", "magi test"]);
9376 run(&["config", "user.email", "magi@example.com"]);
9377 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
9378 run(&["add", "-A"]);
9379 run(&["commit", "-m", "init"]);
9380 }
9381
9382 fn ask_test_home() {
9390 crate::run::pin_test_home();
9391 }
9392
9393 fn runner_at(status: RunStatus) -> Runner {
9396 let mut state = RunState::new(
9397 PathBuf::from("/nonexistent/repo"),
9398 "main".to_owned(),
9399 "deadbeef".to_owned(),
9400 "task".to_owned(),
9401 Config::default(),
9402 );
9403 state.status = status;
9404 Runner {
9405 state,
9406 roles: ResolvedRoles {
9407 implementers: Vec::new(),
9408 judges: Vec::new(),
9409 reviewers: Vec::new(),
9410 fixer: None,
9411 conductor: conductor(),
9412 implementer_roster: Vec::new(),
9413 judge_roster: Vec::new(),
9414 reviewer_roster: Vec::new(),
9415 },
9416 sem: Arc::new(Semaphore::new(1)),
9417 pause: Pause::new(),
9418 interrupt: Pause::new(),
9419 }
9420 }
9421
9422 #[test]
9426 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
9427 crate::run::pin_test_home();
9428 let mut runner = runner_at(RunStatus::Implementing);
9429 let interrupt = Pause::new();
9430 runner.watch_interrupt(interrupt.clone());
9431
9432 interrupt.park_because("task a1b2 asked to run first");
9433
9434 assert!(runner.park_here().expect("park_here"));
9435 assert!(runner.state.parked);
9436 let last = runner.state.events.last().expect("a park event");
9437 assert_eq!(last.node, "park");
9438 assert!(
9439 last.message.contains("task a1b2 asked to run first"),
9440 "expected the interrupt reason in {:?}",
9441 last.message
9442 );
9443 }
9444
9445 #[test]
9453 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
9454 crate::run::pin_test_home();
9455 let mut runner = runner_at(RunStatus::Implementing);
9456 let shutdown = Pause::new();
9457 runner.on_pause(shutdown.clone());
9458 let interrupt = Pause::new();
9459 runner.watch_interrupt(interrupt.clone());
9460
9461 assert!(!runner.park_here().expect("park_here"));
9463 assert!(!runner.state.parked);
9464
9465 interrupt.park_because("test");
9467 assert!(!shutdown.parked());
9468 assert!(runner.park_here().expect("park_here"));
9469 }
9470
9471 #[tokio::test]
9485 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
9486 crate::run::pin_test_home();
9487 let mut runner = runner_at(RunStatus::Implementing);
9488 let interrupt = Pause::new();
9489 runner.watch_interrupt(interrupt.clone());
9490
9491 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
9492 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
9493
9494 let node = async move {
9498 started_tx.send(()).expect("send started");
9499 finish_rx.await.expect("recv finish");
9500 "node finished"
9501 };
9502
9503 let interrupter = async move {
9504 started_rx.await.expect("recv started");
9505 interrupt.park_because("higher-priority task waiting");
9507 tokio::task::yield_now().await;
9511 finish_tx.send(()).expect("send finish");
9512 };
9513
9514 let (node_result, ()) = tokio::join!(node, interrupter);
9515 assert_eq!(
9516 node_result, "node finished",
9517 "the in-flight call ran to completion"
9518 );
9519
9520 assert!(runner.park_here().expect("park_here"));
9523 assert!(runner.state.parked);
9524 }
9525
9526 #[test]
9532 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
9533 crate::run::pin_test_home();
9534 let mut runner = runner_at(RunStatus::Judging);
9535 runner.state.config.agents = vec![conductor()];
9539 runner.state.candidates = vec![Candidate {
9540 index: 0,
9541 label: 'A',
9542 agent: "alpha".to_owned(),
9543 branch: "magi/x/A".to_owned(),
9544 worktree: PathBuf::from("/nonexistent/worktree"),
9545 summary: "did the thing".to_owned(),
9546 stat: "1 file changed".to_owned(),
9547 files: 1,
9548 commits: 1,
9549 empty: false,
9550 failed: None,
9551 verified_noop: None,
9552 duration_ms: 1234,
9553 folded: false,
9554 }];
9555 let run_id = runner.state.id.clone();
9556
9557 let interrupt = Pause::new();
9558 runner.watch_interrupt(interrupt.clone());
9559 interrupt.park_because("task c3d4 asked to run first");
9560 assert!(runner.park_here().expect("park_here"));
9561
9562 let resumed = Runner::resume(&run_id).expect("resume");
9563 assert_eq!(resumed.state.candidates.len(), 1);
9564 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
9565 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
9566 assert_eq!(resumed.state.status, runner.state.status);
9567 assert!(
9568 resumed.state.parked,
9569 "still parked until `execute` actually walks the graph again"
9570 );
9571 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
9572 }
9573
9574 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
9576 let mut q = ask::Question::new(
9577 run.to_owned(),
9578 "implement".to_owned(),
9579 "impl-A".to_owned(),
9580 "Which storage backend should the cache use?".to_owned(),
9581 String::new(),
9582 vec!["SQLite".to_owned(), "Redis".to_owned()],
9583 );
9584 store.put(&mut q).unwrap();
9585 q
9586 }
9587
9588 #[test]
9589 fn a_failed_runs_open_question_is_abandoned() {
9590 ask_test_home();
9591 let store = ask::Questions::open();
9592 let mut runner = runner_at(RunStatus::Failed);
9593 let run = runner.state.id.clone();
9594 let q = ask_open_question(&store, &run);
9595
9596 runner.settle_questions();
9597
9598 let back = store.get(&q.id).unwrap();
9599 assert!(
9600 !back.status.open(),
9601 "the seat that asked died with the run; nobody is left to read an answer"
9602 );
9603 assert!(
9604 back.detail.contains(&run) && back.detail.contains("failed"),
9605 "the reason names what the run became, not just that it is gone: {}",
9606 back.detail
9607 );
9608 }
9609
9610 #[test]
9611 fn a_merged_runs_open_question_is_abandoned_too() {
9612 ask_test_home();
9613 let store = ask::Questions::open();
9614 for status in [RunStatus::Merged, RunStatus::Ready] {
9617 let mut runner = runner_at(status);
9618 let run = runner.state.id.clone();
9619 let q = ask_open_question(&store, &run);
9620
9621 runner.settle_questions();
9622
9623 let back = store.get(&q.id).unwrap();
9624 assert!(
9625 !back.status.open(),
9626 "{status:?} run's question must not outlive the run"
9627 );
9628 }
9629 }
9630
9631 #[test]
9632 fn a_still_resumable_runs_open_question_is_left_alone() {
9633 ask_test_home();
9634 let store = ask::Questions::open();
9635 for status in [RunStatus::Blocked, RunStatus::Stalled] {
9641 let mut runner = runner_at(status);
9642 let run = runner.state.id.clone();
9643 let q = ask_open_question(&store, &run);
9644
9645 runner.settle_questions();
9646
9647 let back = store.get(&q.id).unwrap();
9648 assert!(
9649 back.status.open(),
9650 "{status:?} is still alive; the question must still be waiting"
9651 );
9652 }
9653 }
9654
9655 #[test]
9656 fn settle_questions_never_touches_an_already_answered_question() {
9657 ask_test_home();
9658 let store = ask::Questions::open();
9659 let mut runner = runner_at(RunStatus::Failed);
9660 let run = runner.state.id.clone();
9661 let mut q = ask_open_question(&store, &run);
9662 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
9663 .unwrap();
9664 store.put(&mut q).unwrap();
9665
9666 runner.settle_questions();
9671 runner.settle_questions();
9672
9673 let back = store.get(&q.id).unwrap();
9674 assert_eq!(
9675 back.status,
9676 ask::QuestionStatus::Answered,
9677 "a real answer is a decision on record, never overwritten by a sweep"
9678 );
9679 }
9680
9681 #[tokio::test]
9692 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
9693 crate::run::pin_test_home();
9694 let tmp = tempfile::tempdir().expect("tempdir");
9695 let repo = tmp.path().join("repo");
9696 std::fs::create_dir_all(&repo).unwrap();
9697 init_repo(&repo);
9698
9699 let mut config = Config::default();
9700 config.graph.worktree_root = Some(tmp.path().join("wt"));
9701
9702 let mut state = RunState::new(
9703 repo.clone(),
9704 "main".to_owned(),
9705 "deadbeef".to_owned(),
9706 "task".to_owned(),
9707 config,
9708 );
9709 let root = state.worktree_root();
9710 let wt_a = root.join("cand-A");
9711 let wt_b = root.join("cand-B");
9712 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
9713 .await
9714 .expect("worktree A");
9715 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
9716 .await
9717 .expect("worktree B");
9718
9719 state.candidates = vec![
9720 Candidate {
9721 index: 0,
9722 label: 'A',
9723 agent: "alpha".to_owned(),
9724 branch: "magi/x/A".to_owned(),
9725 worktree: wt_a.clone(),
9726 summary: String::new(),
9727 stat: String::new(),
9728 files: 0,
9729 commits: 0,
9730 empty: false,
9731 failed: None,
9732 verified_noop: None,
9733 duration_ms: 0,
9734 folded: false,
9735 },
9736 Candidate {
9737 index: 1,
9738 label: 'B',
9739 agent: "beta".to_owned(),
9740 branch: "magi/x/B".to_owned(),
9741 worktree: wt_b.clone(),
9742 summary: String::new(),
9743 stat: String::new(),
9744 files: 0,
9745 commits: 0,
9746 empty: false,
9747 failed: None,
9748 verified_noop: None,
9749 duration_ms: 0,
9750 folded: false,
9751 },
9752 ];
9753 state.tally = Some(Tally {
9754 first_choice: BTreeMap::from([('A', 1)]),
9755 borda: BTreeMap::new(),
9756 winner: 'A',
9757 rankings: 1,
9758 unanimous_initial: true,
9759 deliberated: false,
9760 changed_votes: 0,
9761 unanimous_final: true,
9762 tie_break: None,
9763 judges: 1,
9764 present: 1,
9765 quorum: 1,
9766 met_quorum: true,
9767 uncontested: None,
9768 });
9769 state.status = RunStatus::Ready;
9770
9771 fold_run(&mut state, false, &crate::run::home())
9772 .await
9773 .expect("fold_run");
9774
9775 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
9776 assert!(
9777 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9778 "the unmerged winner's branch survives"
9779 );
9780 assert!(
9781 !state.candidates[0].folded,
9782 "the winner is not marked folded"
9783 );
9784
9785 assert!(!wt_b.exists(), "the loser's worktree is removed");
9786 assert!(
9787 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
9788 "the loser's branch is removed"
9789 );
9790 assert!(state.candidates[1].folded, "the loser is marked folded");
9791 }
9792
9793 #[tokio::test]
9796 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
9797 let tmp = tempfile::tempdir().expect("tempdir");
9798 let repo = tmp.path().join("repo");
9799 std::fs::create_dir_all(&repo).unwrap();
9800 init_repo(&repo);
9801 let home = tmp.path().join("home");
9802
9803 let mut config = Config::default();
9804 config.graph.worktree_root = Some(tmp.path().join("wt"));
9805 let mut state = RunState::new(
9806 repo.clone(),
9807 "main".to_owned(),
9808 "deadbeef".to_owned(),
9809 "task".to_owned(),
9810 config,
9811 );
9812 git::git(&repo, &["branch", "magi/x/A", "main"])
9814 .await
9815 .expect("branch");
9816 state.candidates = vec![Candidate {
9817 index: 0,
9818 label: 'A',
9819 agent: "alpha".to_owned(),
9820 branch: "magi/x/A".to_owned(),
9821 worktree: state.worktree_root().join("cand-A"),
9822 summary: String::new(),
9823 stat: String::new(),
9824 files: 0,
9825 commits: 0,
9826 empty: false,
9827 failed: None,
9828 verified_noop: None,
9829 duration_ms: 0,
9830 folded: true,
9831 }];
9832 state.released_to = Some("20260901-000000-new1".to_owned());
9833 state.released_branches = vec!["magi/x/A".to_owned()];
9834
9835 fold_run(&mut state, true, &home).await.expect("fold_run");
9836
9837 assert!(
9838 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9839 "the handed-over branch survives a fold"
9840 );
9841 }
9842
9843 #[tokio::test]
9847 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
9848 let tmp = tempfile::tempdir().expect("tempdir");
9849 let repo = tmp.path().join("repo");
9850 std::fs::create_dir_all(&repo).unwrap();
9851 init_repo(&repo);
9852 let run = |args: &[&str]| {
9853 let out = std::process::Command::new("git")
9854 .quiet()
9855 .args(args)
9856 .current_dir(&repo)
9857 .output()
9858 .expect("spawn git");
9859 assert!(out.status.success(), "git {args:?}");
9860 };
9861 run(&["branch", "magi/x/A"]);
9862 run(&["checkout", "-q", "-b", "magi/x/B"]);
9863 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
9864 run(&["add", "-A"]);
9865 run(&["commit", "-q", "-m", "work"]);
9866 run(&["checkout", "-q", "main"]);
9867
9868 let probe = RunState::new(
9872 repo.clone(),
9873 "main".to_owned(),
9874 "deadbeef".to_owned(),
9875 "task".to_owned(),
9876 Config::default(),
9877 );
9878 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
9879 let bare = tmp.path().join("origin.git");
9880 let out = std::process::Command::new("git")
9881 .quiet()
9882 .args(["init", "-q", "--bare"])
9883 .arg(&bare)
9884 .output()
9885 .expect("spawn git");
9886 assert!(out.status.success(), "git init --bare");
9887 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
9888 run(&["push", "-q", "origin", "main"]);
9889
9890 let mut state = RunState::new(
9891 repo.clone(),
9892 "main".to_owned(),
9893 "deadbeef".to_owned(),
9894 "task".to_owned(),
9895 Config::default(),
9896 );
9897 state.seeds = vec![refs::Seed {
9898 token: "magi/27b2/A".to_owned(),
9899 kind: refs::SeedKind::Unresolved,
9900 sha: String::new(),
9901 branch: true,
9902 detail: "no branch or commit named magi/27b2/A".to_owned(),
9903 }];
9904
9905 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
9906 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
9907 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
9908 let detail = empty_candidate_detail(&state, "main");
9909 assert!(detail.starts_with("empty candidate"), "{detail}");
9910 assert!(detail.contains("magi/27b2/A"), "{detail}");
9911 }
9912
9913 #[tokio::test]
9922 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
9923 let tmp = tempfile::tempdir().expect("tempdir");
9924 let repo = tmp.path().join("repo");
9925 std::fs::create_dir_all(&repo).unwrap();
9926 init_repo(&repo);
9927
9928 let mut config = Config::default();
9929 config.merge.mode = MergeMode::Local;
9930
9931 let mut state = RunState::new(
9932 repo.clone(),
9933 "main".to_owned(),
9934 "deadbeef".to_owned(),
9935 "task".to_owned(),
9936 config,
9937 );
9938 state.candidates = vec![Candidate {
9939 index: 0,
9940 label: 'A',
9941 agent: "alpha".to_owned(),
9942 branch: "does-not-exist".to_owned(),
9943 worktree: repo.clone(),
9944 summary: String::new(),
9945 stat: String::new(),
9946 files: 0,
9947 commits: 0,
9948 empty: false,
9949 failed: None,
9950 verified_noop: None,
9951 duration_ms: 0,
9952 folded: false,
9953 }];
9954 state.tally = Some(Tally {
9955 first_choice: BTreeMap::from([('A', 1)]),
9956 borda: BTreeMap::new(),
9957 winner: 'A',
9958 rankings: 1,
9959 unanimous_initial: true,
9960 deliberated: false,
9961 changed_votes: 0,
9962 unanimous_final: true,
9963 tie_break: None,
9964 judges: 0,
9965 present: 0,
9966 quorum: 0,
9967 met_quorum: true,
9968 uncontested: Some("only candidate A produced a change".to_owned()),
9969 });
9970 state.reviews = vec![ReviewRound {
9971 round: 1,
9972 head: "deadbeef".to_owned(),
9973 verified_head: None,
9974 verified_at: None,
9975 reviews: Vec::new(),
9976 e2e: Vec::new(),
9977 fix: None,
9978 blocking: 0,
9979 answered: 0,
9980 expected: 0,
9981 clean: true,
9982 verify_retried: false,
9983 e2e_deferred: false,
9984 e2e_defer_reason: None,
9985 progressed: false,
9986 vote_split: false,
9987 reconsideration: Vec::new(),
9988 verdict: None,
9989 }];
9990 state.gate = vec![CommandOutcome {
9991 command: "test".to_owned(),
9992 code: Some(0),
9993 output_tail: String::new(),
9994 duration_ms: 0,
9995 resource_blocked: false,
9996 }];
9997 state.gate_ran = true;
9998 state.status = RunStatus::Ready;
10003 state.merge = Some(MergeOutcome {
10004 mode: MergeMode::Local,
10005 ok: false,
10006 detail: "already concluded".to_owned(),
10007 empty: false,
10008 });
10009
10010 let mut runner = Runner {
10011 state,
10012 roles: ResolvedRoles {
10013 implementers: Vec::new(),
10014 judges: Vec::new(),
10015 reviewers: Vec::new(),
10016 fixer: None,
10017 conductor: conductor(),
10018 implementer_roster: Vec::new(),
10019 judge_roster: Vec::new(),
10020 reviewer_roster: Vec::new(),
10021 },
10022 sem: Arc::new(Semaphore::new(1)),
10023 pause: Pause::new(),
10024 interrupt: Pause::new(),
10025 };
10026
10027 runner.merge().await.expect("merge");
10028
10029 assert_eq!(
10030 runner.state.status,
10031 RunStatus::Ready,
10032 "a concluded run's status must not change on reentry"
10033 );
10034 assert_eq!(
10035 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10036 Some("already concluded"),
10037 "merge must not run again once the node already recorded an outcome"
10038 );
10039 }
10040
10041 #[tokio::test]
10050 async fn merge_refuses_a_gate_that_has_not_actually_run() {
10051 let tmp = tempfile::tempdir().expect("tempdir");
10052 let repo = tmp.path().join("repo");
10053 std::fs::create_dir_all(&repo).unwrap();
10054 init_repo(&repo);
10055
10056 let mut config = Config::default();
10057 config.merge.mode = MergeMode::Local;
10058
10059 let mut state = RunState::new(
10060 repo.clone(),
10061 "main".to_owned(),
10062 "deadbeef".to_owned(),
10063 "task".to_owned(),
10064 config,
10065 );
10066 state.candidates = vec![Candidate {
10067 index: 0,
10068 label: 'A',
10069 agent: "alpha".to_owned(),
10070 branch: "does-not-exist".to_owned(),
10071 worktree: repo.clone(),
10072 summary: String::new(),
10073 stat: String::new(),
10074 files: 0,
10075 commits: 0,
10076 empty: false,
10077 failed: None,
10078 verified_noop: None,
10079 duration_ms: 0,
10080 folded: false,
10081 }];
10082 state.tally = Some(Tally {
10083 first_choice: BTreeMap::from([('A', 1)]),
10084 borda: BTreeMap::new(),
10085 winner: 'A',
10086 rankings: 1,
10087 unanimous_initial: true,
10088 deliberated: false,
10089 changed_votes: 0,
10090 unanimous_final: true,
10091 tie_break: None,
10092 judges: 0,
10093 present: 0,
10094 quorum: 0,
10095 met_quorum: true,
10096 uncontested: Some("only candidate A produced a change".to_owned()),
10097 });
10098 state.reviews = vec![ReviewRound {
10099 round: 1,
10100 head: "deadbeef".to_owned(),
10101 verified_head: None,
10102 verified_at: None,
10103 reviews: Vec::new(),
10104 e2e: Vec::new(),
10105 fix: None,
10106 blocking: 0,
10107 answered: 0,
10108 expected: 0,
10109 clean: true,
10110 verify_retried: false,
10111 e2e_deferred: false,
10112 e2e_defer_reason: None,
10113 progressed: false,
10114 vote_split: false,
10115 reconsideration: Vec::new(),
10116 verdict: None,
10117 }];
10118 state.gate = Vec::new();
10120 state.gate_ran = false;
10121 state.status = RunStatus::Gating;
10122
10123 let mut runner = Runner {
10124 state,
10125 roles: ResolvedRoles {
10126 implementers: Vec::new(),
10127 judges: Vec::new(),
10128 reviewers: Vec::new(),
10129 fixer: None,
10130 conductor: conductor(),
10131 implementer_roster: Vec::new(),
10132 judge_roster: Vec::new(),
10133 reviewer_roster: Vec::new(),
10134 },
10135 sem: Arc::new(Semaphore::new(1)),
10136 pause: Pause::new(),
10137 interrupt: Pause::new(),
10138 };
10139
10140 runner.merge().await.expect("merge");
10141
10142 assert!(
10143 runner.state.merge.is_none(),
10144 "an empty gate must never be read as a passing one: {:?}",
10145 runner.state.merge
10146 );
10147 }
10148
10149 #[tokio::test]
10156 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
10157 let tmp = tempfile::tempdir().expect("tempdir");
10158 let repo = tmp.path().join("repo");
10159 std::fs::create_dir_all(&repo).unwrap();
10160 init_repo(&repo);
10161
10162 let config = Config::default();
10164
10165 let mut state = RunState::new(
10166 repo.clone(),
10167 "main".to_owned(),
10168 "deadbeef".to_owned(),
10169 "task".to_owned(),
10170 config,
10171 );
10172 state.candidates = vec![Candidate {
10173 index: 0,
10174 label: 'A',
10175 agent: "alpha".to_owned(),
10176 branch: "does-not-exist".to_owned(),
10177 worktree: repo.clone(),
10178 summary: String::new(),
10179 stat: String::new(),
10180 files: 0,
10181 commits: 0,
10182 empty: false,
10183 failed: None,
10184 verified_noop: None,
10185 duration_ms: 0,
10186 folded: false,
10187 }];
10188 state.tally = Some(Tally {
10189 first_choice: BTreeMap::from([('A', 1)]),
10190 borda: BTreeMap::new(),
10191 winner: 'A',
10192 rankings: 1,
10193 unanimous_initial: true,
10194 deliberated: false,
10195 changed_votes: 0,
10196 unanimous_final: true,
10197 tie_break: None,
10198 judges: 0,
10199 present: 0,
10200 quorum: 0,
10201 met_quorum: true,
10202 uncontested: Some("only candidate A produced a change".to_owned()),
10203 });
10204 state.reviews = vec![ReviewRound {
10205 round: 1,
10206 head: "deadbeef".to_owned(),
10207 verified_head: None,
10208 verified_at: None,
10209 reviews: Vec::new(),
10210 e2e: Vec::new(),
10211 fix: None,
10212 blocking: 0,
10213 answered: 0,
10214 expected: 0,
10215 clean: true,
10216 verify_retried: false,
10217 e2e_deferred: false,
10218 e2e_defer_reason: None,
10219 progressed: false,
10220 vote_split: false,
10221 reconsideration: Vec::new(),
10222 verdict: None,
10223 }];
10224
10225 let mut runner = Runner {
10226 state,
10227 roles: ResolvedRoles {
10228 implementers: Vec::new(),
10229 judges: Vec::new(),
10230 reviewers: Vec::new(),
10231 fixer: None,
10232 conductor: conductor(),
10233 implementer_roster: Vec::new(),
10234 judge_roster: Vec::new(),
10235 reviewer_roster: Vec::new(),
10236 },
10237 sem: Arc::new(Semaphore::new(1)),
10238 pause: Pause::new(),
10239 interrupt: Pause::new(),
10240 };
10241
10242 runner.gate().await.expect("gate");
10243 assert!(
10244 runner.state.gate_ran,
10245 "zero configured commands is still a real attempt, not an unrun gate"
10246 );
10247 assert!(runner.state.gate.is_empty());
10248 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
10249 assert_ne!(
10250 runner.state.status,
10251 RunStatus::Blocked,
10252 "a gate with nothing to check must not read as failed"
10253 );
10254
10255 runner.merge().await.expect("merge");
10256 assert_eq!(
10257 runner.state.status,
10258 RunStatus::Ready,
10259 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
10260 );
10261 }
10262
10263 #[tokio::test]
10274 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
10275 crate::run::pin_test_home();
10276 let home = crate::run::home();
10277
10278 let tmp = tempfile::tempdir().expect("tempdir");
10279 let repo = tmp.path().join("repo");
10280 std::fs::create_dir_all(&repo).unwrap();
10281 init_repo(&repo);
10282 let cache_dir = tmp.path().join("target");
10285
10286 let mut config = Config::default();
10287 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
10288 config.graph.timeout_verify = Some(2);
10291
10292 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10293 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10294 .expect("no io error acquiring directly")
10295 {
10296 crate::cache::AcquireOutcome::Acquired(g) => g,
10297 crate::cache::AcquireOutcome::Busy(b) => {
10298 panic!("expected the direct acquire to win the lease first: {b:?}")
10299 }
10300 };
10301
10302 let mut state = RunState::new(
10303 repo.clone(),
10304 "main".to_owned(),
10305 "deadbeef".to_owned(),
10306 "task".to_owned(),
10307 config,
10308 );
10309 state.candidates = vec![Candidate {
10310 index: 0,
10311 label: 'A',
10312 agent: "alpha".to_owned(),
10313 branch: "does-not-exist".to_owned(),
10314 worktree: repo.clone(),
10315 summary: String::new(),
10316 stat: String::new(),
10317 files: 0,
10318 commits: 0,
10319 empty: false,
10320 failed: None,
10321 verified_noop: None,
10322 duration_ms: 0,
10323 folded: false,
10324 }];
10325 state.tally = Some(Tally {
10326 first_choice: BTreeMap::from([('A', 1)]),
10327 borda: BTreeMap::new(),
10328 winner: 'A',
10329 rankings: 1,
10330 unanimous_initial: true,
10331 deliberated: false,
10332 changed_votes: 0,
10333 unanimous_final: true,
10334 tie_break: None,
10335 judges: 0,
10336 present: 0,
10337 quorum: 0,
10338 met_quorum: true,
10339 uncontested: Some("only candidate A produced a change".to_owned()),
10340 });
10341 state.reviews = vec![ReviewRound {
10342 round: 1,
10343 head: "deadbeef".to_owned(),
10344 verified_head: None,
10345 verified_at: None,
10346 reviews: Vec::new(),
10347 e2e: Vec::new(),
10348 fix: None,
10349 blocking: 0,
10350 answered: 0,
10351 expected: 0,
10352 clean: true,
10353 verify_retried: false,
10354 e2e_deferred: false,
10355 e2e_defer_reason: None,
10356 progressed: false,
10357 vote_split: false,
10358 reconsideration: Vec::new(),
10359 verdict: None,
10360 }];
10361
10362 let mut runner = Runner {
10363 state,
10364 roles: ResolvedRoles {
10365 implementers: Vec::new(),
10366 judges: Vec::new(),
10367 reviewers: Vec::new(),
10368 fixer: None,
10369 conductor: conductor(),
10370 implementer_roster: Vec::new(),
10371 judge_roster: Vec::new(),
10372 reviewer_roster: Vec::new(),
10373 },
10374 sem: Arc::new(Semaphore::new(1)),
10375 pause: Pause::new(),
10376 interrupt: Pause::new(),
10377 };
10378
10379 let started = std::time::Instant::now();
10380 runner.gate().await.expect("gate");
10381 assert!(
10382 started.elapsed() < Duration::from_secs(1),
10383 "a gate with nothing to run must never wait on a lease it never needed"
10384 );
10385 assert!(
10386 runner.state.gate_ran,
10387 "zero commands is still a real, immediate attempt"
10388 );
10389 assert!(runner.state.gate.is_empty());
10390 assert_ne!(
10391 runner.state.status,
10392 RunStatus::Blocked,
10393 "must not read as resource-blocked on a lease it never asked for"
10394 );
10395 }
10396
10397 #[tokio::test]
10407 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
10408 crate::run::pin_test_home();
10409
10410 let tmp = tempfile::tempdir().expect("tempdir");
10411 let repo = tmp.path().join("repo");
10412 std::fs::create_dir_all(&repo).unwrap();
10413 init_repo(&repo);
10414
10415 let mut config = Config::default();
10416 config.verify.gate = vec![
10417 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
10418 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
10419 .to_owned(),
10420 ];
10421
10422 let mut state = RunState::new(
10423 repo.clone(),
10424 "main".to_owned(),
10425 "deadbeef".to_owned(),
10426 "task".to_owned(),
10427 config,
10428 );
10429 let run_id = state.id.clone();
10430 state.candidates = vec![Candidate {
10431 index: 0,
10432 label: 'A',
10433 agent: "alpha".to_owned(),
10434 branch: "does-not-exist".to_owned(),
10435 worktree: repo.clone(),
10436 summary: String::new(),
10437 stat: String::new(),
10438 files: 0,
10439 commits: 0,
10440 empty: false,
10441 failed: None,
10442 verified_noop: None,
10443 duration_ms: 0,
10444 folded: false,
10445 }];
10446 state.tally = Some(Tally {
10447 first_choice: BTreeMap::from([('A', 1)]),
10448 borda: BTreeMap::new(),
10449 winner: 'A',
10450 rankings: 1,
10451 unanimous_initial: true,
10452 deliberated: false,
10453 changed_votes: 0,
10454 unanimous_final: true,
10455 tie_break: None,
10456 judges: 0,
10457 present: 0,
10458 quorum: 0,
10459 met_quorum: true,
10460 uncontested: Some("only candidate A produced a change".to_owned()),
10461 });
10462 state.reviews = vec![ReviewRound {
10463 round: 1,
10464 head: "deadbeef".to_owned(),
10465 verified_head: None,
10466 verified_at: None,
10467 reviews: Vec::new(),
10468 e2e: Vec::new(),
10469 fix: None,
10470 blocking: 0,
10471 answered: 0,
10472 expected: 0,
10473 clean: true,
10474 verify_retried: false,
10475 e2e_deferred: false,
10476 e2e_defer_reason: None,
10477 progressed: false,
10478 vote_split: false,
10479 reconsideration: Vec::new(),
10480 verdict: None,
10481 }];
10482
10483 let mut runner = Runner {
10484 state,
10485 roles: ResolvedRoles {
10486 implementers: Vec::new(),
10487 judges: Vec::new(),
10488 reviewers: Vec::new(),
10489 fixer: None,
10490 conductor: conductor(),
10491 implementer_roster: Vec::new(),
10492 judge_roster: Vec::new(),
10493 reviewer_roster: Vec::new(),
10494 },
10495 sem: Arc::new(Semaphore::new(1)),
10496 pause: Pause::new(),
10497 interrupt: Pause::new(),
10498 };
10499
10500 let started_marker = repo.join("started.marker");
10501 let release_marker = repo.join("release.marker");
10502 let poller = tokio::spawn(async move {
10503 for _ in 0..100 {
10508 if started_marker.exists()
10509 && let Ok(s) = crate::run::RunState::load(&run_id)
10510 && let Some(a) = s.active.get("gate")
10511 {
10512 std::fs::write(&release_marker, b"go").expect("release marker");
10513 return Some(a.clone());
10514 }
10515 tokio::time::sleep(Duration::from_millis(50)).await;
10516 }
10517 None
10518 });
10519
10520 runner.gate().await.expect("gate");
10521 let captured = poller.await.expect("poller task");
10522 let captured = captured.expect(
10523 "the poller never saw a `gate` task entry in run.json while the command was \
10524 still blocked on its own release marker",
10525 );
10526
10527 assert_eq!(captured.task.as_deref(), Some("gate"));
10528 assert_eq!(captured.node, "gate");
10529 assert_eq!(captured.index, Some(1));
10530 assert_eq!(captured.total, Some(1));
10531 assert!(
10532 captured
10533 .command
10534 .as_deref()
10535 .is_some_and(|c| c.contains("started.marker")),
10536 "{captured:?}"
10537 );
10538
10539 assert!(
10540 runner.state.active.is_empty(),
10541 "the entry must be cleared once the command actually finished: {:?}",
10542 runner.state.active
10543 );
10544 assert!(runner.state.gate_ran);
10545 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
10546 }
10547
10548 #[tokio::test]
10551 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
10552 use crate::verdict::{Finding, ReviewVote, Severity};
10553 crate::run::pin_test_home();
10554 let tmp = tempfile::tempdir().expect("tempdir");
10555 let repo = tmp.path().join("repo");
10556 std::fs::create_dir_all(&repo).unwrap();
10557 init_repo(&repo);
10558
10559 for (severity, vote, expect) in [
10560 (Severity::Major, ReviewVote::Reject, true),
10561 (Severity::Minor, ReviewVote::Reject, false),
10562 (Severity::Major, ReviewVote::Approve, false),
10563 ] {
10564 let mut round = review_round(false, 1, 1, 1, false, true);
10565 round.reviews = vec![ReviewRecord {
10566 reviewer: 1,
10567 agent: "alpha".to_owned(),
10568 summary: String::new(),
10569 findings: vec![Finding {
10570 id: "R1-1-1".to_owned(),
10571 severity,
10572 file: None,
10573 line: None,
10574 title: "t".to_owned(),
10575 detail: String::new(),
10576 }],
10577 vote: Some(vote),
10578 failed: None,
10579 duration_ms: 0,
10580 attempts: 0,
10581 }];
10582 let mut state = RunState::new(
10583 repo.clone(),
10584 "main".to_owned(),
10585 "deadbeef".to_owned(),
10586 "task".to_owned(),
10587 Config::default(),
10588 );
10589 state.reviews = vec![round];
10590 let mut runner = Runner {
10591 state,
10592 roles: ResolvedRoles {
10593 implementers: Vec::new(),
10594 judges: Vec::new(),
10595 reviewers: Vec::new(),
10596 fixer: None,
10597 conductor: conductor(),
10598 implementer_roster: Vec::new(),
10599 judge_roster: Vec::new(),
10600 reviewer_roster: Vec::new(),
10601 },
10602 sem: Arc::new(Semaphore::new(1)),
10603 pause: Pause::new(),
10604 interrupt: Pause::new(),
10605 };
10606 let shell = runner.state.config.shell();
10607 runner
10608 .stop_reviewing("round budget spent", &shell, &repo)
10609 .await
10610 .expect("stop_reviewing");
10611 assert_eq!(runner.state.status, RunStatus::Gating);
10612 assert_eq!(
10613 runner.state.contested_handoff.is_some(),
10614 expect,
10615 "{severity:?} + {vote:?}"
10616 );
10617 }
10618 }
10619
10620 #[tokio::test]
10633 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
10634 crate::run::pin_test_home();
10635 let home = crate::run::home();
10636
10637 let tmp = tempfile::tempdir().expect("tempdir");
10638 let repo = tmp.path().join("repo");
10639 std::fs::create_dir_all(&repo).unwrap();
10640 init_repo(&repo);
10641 let head = crate::git::rev_parse(&repo, "HEAD")
10642 .await
10643 .expect("rev-parse");
10644 let cache_dir = tmp.path().join("target");
10647
10648 let mut config = Config::default();
10649 config.verify.e2e = vec![format!(
10650 "CARGO_TARGET_DIR='{}' test -f README.md",
10651 cache_dir.display()
10652 )];
10653 config.graph.review_rounds = 1;
10654 config.graph.timeout_verify = Some(2);
10657
10658 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10659 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10660 .expect("no io error acquiring directly")
10661 {
10662 crate::cache::AcquireOutcome::Acquired(g) => g,
10663 crate::cache::AcquireOutcome::Busy(b) => {
10664 panic!("expected the direct acquire to win the lease first: {b:?}")
10665 }
10666 };
10667
10668 let mut state = RunState::new(
10669 repo.clone(),
10670 "main".to_owned(),
10671 head.clone(),
10672 "task".to_owned(),
10673 config,
10674 );
10675 state.candidates = vec![Candidate {
10676 index: 0,
10677 label: 'A',
10678 agent: "alpha".to_owned(),
10679 branch: "does-not-exist".to_owned(),
10680 worktree: repo.clone(),
10681 summary: String::new(),
10682 stat: String::new(),
10683 files: 0,
10684 commits: 0,
10685 empty: false,
10686 failed: None,
10687 verified_noop: None,
10688 duration_ms: 0,
10689 folded: false,
10690 }];
10691 state.tally = Some(Tally {
10692 first_choice: BTreeMap::from([('A', 1)]),
10693 borda: BTreeMap::new(),
10694 winner: 'A',
10695 rankings: 1,
10696 unanimous_initial: true,
10697 deliberated: false,
10698 changed_votes: 0,
10699 unanimous_final: true,
10700 tie_break: None,
10701 judges: 0,
10702 present: 0,
10703 quorum: 0,
10704 met_quorum: true,
10705 uncontested: Some("only candidate A produced a change".to_owned()),
10706 });
10707 state.reviews = vec![ReviewRound {
10711 round: 1,
10712 head: head.clone(),
10713 verified_head: None,
10714 verified_at: None,
10715 reviews: Vec::new(),
10716 e2e: Vec::new(),
10717 fix: None,
10718 blocking: 1,
10719 answered: 1,
10720 expected: 1,
10721 clean: false,
10722 verify_retried: false,
10723 e2e_deferred: true,
10724 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
10725 progressed: false,
10726 vote_split: false,
10727 reconsideration: Vec::new(),
10728 verdict: None,
10729 }];
10730
10731 let mut runner = Runner {
10732 state,
10733 roles: ResolvedRoles {
10734 implementers: Vec::new(),
10735 judges: Vec::new(),
10736 reviewers: Vec::new(),
10737 fixer: None,
10738 conductor: conductor(),
10739 implementer_roster: Vec::new(),
10740 judge_roster: Vec::new(),
10741 reviewer_roster: Vec::new(),
10742 },
10743 sem: Arc::new(Semaphore::new(1)),
10744 pause: Pause::new(),
10745 interrupt: Pause::new(),
10746 };
10747
10748 let shell = runner.state.config.shell();
10749 runner
10750 .stop_reviewing("round budget spent", &shell, &repo)
10751 .await
10752 .expect("stop_reviewing");
10753
10754 let last = runner.state.reviews.last().expect("round record");
10755 assert_eq!(
10756 last.e2e_status(),
10757 E2eStatus::ResourceBlocked,
10758 "the shared cache is still held; the attempt must read as blocked, not deferred or \
10759 failed: {last:?}"
10760 );
10761 assert_eq!(
10762 last.verified_head.as_deref(),
10763 Some(head.as_str()),
10764 "which commit this attempt targeted is known even though nothing finished checking \
10765 it"
10766 );
10767 let first_attempt_at = last
10768 .verified_at
10769 .expect("when this attempt ran is known too");
10770 assert_ne!(
10771 runner.state.status,
10772 RunStatus::Blocked,
10773 "contention is evidence about the machine, not the patch — it must not settle the \
10774 run as blocked: {:?}",
10775 runner.state.status
10776 );
10777 assert!(
10778 !runner
10779 .state
10780 .events
10781 .iter()
10782 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
10783 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
10784 runner.state.events
10785 );
10786
10787 runner
10792 .stop_reviewing("round budget spent", &shell, &repo)
10793 .await
10794 .expect("stop_reviewing retry");
10795 assert_eq!(
10796 runner.state.reviews.len(),
10797 1,
10798 "no new round was started: {:?}",
10799 runner.state.reviews
10800 );
10801 let last = runner.state.reviews.last().expect("round record");
10802 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
10803 assert!(
10804 last.verified_at.expect("still known") > first_attempt_at,
10805 "a second reentry must be a fresh attempt, not a stale copy of the first"
10806 );
10807 assert_ne!(runner.state.status, RunStatus::Blocked);
10808
10809 held.release();
10810 }
10811
10812 #[tokio::test]
10824 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
10825 crate::run::pin_test_home();
10826 let home = crate::run::home();
10827
10828 let tmp = tempfile::tempdir().expect("tempdir");
10829 let repo = tmp.path().join("repo");
10830 std::fs::create_dir_all(&repo).unwrap();
10831 init_repo(&repo);
10832 let head = crate::git::rev_parse(&repo, "HEAD")
10833 .await
10834 .expect("rev-parse");
10835 let cache_dir = tmp.path().join("target");
10836
10837 let mut config = Config::default();
10838 config.verify.e2e = vec![format!(
10839 "CARGO_TARGET_DIR='{}' test -f README.md",
10840 cache_dir.display()
10841 )];
10842 config.graph.review_rounds = 1;
10843 config.graph.timeout_verify = Some(2);
10844
10845 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10846 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10847 .expect("no io error acquiring directly")
10848 {
10849 crate::cache::AcquireOutcome::Acquired(g) => g,
10850 crate::cache::AcquireOutcome::Busy(b) => {
10851 panic!("expected the direct acquire to win the lease first: {b:?}")
10852 }
10853 };
10854
10855 let mut state = RunState::new(
10856 repo.clone(),
10857 "main".to_owned(),
10858 head.clone(),
10859 "task".to_owned(),
10860 config,
10861 );
10862 state.candidates = vec![Candidate {
10863 index: 0,
10864 label: 'A',
10865 agent: "alpha".to_owned(),
10866 branch: "does-not-exist".to_owned(),
10867 worktree: repo.clone(),
10868 summary: String::new(),
10869 stat: String::new(),
10870 files: 0,
10871 commits: 0,
10872 empty: false,
10873 failed: None,
10874 verified_noop: None,
10875 duration_ms: 0,
10876 folded: false,
10877 }];
10878 state.tally = Some(Tally {
10879 first_choice: BTreeMap::from([('A', 1)]),
10880 borda: BTreeMap::new(),
10881 winner: 'A',
10882 rankings: 1,
10883 unanimous_initial: true,
10884 deliberated: false,
10885 changed_votes: 0,
10886 unanimous_final: true,
10887 tie_break: None,
10888 judges: 0,
10889 present: 0,
10890 quorum: 0,
10891 met_quorum: true,
10892 uncontested: Some("only candidate A produced a change".to_owned()),
10893 });
10894 state.reviews = vec![ReviewRound {
10898 round: 1,
10899 head: head.clone(),
10900 verified_head: Some(head.clone()),
10901 verified_at: Some(jiff::Timestamp::now()),
10902 reviews: Vec::new(),
10903 e2e: vec![CommandOutcome {
10904 command: format!(
10905 "CARGO_TARGET_DIR='{}' test -f README.md",
10906 cache_dir.display()
10907 ),
10908 code: None,
10909 output_tail: "waiting for the shared build cache".to_owned(),
10910 duration_ms: 0,
10911 resource_blocked: true,
10912 }],
10913 fix: None,
10914 blocking: 1,
10915 answered: 1,
10916 expected: 1,
10917 clean: false,
10918 verify_retried: false,
10919 e2e_deferred: false,
10920 e2e_defer_reason: None,
10921 progressed: false,
10922 vote_split: false,
10923 reconsideration: Vec::new(),
10924 verdict: None,
10925 }];
10926
10927 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
10928 let mut runner = Runner {
10929 state,
10930 roles: ResolvedRoles {
10931 implementers: Vec::new(),
10932 judges: Vec::new(),
10933 reviewers: Vec::new(),
10934 fixer: None,
10935 conductor: conductor(),
10936 implementer_roster: Vec::new(),
10937 judge_roster: Vec::new(),
10938 reviewer_roster: Vec::new(),
10939 },
10940 sem: Arc::new(Semaphore::new(1)),
10941 pause: Pause::new(),
10942 interrupt: Pause::new(),
10943 };
10944
10945 runner.review_loop().await.expect("review_loop");
10950
10951 assert_eq!(
10952 runner.state.reviews.len(),
10953 1,
10954 "no new round was started on top of the unresolved one: {:?}",
10955 runner.state.reviews
10956 );
10957 let last = &runner.state.reviews[0];
10958 assert_eq!(
10959 last.e2e_status(),
10960 E2eStatus::ResourceBlocked,
10961 "still contended: {last:?}"
10962 );
10963 assert!(
10964 last.verified_at.expect("still known") > first_attempt_at,
10965 "review_loop must have actually retried the check, not left it exactly as found"
10966 );
10967 assert_ne!(
10968 runner.state.status,
10969 RunStatus::Blocked,
10970 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
10971 runner.state.status
10972 );
10973
10974 held.release();
10975 }
10976
10977 #[tokio::test]
10978 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
10979 crate::run::pin_test_home();
10980 let tmp = tempfile::tempdir().expect("tempdir");
10981 let repo = tmp.path().join("repo");
10982 std::fs::create_dir_all(&repo).unwrap();
10983 init_repo(&repo);
10984
10985 let mut config = Config::default();
10986 config.merge.mode = MergeMode::Pr;
10987 config.graph.land = true;
10988 config.graph.land_approval = false;
10989
10990 let mut state = RunState::new(
10991 repo.clone(),
10992 "main".to_owned(),
10993 "deadbeef".to_owned(),
10994 "task".to_owned(),
10995 config,
10996 );
10997 state.candidates = vec![Candidate {
10998 index: 0,
10999 label: 'A',
11000 agent: "alpha".to_owned(),
11001 branch: "does-not-exist".to_owned(),
11002 worktree: repo.clone(),
11003 summary: String::new(),
11004 stat: String::new(),
11005 files: 0,
11006 commits: 0,
11007 empty: false,
11008 failed: None,
11009 verified_noop: None,
11010 duration_ms: 0,
11011 folded: false,
11012 }];
11013 state.tally = Some(Tally {
11014 first_choice: BTreeMap::from([('A', 1)]),
11015 borda: BTreeMap::new(),
11016 winner: 'A',
11017 rankings: 1,
11018 unanimous_initial: true,
11019 deliberated: false,
11020 changed_votes: 0,
11021 unanimous_final: true,
11022 tie_break: None,
11023 judges: 0,
11024 present: 0,
11025 quorum: 0,
11026 met_quorum: true,
11027 uncontested: Some("only candidate A produced a change".to_owned()),
11028 });
11029 state.reviews = vec![ReviewRound {
11030 round: 1,
11031 head: "deadbeef".to_owned(),
11032 verified_head: None,
11033 verified_at: None,
11034 reviews: Vec::new(),
11035 e2e: Vec::new(),
11036 fix: None,
11037 blocking: 0,
11038 answered: 0,
11039 expected: 0,
11040 clean: true,
11041 verify_retried: false,
11042 e2e_deferred: false,
11043 e2e_defer_reason: None,
11044 progressed: false,
11045 vote_split: false,
11046 reconsideration: Vec::new(),
11047 verdict: None,
11048 }];
11049 state.gate = vec![CommandOutcome {
11050 command: "test".to_owned(),
11051 code: Some(0),
11052 output_tail: String::new(),
11053 duration_ms: 0,
11054 resource_blocked: false,
11055 }];
11056 state.gate_ran = true;
11057 state.status = RunStatus::Landing;
11061 state.merge = Some(MergeOutcome {
11062 mode: MergeMode::Pr,
11063 ok: true,
11064 detail: "https://example.invalid/x/y/pull/1".to_owned(),
11065 empty: false,
11066 });
11067
11068 ask_test_home();
11072 let store = ask::Questions::open();
11073 let q = ask_open_question(&store, &state.id);
11074
11075 let mut runner = Runner {
11076 state,
11077 roles: ResolvedRoles {
11078 implementers: Vec::new(),
11079 judges: Vec::new(),
11080 reviewers: Vec::new(),
11081 fixer: None,
11082 conductor: conductor(),
11083 implementer_roster: Vec::new(),
11084 judge_roster: Vec::new(),
11085 reviewer_roster: Vec::new(),
11086 },
11087 sem: Arc::new(Semaphore::new(1)),
11088 pause: Pause::new(),
11089 interrupt: Pause::new(),
11090 };
11091
11092 runner.execute().await.expect("execute");
11097
11098 assert_eq!(
11099 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
11100 Some("https://example.invalid/x/y/pull/1"),
11101 "reentry must not push again or open a second pull request over the \
11102 one `land` is already watching"
11103 );
11104 assert_ne!(
11105 runner.state.status,
11106 RunStatus::Landing,
11107 "land could not actually reach the fake pull request, so it must \
11108 have given up rather than left the run silently parked forever"
11109 );
11110 assert_eq!(runner.state.status, RunStatus::Blocked);
11114 assert!(
11115 store.get(&q.id).unwrap().status.open(),
11116 "Blocked is still alive; settle_questions must have been a no-op here"
11117 );
11118 }
11119
11120 fn state_with_round(round: ReviewRound) -> RunState {
11121 let mut s = RunState::new(
11122 PathBuf::from("/repo"),
11123 "main".to_owned(),
11124 "abc1234".to_owned(),
11125 "add retries".to_owned(),
11126 Config::default(),
11127 );
11128 s.reviews = vec![round];
11129 s
11130 }
11131
11132 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
11133 crate::verdict::Finding {
11134 id: id.to_owned(),
11135 severity,
11136 file: None,
11137 line: None,
11138 title: title.to_owned(),
11139 detail: String::new(),
11140 }
11141 }
11142
11143 #[test]
11144 fn pr_body_names_open_findings_and_declined_ones() {
11145 let round = ReviewRound {
11146 round: 2,
11147 head: "deadbee".to_owned(),
11148 verified_head: None,
11149 verified_at: None,
11150 reviews: vec![ReviewRecord {
11151 attempts: 0,
11152 reviewer: 1,
11153 agent: "alpha".to_owned(),
11154 summary: String::new(),
11155 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
11156 vote: None,
11157 failed: None,
11158 duration_ms: 0,
11159 }],
11160 e2e: vec![CommandOutcome {
11161 command: "cargo test".to_owned(),
11162 code: Some(0),
11163 output_tail: String::new(),
11164 duration_ms: 0,
11165 resource_blocked: false,
11166 }],
11167 verify_retried: false,
11168 e2e_deferred: false,
11169 e2e_defer_reason: None,
11170 fix: Some(FixRecord {
11171 agent: "alpha".to_owned(),
11172 addressed: Vec::new(),
11173 rejected: vec![crate::verdict::Rejection {
11174 id: "R1-1-1".to_owned(),
11175 why: "not reachable from any caller".to_owned(),
11176 }],
11177 notes: String::new(),
11178 committed: true,
11179 failed: None,
11180 duration_ms: 0,
11181 continuation: None,
11182 }),
11183 blocking: 0,
11184 answered: 1,
11185 expected: 1,
11186 clean: false,
11187 progressed: true,
11188 vote_split: false,
11189 reconsideration: Vec::new(),
11190 verdict: None,
11191 };
11192 let state = state_with_round(round);
11193 let body = pr_message(&state, 'A').body;
11194
11195 assert!(body.contains("add retries"), "the task must still be there");
11196 assert!(body.contains("R2-1-1"), "{body}");
11197 assert!(body.contains("unused import"), "{body}");
11198 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
11199 assert!(
11200 body.contains("not reachable from any caller"),
11201 "the reason it was declined: {body}"
11202 );
11203 }
11204
11205 #[test]
11206 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
11207 let round = ReviewRound {
11208 round: 1,
11209 head: "deadbee".to_owned(),
11210 verified_head: None,
11211 verified_at: None,
11212 reviews: vec![ReviewRecord {
11213 attempts: 0,
11214 reviewer: 1,
11215 agent: "alpha".to_owned(),
11216 summary: String::new(),
11217 findings: Vec::new(),
11218 vote: None,
11219 failed: None,
11220 duration_ms: 0,
11221 }],
11222 e2e: Vec::new(),
11223 verify_retried: false,
11224 e2e_deferred: false,
11225 e2e_defer_reason: None,
11226 fix: None,
11227 blocking: 0,
11228 answered: 1,
11229 expected: 1,
11230 clean: true,
11231 progressed: false,
11232 vote_split: false,
11233 reconsideration: Vec::new(),
11234 verdict: None,
11235 };
11236 let state = state_with_round(round);
11237 let body = pr_message(&state, 'A').body;
11238 assert!(!body.contains("Open review findings"), "{body}");
11239 assert!(!body.contains("Declined"), "{body}");
11240 }
11241
11242 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
11243 let mut state = RunState::new(
11244 PathBuf::from("/repo"),
11245 "main".to_owned(),
11246 "abc1234".to_owned(),
11247 instruction.to_owned(),
11248 Config::default(),
11249 );
11250 state.candidates.push(Candidate {
11251 index: 0,
11252 label: 'A',
11253 agent: "alpha".to_owned(),
11254 branch: "magi/x/A".to_owned(),
11255 worktree: PathBuf::from("/wt"),
11256 summary: summary.to_owned(),
11257 stat: String::new(),
11258 files: 1,
11259 commits: 1,
11260 empty: false,
11261 failed: None,
11262 verified_noop: None,
11263 folded: false,
11264 duration_ms: 0,
11265 });
11266 state
11267 }
11268
11269 fn review_state(subjects: &[&str]) -> RunState {
11270 let mut state = state_with_summary(
11271 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
11272 "",
11273 );
11274 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
11275 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
11276 state
11277 }
11278
11279 #[tokio::test]
11282 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
11283 ask_test_home();
11284 let tmp = tempfile::tempdir().unwrap();
11285 let repo = tmp.path().join("repo");
11286 std::fs::create_dir_all(&repo).unwrap();
11287 init_repo(&repo);
11288 let origin = tmp.path().join("origin.git");
11289 let g = |dir: &Path, args: &[&str]| {
11290 let out = std::process::Command::new("git")
11291 .args(args)
11292 .current_dir(dir)
11293 .quiet()
11294 .output()
11295 .expect("spawn git");
11296 assert!(
11297 out.status.success(),
11298 "git {args:?}: {}",
11299 String::from_utf8_lossy(&out.stderr)
11300 );
11301 };
11302 g(
11303 tmp.path(),
11304 &[
11305 "clone",
11306 "--bare",
11307 "-q",
11308 repo.to_str().unwrap(),
11309 origin.to_str().unwrap(),
11310 ],
11311 );
11312 g(
11313 &repo,
11314 &["remote", "add", "origin", origin.to_str().unwrap()],
11315 );
11316 let c1 = git::rev_parse(&repo, "main").await.unwrap();
11317
11318 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
11320 g(&repo, &["add", "-A"]);
11321 g(
11322 &repo,
11323 &["commit", "-q", "-m", "chore(deps): update a crate"],
11324 );
11325 g(&repo, &["push", "-q", "origin", "main"]);
11326 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
11327 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
11328 g(&repo, &["add", "-A"]);
11329 g(
11330 &repo,
11331 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
11332 );
11333 g(&repo, &["checkout", "-q", "main"]);
11334
11335 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
11336 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
11337 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
11339 assert_eq!(fallback, c1);
11340
11341 let mut state = review_state(&[
11342 "chore(deps): update a crate",
11343 "fix(daemon): apply a chosen action",
11344 ]);
11345 state.repo = repo.clone();
11346 state.base_branch = "main".to_owned();
11347 state.base_commit = c1;
11348 refresh_reviewed_commits(&mut state, "feat/own").await;
11349 assert_eq!(
11350 state.reviewed_commits,
11351 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
11352 );
11353 assert_eq!(
11354 review_title(&state).as_deref(),
11355 Some("fix(daemon): apply a chosen action")
11356 );
11357 assert_eq!(
11358 leaked_subjects(&state, "feat/own").await,
11359 Some(vec!["chore(deps): update a crate".to_owned()])
11360 );
11361 state.config.merge.remote = "nowhere".to_owned();
11364 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
11365 }
11366
11367 #[test]
11368 fn pr_message_review_single_commit_uses_its_subject() {
11369 let state = review_state(&["feat(nats): per-role user"]);
11370 let m = pr_message(&state, 'A');
11371 assert_eq!(m.title, "feat(nats): per-role user");
11372 assert!(!m.body.contains("Review the work already"), "{}", m.body);
11373 assert!(m.body.contains("## Commits under review"), "{}", m.body);
11374 }
11375
11376 #[test]
11377 fn pr_message_review_multi_commit_takes_the_oldest() {
11378 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
11379 let m = pr_message(&state, 'A');
11380 assert_eq!(m.title, "feat: the change");
11381 for s in ["feat: the change", "fix: typo", "fix: again"] {
11382 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
11383 }
11384 }
11385
11386 #[test]
11387 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
11388 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
11389 let mut state = review_state(&[first, "fix: later fixup"]);
11390 state.candidates[0].branch = "機能/ログイン".to_owned();
11391 let m = pr_message(&state, 'A');
11392 assert!(
11393 m.title.starts_with("chore: land candidate A of run"),
11394 "{}",
11395 m.title
11396 );
11397 }
11398 }
11399
11400 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
11401 BranchFacts {
11402 commits: commits
11403 .iter()
11404 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
11405 .collect(),
11406 stat: stat.to_owned(),
11407 }
11408 }
11409
11410 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
11411 let mut r = review_round(true, 0, 1, 1, true, true);
11412 r.round = round;
11413 r.fix = notes.map(|n| FixRecord {
11414 agent: "fixer".to_owned(),
11415 addressed: Vec::new(),
11416 rejected: Vec::new(),
11417 notes: n.to_owned(),
11418 committed: true,
11419 failed: None,
11420 duration_ms: 0,
11421 continuation: None,
11422 });
11423 r
11424 }
11425
11426 #[test]
11427 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
11428 let state = review_state(&["ignored"]);
11429 let f = facts(
11430 &[
11431 (
11432 "fix(login): resolve PATH on macOS",
11433 "Login shells skip rc files.",
11434 ),
11435 ("fix: address review", ""),
11436 ],
11437 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
11438 );
11439 let m = pr_message_with(&state, 'A', Some(&f));
11440 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
11441 assert!(
11442 m.body
11443 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
11444 "{}",
11445 m.body
11446 );
11447 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
11448 assert!(m.body.contains("## Diff stat"), "{}", m.body);
11449 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
11450 for banned in [
11451 "Review the work already",
11452 "no task statement",
11453 "Original task",
11454 ] {
11455 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
11456 }
11457 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
11458 }
11459
11460 #[test]
11461 fn pr_message_review_truncates_a_huge_first_commit_body() {
11462 let state = review_state(&["ignored"]);
11463 let f = facts(
11464 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
11465 "s",
11466 );
11467 let m = pr_message_with(&state, 'A', Some(&f));
11468 assert!(m.body.len() < 30_000, "{}", m.body.len());
11469 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
11470 assert!(
11471 m.body.contains("- ... 1 more commit(s)"),
11472 "{}",
11473 m.body.len()
11474 );
11475 assert!(m.body.ends_with("magi:candidate-a\n"));
11476 }
11477
11478 #[test]
11479 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
11480 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
11481 assert_eq!(m.title, "feat: x");
11482 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
11483 assert!(!m.body.contains("Diff stat"), "{}", m.body);
11484 assert!(!m.body.contains("no task statement"), "{}", m.body);
11485 }
11486
11487 #[test]
11488 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
11489 let mut state = review_state(&["日本語の件名"]);
11490 state.candidates[0].branch = "fix/macos-login-path".to_owned();
11491 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
11492 state.candidates[0].branch = "機能/ログイン".to_owned();
11493 assert!(
11494 pr_message(&state, 'A')
11495 .title
11496 .starts_with("chore: land candidate A")
11497 );
11498 }
11499
11500 #[test]
11501 fn pr_message_review_fixes_survive_a_clean_final_round() {
11502 let mut state = review_state(&["feat: x"]);
11503 state.reviews = vec![
11504 round_with_notes(1, Some("handled the PATH case")),
11505 round_with_notes(2, None),
11506 ];
11507 let body = pr_message(&state, 'A').body;
11508 assert!(
11509 body.contains("## Review fixes\n\nhandled the PATH case\n"),
11510 "{body}"
11511 );
11512 assert!(!body.contains("### Round"), "{body}");
11513
11514 state.reviews = vec![
11515 round_with_notes(1, Some("first fix")),
11516 round_with_notes(2, Some("")),
11517 round_with_notes(3, Some("second fix")),
11518 round_with_notes(4, None),
11519 ];
11520 let body = pr_message(&state, 'A').body;
11521 assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
11522 assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
11523 assert!(!body.contains("### Round 2"), "{body}");
11524 }
11525
11526 #[test]
11527 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
11528 let state = state_with_summary(
11529 "add retries to the client",
11530 "TITLE: feat: retries\n\nDid it.",
11531 );
11532 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
11533 assert_eq!(m.title, "feat: retries");
11534 assert!(
11535 m.body.contains("<summary>Original task</summary>"),
11536 "{}",
11537 m.body
11538 );
11539 assert!(!m.body.contains("Commits under review"), "{}", m.body);
11540 assert!(
11541 m.body
11542 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
11543 "{}",
11544 m.body
11545 );
11546 }
11547
11548 #[test]
11549 fn pr_message_review_bounds_a_long_english_subject() {
11550 let long = format!("feat: {}", "word ".repeat(100));
11551 let m = pr_message(&review_state(&[&long]), 'A');
11552 assert!(m.title.starts_with("feat: word"), "{}", m.title);
11553 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11554 }
11555
11556 #[test]
11557 fn pr_message_implementation_run_is_unchanged_by_review_support() {
11558 let state = state_with_summary("add retries\n\ndetails", "- did some things");
11559 let m = pr_message(&state, 'A');
11560 assert_eq!(m.title, "add retries");
11561 assert!(m.body.contains("<summary>Original task</summary>"));
11562 assert!(!m.body.contains("Commits under review"));
11563 assert_eq!(landing_subject_source(&state), state.instruction);
11564 }
11565
11566 #[test]
11567 fn review_run_squash_subject_is_the_change_not_the_prompt() {
11568 let state = review_state(&["feat: the change", "fix: typo"]);
11569 let source = landing_subject_source(&state);
11570 assert_eq!(land::merge_subject("", &source), "feat: the change");
11571 assert_eq!(
11572 land::merge_subject("magi: candidate A (uncommitted work)", &source),
11573 "feat: the change"
11574 );
11575 assert_eq!(
11577 land::merge_subject("feat: renamed by hand", &source),
11578 "feat: renamed by hand"
11579 );
11580 let blank = review_state(&["日本語"]);
11581 assert!(
11582 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
11583 );
11584 }
11585
11586 #[test]
11587 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
11588 let state = review_state(&["feat: the change"]);
11589 let source = landing_subject_source(&state);
11590 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
11591 assert_eq!(
11592 land::merge_subject(landing_title(&state, old), &source),
11593 "feat: the change"
11594 );
11595 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
11596 let task = state_with_summary("add retries", "");
11597 assert_eq!(landing_title(&task, old), old);
11598 }
11599
11600 #[test]
11601 fn pr_message_describes_the_change_not_the_task() {
11602 let state = state_with_summary(
11603 "今回やってほしいこと: results projector を直す",
11604 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
11605 );
11606 let m = pr_message(&state, 'A');
11607 assert_eq!(m.title, "fix(web): batch the runs list reads");
11608 assert!(
11609 m.body.starts_with("## Summary\n\n- reads run.json once"),
11610 "{}",
11611 m.body
11612 );
11613 assert!(!m.body.contains("TITLE:"), "{}", m.body);
11614 let task_at = m.body.find("今回やってほしいこと").unwrap();
11615 let details_at = m.body.find("<details>").unwrap();
11616 assert!(
11617 details_at < task_at,
11618 "the task lives inside <details>: {}",
11619 m.body
11620 );
11621 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
11622 assert!(m.body.contains("magi:candidate-a"));
11623 }
11624
11625 #[test]
11626 fn pr_message_falls_back_to_the_task_without_a_title_line() {
11627 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
11628 let m = pr_message(&state, 'A');
11629 assert_eq!(m.title, "add retries");
11630 assert!(
11631 m.body.contains("## Summary\n\n- did some things"),
11632 "{}",
11633 m.body
11634 );
11635
11636 let none = RunState::new(
11637 PathBuf::from("/repo"),
11638 "main".to_owned(),
11639 "abc1234".to_owned(),
11640 "add retries".to_owned(),
11641 Config::default(),
11642 );
11643 let m = pr_message(&none, 'A');
11644 assert_eq!(m.title, "add retries");
11645 assert!(!m.body.contains("## Summary"), "{}", m.body);
11646 }
11647
11648 #[test]
11649 fn pr_message_refuses_the_candidate_commit_subject() {
11650 for bad in [
11651 "TITLE: magi: candidate A (uncommitted work)",
11652 "TITLE: chore: stuff (uncommitted work)",
11653 "TITLE: ",
11654 ] {
11655 let state = state_with_summary("add retries", bad);
11656 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
11657 }
11658 }
11659
11660 #[test]
11661 fn pr_message_bounds_a_very_long_task_and_title() {
11662 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
11663 let state = state_with_summary(&long, "- nothing");
11664 let m = pr_message(&state, 'A');
11665 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11666 assert!(!m.title.contains('\n'));
11667
11668 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
11669 let m = pr_message(&state, 'A');
11670 assert!(m.title.starts_with("feat: "));
11671 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11672 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
11673 }
11674
11675 fn long_title_of(instruction: &str) -> String {
11676 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
11677 }
11678
11679 #[test]
11680 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
11681 let first = "Make the landing path keep a readable title for long tasks";
11682 let line = format!(
11683 "{first}. {}",
11684 "And then keep going with more words ".repeat(20)
11685 );
11686 let t = long_title_of(&line);
11687 assert_eq!(t, first);
11688 assert!(!t.starts_with("chore: land"));
11689 }
11690
11691 #[test]
11692 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
11693 let line = "word ".repeat(200);
11694 let t = long_title_of(&line);
11695 assert!(t.ends_with("word..."), "{t}");
11696 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
11697 }
11698
11699 #[test]
11700 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
11701 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
11702 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
11703 }
11704 }
11705
11706 #[test]
11707 fn pr_message_title_limit_is_exact() {
11708 let at = "a".repeat(PR_TITLE_MAX);
11709 assert_eq!(long_title_of(&at), at);
11710 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
11711 assert!(over.ends_with("..."), "{over}");
11712 assert_eq!(over.chars().count(), PR_TITLE_MAX);
11713 }
11714
11715 #[test]
11716 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
11717 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
11718 let t = long_title_of(&line);
11719 assert!(t.ends_with("..."), "{t}");
11720 assert!(t.is_ascii(), "{t}");
11721 }
11722
11723 #[test]
11724 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
11725 let line = format!(
11726 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
11727 "plus more filler words ".repeat(20)
11728 );
11729 let t = long_title_of(&line);
11730 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
11731 }
11732
11733 #[test]
11734 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
11735 let line = format!("\n\n# {}", "title words ".repeat(40));
11736 let state = state_with_summary(&line, "- nothing");
11737 let m = pr_message(&state, 'A');
11738 assert!(m.title.starts_with("title words"), "{}", m.title);
11739 assert_eq!(
11740 land::merge_subject(&m.title, &landing_subject_source(&state)),
11741 m.title
11742 );
11743 assert_eq!(
11745 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
11746 "feat: renamed by hand"
11747 );
11748 }
11749
11750 #[tokio::test]
11751 async fn github_text_rewrite_is_bounded_and_falls_back() {
11752 crate::run::pin_test_home();
11753 for (reply, accepted) in [
11754 (
11755 "TITLE: fix: retries\nAdd retries for failed requests.",
11756 true,
11757 ),
11758 (
11759 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
11760 false,
11761 ),
11762 ("TITLE: fix: retries\nUse token=secret", false),
11763 ] {
11764 let dir = tempfile::tempdir().unwrap();
11765 let mut runner = runner_at(RunStatus::Gating);
11766 runner.state = state_with_summary(
11767 "add retries",
11768 "TITLE: fix: retries\n日本語の説明を書きます。",
11769 );
11770 runner.state.repo = dir.path().to_owned();
11771 runner.state.candidates[0].worktree = dir.path().to_owned();
11772 let mut author = spec("alpha");
11773 author.command = vec![
11774 "sh".into(),
11775 "-c".into(),
11776 "printf '%s' \"$REWRITE_REPLY\"".into(),
11777 ];
11778 author.env.insert("REWRITE_REPLY".into(), reply.into());
11779 runner.state.config.agents = vec![author];
11780 let winner = runner.state.candidates[0].clone();
11781 let message = runner.guarded_pr_message(&winner, None, true).await;
11782 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
11783 assert_eq!(
11784 message.body.contains("Add retries for failed requests."),
11785 accepted
11786 );
11787 assert_eq!(
11788 runner.state.seats["impl-A"].turns, 1,
11789 "only one rewrite invocation"
11790 );
11791 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
11792 if !accepted {
11793 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
11794 assert!(
11795 message
11796 .body
11797 .contains(&format!("magi:run/{}", runner.state.id))
11798 );
11799 }
11800 }
11801 }
11802
11803 #[test]
11804 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
11805 let mut state = state_with_summary(
11809 "add retries",
11810 "TITLE: fix(web): batch reads\n- reads run.json once",
11811 );
11812 state.config.graph.language = "ja".to_owned();
11813 let m = pr_message(&state, 'A');
11814 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
11815 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
11816
11817 let task = "今回やってほしいこと: results projector を直す";
11820 let mut state = state_with_summary(task, "- no title line");
11821 state.config.graph.language = "ja".to_owned();
11822 let m = pr_message(&state, 'A');
11823 assert_eq!(
11824 m.title,
11825 format!("chore: land candidate A of run {}", state.id)
11826 );
11827 assert!(
11828 m.body.contains(&format!(
11829 "<summary>Original task</summary>\n\n{task}\n\n</details>"
11830 )),
11831 "{}",
11832 m.body
11833 );
11834 }
11835
11836 #[test]
11837 fn pr_message_scrubs_home_paths_and_addresses() {
11838 let state = state_with_summary(
11839 "fix it in /Users/someone/src/x",
11840 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
11841 );
11842 let m = pr_message(&state, 'A');
11843 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
11844 assert!(!m.body.contains(leak), "{}", m.body);
11845 }
11846 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
11847 }
11848
11849 #[test]
11850 fn pr_message_survives_a_task_that_closes_details() {
11851 let state = state_with_summary("a </details> b", "TITLE: fix: x");
11852 let m = pr_message(&state, 'A');
11853 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
11854 }
11855
11856 #[test]
11857 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
11858 let cmd = manual_merge_command(
11859 MergeStyle::Squash,
11860 Path::new("/repo"),
11861 "b",
11862 "fix: \"quoted\" $(x) `y`\n\nbody",
11863 );
11864 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
11865 }
11866
11867 #[test]
11868 fn manual_merge_command_matches_the_configured_style() {
11869 let repo = Path::new("/repo");
11870 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
11871
11872 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
11873 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
11874
11875 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
11876 assert_eq!(
11877 squash,
11878 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
11879 \"Merge magi run 0832 (candidate A)\""
11880 );
11881
11882 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
11883 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
11884 }
11885
11886 #[test]
11887 fn a_nudge_gets_a_quarter_of_the_budget() {
11888 assert_eq!(retry_budget(secs(1200), true), secs(300));
11890 assert_eq!(retry_budget(secs(3600), true), secs(900));
11891 }
11892
11893 #[test]
11894 fn a_resent_prompt_keeps_the_whole_budget() {
11895 assert_eq!(retry_budget(secs(1200), false), secs(1200));
11898 assert_eq!(retry_budget(secs(60), false), secs(60));
11899 }
11900
11901 #[test]
11902 fn the_floor_never_exceeds_the_original_budget() {
11903 assert_eq!(retry_budget(secs(60), true), secs(60));
11907 assert_eq!(retry_budget(secs(480), true), secs(120));
11908 assert_eq!(retry_budget(secs(0), true), secs(0));
11909 }
11910
11911 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
11912 agent::CommandEvidence {
11913 id: "item1".to_owned(),
11914 description: "cargo test".to_owned(),
11915 exit_code,
11916 result_summary: String::new(),
11917 source: "codex".to_owned(),
11918 }
11919 }
11920
11921 #[test]
11922 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
11923 assert!(!has_unconfirmed_command(&[]));
11927 }
11928
11929 #[test]
11930 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
11931 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
11935 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
11936 assert!(!has_unconfirmed_command(&[
11937 evidence(Some(0)),
11938 evidence(Some(101))
11939 ]));
11940 }
11941
11942 #[test]
11943 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
11944 assert!(has_unconfirmed_command(&[
11945 evidence(Some(0)),
11946 evidence(None)
11947 ]));
11948 }
11949
11950 #[test]
11951 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
11952 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11953 assert_eq!(
11954 verified_noop_claim(true, &[], text).as_deref(),
11955 Some("already fixed by b32cfc4, on main.")
11956 );
11957 }
11958
11959 #[test]
11960 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
11961 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11964 assert!(verified_noop_claim(false, &[], text).is_none());
11965 }
11966
11967 #[test]
11968 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
11969 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11970 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
11971 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
11973 }
11974
11975 #[test]
11976 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
11977 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
11978 }
11979
11980 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
11983 runner.state.candidates = shape
11984 .iter()
11985 .enumerate()
11986 .map(|(i, &(empty, verified))| Candidate {
11987 index: i,
11988 label: (b'A' + i as u8) as char,
11989 agent: "sonnet".to_owned(),
11990 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
11991 worktree: PathBuf::from(format!("/wt/{i}")),
11992 summary: String::new(),
11993 stat: String::new(),
11994 files: 0,
11995 commits: 0,
11996 empty,
11997 failed: None,
11998 verified_noop: verified.map(str::to_owned),
11999 duration_ms: 0,
12000 folded: false,
12001 })
12002 .collect();
12003 }
12004
12005 #[test]
12006 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
12007 ask_test_home();
12008 let mut runner = runner_at(RunStatus::Implementing);
12009 set_candidates(
12010 &mut runner,
12011 &[
12012 (true, Some("already on main at b32cfc4")),
12013 (true, Some("same fix, see the existing test")),
12014 ],
12015 );
12016
12017 runner
12018 .after_implement()
12019 .expect("a verified no-op is not an error");
12020
12021 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
12022 }
12023
12024 #[test]
12025 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
12026 ask_test_home();
12027 let mut runner = runner_at(RunStatus::Implementing);
12028 set_candidates(
12032 &mut runner,
12033 &[(true, Some("already on main at b32cfc4")), (true, None)],
12034 );
12035
12036 let err = runner
12037 .after_implement()
12038 .expect_err("an unverified empty candidate must still fail the run");
12039
12040 assert!(
12041 err.to_string().contains("no candidate produced a change"),
12042 "{err}"
12043 );
12044 assert_eq!(runner.state.status, RunStatus::Failed);
12045 }
12046
12047 #[test]
12048 fn after_implement_still_fails_an_ordinary_all_empty_run() {
12049 ask_test_home();
12050 let mut runner = runner_at(RunStatus::Implementing);
12051 set_candidates(&mut runner, &[(true, None), (true, None)]);
12052
12053 let err = runner
12054 .after_implement()
12055 .expect_err("no candidate declared anything; this is an ordinary failure");
12056
12057 assert!(
12058 err.to_string().contains("no candidate produced a change"),
12059 "{err}"
12060 );
12061 assert_eq!(runner.state.status, RunStatus::Failed);
12062 }
12063}