1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::run::{
45 BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
46 MergeOutcome, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus,
47 Tally, VoteRecord, tail, write_artifact,
48};
49use crate::verdict::{
50 self, FinalVote, FixReport, Position, Proposal, Ranking, Review, ReviewRevote, ReviewVote,
51 Severity,
52};
53
54const OUTPUT_TAIL: usize = 8_000;
56
57const EVENT_OUTPUT_TAIL: usize = 2_000;
60
61pub(crate) const STAGNANT_LIMIT: usize = 2;
75
76const BASE_SYNC_ROUNDS: usize = 4;
89
90#[derive(Clone)]
96struct SeatJob {
97 spec: AgentSpec,
98 seat: SeatState,
99 cwd: PathBuf,
100 prompt: String,
101 timeout: Duration,
102 allow_write: bool,
103 sessions: bool,
104 artifacts: PathBuf,
105 stem: String,
106}
107
108enum AgentOutcome {
120 Ok(AgentOutput),
122 Quota(AgentOutput),
124 Dropped(AgentOutput),
127 Failed(String),
129}
130
131#[derive(Debug, Clone, Default)]
158pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
159
160impl Pause {
161 #[must_use]
163 pub fn new() -> Self {
164 Self::default()
165 }
166
167 pub fn park(&self) {
169 self.0.store(true, Ordering::SeqCst);
170 }
171
172 pub fn park_because(&self, reason: impl Into<String>) {
178 let mut reason_guard = self
179 .1
180 .lock()
181 .unwrap_or_else(std::sync::PoisonError::into_inner);
182 if reason_guard.is_none() {
183 *reason_guard = Some(reason.into());
184 }
185 drop(reason_guard);
186 self.park();
187 }
188
189 #[must_use]
191 pub fn parked(&self) -> bool {
192 self.0.load(Ordering::SeqCst)
193 }
194
195 #[must_use]
197 pub fn reason(&self) -> Option<String> {
198 self.1
199 .lock()
200 .unwrap_or_else(std::sync::PoisonError::into_inner)
201 .clone()
202 }
203}
204
205pub struct Runner {
207 pub state: RunState,
209 roles: ResolvedRoles,
210 sem: Arc<Semaphore>,
211 pause: Pause,
215 interrupt: Pause,
221}
222
223async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
243 let tracking = format!("{remote}/{base_branch}");
244 let fetched = git::fetch(repo, remote, base_branch).await;
245 if let Ok(out) = &fetched
246 && out.ok()
247 && git::rev_exists(repo, &tracking).await
248 {
249 return git::rev_parse(repo, &tracking).await;
250 }
251 let why = match &fetched {
252 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
253 Ok(_) => format!("{remote} has no {base_branch}"),
254 Err(e) => e.to_string(),
255 };
256 tracing::warn!(
257 "could not read {tracking} ({why}); branching off the local \
258 {base_branch} instead, which may be behind"
259 );
260 git::rev_parse(repo, base_branch).await.with_context(|| {
261 format!(
262 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
263 branch that exists"
264 )
265 })
266}
267
268impl Runner {
269 pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
271 let repo = git::toplevel(repo).await?;
272 let missing = agent::missing_programs(&config.agents);
273 if !missing.is_empty() {
274 bail!(
275 "these agent programs are not on PATH: {}. Fix the roster in \
276 magi.toml or install them.",
277 missing.join(", ")
278 );
279 }
280 let base_branch = match config.merge.base.clone() {
281 Some(b) => b,
282 None => git::current_branch(&repo)
283 .await?
284 .context("HEAD is detached; set [merge] base in magi.toml")?,
285 };
286 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
287 if !git::is_clean(&repo).await? {
291 tracing::warn!(
292 "{} has uncommitted changes; they are not part of this run, \
293 which branches off {base_branch} ({})",
294 repo.display(),
295 &base_commit[..base_commit.len().min(8)]
296 );
297 }
298 let roles = config.resolve_roles()?;
299 let max_parallel = config.graph.max_parallel.max(1);
300 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
301 state.event("start", format!("run {} created", state.id));
302 state.save()?;
303 Ok(Self {
304 state,
305 roles,
306 sem: Arc::new(Semaphore::new(max_parallel)),
307 pause: Pause::new(),
308 interrupt: Pause::new(),
309 })
310 }
311
312 pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
326 let repo = git::toplevel(repo).await?;
327 let missing = agent::missing_programs(&config.agents);
328 if !missing.is_empty() {
329 bail!(
330 "these agent programs are not on PATH: {}. Fix the roster in \
331 magi.toml or install them.",
332 missing.join(", ")
333 );
334 }
335 if !git::branch_exists(&repo, branch).await? {
336 bail!("no branch `{branch}` in {}", repo.display());
337 }
338 let base_branch = match config.merge.base.clone() {
339 Some(b) => b,
340 None => git::current_branch(&repo)
341 .await?
342 .context("HEAD is detached; set [merge] base in magi.toml")?,
343 };
344 if base_branch == branch {
345 bail!("`{branch}` is the base branch; there is nothing to review against");
346 }
347 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
348
349 let roles = config.resolve_roles()?;
350 let max_parallel = config.graph.max_parallel.max(1);
351 let log = git::log_oneline(&repo, &base_commit, branch)
354 .await
355 .unwrap_or_default();
356 let instruction = format!(
357 "Review the work already on branch `{branch}`. There is no task \
358 statement: what the change claims to do is whatever its commits \
359 say.\n\n{}",
360 if log.trim().is_empty() {
361 "(no commit messages)"
362 } else {
363 log.trim()
364 }
365 );
366 let mut state = RunState::new(
367 repo.clone(),
368 base_branch,
369 base_commit.clone(),
370 instruction,
371 config,
372 );
373
374 let worktree = state.worktree_root().join("under-review");
377 if let Some(parent) = worktree.parent() {
378 tokio::fs::create_dir_all(parent).await.ok();
379 }
380 let path = worktree.to_string_lossy().to_string();
381 git::git(&repo, &["worktree", "add", &path, branch])
382 .await
383 .with_context(|| {
384 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
385 })?;
386
387 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
388 .await
389 .unwrap_or(0);
390 if commits == 0 {
391 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
392 }
393 let files = git::changed_files(&worktree, &base_commit, "HEAD")
394 .await
395 .map(|f| f.len())
396 .unwrap_or(0);
397 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
398 .await
399 .unwrap_or_default();
400
401 state.candidates.push(Candidate {
402 index: 0,
403 label: 'A',
404 agent: "(existing branch)".to_owned(),
407 branch: branch.to_owned(),
408 worktree,
409 summary: String::new(),
410 stat,
411 files,
412 commits,
413 empty: false,
414 failed: None,
415 duration_ms: 0,
416 folded: false,
417 });
418 state.tally = Some(Tally {
419 first_choice: BTreeMap::from([('A', 0)]),
420 borda: BTreeMap::new(),
421 winner: 'A',
422 rankings: 0,
423 unanimous_initial: false,
424 deliberated: false,
425 changed_votes: 0,
426 unanimous_final: false,
427 tie_break: None,
428 judges: 0,
432 present: 0,
433 quorum: 0,
434 met_quorum: true,
435 uncontested: Some("review-only run: nothing competed".to_owned()),
436 });
437 state.status = RunStatus::Reviewing;
438 state.event(
439 "start",
440 format!(
441 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
442 state.id
443 ),
444 );
445 state.save()?;
446 Ok(Self {
447 state,
448 roles,
449 sem: Arc::new(Semaphore::new(max_parallel)),
450 pause: Pause::new(),
451 interrupt: Pause::new(),
452 })
453 }
454
455 pub fn resume(id: &str) -> Result<Self> {
457 let state = RunState::load(id)?;
458 let roles = state.config.resolve_roles()?;
459 let max_parallel = state.config.graph.max_parallel.max(1);
460 Ok(Self {
461 state,
462 roles,
463 sem: Arc::new(Semaphore::new(max_parallel)),
464 pause: Pause::new(),
465 interrupt: Pause::new(),
466 })
467 }
468
469 pub async fn execute(&mut self) -> Result<()> {
471 self.state.parked = false;
476 if self.state.clear_active() {
483 self.state.save()?;
484 }
485 if self.state.status == RunStatus::Stalled {
498 if self.recover_stall().await? {
499 self.finish_after_tally().await?;
500 } else {
501 self.state.save()?;
503 }
504 return Ok(());
505 }
506 if self.state.status == RunStatus::Landing {
516 self.run_land().await?;
517 self.settle_questions();
522 return Ok(());
523 }
524 self.prep().await?;
525 if self.park_here()? {
526 return Ok(());
527 }
528 self.advise().await?;
529 if self.park_here()? {
530 return Ok(());
531 }
532 self.implement().await?;
533 if self.park_here()? {
534 return Ok(());
535 }
536 self.judge().await?;
537 if self.park_here()? {
538 return Ok(());
539 }
540 self.deliberate().await?;
541 if self.park_here()? {
542 return Ok(());
543 }
544 self.vote().await?;
545 if self.park_here()? {
546 return Ok(());
547 }
548 self.tally()?;
549 if self.state.status == RunStatus::Stalled {
554 self.state.save()?;
558 return Ok(());
559 }
560 self.finish_after_tally().await?;
561 Ok(())
562 }
563
564 fn park_here(&mut self) -> Result<bool> {
571 if !self.pause.parked() && !self.interrupt.parked() {
576 return Ok(false);
577 }
578 let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
579 Some(reason) => format!(
580 "parked after `{}` ({reason}) — resume to carry on from here",
581 self.state.status.as_str()
582 ),
583 None => format!(
584 "parked after `{}` — resume to carry on from here",
585 self.state.status.as_str()
586 ),
587 };
588 self.state.event("park", why);
589 self.state.parked = true;
590 self.state.save()?;
591 Ok(true)
592 }
593
594 pub fn on_pause(&mut self, pause: Pause) {
596 self.pause = pause;
597 }
598
599 pub fn watch_interrupt(&mut self, pause: Pause) {
605 self.interrupt = pause;
606 }
607
608 fn settle_questions(&mut self) {
626 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
627 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
628 }
629 }
630
631 async fn finish_after_tally(&mut self) -> Result<()> {
634 self.fold_losers().await?;
635 self.sync_to_base().await?;
640 self.review_loop().await?;
641 self.sync_to_base().await?;
642 self.gate().await?;
643 self.merge().await?;
644 self.state.save()?;
645 Ok(())
646 }
647
648 async fn prep(&mut self) -> Result<()> {
651 if !self.state.candidates.is_empty() {
652 return Ok(());
653 }
654 self.state.status = RunStatus::Prep;
655 let repo = self.state.repo.clone();
656 let base = self.state.base_commit.clone();
657 let root = self.state.worktree_root();
658 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
659
660 let hooks_dir = self.state.dir().join("hooks");
663 if self.state.config.blind.commit_msg_hook {
664 std::fs::create_dir_all(&hooks_dir)
665 .with_context(|| format!("create {}", hooks_dir.display()))?;
666 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
667 let path = hooks_dir.join("commit-msg");
668 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
669 make_executable(&path)?;
670 git::acquire_worktree_config(&repo).await?;
678 self.state.enabled_worktree_config = true;
679 }
680
681 for (index, (spec, label)) in self
682 .roles
683 .implementers
684 .clone()
685 .into_iter()
686 .zip(labels)
687 .enumerate()
688 {
689 let branch = self.state.branch_for(label);
690 let worktree = root.join(format!("cand-{label}"));
691 git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
692 if self.state.config.blind.commit_msg_hook {
693 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
694 }
695 git::local_exclude(&worktree, "/.magi/").await?;
696 self.state.candidates.push(Candidate {
697 index,
698 label,
699 agent: spec.id.clone(),
700 branch,
701 worktree,
702 summary: String::new(),
703 stat: String::new(),
704 files: 0,
705 commits: 0,
706 empty: false,
707 failed: None,
708 duration_ms: 0,
709 folded: false,
710 });
711 }
712
713 for j in 1..=self.roles.judges.len() {
714 let wt = root.join(format!("judge-{j}"));
715 if !wt.exists() {
716 git::worktree_add_detached(&repo, &wt, &base).await?;
717 }
718 }
719
720 if self.state.config.graph.advise {
728 for k in 1..=self.state.config.graph.advisors {
729 let wt = root.join(format!("advisor-{k}"));
730 if !wt.exists() {
731 git::worktree_add_detached(&repo, &wt, &base).await?;
732 }
733 }
734 }
735
736 let authors: Vec<&str> = self
741 .roles
742 .implementers
743 .iter()
744 .map(|a| a.id.as_str())
745 .collect();
746 let overlap: Vec<String> = self
747 .roles
748 .judges
749 .iter()
750 .enumerate()
751 .filter(|(_, j)| authors.contains(&j.id.as_str()))
752 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
753 .collect();
754 if !overlap.is_empty() {
755 let note = format!(
756 "{} also authored a candidate; blind, but the panel is less \
757 independent than {} distinct agents would be",
758 overlap.join(", "),
759 self.roles.judges.len()
760 );
761 self.state.event("prep", note);
762 }
763
764 self.state.event(
765 "prep",
766 format!(
767 "{} candidates, {} judges, base {} ({})",
768 self.state.candidates.len(),
769 self.roles.judges.len(),
770 &self.state.base_commit[..7.min(self.state.base_commit.len())],
771 self.state.base_branch
772 ),
773 );
774 self.state.status = RunStatus::Implementing;
775 self.state.save()?;
776 Ok(())
777 }
778
779 async fn advise(&mut self) -> Result<()> {
812 let implement_untouched = self
813 .state
814 .candidates
815 .iter()
816 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
817 if !self.state.config.graph.advise || self.state.advise_attempted {
818 return Ok(());
819 }
820 if !implement_untouched {
821 self.state.event(
822 "advise",
823 "skipping the design-deliberation stage: at least one \
824 candidate already shows implementation progress, so this \
825 run is past the point the stage exists to run before"
826 .to_owned(),
827 );
828 self.state.advise_attempted = true;
829 self.state.save()?;
830 return Ok(());
831 }
832 let run_id = self.state.id.clone();
833 let prompts = self.state.config.prompts.clone();
834 let instruction = self.state.instruction.clone();
835 let language = self.state.config.graph.language.clone();
836 let root = self.state.worktree_root();
837 let n = self.state.config.graph.advisors;
838 let where_recorded = self.state.dir().join("run.json");
839
840 let seats = match self.state.config.advisors() {
841 Ok(seats) if !seats.is_empty() => seats,
842 Ok(_) => {
843 self.state.event(
844 "advise",
845 format!(
846 "[graph] advisors is 0; skipping the design-deliberation \
847 stage and continuing without a synthesis brief (see {})",
848 where_recorded.display()
849 ),
850 );
851 self.state.advise_attempted = true;
852 self.state.save()?;
853 return Ok(());
854 }
855 Err(e) => {
856 self.state.event(
857 "advise",
858 format!(
859 "could not resolve advisor seats ({e:#}); continuing \
860 without a design-deliberation brief (see {})",
861 where_recorded.display()
862 ),
863 );
864 self.state.advise_attempted = true;
865 self.state.save()?;
866 return Ok(());
867 }
868 };
869
870 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
871 let artifacts = agent::artifacts_dir(&self.state.dir());
872 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
873
874 let mut jobs = Vec::new();
875 for (i, spec) in seats.iter().cloned().enumerate() {
876 let seat_key = format!("advisor-{}", i + 1);
877 let seat = self.seat(&seat_key, &spec.id);
878 jobs.push(SeatJob {
879 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
880 spec,
881 seat,
882 cwd: worktrees[i % worktrees.len()].clone(),
883 timeout,
884 allow_write: false,
885 sessions: false,
886 artifacts: artifacts.clone(),
887 stem: seat_key,
888 });
889 }
890
891 self.state.event(
892 "advise",
893 format!(
894 "{} advisor seat(s) sketching a design in parallel",
895 jobs.len()
896 ),
897 );
898 let mut quota_losses = Vec::new();
899 let cache = self.state.config.cache_dir();
900 let ctx = WaveCtx {
901 run: &run_id,
902 node: "advise",
903 prompts: &prompts,
904 cache: cache.as_deref(),
905 };
906 let results = ask_json_wave::<Proposal>(
907 jobs,
908 Arc::clone(&self.sem),
909 self.state.config.graph.retries,
910 &ctx,
911 &mut quota_losses,
912 &mut self.state,
913 &|p: &Proposal| p.validate(),
914 )
915 .await;
916 self.state.quota.extend(quota_losses);
917
918 let mut records = Vec::with_capacity(results.len());
919 for (i, (seat, res)) in results.into_iter().enumerate() {
920 let agent_id = seat.agent.clone();
921 self.state.seats.insert(seat.key.clone(), seat);
922 match res {
923 Ok((proposal, out)) => {
924 self.state
925 .event("advise", format!("advisor-{} proposed a design", i + 1));
926 records.push(advise::AdvisorRecord::proposed(
927 i + 1,
928 agent_id,
929 proposal,
930 out.duration_ms,
931 ));
932 }
933 Err(e) => {
934 self.state.event(
935 "advise",
936 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
937 );
938 records.push(advise::AdvisorRecord::failed(
939 i + 1,
940 agent_id,
941 e.to_string(),
942 ));
943 }
944 }
945 }
946
947 let mut advice = advise::Advice {
948 records,
949 synthesis: None,
950 };
951 if advice.proposals().is_empty() {
952 self.state.event(
953 "advise",
954 "no advisor produced a usable proposal; continuing without a \
955 synthesis brief"
956 .to_owned(),
957 );
958 } else {
959 match self
960 .synthesize_brief(
961 &advice,
962 &instruction,
963 &language,
964 &worktrees[0],
965 &artifacts,
966 &run_id,
967 &prompts,
968 cache.as_deref(),
969 )
970 .await
971 {
972 Ok(Some(text)) => {
973 self.state.event(
974 "advise",
975 "synthesized a design brief for the implementer".to_owned(),
976 );
977 advice.synthesis = Some(text);
978 }
979 Ok(None) => {
980 self.state.event(
981 "advise",
982 "the synthesis seat produced nothing usable; continuing \
983 without a design brief"
984 .to_owned(),
985 );
986 }
987 Err(e) => {
988 self.state.event(
989 "advise",
990 format!("could not synthesize a design brief: {e:#}"),
991 );
992 }
993 }
994 }
995 advise::apply_reflection(&mut advice);
996
997 self.state.advice = Some(advice);
998 self.state.advise_attempted = true;
999 self.state.save()?;
1000 Ok(())
1001 }
1002
1003 #[allow(clippy::too_many_arguments)]
1014 async fn synthesize_brief(
1015 &mut self,
1016 advice: &advise::Advice,
1017 instruction: &str,
1018 language: &str,
1019 cwd: &Path,
1020 artifacts: &Path,
1021 run_id: &str,
1022 prompts: &Prompts,
1023 cache: Option<&Path>,
1024 ) -> Result<Option<String>> {
1025 let spec = agent::pick(&self.state.config.agents, None, &agent::installed)?;
1026 let mut seat = self.seat("advise-synthesis", &spec.id);
1027 let proposals = advice.proposals();
1028 let mut prompt = prompt::with_overlay(
1029 prompt::synthesize_brief(instruction, &proposals, language),
1030 prompts.overlay("advise"),
1031 );
1032 if cache.is_some() {
1033 prompt.push('\n');
1034 prompt.push_str(&prompt::build_cache_note("advise"));
1035 }
1036 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1037 let out = agent::invoke(
1038 &spec,
1039 &mut seat,
1040 &Invocation {
1041 cwd,
1042 prompt: &prompt,
1043 timeout,
1044 allow_write: false,
1045 sessions: false,
1046 artifacts,
1047 stem: "advise-synthesis",
1048 run: run_id,
1049 node: "advise",
1050 cache_dir: cache,
1051 attachments: &[],
1052 },
1053 )
1054 .await?;
1055 self.state.seats.insert(seat.key.clone(), seat);
1056 if !out.usable() {
1057 return Ok(None);
1058 }
1059 let text =
1060 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1061 Ok((!text.trim().is_empty()).then_some(text))
1062 }
1063
1064 async fn implement(&mut self) -> Result<()> {
1067 let run_id = self.state.id.clone();
1072 let prompts = self.state.config.prompts.clone();
1073 let todo: Vec<usize> = self
1074 .state
1075 .candidates
1076 .iter()
1077 .enumerate()
1078 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1079 .map(|(i, _)| i)
1080 .collect();
1081 if todo.is_empty() {
1082 return self.after_implement();
1083 }
1084 self.state.status = RunStatus::Implementing;
1085
1086 let language = self.state.config.graph.language.clone();
1087 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1088 let sessions = self.state.config.graph.sessions;
1089 let artifacts = agent::artifacts_dir(&self.state.dir());
1090 let brief = self
1094 .state
1095 .advice
1096 .as_ref()
1097 .and_then(|a| a.synthesis.as_deref())
1098 .map(str::to_owned);
1099
1100 let mut jobs = Vec::new();
1101 for &i in &todo {
1102 let (index, label, worktree) = {
1103 let c = &self.state.candidates[i];
1104 (c.index, c.label, c.worktree.clone())
1105 };
1106 let spec = self.roles.implementers[index].clone();
1107 let seat_key = format!("impl-{label}");
1108 let seat = self.seat(&seat_key, &spec.id);
1109 let instruction = self.state.instruction.clone();
1110 jobs.push(SeatJob {
1111 spec,
1112 seat,
1113 prompt: prompt::implement(
1114 &instruction,
1115 &worktree.to_string_lossy(),
1116 &language,
1117 brief.as_deref(),
1118 ),
1119 cwd: worktree,
1120 timeout,
1121 allow_write: true,
1122 sessions,
1123 artifacts: artifacts.clone(),
1124 stem: format!("impl-{label}"),
1125 });
1126 }
1127
1128 self.state.event(
1129 "implement",
1130 format!("{} candidates in parallel", jobs.len()),
1131 );
1132 let sent = jobs.clone();
1135 let cache = self.state.config.cache_dir();
1136 let ctx = WaveCtx {
1137 run: &run_id,
1138 node: "implement",
1139 prompts: &prompts,
1140 cache: cache.as_deref(),
1141 };
1142 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1143 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1144 .await;
1145
1146 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1147 let seat_key = seat.key.clone();
1148 self.state.seats.insert(seat.key.clone(), seat);
1149 let label = self.state.candidates[i].label;
1150 let worktree = self.state.candidates[i].worktree.clone();
1151 let base = self.state.base_commit.clone();
1152
1153 let (summary, duration, failed) = match out {
1154 AgentOutcome::Ok(o) => {
1155 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1156 let failed = (!o.usable()).then(|| {
1157 if o.timed_out {
1158 "agent timed out".to_owned()
1159 } else {
1160 format!("agent exited with {:?}", o.exit_code)
1161 }
1162 });
1163 (text, o.duration_ms, failed)
1164 }
1165 AgentOutcome::Dropped(o) => {
1171 let why = o
1172 .dropped
1173 .as_ref()
1174 .map(|d| d.why.as_str())
1175 .unwrap_or("the CLI ended the stream without delivering its answer");
1176 (
1177 String::new(),
1178 o.duration_ms,
1179 Some(format!("the CLI dropped the stream ({why})")),
1180 )
1181 }
1182 AgentOutcome::Quota(o) => {
1183 self.state.quota.push(QuotaLoss {
1184 seat: seat_key,
1185 node: "implement".to_owned(),
1186 at: Timestamp::now(),
1187 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1188 });
1189 (
1190 String::new(),
1191 o.duration_ms,
1192 Some("rate limited (quota); produced no change".to_owned()),
1193 )
1194 }
1195 AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
1196 };
1197
1198 let rescued = git::commit_all(
1201 &worktree,
1202 &format!("magi: candidate {label} (uncommitted work)"),
1203 )
1204 .await
1205 .unwrap_or(false);
1206 let commits = git::commits_ahead(&worktree, &base, "HEAD")
1207 .await
1208 .unwrap_or(0);
1209 let patch = git::diff(&worktree, &base, "HEAD")
1210 .await
1211 .unwrap_or_default();
1212 let stat = git::diff_stat(&worktree, &base, "HEAD")
1213 .await
1214 .unwrap_or_default();
1215 let files = git::changed_files(&worktree, &base, "HEAD")
1216 .await
1217 .map(|f| f.len())
1218 .unwrap_or(0);
1219 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1220
1221 let c = &mut self.state.candidates[i];
1222 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1223 c.stat = stat;
1224 c.files = files;
1225 c.commits = commits;
1226 c.duration_ms = duration;
1227 c.empty = commits == 0 || patch.trim().is_empty();
1228 c.failed = match failed {
1231 Some(_) if c.empty => failed,
1232 _ => None,
1233 };
1234 let note = match (&c.failed, c.empty, rescued) {
1235 (Some(e), _, _) => format!("candidate {label}: {e}"),
1236 (None, true, _) => format!("candidate {label}: no change produced"),
1237 (None, false, true) => {
1238 format!(
1239 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1240 )
1241 }
1242 (None, false, false) => {
1243 format!("candidate {label}: {files} files, {commits} commits")
1244 }
1245 };
1246 self.state.event("implement", note);
1247 self.state.save()?;
1248 }
1249
1250 self.after_implement()
1251 }
1252
1253 async fn resume_undelivered(
1281 &mut self,
1282 results: &mut [(usize, SeatState, AgentOutcome)],
1283 sent: &[SeatJob],
1284 prompts: &Prompts,
1285 run_id: &str,
1286 ) {
1287 for (wi, seat, out) in results.iter_mut() {
1288 let Some(dropped) = (match &*out {
1289 AgentOutcome::Dropped(o) => o.dropped.clone(),
1290 _ => None,
1291 }) else {
1292 continue;
1293 };
1294 let Some(job) = sent.get(*wi) else { continue };
1295 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1297 self.state.event(
1298 "implement",
1299 format!(
1300 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1301 work is in the tree",
1302 seat.key, dropped.output_tokens, dropped.why
1303 ),
1304 );
1305 continue;
1306 }
1307 if !has_context(&job.spec, seat, job.sessions) {
1315 self.state.event(
1316 "implement",
1317 format!(
1318 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1319 is no session left to resume",
1320 seat.key, dropped.output_tokens, dropped.why
1321 ),
1322 );
1323 continue;
1324 }
1325 self.state.event(
1326 "implement",
1327 format!(
1328 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1329 conversation",
1330 seat.key, dropped.output_tokens, dropped.why
1331 ),
1332 );
1333 let mut retry = job.clone();
1334 retry.seat = seat.clone();
1335 retry.prompt = prompt::resume_after_drop(&dropped.why);
1336 retry.timeout = retry_budget(job.timeout, true);
1337 retry.stem = format!("{}-resume", job.stem);
1338 let cache = self.state.config.cache_dir();
1339 let ctx = WaveCtx {
1340 run: run_id,
1341 node: "implement",
1342 prompts,
1343 cache: cache.as_deref(),
1344 };
1345 let (resumed_seat, resumed) =
1346 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1347 *seat = resumed_seat;
1348 *out = resumed;
1349 }
1350 }
1351
1352 fn after_implement(&mut self) -> Result<()> {
1353 if self.state.leaks.is_empty() {
1355 let cfg = self.state.config.blind.clone();
1356 let mut leaks = Vec::new();
1357 for c in &self.state.candidates {
1358 let Some(patch) =
1359 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1360 else {
1361 continue;
1362 };
1363 leaks.extend(blind::scan(
1364 &format!("candidate {} patch", c.label),
1365 &patch,
1366 &cfg.vendor_tokens,
1367 ));
1368 }
1369 if !leaks.is_empty() {
1370 let summary = leaks
1371 .iter()
1372 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
1373 .collect::<Vec<_>>()
1374 .join(", ");
1375 match cfg.on_leak {
1376 LeakPolicy::Fail => {
1377 self.state.status = RunStatus::Failed;
1378 self.state
1379 .event("blind", format!("vendor text in a patch: {summary}"));
1380 self.state.leaks = leaks;
1381 self.state.save()?;
1382 self.settle_questions();
1383 bail!(
1384 "blind.on_leak = \"fail\" and vendor text reached a \
1385 judged patch: {summary}"
1386 );
1387 }
1388 LeakPolicy::Redact => self.state.event(
1389 "blind",
1390 format!("redacting vendor text for judging: {summary}"),
1391 ),
1392 LeakPolicy::Warn => self.state.event(
1393 "blind",
1394 format!("vendor text present in a judged patch (shown as-is): {summary}"),
1395 ),
1396 }
1397 self.state.leaks = leaks;
1398 }
1399 }
1400
1401 if self.state.viable().is_empty() {
1402 self.state.status = RunStatus::Failed;
1403 self.state.save()?;
1404 self.settle_questions();
1405 bail!("no candidate produced a change; nothing to judge");
1406 }
1407 self.state.status = RunStatus::Judging;
1408 self.state.save()?;
1409 Ok(())
1410 }
1411
1412 async fn judge(&mut self) -> Result<()> {
1415 let run_id = self.state.id.clone();
1420 let prompts = self.state.config.prompts.clone();
1421 if !self.state.judgements.is_empty() || self.state.judge_skipped {
1422 return Ok(());
1423 }
1424 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1425 if viable.len() == 1 {
1426 self.state.judge_skipped = true;
1433 self.state.event(
1434 "judge",
1435 format!(
1436 "only candidate {} produced a change; judging skipped",
1437 viable[0].label
1438 ),
1439 );
1440 self.state.save()?;
1441 return Ok(());
1442 }
1443 self.state.status = RunStatus::Judging;
1444
1445 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1446 let language = self.state.config.graph.language.clone();
1447 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1448 let sessions = self.state.config.graph.sessions;
1449 let artifacts = agent::artifacts_dir(&self.state.dir());
1450 let root = self.state.worktree_root();
1451 let base_short = short(&self.state.base_commit);
1452
1453 let mut jobs = Vec::new();
1454 let mut orders = Vec::new();
1455 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1456 let order = blind::presentation_order(viable.len(), j, self.state.seed);
1457 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1458 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
1459 let seat_key = format!("judge-{}", j + 1);
1460 let seat = self.seat(&seat_key, &spec.id);
1461 jobs.push(SeatJob {
1462 prompt: prompt::judge(
1463 &self.state.instruction,
1464 &views,
1465 self.roles.judges.len(),
1466 &base_short,
1467 &language,
1468 ),
1469 spec,
1470 seat,
1471 cwd: root.join(format!("judge-{}", j + 1)),
1472 timeout,
1473 allow_write: false,
1474 sessions,
1475 artifacts: artifacts.clone(),
1476 stem: format!("judge-{}", j + 1),
1477 });
1478 }
1479
1480 self.state.event(
1481 "judge",
1482 format!(
1483 "{} judges ranking {} candidates blind",
1484 jobs.len(),
1485 viable.len()
1486 ),
1487 );
1488 let labels_for_check = labels.clone();
1489 let mut quota_losses = Vec::new();
1490 let cache = self.state.config.cache_dir();
1491 let ctx = WaveCtx {
1492 run: &run_id,
1493 node: "judge",
1494 prompts: &prompts,
1495 cache: cache.as_deref(),
1496 };
1497 let results = ask_json_wave::<Ranking>(
1498 jobs,
1499 Arc::clone(&self.sem),
1500 self.state.config.graph.retries,
1501 &ctx,
1502 &mut quota_losses,
1503 &mut self.state,
1504 &move |r: &Ranking| r.validate(&labels_for_check),
1505 )
1506 .await;
1507 self.state.quota.extend(quota_losses);
1508
1509 for (j, (seat, res)) in results.into_iter().enumerate() {
1510 let agent_id = seat.agent.clone();
1511 self.state.seats.insert(seat.key.clone(), seat);
1512 let mut record = Judgement {
1513 judge: j + 1,
1514 seat: format!("judge-{}", j + 1),
1515 agent: agent_id,
1516 ranking: Vec::new(),
1517 reasons: BTreeMap::new(),
1518 confidence: None,
1519 order: orders[j].clone(),
1520 failed: None,
1521 duration_ms: 0,
1522 };
1523 match res {
1524 Ok((ranking, out)) => {
1525 record.ranking = ranking.normalized();
1526 record.reasons = ranking.reasons;
1527 record.confidence = ranking.confidence;
1528 record.duration_ms = out.duration_ms;
1529 self.state.event(
1530 "judge",
1531 format!(
1532 "judge {} ranked {}",
1533 j + 1,
1534 record.ranking.iter().collect::<String>()
1535 ),
1536 );
1537 }
1538 Err(e) => {
1539 record.failed = Some(e.to_string());
1540 self.state
1541 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1542 }
1543 }
1544 self.state.judgements.push(record);
1545 self.state.save()?;
1546 }
1547 Ok(())
1548 }
1549
1550 async fn deliberate(&mut self) -> Result<()> {
1553 let run_id = self.state.id.clone();
1558 let prompts = self.state.config.prompts.clone();
1559 if !self.state.deliberation.is_empty() {
1560 return Ok(());
1561 }
1562 let tops: Vec<char> = self
1563 .state
1564 .judgements
1565 .iter()
1566 .filter_map(|j| j.ranking.first().copied())
1567 .collect();
1568 let rounds = self.state.config.graph.deliberate_rounds;
1569 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1570 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1571 self.state.event(
1572 "deliberate",
1573 format!("judges agreed on {} outright; no deliberation", tops[0]),
1574 );
1575 }
1576 self.state.status = RunStatus::Voting;
1577 self.state.save()?;
1578 return Ok(());
1579 }
1580
1581 self.state.status = RunStatus::Deliberating;
1582 self.state.event(
1583 "deliberate",
1584 format!(
1585 "split: first choices were {} — opening {rounds} round(s)",
1586 tops.iter().collect::<String>()
1587 ),
1588 );
1589
1590 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1591 let language = self.state.config.graph.language.clone();
1592 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1593 let sessions = self.state.config.graph.sessions;
1594 let artifacts = agent::artifacts_dir(&self.state.dir());
1595 let root = self.state.worktree_root();
1596 let base_short = short(&self.state.base_commit);
1597
1598 for round in 1..=rounds {
1602 let mut turns: Vec<DeliberationTurn> = Vec::new();
1603 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1604 if self.state.judgements[j].failed.is_some() {
1605 continue;
1606 }
1607 let seat_key = format!("judge-{}", j + 1);
1608 let mut seat = self.seat(&seat_key, &spec.id);
1609 let transcript = self.transcript(&turns, j);
1610 let context = if has_context(&spec, &seat, sessions) {
1611 None
1612 } else {
1613 Some(self.candidate_block(&viable, &base_short))
1614 };
1615 let text = prompt::deliberate(
1616 &self.state.instruction,
1617 context.as_deref(),
1618 &transcript,
1619 round,
1620 rounds,
1621 &language,
1622 );
1623 let job = SeatJob {
1624 spec,
1625 seat: seat.clone(),
1626 prompt: text,
1627 cwd: root.join(format!("judge-{}", j + 1)),
1628 timeout,
1629 allow_write: false,
1630 sessions,
1631 artifacts: artifacts.clone(),
1632 stem: format!("delib-{round}-judge-{}", j + 1),
1633 };
1634 let cache = self.state.config.cache_dir();
1635 let ctx = WaveCtx {
1636 run: &run_id,
1637 node: "deliberate",
1638 prompts: &prompts,
1639 cache: cache.as_deref(),
1640 };
1641 let (updated, out) =
1642 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1643 seat = updated;
1644 let agent_id = seat.agent.clone();
1645 let seat_key = seat.key.clone();
1646 self.state.seats.insert(seat.key.clone(), seat);
1647 let body = match out {
1648 AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1649 AgentOutcome::Dropped(o) => {
1653 let why =
1654 o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1655 "the CLI ended the stream without delivering its answer",
1656 );
1657 self.state.event(
1658 "deliberate",
1659 format!(
1660 "judge {} skipped: the CLI dropped the stream ({why})",
1661 j + 1
1662 ),
1663 );
1664 continue;
1665 }
1666 AgentOutcome::Quota(o) => {
1667 self.state.quota.push(QuotaLoss {
1668 seat: seat_key,
1669 node: "deliberate".to_owned(),
1670 at: Timestamp::now(),
1671 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1672 });
1673 self.state.event(
1674 "deliberate",
1675 format!("judge {} skipped: rate limited (quota)", j + 1),
1676 );
1677 continue;
1678 }
1679 AgentOutcome::Failed(e) => {
1680 self.state
1681 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1682 continue;
1683 }
1684 };
1685 let tentative = verdict::extract_json::<Position>(&body)
1686 .ok()
1687 .and_then(|p| p.tentative)
1688 .and_then(|s| s.trim().chars().next())
1689 .map(|c| c.to_ascii_uppercase());
1690 self.state.event(
1691 "deliberate",
1692 format!(
1693 "round {round}: judge {} now favours {}",
1694 j + 1,
1695 tentative.map_or("—".to_owned(), |c| c.to_string())
1696 ),
1697 );
1698 turns.push(DeliberationTurn {
1699 judge: j + 1,
1700 agent: agent_id,
1701 body: blind::sanitize_prose(&body, &self.state.config.blind),
1702 tentative,
1703 });
1704 }
1705 self.state
1706 .deliberation
1707 .push(DeliberationRound { round, turns });
1708 self.state.save()?;
1709 }
1710
1711 self.state.status = RunStatus::Voting;
1712 self.state.save()?;
1713 Ok(())
1714 }
1715
1716 async fn vote(&mut self) -> Result<()> {
1719 let run_id = self.state.id.clone();
1724 let prompts = self.state.config.prompts.clone();
1725 if !self.state.votes.is_empty() {
1726 return Ok(());
1727 }
1728 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1729 if viable.len() == 1 {
1730 return Ok(());
1731 }
1732 self.state.status = RunStatus::Voting;
1733
1734 let language = self.state.config.graph.language.clone();
1735 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1736 let sessions = self.state.config.graph.sessions;
1737 let artifacts = agent::artifacts_dir(&self.state.dir());
1738 let root = self.state.worktree_root();
1739 let base_short = short(&self.state.base_commit);
1740 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1741
1742 let mut jobs = Vec::new();
1743 let mut seats_at = Vec::new();
1744 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1745 if self
1746 .state
1747 .judgements
1748 .get(j)
1749 .is_some_and(|r| r.failed.is_some())
1750 {
1751 continue;
1752 }
1753 let seat_key = format!("judge-{}", j + 1);
1754 let seat = self.seat(&seat_key, &spec.id);
1755 let mut text = prompt::final_vote(&viable, &language);
1756 if !has_context(&spec, &seat, sessions) {
1757 text = format!(
1758 "{}\n\n# Candidates\n\n{}",
1759 text,
1760 self.candidate_block(&candidates, &base_short)
1761 );
1762 }
1763 jobs.push(SeatJob {
1764 spec,
1765 seat,
1766 prompt: text,
1767 cwd: root.join(format!("judge-{}", j + 1)),
1768 timeout,
1769 allow_write: false,
1770 sessions,
1771 artifacts: artifacts.clone(),
1772 stem: format!("vote-judge-{}", j + 1),
1773 });
1774 seats_at.push(j);
1775 }
1776
1777 self.state.event(
1778 "vote",
1779 format!(
1780 "collecting {} final votes one by one, privately",
1781 jobs.len()
1782 ),
1783 );
1784 let allowed = viable.clone();
1785 let mut quota_losses = Vec::new();
1786 let cache = self.state.config.cache_dir();
1787 let ctx = WaveCtx {
1788 run: &run_id,
1789 node: "vote",
1790 prompts: &prompts,
1791 cache: cache.as_deref(),
1792 };
1793 let results = ask_json_wave::<FinalVote>(
1794 jobs,
1795 Arc::clone(&self.sem),
1796 self.state.config.graph.retries,
1797 &ctx,
1798 &mut quota_losses,
1799 &mut self.state,
1800 &move |v: &FinalVote| match v.label() {
1801 Some(c) if allowed.contains(&c) => Ok(()),
1802 other => bail!("vote {other:?} is not one of {allowed:?}"),
1803 },
1804 )
1805 .await;
1806 self.state.quota.extend(quota_losses);
1807
1808 for (&j, (seat, res)) in seats_at.iter().zip(results) {
1809 let agent_id = seat.agent.clone();
1810 self.state.seats.insert(seat.key.clone(), seat);
1811 let initial = self
1812 .state
1813 .judgements
1814 .get(j)
1815 .and_then(|r| r.ranking.first().copied());
1816 let mut record = VoteRecord {
1817 judge: j + 1,
1818 agent: agent_id,
1819 vote: None,
1820 reason: String::new(),
1821 changed: false,
1822 };
1823 match res {
1824 Ok((v, _)) => {
1825 record.vote = v.label();
1826 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1827 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1828 self.state.event(
1829 "vote",
1830 format!(
1831 "judge {} voted {}{}",
1832 j + 1,
1833 record.vote.unwrap_or('?'),
1834 if record.changed { " (changed)" } else { "" }
1835 ),
1836 );
1837 }
1838 Err(e) => {
1839 self.state
1840 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1841 }
1842 }
1843 self.state.votes.push(record);
1844 self.state.save()?;
1845 }
1846 Ok(())
1847 }
1848
1849 fn tally(&mut self) -> Result<()> {
1852 if self.state.tally.is_some() {
1853 return Ok(());
1854 }
1855 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1856 let tops: Vec<char> = self
1857 .state
1858 .judgements
1859 .iter()
1860 .filter_map(|j| j.ranking.first().copied())
1861 .collect();
1862 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1863
1864 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1867 let mut cast: Vec<char> = Vec::new();
1868 for (i, j) in self.state.judgements.iter().enumerate() {
1869 let vote = self
1870 .state
1871 .votes
1872 .iter()
1873 .find(|v| v.judge == i + 1)
1874 .and_then(|v| v.vote)
1875 .or_else(|| j.ranking.first().copied());
1876 if let Some(v) = vote {
1877 *first_choice.entry(v).or_insert(0) += 1;
1878 cast.push(v);
1879 }
1880 }
1881
1882 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1883 for j in &self.state.judgements {
1884 let n = j.ranking.len();
1885 for (pos, label) in j.ranking.iter().enumerate() {
1886 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1887 }
1888 }
1889
1890 let best = first_choice.values().copied().max().unwrap_or(0);
1891 let mut leaders: Vec<char> = first_choice
1892 .iter()
1893 .filter(|(_, v)| **v == best)
1894 .map(|(k, _)| *k)
1895 .collect();
1896 let mut tie_break = None;
1897 if leaders.len() > 1 {
1898 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1899 let borda_leaders: Vec<char> = leaders
1900 .iter()
1901 .copied()
1902 .filter(|l| borda[l] == top_borda)
1903 .collect();
1904 tie_break = Some(if borda_leaders.len() == 1 {
1905 format!(
1906 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1907 leaders.len()
1908 )
1909 } else {
1910 format!(
1911 "{} way tie on both first-choice votes and Borda points, broken by label order",
1912 leaders.len()
1913 )
1914 });
1915 leaders = borda_leaders;
1916 leaders.sort_unstable();
1917 }
1918 let winner = *leaders
1919 .first()
1920 .or(viable.first())
1921 .context("no candidate to declare a winner from")?;
1922
1923 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1924 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1925 let deliberated = !self.state.deliberation.is_empty();
1926
1927 let quota_seats: std::collections::BTreeSet<&str> =
1931 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1932 let mut present = 0usize;
1933 for (i, j) in self.state.judgements.iter().enumerate() {
1934 if quota_seats.contains(j.seat.as_str()) {
1935 continue;
1936 }
1937 let ranked = !j.ranking.is_empty() && j.failed.is_none();
1938 let voted = self
1939 .state
1940 .votes
1941 .iter()
1942 .any(|v| v.judge == i + 1 && v.vote.is_some());
1943 if ranked || voted {
1944 present += 1;
1945 }
1946 }
1947 let needs_quorum = viable.len() > 1;
1953 let judges_total = if needs_quorum {
1954 self.roles.judges.len()
1955 } else {
1956 0
1957 };
1958 let quorum = if needs_quorum {
1959 judges_total / 2 + 1
1960 } else {
1961 0
1962 };
1963 let met_quorum = !needs_quorum || present >= quorum;
1964 let uncontested = (!needs_quorum).then(|| {
1965 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1966 });
1967
1968 self.state.event(
1969 "tally",
1970 match &uncontested {
1971 Some(reason) => format!("winner {winner} — {reason}"),
1972 None => format!(
1973 "winner {winner} — votes {} | initial {} | {} changed | \
1974 {present}/{judges_total} judges{}",
1975 first_choice
1976 .iter()
1977 .map(|(k, v)| format!("{k}:{v}"))
1978 .collect::<Vec<_>>()
1979 .join(" "),
1980 if unanimous_initial {
1981 "unanimous"
1982 } else {
1983 "split"
1984 },
1985 changed_votes,
1986 if met_quorum {
1987 String::new()
1988 } else {
1989 format!(" — below quorum ({quorum} required)")
1990 },
1991 ),
1992 },
1993 );
1994 if !met_quorum {
1995 self.state.event(
1996 "stall",
1997 format!(
1998 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1999 the run stops here, resumable"
2000 ),
2001 );
2002 }
2003 self.state.tally = Some(Tally {
2004 first_choice,
2005 borda,
2006 winner,
2007 rankings: tops.len(),
2008 unanimous_initial,
2009 deliberated,
2010 changed_votes,
2011 unanimous_final,
2012 tie_break,
2013 judges: judges_total,
2014 present,
2015 quorum,
2016 met_quorum,
2017 uncontested,
2018 });
2019 self.state.status = if met_quorum {
2020 RunStatus::Reviewing
2021 } else {
2022 RunStatus::Stalled
2023 };
2024 self.state.save()?;
2025 Ok(())
2026 }
2027
2028 #[allow(clippy::too_many_lines)]
2049 async fn recover_stall(&mut self) -> Result<bool> {
2050 let run_id = self.state.id.clone();
2055 let prompts = self.state.config.prompts.clone();
2056 let quota_seats: BTreeSet<&str> =
2061 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2062 let absent: Vec<String> = self
2063 .state
2064 .judgements
2065 .iter()
2066 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2067 .map(|j| j.seat.clone())
2068 .collect();
2069 if absent.is_empty() {
2070 return Ok(false);
2071 }
2072 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2073 if viable.len() <= 1 {
2074 return Ok(false);
2075 }
2076 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2077 let language = self.state.config.graph.language.clone();
2078 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2079 let sessions = self.state.config.graph.sessions;
2080 let artifacts = agent::artifacts_dir(&self.state.dir());
2081 let root = self.state.worktree_root();
2082 let base_short = short(&self.state.base_commit);
2083 let candidates: Vec<Candidate> = viable.clone();
2084
2085 let mut positions: Vec<usize> = absent
2087 .iter()
2088 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2089 .collect();
2090 if positions.is_empty() {
2091 return Ok(false);
2092 }
2093 positions.sort_unstable();
2094 positions.dedup();
2095
2096 let mut judge_jobs = Vec::new();
2098 for &j in &positions {
2099 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2100 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2101 let seat_key = format!("judge-{}", j + 1);
2102 let spec = self.roles.judges[j].clone();
2103 let seat = self.seat(&seat_key, &spec.id);
2104 judge_jobs.push(SeatJob {
2105 spec,
2106 seat,
2107 prompt: prompt::judge(
2108 &self.state.instruction,
2109 &views,
2110 self.roles.judges.len(),
2111 &base_short,
2112 &language,
2113 ),
2114 cwd: root.join(seat_key),
2115 timeout,
2116 allow_write: false,
2117 sessions,
2118 artifacts: artifacts.clone(),
2119 stem: format!("judge-{}-recover", j + 1),
2120 });
2121 }
2122
2123 let labels_for_check = labels.clone();
2124 let mut judge_losses = Vec::new();
2125 let retries = self.state.config.graph.retries;
2126 let cache = self.state.config.cache_dir();
2127 let ctx = WaveCtx {
2128 run: &run_id,
2129 node: "judge",
2130 prompts: &prompts,
2131 cache: cache.as_deref(),
2132 };
2133 let results = ask_json_wave::<Ranking>(
2134 judge_jobs,
2135 Arc::clone(&self.sem),
2136 retries,
2137 &ctx,
2138 &mut judge_losses,
2139 &mut self.state,
2140 &move |r: &Ranking| r.validate(&labels_for_check),
2141 )
2142 .await;
2143
2144 let mut recovered: BTreeSet<usize> = BTreeSet::new();
2146 for (&j, (seat, res)) in positions.iter().zip(results) {
2147 self.state.seats.insert(seat.key.clone(), seat);
2148 let record = &mut self.state.judgements[j];
2149 match res {
2150 Ok((ranking, out)) => {
2151 record.ranking = ranking.normalized();
2152 record.reasons = ranking.reasons;
2153 record.confidence = ranking.confidence;
2154 record.failed = None;
2155 record.duration_ms = out.duration_ms;
2156 recovered.insert(j);
2157 self.state.event(
2158 "recover",
2159 format!("judge {} ranked again after the limit", j + 1),
2160 );
2161 }
2162 Err(e) => {
2163 self.state
2164 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2165 }
2166 }
2167 }
2168
2169 let mut vote_jobs = Vec::new();
2171 let mut vote_pos: Vec<usize> = Vec::new();
2172 for &j in &recovered {
2173 let seat_key = format!("judge-{}", j + 1);
2174 let spec = self.roles.judges[j].clone();
2175 let seat = self.seat(&seat_key, &spec.id);
2176 let mut text = prompt::final_vote(&labels, &language);
2177 if !has_context(&spec, &seat, sessions) {
2178 text = format!(
2179 "{}\n\n# Candidates\n\n{}",
2180 text,
2181 self.candidate_block(&candidates, &base_short)
2182 );
2183 }
2184 vote_jobs.push(SeatJob {
2185 spec,
2186 seat,
2187 prompt: text,
2188 cwd: root.join(seat_key),
2189 timeout,
2190 allow_write: false,
2191 sessions,
2192 artifacts: artifacts.clone(),
2193 stem: format!("vote-judge-{}-recover", j + 1),
2194 });
2195 vote_pos.push(j);
2196 }
2197 let allowed = labels.clone();
2198 let mut vote_losses = Vec::new();
2199 let vote_retries = self.state.config.graph.retries;
2200 let vote_cache = self.state.config.cache_dir();
2201 let ctx = WaveCtx {
2202 run: &run_id,
2203 node: "vote",
2204 prompts: &prompts,
2205 cache: vote_cache.as_deref(),
2206 };
2207 let votes = ask_json_wave::<FinalVote>(
2208 vote_jobs,
2209 Arc::clone(&self.sem),
2210 vote_retries,
2211 &ctx,
2212 &mut vote_losses,
2213 &mut self.state,
2214 &move |v: &FinalVote| match v.label() {
2215 Some(c) if allowed.contains(&c) => Ok(()),
2216 other => bail!("vote {other:?} is not one of {allowed:?}"),
2217 },
2218 )
2219 .await;
2220 for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
2221 let agent_id = seat.agent.clone();
2222 self.state.seats.insert(seat.key.clone(), seat);
2223 match res {
2224 Ok((v, _)) => {
2225 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2226 rec.vote = v.label();
2227 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2228 } else {
2229 self.state.votes.push(VoteRecord {
2230 judge: j + 1,
2231 agent: agent_id,
2232 vote: v.label(),
2233 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2234 changed: false,
2235 });
2236 }
2237 self.state.event(
2238 "recover",
2239 format!("judge {} voted again after the limit", j + 1),
2240 );
2241 }
2242 Err(e) => {
2243 self.state
2244 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2245 }
2246 }
2247 }
2248
2249 if !recovered.is_empty() {
2253 let recovered_keys: BTreeSet<String> = recovered
2254 .iter()
2255 .map(|&j| format!("judge-{}", j + 1))
2256 .collect();
2257 self.state
2258 .quota
2259 .retain(|q| !recovered_keys.contains(&q.seat));
2260 }
2261
2262 self.state.tally = None;
2264 self.tally()?;
2265 Ok(self
2266 .state
2267 .tally
2268 .as_ref()
2269 .map(|t| t.met_quorum)
2270 .unwrap_or(false))
2271 }
2272
2273 async fn fold_losers(&mut self) -> Result<()> {
2276 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2277 return Ok(());
2278 };
2279 let repo = self.state.repo.clone();
2280 let mut folded = Vec::new();
2281 for i in 0..self.state.candidates.len() {
2282 let c = &self.state.candidates[i];
2283 if c.label == winner || c.folded {
2284 continue;
2285 }
2286 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2287 git::worktree_remove(&repo, &wt).await.ok();
2288 git::branch_delete(&repo, &branch).await.ok();
2289 self.state.candidates[i].folded = true;
2290 folded.push(label.to_string());
2291 }
2292 let root = self.state.worktree_root();
2294 for j in 1..=self.roles.judges.len() {
2295 let wt = root.join(format!("judge-{j}"));
2296 if wt.exists() {
2297 git::worktree_remove(&repo, &wt).await.ok();
2298 }
2299 }
2300 if self.state.config.graph.advise {
2303 for k in 1..=self.state.config.graph.advisors {
2304 let wt = root.join(format!("advisor-{k}"));
2305 if wt.exists() {
2306 git::worktree_remove(&repo, &wt).await.ok();
2307 }
2308 }
2309 }
2310 if !folded.is_empty() {
2311 self.state
2312 .event("fold", format!("folded candidates {}", folded.join(", ")));
2313 self.state.save()?;
2314 }
2315 Ok(())
2316 }
2317
2318 async fn sync_to_base(&mut self) -> Result<()> {
2348 if self
2349 .state
2350 .base_sync
2351 .as_ref()
2352 .is_some_and(|s| s.conflict.is_some())
2353 {
2354 return Ok(());
2355 }
2356 let Some(winner) = self.state.winner().cloned() else {
2357 return Ok(());
2358 };
2359
2360 let repo = self.state.repo.clone();
2361 let remote = self.state.config.merge.remote.clone();
2362 let base_branch = self.state.base_branch.clone();
2363 let tracking = format!("{remote}/{base_branch}");
2364
2365 git::fetch(&repo, &remote, &base_branch).await.ok();
2366 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
2370 return Ok(());
2371 };
2372
2373 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2374 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
2375 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
2376
2377 if behind == 0 {
2378 self.state.base_sync = Some(BaseSync {
2379 tip,
2380 behind: 0,
2381 attempts,
2382 conflict: None,
2383 });
2384 self.state.save()?;
2385 return Ok(());
2386 }
2387
2388 if attempts >= BASE_SYNC_ROUNDS {
2389 let why = format!(
2390 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
2391 rebase(s); rebasing again would only race it",
2392 winner.branch
2393 );
2394 self.state.status = RunStatus::Blocked;
2395 self.state.base_sync = Some(BaseSync {
2396 tip,
2397 behind,
2398 attempts,
2399 conflict: Some(why.clone()),
2400 });
2401 self.state.event("land", why);
2402 self.state.save()?;
2403 return Ok(());
2404 }
2405
2406 self.state.event(
2407 "land",
2408 format!(
2409 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
2410 winner.branch
2411 ),
2412 );
2413 self.state.save()?;
2414
2415 let scratch = self.state.dir().join("base-sync");
2416 let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
2417 let attempts = attempts + 1;
2418 match rebased {
2419 Ok(None) => {
2420 git::sync_to_head(&winner.worktree).await?;
2424 self.state.base_sync = Some(BaseSync {
2425 tip: tip.clone(),
2426 behind: 0,
2427 attempts,
2428 conflict: None,
2429 });
2430 self.state
2431 .event("land", format!("rebased {} onto {tracking}", winner.branch));
2432 }
2433 Ok(Some(conflict)) => {
2434 let why = format!(
2435 "{} conflicts with {tracking} and did not rebase: {}",
2436 winner.branch,
2437 conflict.chars().take(600).collect::<String>()
2438 );
2439 self.state.status = RunStatus::Blocked;
2440 self.state.base_sync = Some(BaseSync {
2441 tip,
2442 behind,
2443 attempts,
2444 conflict: Some(why.clone()),
2445 });
2446 self.state.event("land", why);
2447 }
2448 Err(e) => {
2449 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
2450 self.state.status = RunStatus::Blocked;
2451 self.state.base_sync = Some(BaseSync {
2452 tip,
2453 behind,
2454 attempts,
2455 conflict: Some(why.clone()),
2456 });
2457 self.state.event("land", why);
2458 }
2459 }
2460 self.state.save()?;
2461 Ok(())
2462 }
2463
2464 fn landing_base(&self) -> String {
2474 self.state
2475 .base_sync
2476 .as_ref()
2477 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2478 }
2479
2480 async fn review_loop(&mut self) -> Result<()> {
2483 if self
2488 .state
2489 .base_sync
2490 .as_ref()
2491 .is_some_and(|s| s.conflict.is_some())
2492 {
2493 return Ok(());
2494 }
2495 let run_id = self.state.id.clone();
2500 let prompts = self.state.config.prompts.clone();
2501 let Some(winner) = self.state.winner().cloned() else {
2502 return Ok(());
2503 };
2504 let max_rounds = self.state.config.graph.review_rounds;
2505 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
2515 self.state.status = status;
2516 self.state.save()?;
2517 return Ok(());
2518 }
2519 self.state.status = RunStatus::Reviewing;
2520
2521 let repo = self.state.repo.clone();
2522 let root = self.state.worktree_root();
2523 let language = self.state.config.graph.language.clone();
2524 let sessions = self.state.config.graph.sessions;
2525 let artifacts = agent::artifacts_dir(&self.state.dir());
2526 let base = self.landing_base();
2527 let base_short = short(&base);
2528 let reviewers = self.roles.reviewers.clone();
2529 let shell = self.state.config.shell();
2530
2531 let mut prev_e2e: Option<String> = None;
2532 for round in (self.state.reviews.len() + 1)..=max_rounds {
2533 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2534 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2535 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2536
2537 let mut jobs = Vec::new();
2541 for (r, spec) in reviewers.iter().cloned().enumerate() {
2542 let wt = root.join(format!("review-{}", r + 1));
2543 if wt.exists() {
2544 git::reset_detached(&wt, &head).await?;
2545 } else {
2546 git::worktree_add_detached(&repo, &wt, &head).await?;
2547 }
2548 let seat_key = format!("review-{}", r + 1);
2549 let seat = self.seat(&seat_key, &spec.id);
2550 jobs.push(SeatJob {
2551 prompt: prompt::review(&prompt::ReviewCtx {
2552 instruction: &self.state.instruction,
2553 branch: &winner.branch,
2554 base_short: &base_short,
2555 stat: &stat,
2556 patch: &patch,
2557 e2e: prev_e2e.as_deref(),
2558 reviewers: reviewers.len(),
2559 round,
2560 rounds: max_rounds,
2561 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2564 lens: Lens::for_seat(r),
2565 language: &language,
2566 }),
2567 spec,
2568 seat,
2569 cwd: wt,
2570 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2571 allow_write: false,
2572 sessions,
2573 artifacts: artifacts.clone(),
2574 stem: format!("review-{round}-{}", r + 1),
2575 });
2576 }
2577
2578 self.state.event(
2579 "review",
2580 format!(
2581 "round {round}: {} reviewers on {}",
2582 jobs.len(),
2583 short(&head)
2584 ),
2585 );
2586 let mut quota_losses = Vec::new();
2587 let review_retries = self.state.config.graph.retries;
2588 let review_cache = self.state.config.cache_dir();
2589 let ctx = WaveCtx {
2590 run: &run_id,
2591 node: "review",
2592 prompts: &prompts,
2593 cache: review_cache.as_deref(),
2594 };
2595 let results = ask_json_wave::<Review>(
2596 jobs,
2597 Arc::clone(&self.sem),
2598 review_retries,
2599 &ctx,
2600 &mut quota_losses,
2601 &mut self.state,
2602 &|_: &Review| Ok(()),
2603 )
2604 .await;
2605 let round_quota_missing = quota_losses.len();
2609 self.state.quota.extend(quota_losses);
2610
2611 let mut records = Vec::new();
2612 let mut all_findings = Vec::new();
2613 for (r, (seat, res)) in results.into_iter().enumerate() {
2614 let agent_id = seat.agent.clone();
2615 self.state.seats.insert(seat.key.clone(), seat);
2616 let mut record = ReviewRecord {
2617 reviewer: r + 1,
2618 agent: agent_id,
2619 summary: String::new(),
2620 findings: Vec::new(),
2621 vote: None,
2622 failed: None,
2623 duration_ms: 0,
2624 };
2625 match res {
2626 Ok((review, out)) => {
2627 record.summary =
2635 blind::sanitize_prose(&review.summary, &self.state.config.blind);
2636 record.vote = Some(review.vote);
2637 record.duration_ms = out.duration_ms;
2638 for (n, mut f) in review.findings.into_iter().enumerate() {
2639 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2642 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
2643 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
2644 f.file = f
2650 .file
2651 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
2652 all_findings.push(f.clone());
2653 record.findings.push(f);
2654 }
2655 self.state.event(
2656 "review",
2657 format!(
2658 "round {round}: reviewer {} voted {} with {} finding(s)",
2659 r + 1,
2660 review.vote.label(),
2661 record.findings.len()
2662 ),
2663 );
2664 }
2665 Err(e) => {
2666 record.failed = Some(e.to_string());
2667 self.state.event(
2668 "review",
2669 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2670 );
2671 }
2672 }
2673 records.push(record);
2674 }
2675
2676 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
2683 let vote_split =
2684 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
2685 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
2686 if vote_split {
2687 self.state.event(
2688 "review",
2689 format!(
2690 "round {round}: votes split ({}) — one round of reconsideration",
2691 initial_votes
2692 .iter()
2693 .map(|v| v.label())
2694 .collect::<Vec<_>>()
2695 .join(", ")
2696 ),
2697 );
2698 let panel: Vec<ReviewSeatReport<'_>> = records
2701 .iter()
2702 .filter_map(|r| {
2703 r.vote.map(|vote| ReviewSeatReport {
2704 reviewer: r.reviewer,
2705 vote,
2706 summary: &r.summary,
2707 findings: &r.findings,
2708 })
2709 })
2710 .collect();
2711
2712 let mut jobs = Vec::new();
2713 let mut seats_at = Vec::new();
2714 for (r, spec) in reviewers.iter().cloned().enumerate() {
2715 if records[r].vote.is_none() {
2719 continue;
2720 }
2721 let wt = root.join(format!("review-{}", r + 1));
2722 let seat_key = format!("review-{}", r + 1);
2723 let seat = self.seat(&seat_key, &spec.id);
2724 let patch_ctx = if has_context(&spec, &seat, sessions) {
2729 None
2730 } else {
2731 Some(ReviewPatch {
2732 branch: &winner.branch,
2733 base_short: &base_short,
2734 stat: &stat,
2735 patch: &patch,
2736 })
2737 };
2738 let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
2739 instruction: &self.state.instruction,
2740 reviewer: r + 1,
2741 lens: Lens::for_seat(r),
2742 panel: &panel,
2743 patch: patch_ctx,
2744 round,
2745 rounds: max_rounds,
2746 language: &language,
2747 });
2748 jobs.push(SeatJob {
2749 prompt,
2750 spec,
2751 seat,
2752 cwd: wt,
2753 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2754 allow_write: false,
2755 sessions,
2756 artifacts: artifacts.clone(),
2757 stem: format!("review-{round}-reconsider-{}", r + 1),
2758 });
2759 seats_at.push(r);
2760 }
2761
2762 let mut recon_quota_losses = Vec::new();
2763 let recon_cache = self.state.config.cache_dir();
2764 let recon_ctx = WaveCtx {
2765 run: &run_id,
2766 node: "review",
2767 prompts: &prompts,
2768 cache: recon_cache.as_deref(),
2769 };
2770 let recon_results = ask_json_wave::<ReviewRevote>(
2771 jobs,
2772 Arc::clone(&self.sem),
2773 review_retries,
2774 &recon_ctx,
2775 &mut recon_quota_losses,
2776 &mut self.state,
2777 &|_: &ReviewRevote| Ok(()),
2778 )
2779 .await;
2780 self.state.quota.extend(recon_quota_losses);
2781
2782 for (&r, (seat, res)) in seats_at.iter().zip(recon_results) {
2783 let agent_id = seat.agent.clone();
2784 self.state.seats.insert(seat.key.clone(), seat);
2785 let mut rec = ReviewRevoteRecord {
2786 reviewer: r + 1,
2787 agent: agent_id,
2788 vote: None,
2789 reason: String::new(),
2790 failed: None,
2791 };
2792 match res {
2793 Ok((rv, _)) => {
2794 rec.vote = Some(rv.vote);
2795 rec.reason =
2796 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
2797 self.state.event(
2798 "review",
2799 format!(
2800 "round {round}: reviewer {} revoted {}",
2801 r + 1,
2802 rv.vote.label()
2803 ),
2804 );
2805 }
2806 Err(e) => {
2807 rec.failed = Some(e.to_string());
2808 self.state.event(
2809 "review",
2810 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
2811 );
2812 }
2813 }
2814 reconsideration.push(rec);
2815 }
2816 } else if initial_votes.len() > 1 {
2817 self.state.event(
2818 "review",
2819 format!(
2820 "round {round}: votes agreed ({}) — no reconsideration",
2821 initial_votes[0].label()
2822 ),
2823 );
2824 }
2825
2826 let final_votes: Vec<ReviewVote> = records
2830 .iter()
2831 .filter_map(|r| {
2832 reconsideration
2833 .iter()
2834 .find(|rv| rv.reviewer == r.reviewer)
2835 .and_then(|rv| rv.vote)
2836 .or(r.vote)
2837 })
2838 .collect();
2839 let round_verdict = ReviewVote::worst(final_votes);
2840
2841 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2842 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
2843 let defer_e2e =
2854 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
2855 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
2856 let reason =
2857 format!("{blocking} blocking finding(s) already required a fix this round");
2858 self.state.event(
2859 "verify",
2860 format!(
2861 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
2862 {}); it will run once a round has none left",
2863 short(&head)
2864 ),
2865 );
2866 (Vec::new(), false, true, Some(reason))
2867 } else {
2868 let e2e_commands = self.state.config.verify.e2e.clone();
2869 let (e2e, verify_retried) = run_e2e_with_retry(
2870 &mut self.state,
2871 &shell,
2872 &e2e_commands,
2873 &winner.worktree,
2874 verify_timeout,
2875 &format!("round {round}"),
2876 )
2877 .await;
2878 (e2e, verify_retried, false, None)
2879 };
2880
2881 let e2e_failures: String = e2e
2882 .iter()
2883 .filter(|o| !o.ok())
2884 .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2885 .collect();
2886
2887 let expected = records.len();
2888 let answered = records.iter().filter(|r| r.failed.is_none()).count();
2889 let incomplete = answered < expected;
2890 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
2891 let policy = self.state.config.graph.incomplete_review;
2892 let clean = round_is_clean(
2893 blocking,
2894 e2e_ok,
2895 answered,
2896 expected,
2897 round_quota_missing,
2898 policy,
2899 );
2900
2901 let mut round_record = ReviewRound {
2902 round,
2903 head: head.clone(),
2904 verified_head: None,
2905 reviews: records,
2906 e2e,
2907 verify_retried,
2908 e2e_deferred,
2909 e2e_defer_reason,
2910 fix: None,
2911 blocking,
2912 answered,
2913 expected,
2914 clean,
2915 progressed: false,
2916 vote_split,
2917 reconsideration,
2918 verdict: round_verdict,
2919 };
2920
2921 if incomplete {
2922 let missing: Vec<String> = round_record
2923 .reviews
2924 .iter()
2925 .filter(|r| r.failed.is_some())
2926 .map(|r| format!("review-{}", r.reviewer))
2927 .collect();
2928 self.state.event(
2929 "review",
2930 format!(
2931 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
2932 missing.join(", ")
2933 ),
2934 );
2935 }
2936
2937 if clean {
2938 self.state.event(
2939 "review",
2940 if incomplete && policy == IncompleteReviewPolicy::Warn {
2941 format!(
2942 "round {round}: clean (warn policy, incomplete panel) — no \
2943 blocking findings from the seats that answered, verification green"
2944 )
2945 } else if incomplete {
2946 format!(
2947 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
2948 quorum) — no blocking findings from the seats that answered, \
2949 verification green",
2950 expected - answered
2951 )
2952 } else {
2953 format!("round {round}: clean — no blocking findings, verification green")
2954 },
2955 );
2956 self.state.reviews.push(round_record);
2957 self.state.status = RunStatus::Gating;
2958 self.state.save()?;
2959 return Ok(());
2960 }
2961
2962 if incomplete && blocking == 0 && e2e_ok {
2970 self.state.reviews.push(round_record);
2971 self.state.save()?;
2972 if round == max_rounds {
2973 self.state.status = RunStatus::Blocked;
2974 self.state.event(
2975 "review",
2976 format!(
2977 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
2978 refusing to call it clean",
2979 expected - answered
2980 ),
2981 );
2982 return Ok(());
2983 }
2984 prev_e2e = None;
2985 continue;
2986 }
2987
2988 if round == max_rounds {
2989 self.state.reviews.push(round_record);
2990 return self
2991 .stop_reviewing(
2992 &format!(
2993 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
2994 ),
2995 &shell,
2996 &winner.worktree,
2997 )
2998 .await;
2999 }
3000
3001 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3004 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3005 _ => (
3006 self.state
3007 .config
3008 .agent(&winner.agent)
3009 .cloned()
3010 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3011 format!("impl-{}", winner.label),
3012 ),
3013 };
3014 let seat = self.seat(&fix_seat_key, &fix_spec.id);
3015 let blocking_findings: Vec<_> = all_findings
3016 .iter()
3017 .filter(|f| f.severity.blocks())
3018 .cloned()
3019 .collect();
3020 let job = SeatJob {
3021 prompt: prompt::fix(
3022 &self.state.instruction,
3023 &blocking_findings,
3024 (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
3025 e2e_deferred,
3026 round,
3027 max_rounds,
3028 &language,
3029 ),
3030 spec: fix_spec.clone(),
3031 seat,
3032 cwd: winner.worktree.clone(),
3033 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3034 allow_write: true,
3035 sessions,
3036 artifacts: artifacts.clone(),
3037 stem: format!("fix-{round}"),
3038 };
3039 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
3040 let cache = self.state.config.cache_dir();
3041 let ctx = WaveCtx {
3042 run: &run_id,
3043 node: "fix",
3044 prompts: &prompts,
3045 cache: cache.as_deref(),
3046 };
3047 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3048 let agent_id = seat.agent.clone();
3049 let seat_key = seat.key.clone();
3050 self.state.seats.insert(seat.key.clone(), seat);
3051
3052 let mut fix = FixRecord {
3053 agent: agent_id,
3054 addressed: Vec::new(),
3055 rejected: Vec::new(),
3056 notes: String::new(),
3057 committed: false,
3058 failed: None,
3059 duration_ms: 0,
3060 };
3061 match out {
3062 AgentOutcome::Ok(o) => {
3063 fix.duration_ms = o.duration_ms;
3064 match verdict::extract_json::<FixReport>(&o.text) {
3065 Ok(report) => {
3066 fix.addressed = report.addressed;
3067 fix.rejected = report.rejected;
3068 fix.notes =
3069 blind::sanitize_prose(&report.notes, &self.state.config.blind);
3070 }
3071 Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
3072 }
3073 }
3074 AgentOutcome::Dropped(o) => {
3076 fix.duration_ms = o.duration_ms;
3077 let why = o
3078 .dropped
3079 .as_ref()
3080 .map(|d| d.why.as_str())
3081 .unwrap_or("the CLI ended the stream without delivering its answer");
3082 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3083 }
3084 AgentOutcome::Quota(o) => {
3085 self.state.quota.push(QuotaLoss {
3086 seat: seat_key,
3087 node: "fix".to_owned(),
3088 at: Timestamp::now(),
3089 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3090 });
3091 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3092 }
3093 AgentOutcome::Failed(e) => fix.failed = Some(e),
3094 }
3095 git::commit_all(
3096 &winner.worktree,
3097 &format!("magi: review round {round} fixes (uncommitted work)"),
3098 )
3099 .await
3100 .ok();
3101 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
3102 fix.committed = after != before;
3103 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
3111 let progressed = diff_after != patch;
3112 let commit_note = if fix.committed {
3113 "committed"
3114 } else {
3115 "NO new commit"
3116 };
3117 let tree_note = if progressed {
3118 "changed vs base"
3119 } else {
3120 "unchanged vs base"
3121 };
3122 self.state.event(
3123 "fix",
3124 match &fix.failed {
3125 Some(reason) => {
3131 format!(
3132 "round {round}: fixer's adoption report was lost ({reason}); \
3133 {commit_note}, tree {tree_note}"
3134 )
3135 }
3136 None => format!(
3137 "round {round}: {} addressed, {} rejected, {commit_note}, tree {tree_note}",
3138 fix.addressed.len(),
3139 fix.rejected.len(),
3140 ),
3141 },
3142 );
3143 round_record.fix = Some(fix);
3144 round_record.progressed = progressed;
3145 self.state.reviews.push(round_record);
3146 self.state.save()?;
3147
3148 prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
3149
3150 let streak = self
3151 .state
3152 .reviews
3153 .iter()
3154 .rev()
3155 .take_while(|r| !r.progressed)
3156 .count();
3157 if streak >= STAGNANT_LIMIT {
3158 return self
3159 .stop_reviewing(
3160 &format!(
3161 "the tree has not moved against base for {streak} round(s) in a row"
3162 ),
3163 &shell,
3164 &winner.worktree,
3165 )
3166 .await;
3167 }
3168 }
3169 Ok(())
3170 }
3171
3172 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
3196 let round_idx = self.state.reviews.len() - 1;
3197 let needs_catchup_run = {
3198 let last = &self.state.reviews[round_idx];
3199 last.e2e.is_empty() && last.e2e_deferred
3200 };
3201 if needs_catchup_run {
3202 let round = self.state.reviews[round_idx].round;
3203 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3204 let commands = self.state.config.verify.e2e.clone();
3205 let verified_head = git::rev_parse(worktree, "HEAD").await?;
3206 let (outcomes, verify_retried) = run_e2e_with_retry(
3207 &mut self.state,
3208 shell,
3209 &commands,
3210 worktree,
3211 timeout,
3212 &format!("round {round}: deferred e2e, now catching up before the final decision"),
3213 )
3214 .await;
3215 let last = &mut self.state.reviews[round_idx];
3216 last.e2e = outcomes;
3217 last.verify_retried = verify_retried;
3218 last.e2e_deferred = false;
3219 if verified_head != last.head {
3220 last.verified_head = Some(verified_head);
3221 }
3222 }
3223 let last = &self.state.reviews[round_idx];
3224 let red: Vec<String> = last
3225 .e2e
3226 .iter()
3227 .filter(|o| !o.ok())
3228 .map(|o| {
3229 format!(
3230 "`{}` -> {:?}\n{}",
3231 o.command,
3232 o.code,
3233 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3234 )
3235 })
3236 .collect();
3237 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
3238
3239 if red.is_empty() {
3240 self.state.event(
3241 "review",
3242 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
3243 );
3244 self.state.status = RunStatus::Gating;
3245 } else {
3246 self.state
3247 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
3248 self.state.status = RunStatus::Blocked;
3249 }
3250 self.state.save()?;
3251 Ok(())
3252 }
3253
3254 async fn gate(&mut self) -> Result<()> {
3257 if self.state.status == RunStatus::Failed
3269 || self
3270 .state
3271 .base_sync
3272 .as_ref()
3273 .is_some_and(|s| s.conflict.is_some())
3274 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3275 != Some(RunStatus::Gating)
3276 {
3277 return Ok(());
3278 }
3279 if !self.state.gate.is_empty() {
3280 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
3288 self.state.status = RunStatus::Blocked;
3289 self.state.save()?;
3290 }
3291 return Ok(());
3292 }
3293 let Some(winner) = self.state.winner().cloned() else {
3294 return Ok(());
3295 };
3296 self.state.status = RunStatus::Gating;
3297 let shell = self.state.config.shell();
3298 let outcomes = run_commands(
3299 &shell,
3300 &self.state.config.verify.gate,
3301 &winner.worktree,
3302 Duration::from_secs(self.state.config.graph.verify_timeout()),
3303 )
3304 .await;
3305 for o in &outcomes {
3306 self.state.event(
3307 "gate",
3308 format!(
3309 "`{}` -> {}",
3310 o.command,
3311 if o.ok() {
3312 "pass".to_owned()
3313 } else {
3314 format!(
3315 "FAIL ({:?})\n{}",
3316 o.code,
3317 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3318 )
3319 }
3320 ),
3321 );
3322 }
3323 let passed = outcomes.iter().all(CommandOutcome::ok);
3324 self.state.gate = outcomes;
3325 if !passed {
3326 self.state.status = RunStatus::Blocked;
3327 self.state.event("gate", "gate failed; not merging");
3328 }
3329 self.state.save()?;
3330 Ok(())
3331 }
3332
3333 async fn merge(&mut self) -> Result<()> {
3336 if self
3351 .state
3352 .base_sync
3353 .as_ref()
3354 .is_some_and(|s| s.conflict.is_some())
3355 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3356 != Some(RunStatus::Gating)
3357 || self.state.gate.iter().any(|o| !o.ok())
3358 {
3359 return Ok(());
3360 }
3361 if self.state.merge.is_some() {
3370 return Ok(());
3371 }
3372 let Some(winner) = self.state.winner().cloned() else {
3373 return Ok(());
3374 };
3375 let repo = self.state.repo.clone();
3376 let base = self.state.base_branch.clone();
3377 let mode = self.state.config.merge.mode;
3378 let style = self.state.config.merge.style;
3379 let message = pr_body(&self.state, winner.label);
3380
3381 let outcome = match mode {
3382 MergeMode::None => MergeOutcome {
3383 mode,
3384 ok: true,
3385 detail: manual_merge_command(style, &repo, &winner.branch, &message),
3386 },
3387 MergeMode::Local => {
3388 let on = git::current_branch(&repo).await?;
3389 if on.as_deref() != Some(base.as_str()) {
3390 MergeOutcome {
3391 mode,
3392 ok: false,
3393 detail: format!(
3394 "{} has {} checked out, not the base branch {base}",
3395 repo.display(),
3396 on.unwrap_or_else(|| "a detached HEAD".to_owned())
3397 ),
3398 }
3399 } else if !git::is_clean(&repo).await? {
3400 MergeOutcome {
3401 mode,
3402 ok: false,
3403 detail: format!("{} is dirty; refusing to merge", repo.display()),
3404 }
3405 } else {
3406 let out = match style {
3407 MergeStyle::Merge => {
3408 git::merge_no_ff(&repo, &winner.branch, &message).await?
3409 }
3410 MergeStyle::Squash => {
3411 git::merge_squash(&repo, &winner.branch, &message).await?
3412 }
3413 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
3414 };
3415 MergeOutcome {
3416 mode,
3417 ok: out.ok(),
3418 detail: if out.ok() { out.stdout } else { out.stderr },
3419 }
3420 }
3421 }
3422 MergeMode::Pr => {
3423 let remote = self.state.config.merge.remote.clone();
3424 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
3425 if !pushed.ok() {
3426 MergeOutcome {
3427 mode,
3428 ok: false,
3429 detail: pushed.stderr,
3430 }
3431 } else {
3432 let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
3433 match out {
3434 Ok(url) => MergeOutcome {
3435 mode,
3436 ok: true,
3437 detail: url,
3438 },
3439 Err(e) => MergeOutcome {
3440 mode,
3441 ok: false,
3442 detail: e.to_string(),
3443 },
3444 }
3445 }
3446 }
3447 };
3448
3449 self.state.status = match (mode, outcome.ok) {
3450 (MergeMode::None, _) => RunStatus::Ready,
3451 (_, true) => RunStatus::Merged,
3452 (_, false) => RunStatus::Blocked,
3453 };
3454 self.state.event(
3455 "merge",
3456 format!(
3457 "{:?}: {}",
3458 mode,
3459 outcome.detail.lines().next().unwrap_or("")
3460 ),
3461 );
3462 self.state.merge = Some(outcome);
3463 self.state.save()?;
3464
3465 if self.state.config.graph.land
3471 && mode == MergeMode::Pr
3472 && self.state.status == RunStatus::Merged
3473 {
3474 self.run_land().await?;
3475 }
3476 self.settle_questions();
3481 Ok(())
3482 }
3483
3484 async fn run_land(&mut self) -> Result<()> {
3495 let url = self
3496 .state
3497 .merge
3498 .as_ref()
3499 .map(|m| m.detail.clone())
3500 .unwrap_or_default();
3501 let url = url.lines().next().unwrap_or("").trim().to_owned();
3502 if !url.starts_with("http") {
3503 return Ok(());
3504 }
3505 match land::land(&mut self.state, &url).await {
3508 Ok(pr) if self.state.parked => {
3509 let _ = pr;
3513 }
3514 Ok(pr) => {
3515 self.state.status = match pr.state {
3516 land::PrLifecycle::Merged => RunStatus::Merged,
3517 _ => RunStatus::Blocked,
3518 };
3519 if bump::should_release_bump(self.state.status)
3526 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
3527 {
3528 self.state
3529 .event("bump", format!("release bump skipped: {e:#}"));
3530 }
3531 self.state.save()?;
3532 }
3533 Err(e) => {
3534 self.state.status = RunStatus::Blocked;
3535 self.state.event("land", format!("gave up: {e}"));
3536 self.state.save()?;
3537 }
3538 }
3539 Ok(())
3540 }
3541
3542 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
3546 if let Some(existing) = self.state.seats.get(key)
3547 && existing.agent == agent
3548 {
3549 return existing.clone();
3550 }
3551 let fresh = SeatState::new(key, agent, self.state.seed);
3552 self.state.seats.insert(key.to_owned(), fresh.clone());
3553 fresh
3554 }
3555
3556 fn view(&self, c: &Candidate) -> CandidateView {
3558 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3559 .unwrap_or_default();
3560 let (patch, _) = blind::sanitize_patch(
3561 &format!("candidate {} patch", c.label),
3562 &raw,
3563 &self.state.config.blind,
3564 );
3565 CandidateView {
3566 label: c.label,
3567 branch: c.branch.clone(),
3568 summary: c.summary.clone(),
3569 stat: c.stat.clone(),
3570 patch,
3571 }
3572 }
3573
3574 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
3576 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
3577 prompt::judge(
3578 "(see above)",
3579 &views,
3580 self.roles.judges.len(),
3581 base_short,
3582 "en",
3583 )
3584 }
3585
3586 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
3593 let mut turns = Vec::new();
3594 for j in &self.state.judgements {
3595 if j.ranking.is_empty() {
3596 continue;
3597 }
3598 let reasons = j
3599 .reasons
3600 .iter()
3601 .map(|(k, v)| format!("- {k}: {v}"))
3602 .collect::<Vec<_>>()
3603 .join("\n");
3604 turns.push(Turn {
3605 who: format!("Judge {} (opening ranking)", j.judge),
3606 is_self: j.judge == self_idx + 1,
3607 body: format!(
3608 "Ranked {}{}{reasons}",
3609 j.ranking.iter().collect::<String>(),
3610 if reasons.is_empty() {
3611 ""
3612 } else {
3613 ", because:\n"
3614 }
3615 ),
3616 });
3617 }
3618 for t in self
3619 .state
3620 .deliberation
3621 .iter()
3622 .flat_map(|r| r.turns.iter())
3623 .chain(current)
3624 {
3625 turns.push(Turn {
3626 who: format!("Judge {}", t.judge),
3627 is_self: t.judge == self_idx + 1,
3628 body: t.body.clone(),
3629 });
3630 }
3631 turns
3632 }
3633}
3634
3635fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
3637 agent::has_session(spec.kind, seat, sessions)
3638}
3639
3640fn short(commit: &str) -> String {
3641 commit.chars().take(7).collect()
3642}
3643
3644fn make_executable(path: &Path) -> Result<()> {
3645 #[cfg(unix)]
3646 {
3647 use std::os::unix::fs::PermissionsExt as _;
3648 let mut perms = std::fs::metadata(path)?.permissions();
3649 perms.set_mode(0o755);
3650 std::fs::set_permissions(path, perms)?;
3651 }
3652 #[cfg(not(unix))]
3653 {
3654 let _ = path;
3655 }
3656 Ok(())
3657}
3658
3659struct WaveCtx<'a> {
3666 run: &'a str,
3669 node: &'a str,
3671 prompts: &'a Prompts,
3672 cache: Option<&'a Path>,
3674}
3675
3676async fn run_one(
3678 job: SeatJob,
3679 sem: Arc<Semaphore>,
3680 ctx: &WaveCtx<'_>,
3681 state: &mut RunState,
3682 attempt: usize,
3683) -> (SeatState, AgentOutcome) {
3684 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
3685 .await
3686 .pop()
3687 .expect("one job in, one result out");
3688 (seat, out)
3689}
3690
3691async fn wave(
3697 jobs: Vec<SeatJob>,
3698 sem: Arc<Semaphore>,
3699 ctx: &WaveCtx<'_>,
3700 state: &mut RunState,
3701 attempt: usize,
3702) -> Vec<(usize, SeatState, AgentOutcome)> {
3703 let WaveCtx {
3704 run,
3705 node,
3706 prompts,
3707 cache,
3708 } = *ctx;
3709 for job in &jobs {
3710 state.seat_started(node, &job.seat.key, job.timeout, attempt);
3711 }
3712 if let Err(e) = state.save() {
3713 tracing::warn!("could not persist in-progress seats: {e:#}");
3718 }
3719 let mut set = tokio::task::JoinSet::new();
3720 let overlay = prompts.overlay(node);
3721 for (i, mut job) in jobs.into_iter().enumerate() {
3722 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
3723 if cache.is_some() {
3724 job.prompt.push('\n');
3725 job.prompt.push_str(&prompt::build_cache_note(node));
3726 }
3727 let sem = Arc::clone(&sem);
3728 let run = run.to_owned();
3729 let node = node.to_owned();
3730 let cache = cache.map(Path::to_path_buf);
3731 set.spawn(async move {
3732 let _permit = sem.acquire().await;
3733 let mut seat = job.seat;
3734 let out = agent::invoke(
3735 &job.spec,
3736 &mut seat,
3737 &Invocation {
3738 cwd: &job.cwd,
3739 prompt: &job.prompt,
3740 timeout: job.timeout,
3741 allow_write: job.allow_write,
3742 sessions: job.sessions,
3743 artifacts: &job.artifacts,
3744 stem: &job.stem,
3745 run: &run,
3746 node: &node,
3747 cache_dir: cache.as_deref(),
3748 attachments: &[],
3749 },
3750 )
3751 .await;
3752 let out = match out {
3753 Ok(o) if o.usable() => AgentOutcome::Ok(o),
3754 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
3755 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
3763 Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
3764 Ok(o) => AgentOutcome::Failed(format!(
3765 "exited with {:?} and no usable output",
3766 o.exit_code
3767 )),
3768 Err(e) => AgentOutcome::Failed(e.to_string()),
3769 };
3770 (i, seat, out)
3771 });
3772 }
3773 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
3774 while let Some(joined) = set.join_next().await {
3775 let (i, seat, out) = match joined {
3776 Ok(v) => v,
3777 Err(e) => {
3781 tracing::error!("agent task panicked: {e}");
3782 continue;
3783 }
3784 };
3785 state.seat_finished(&seat.key);
3786 if let Err(e) = state.save() {
3787 tracing::warn!("could not persist a seat's completion: {e:#}");
3788 }
3789 if collected.len() <= i {
3790 collected.resize_with(i + 1, || None);
3791 }
3792 collected[i] = Some((i, seat, out));
3793 }
3794 if state
3800 .active
3801 .values()
3802 .any(|a| a.node == node && a.attempt == attempt)
3803 {
3804 state
3805 .active
3806 .retain(|_, a| !(a.node == node && a.attempt == attempt));
3807 if let Err(e) = state.save() {
3808 tracing::warn!("could not persist the end of a wave: {e:#}");
3809 }
3810 }
3811 collected.into_iter().flatten().collect()
3812}
3813
3814fn round_is_clean(
3835 blocking: usize,
3836 e2e_ok: bool,
3837 answered: usize,
3838 expected: usize,
3839 quota_missing: usize,
3840 policy: IncompleteReviewPolicy,
3841) -> bool {
3842 if blocking != 0 || !e2e_ok {
3843 return false;
3844 }
3845 if answered == expected || policy == IncompleteReviewPolicy::Warn {
3846 return true;
3847 }
3848 answered > 0 && expected - answered <= quota_missing
3849}
3850
3851fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
3868 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
3869 return Some(RunStatus::Gating);
3870 }
3871 let last = reviews.last()?;
3872 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
3873 if reviews.len() < max_rounds && !stagnant {
3874 return None;
3875 }
3876 Some(if last.incomplete() && last.blocking == 0 {
3877 RunStatus::Blocked
3878 } else if last.e2e.iter().all(CommandOutcome::ok) {
3879 RunStatus::Gating
3880 } else {
3881 RunStatus::Blocked
3882 })
3883}
3884
3885fn retry_budget(full: Duration, nudged: bool) -> Duration {
3900 if nudged {
3901 (full / 4).max(Duration::from_secs(120)).min(full)
3902 } else {
3903 full
3904 }
3905}
3906
3907#[allow(clippy::too_many_arguments)]
3920async fn ask_json_wave<T>(
3921 jobs: Vec<SeatJob>,
3922 sem: Arc<Semaphore>,
3923 retries: usize,
3924 ctx: &WaveCtx<'_>,
3925 losses: &mut Vec<QuotaLoss>,
3926 state: &mut RunState,
3927 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
3928) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
3929where
3930 T: serde::de::DeserializeOwned + Send + 'static,
3931{
3932 let n = jobs.len();
3933 let originals: Vec<SeatJob> = jobs;
3934 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
3935 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
3936 let mut pending: Vec<usize> = (0..n).collect();
3937
3938 for attempt in 0..=retries {
3939 if pending.is_empty() {
3940 break;
3941 }
3942 let mut batch = Vec::with_capacity(pending.len());
3943 for &i in &pending {
3944 let src = &originals[i];
3945 let (prompt, timeout) = if attempt == 0 {
3948 (src.prompt.clone(), src.timeout)
3949 } else {
3950 let why = done[i]
3951 .as_ref()
3952 .and_then(|r| r.as_ref().err().map(ToString::to_string))
3953 .unwrap_or_else(|| "no parsable answer".to_owned());
3954 let nudge = prompt::nudge(&why);
3955 let nudged = has_context(&src.spec, &seats[i], src.sessions);
3956 let prompt = if nudged {
3957 nudge
3958 } else {
3959 format!("{}\n\n---\n\n{}", src.prompt, nudge)
3960 };
3961 (prompt, retry_budget(src.timeout, nudged))
3962 };
3963 batch.push(SeatJob {
3964 spec: src.spec.clone(),
3965 seat: seats[i].clone(),
3966 cwd: src.cwd.clone(),
3967 prompt,
3968 timeout,
3969 allow_write: src.allow_write,
3970 sessions: src.sessions,
3971 artifacts: src.artifacts.clone(),
3972 stem: if attempt == 0 {
3973 src.stem.clone()
3974 } else {
3975 format!("{}-retry{attempt}", src.stem)
3976 },
3977 });
3978 }
3979
3980 if attempt > 0 {
3981 let seats_out: Vec<&str> = pending
3982 .iter()
3983 .map(|&i| originals[i].seat.key.as_str())
3984 .collect();
3985 state.event(
3986 ctx.node,
3987 format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3988 );
3989 }
3990 let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3991 let mut still = Vec::new();
3992 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3993 seats[i] = seat;
3994 let (parsed, quota) = match out {
3995 AgentOutcome::Ok(o) => (
3996 match verdict::extract_json::<T>(&o.text) {
3997 Ok(v) => match validate(&v) {
3998 Ok(()) => Ok((v, o)),
3999 Err(e) => Err(e),
4000 },
4001 Err(e) => Err(e),
4002 },
4003 false,
4004 ),
4005 AgentOutcome::Quota(o) => {
4006 losses.push(QuotaLoss {
4007 seat: originals[i].seat.key.clone(),
4008 node: ctx.node.to_owned(),
4009 at: Timestamp::now(),
4010 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4011 });
4012 (
4013 Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
4014 true,
4015 )
4016 }
4017 AgentOutcome::Dropped(o) => {
4022 let why = o
4023 .dropped
4024 .as_ref()
4025 .map(|d| d.why.as_str())
4026 .unwrap_or("the CLI ended the stream without delivering its answer");
4027 (
4028 Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
4029 false,
4030 )
4031 }
4032 AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
4033 };
4034 let failed = parsed.is_err();
4035 done[i] = Some(parsed);
4036 if failed && !quota {
4039 still.push(i);
4040 }
4041 }
4042 pending = still;
4043 }
4044
4045 seats
4046 .into_iter()
4047 .zip(done)
4048 .map(|(seat, res)| {
4049 (
4050 seat,
4051 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
4052 )
4053 })
4054 .collect()
4055}
4056
4057fn e2e_outcome_label(o: &CommandOutcome) -> String {
4061 if o.ok() {
4062 return "pass".to_owned();
4063 }
4064 let reason = if o.build_failed() {
4065 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
4066 } else {
4067 format!("FAIL ({:?})", o.code)
4068 };
4069 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
4070}
4071
4072async fn run_e2e_with_retry(
4080 state: &mut RunState,
4081 shell: &[String],
4082 commands: &[String],
4083 worktree: &Path,
4084 timeout: Duration,
4085 context: &str,
4086) -> (Vec<CommandOutcome>, bool) {
4087 let mut e2e = run_commands(shell, commands, worktree, timeout).await;
4088 for o in &e2e {
4089 state.event(
4090 "verify",
4091 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
4092 );
4093 }
4094 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
4098 if verify_retried {
4099 state.event(
4100 "verify",
4101 format!(
4102 "{context}: verify could not build/link, not a test result — retrying once \
4103 before concluding"
4104 ),
4105 );
4106 e2e = run_commands(shell, commands, worktree, timeout).await;
4107 for o in &e2e {
4108 state.event(
4109 "verify",
4110 format!(
4111 "{context}: retry `{}` -> {}",
4112 o.command,
4113 e2e_outcome_label(o)
4114 ),
4115 );
4116 }
4117 }
4118 (e2e, verify_retried)
4119}
4120
4121async fn run_commands(
4123 shell: &[String],
4124 commands: &[String],
4125 cwd: &Path,
4126 timeout: Duration,
4127) -> Vec<CommandOutcome> {
4128 let mut out = Vec::new();
4129 for command in commands {
4130 let started = Instant::now();
4131 let mut cmd = tokio::process::Command::new(&shell[0]);
4132 cmd.quiet();
4133 cmd.args(&shell[1..])
4134 .arg(command)
4135 .current_dir(cwd)
4136 .stdin(std::process::Stdio::null())
4137 .stdout(std::process::Stdio::piped())
4138 .stderr(std::process::Stdio::piped())
4139 .kill_on_drop(true);
4140 let spawned = cmd.spawn();
4141 let (code, body) = match spawned {
4142 Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
4143 Ok(Ok(o)) => {
4144 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
4145 body.push_str(&String::from_utf8_lossy(&o.stderr));
4146 (o.status.code(), body)
4147 }
4148 Ok(Err(e)) => (None, format!("failed to run: {e}")),
4149 Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
4150 },
4151 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
4152 };
4153 out.push(CommandOutcome {
4154 command: command.clone(),
4155 code,
4156 output_tail: tail(&body, OUTPUT_TAIL),
4157 duration_ms: started.elapsed().as_millis() as u64,
4158 });
4159 }
4160 out
4161}
4162
4163fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
4175 let repo = repo.display();
4176 match style {
4177 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
4178 MergeStyle::Squash => {
4179 let subject = message.lines().next().unwrap_or(branch);
4180 format!(
4181 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
4182 )
4183 }
4184 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
4185 }
4186}
4187
4188fn pr_body(state: &RunState, winner: char) -> String {
4214 let instruction = state.instruction.trim_start();
4215 let mut message = if instruction.is_empty() {
4216 "(empty task)".to_owned()
4217 } else {
4218 instruction.to_owned()
4219 };
4220
4221 let open = state.open_findings();
4222 if !open.is_empty() {
4223 message.push_str("\n\n## Open review findings\n\n");
4224 for f in &open {
4225 message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
4226 }
4227 }
4228
4229 if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
4230 && !fix.rejected.is_empty()
4231 {
4232 message.push_str("\n## Declined by the fixer\n\n");
4233 for r in &fix.rejected {
4234 message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
4235 }
4236 }
4237
4238 message.push_str(&format!(
4239 "\n\n---\nmagi:run/{} magi:candidate-{}\n",
4240 state.id,
4241 winner.to_ascii_lowercase()
4242 ));
4243
4244 message
4245}
4246
4247async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
4249 let title = body.lines().next().unwrap_or("magi run").to_owned();
4250 let out = tokio::process::Command::new("gh")
4251 .args([
4252 "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
4253 ])
4254 .current_dir(cwd)
4255 .quiet()
4256 .stdin(std::process::Stdio::null())
4257 .output()
4258 .await
4259 .context("spawn gh")?;
4260 if out.status.success() {
4261 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
4262 } else {
4263 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
4264 }
4265}
4266
4267pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
4269 let repo = state.repo.clone();
4270 let root = state.worktree_root();
4271 let winner = state.tally.as_ref().map(|t| t.winner);
4272 let mut removed = Vec::new();
4273
4274 for i in 0..state.candidates.len() {
4275 let c = state.candidates[i].clone();
4276 let is_winner = Some(c.label) == winner;
4277 if is_winner && !drop_winner {
4278 continue;
4279 }
4280 if c.worktree.exists() {
4281 git::worktree_remove(&repo, &c.worktree).await.ok();
4282 removed.push(c.worktree.to_string_lossy().into_owned());
4283 }
4284 if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
4285 git::branch_delete(&repo, &c.branch).await.ok();
4286 removed.push(c.branch.clone());
4287 }
4288 state.candidates[i].folded = true;
4289 }
4290
4291 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
4292 let path = name.path();
4293 let keep = !drop_winner
4294 && winner.is_some_and(|w| {
4295 path.file_name()
4296 .is_some_and(|n| n == format!("cand-{w}").as_str())
4297 });
4298 if keep {
4299 continue;
4300 }
4301 git::worktree_remove(&repo, &path).await.ok();
4302 removed.push(path.to_string_lossy().into_owned());
4303 }
4304
4305 remove_if_empty(&root);
4314
4315 if state.enabled_worktree_config && drop_winner {
4316 git::release_worktree_config(&repo).await.ok();
4320 state.enabled_worktree_config = false;
4321 }
4322 state.save()?;
4323 Ok(removed)
4324}
4325
4326fn remove_if_empty(dir: &Path) {
4337 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
4338 std::fs::remove_dir(dir).ok();
4339 }
4340}
4341
4342pub fn worst_open(state: &RunState) -> Option<Severity> {
4344 state
4345 .reviews
4346 .last()?
4347 .reviews
4348 .iter()
4349 .flat_map(|r| r.findings.iter())
4350 .map(|f| f.severity)
4351 .max()
4352}
4353
4354#[cfg(test)]
4355mod tests {
4356 use super::*;
4357 use std::collections::BTreeMap;
4358 use std::time::Duration;
4359
4360 fn conductor() -> AgentSpec {
4361 AgentSpec {
4362 id: "conductor".to_owned(),
4363 kind: crate::config::AgentKind::Command,
4364 model: None,
4365 command: vec!["true".to_owned()],
4366 extra_args: Vec::new(),
4367 env: BTreeMap::new(),
4368 prompt_delivery: None,
4369 }
4370 }
4371
4372 #[test]
4373 fn remove_if_empty_only_ever_takes_a_bare_directory() {
4374 let dir = tempfile::tempdir().unwrap();
4375 let bay = dir.path().join("ffff");
4376
4377 remove_if_empty(&bay);
4379 assert!(!bay.exists());
4380
4381 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
4384 remove_if_empty(&bay);
4385 assert!(bay.exists(), "non-empty directory must survive");
4386
4387 std::fs::remove_dir(bay.join("cand-A")).unwrap();
4389 remove_if_empty(&bay);
4390 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
4391 }
4392
4393 #[test]
4402 fn a_full_panel_that_found_nothing_is_clean() {
4403 assert!(round_is_clean(
4404 0,
4405 true,
4406 2,
4407 2,
4408 0,
4409 IncompleteReviewPolicy::Block
4410 ));
4411 }
4412
4413 #[test]
4414 fn a_missing_seat_is_never_clean_under_the_default_policy() {
4415 assert!(!round_is_clean(
4416 0,
4417 true,
4418 1,
4419 2,
4420 0,
4421 IncompleteReviewPolicy::Block
4422 ));
4423 }
4424
4425 #[test]
4426 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
4427 assert!(!round_is_clean(
4428 1,
4429 true,
4430 1,
4431 2,
4432 0,
4433 IncompleteReviewPolicy::Warn
4434 ));
4435 }
4436
4437 #[test]
4438 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
4439 assert!(round_is_clean(
4440 0,
4441 true,
4442 1,
4443 2,
4444 0,
4445 IncompleteReviewPolicy::Warn
4446 ));
4447 }
4448
4449 #[test]
4450 fn a_full_panel_with_an_open_finding_is_not_clean() {
4451 assert!(!round_is_clean(
4452 1,
4453 true,
4454 2,
4455 2,
4456 0,
4457 IncompleteReviewPolicy::Block
4458 ));
4459 }
4460
4461 #[test]
4462 fn a_full_panel_with_a_red_e2e_is_not_clean() {
4463 assert!(!round_is_clean(
4464 0,
4465 false,
4466 2,
4467 2,
4468 0,
4469 IncompleteReviewPolicy::Block
4470 ));
4471 }
4472
4473 #[test]
4480 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
4481 assert!(round_is_clean(
4484 0,
4485 true,
4486 1,
4487 2,
4488 1,
4489 IncompleteReviewPolicy::Block
4490 ));
4491 }
4492
4493 #[test]
4494 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
4495 assert!(!round_is_clean(
4498 0,
4499 true,
4500 1,
4501 2,
4502 0,
4503 IncompleteReviewPolicy::Block
4504 ));
4505 }
4506
4507 #[test]
4508 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
4509 assert!(!round_is_clean(
4510 1,
4511 true,
4512 1,
4513 2,
4514 1,
4515 IncompleteReviewPolicy::Block
4516 ));
4517 assert!(!round_is_clean(
4518 0,
4519 false,
4520 1,
4521 2,
4522 1,
4523 IncompleteReviewPolicy::Block
4524 ));
4525 }
4526
4527 #[test]
4528 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
4529 assert!(!round_is_clean(
4533 0,
4534 true,
4535 0,
4536 2,
4537 2,
4538 IncompleteReviewPolicy::Block
4539 ));
4540 }
4541
4542 fn review_round(
4548 clean: bool,
4549 blocking: usize,
4550 answered: usize,
4551 expected: usize,
4552 progressed: bool,
4553 e2e_ok: bool,
4554 ) -> ReviewRound {
4555 ReviewRound {
4556 round: 1,
4557 head: "h".to_owned(),
4558 verified_head: None,
4559 reviews: Vec::new(),
4560 e2e: vec![CommandOutcome {
4561 command: "test".to_owned(),
4562 code: Some(if e2e_ok { 0 } else { 1 }),
4563 output_tail: String::new(),
4564 duration_ms: 0,
4565 }],
4566 verify_retried: false,
4567 e2e_deferred: false,
4568 e2e_defer_reason: None,
4569 fix: None,
4570 blocking,
4571 answered,
4572 expected,
4573 clean,
4574 progressed,
4575 vote_split: false,
4576 reconsideration: Vec::new(),
4577 verdict: None,
4578 }
4579 }
4580
4581 #[test]
4582 fn review_conclusion_is_none_when_nothing_has_run() {
4583 assert_eq!(review_conclusion(&[], 3), None);
4584 }
4585
4586 #[test]
4587 fn review_conclusion_is_none_while_rounds_remain() {
4588 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
4589 assert_eq!(review_conclusion(&rounds, 3), None);
4590 }
4591
4592 #[test]
4593 fn review_conclusion_is_gating_once_a_round_is_clean() {
4594 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
4595 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
4596 }
4597
4598 #[test]
4599 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
4600 let rounds = vec![
4601 review_round(false, 1, 2, 2, true, true),
4602 review_round(false, 1, 2, 2, true, true),
4603 ];
4604 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
4605 }
4606
4607 #[test]
4608 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
4609 let rounds = vec![
4610 review_round(false, 1, 2, 2, true, true),
4611 review_round(false, 1, 2, 2, true, false),
4612 ];
4613 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
4614 }
4615
4616 #[test]
4617 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
4618 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
4620 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
4621 }
4622
4623 #[test]
4624 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
4625 let rounds = vec![
4626 review_round(false, 1, 2, 2, false, true),
4627 review_round(false, 1, 2, 2, false, true),
4628 ];
4629 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
4630 }
4631
4632 fn secs(n: u64) -> Duration {
4633 Duration::from_secs(n)
4634 }
4635
4636 fn init_repo(dir: &Path) {
4639 let run = |args: &[&str]| {
4640 let out = std::process::Command::new("git")
4641 .args(args)
4642 .current_dir(dir)
4643 .quiet()
4644 .output()
4645 .expect("spawn git");
4646 assert!(
4647 out.status.success(),
4648 "git {args:?} failed: {}",
4649 String::from_utf8_lossy(&out.stderr)
4650 );
4651 };
4652 run(&["init", "-b", "main"]);
4653 run(&["config", "user.name", "magi test"]);
4654 run(&["config", "user.email", "magi@example.com"]);
4655 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
4656 run(&["add", "-A"]);
4657 run(&["commit", "-m", "init"]);
4658 }
4659
4660 fn ask_test_home() {
4668 crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
4669 }
4670
4671 fn runner_at(status: RunStatus) -> Runner {
4674 let mut state = RunState::new(
4675 PathBuf::from("/nonexistent/repo"),
4676 "main".to_owned(),
4677 "deadbeef".to_owned(),
4678 "task".to_owned(),
4679 Config::default(),
4680 );
4681 state.status = status;
4682 Runner {
4683 state,
4684 roles: ResolvedRoles {
4685 implementers: Vec::new(),
4686 judges: Vec::new(),
4687 reviewers: Vec::new(),
4688 fixer: None,
4689 conductor: conductor(),
4690 },
4691 sem: Arc::new(Semaphore::new(1)),
4692 pause: Pause::new(),
4693 interrupt: Pause::new(),
4694 }
4695 }
4696
4697 #[test]
4701 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
4702 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
4703 let mut runner = runner_at(RunStatus::Implementing);
4704 let interrupt = Pause::new();
4705 runner.watch_interrupt(interrupt.clone());
4706
4707 interrupt.park_because("task a1b2 asked to run first");
4708
4709 assert!(runner.park_here().expect("park_here"));
4710 assert!(runner.state.parked);
4711 let last = runner.state.events.last().expect("a park event");
4712 assert_eq!(last.node, "park");
4713 assert!(
4714 last.message.contains("task a1b2 asked to run first"),
4715 "expected the interrupt reason in {:?}",
4716 last.message
4717 );
4718 }
4719
4720 #[test]
4728 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
4729 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
4730 let mut runner = runner_at(RunStatus::Implementing);
4731 let shutdown = Pause::new();
4732 runner.on_pause(shutdown.clone());
4733 let interrupt = Pause::new();
4734 runner.watch_interrupt(interrupt.clone());
4735
4736 assert!(!runner.park_here().expect("park_here"));
4738 assert!(!runner.state.parked);
4739
4740 interrupt.park_because("test");
4742 assert!(!shutdown.parked());
4743 assert!(runner.park_here().expect("park_here"));
4744 }
4745
4746 #[tokio::test]
4760 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
4761 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
4762 let mut runner = runner_at(RunStatus::Implementing);
4763 let interrupt = Pause::new();
4764 runner.watch_interrupt(interrupt.clone());
4765
4766 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
4767 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
4768
4769 let node = async move {
4773 started_tx.send(()).expect("send started");
4774 finish_rx.await.expect("recv finish");
4775 "node finished"
4776 };
4777
4778 let interrupter = async move {
4779 started_rx.await.expect("recv started");
4780 interrupt.park_because("higher-priority task waiting");
4782 tokio::task::yield_now().await;
4786 finish_tx.send(()).expect("send finish");
4787 };
4788
4789 let (node_result, ()) = tokio::join!(node, interrupter);
4790 assert_eq!(
4791 node_result, "node finished",
4792 "the in-flight call ran to completion"
4793 );
4794
4795 assert!(runner.park_here().expect("park_here"));
4798 assert!(runner.state.parked);
4799 }
4800
4801 #[test]
4807 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
4808 crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
4809 let mut runner = runner_at(RunStatus::Judging);
4810 runner.state.config.agents = vec![conductor()];
4814 runner.state.candidates = vec![Candidate {
4815 index: 0,
4816 label: 'A',
4817 agent: "alpha".to_owned(),
4818 branch: "magi/x/A".to_owned(),
4819 worktree: PathBuf::from("/nonexistent/worktree"),
4820 summary: "did the thing".to_owned(),
4821 stat: "1 file changed".to_owned(),
4822 files: 1,
4823 commits: 1,
4824 empty: false,
4825 failed: None,
4826 duration_ms: 1234,
4827 folded: false,
4828 }];
4829 let run_id = runner.state.id.clone();
4830
4831 let interrupt = Pause::new();
4832 runner.watch_interrupt(interrupt.clone());
4833 interrupt.park_because("task c3d4 asked to run first");
4834 assert!(runner.park_here().expect("park_here"));
4835
4836 let resumed = Runner::resume(&run_id).expect("resume");
4837 assert_eq!(resumed.state.candidates.len(), 1);
4838 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
4839 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
4840 assert_eq!(resumed.state.status, runner.state.status);
4841 assert!(
4842 resumed.state.parked,
4843 "still parked until `execute` actually walks the graph again"
4844 );
4845 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
4846 }
4847
4848 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
4850 let mut q = ask::Question::new(
4851 run.to_owned(),
4852 "implement".to_owned(),
4853 "impl-A".to_owned(),
4854 "Which storage backend should the cache use?".to_owned(),
4855 String::new(),
4856 vec!["SQLite".to_owned(), "Redis".to_owned()],
4857 );
4858 store.put(&mut q).unwrap();
4859 q
4860 }
4861
4862 #[test]
4863 fn a_failed_runs_open_question_is_abandoned() {
4864 ask_test_home();
4865 let store = ask::Questions::open();
4866 let mut runner = runner_at(RunStatus::Failed);
4867 let run = runner.state.id.clone();
4868 let q = ask_open_question(&store, &run);
4869
4870 runner.settle_questions();
4871
4872 let back = store.get(&q.id).unwrap();
4873 assert!(
4874 !back.status.open(),
4875 "the seat that asked died with the run; nobody is left to read an answer"
4876 );
4877 assert!(
4878 back.detail.contains(&run) && back.detail.contains("failed"),
4879 "the reason names what the run became, not just that it is gone: {}",
4880 back.detail
4881 );
4882 }
4883
4884 #[test]
4885 fn a_merged_runs_open_question_is_abandoned_too() {
4886 ask_test_home();
4887 let store = ask::Questions::open();
4888 for status in [RunStatus::Merged, RunStatus::Ready] {
4891 let mut runner = runner_at(status);
4892 let run = runner.state.id.clone();
4893 let q = ask_open_question(&store, &run);
4894
4895 runner.settle_questions();
4896
4897 let back = store.get(&q.id).unwrap();
4898 assert!(
4899 !back.status.open(),
4900 "{status:?} run's question must not outlive the run"
4901 );
4902 }
4903 }
4904
4905 #[test]
4906 fn a_still_resumable_runs_open_question_is_left_alone() {
4907 ask_test_home();
4908 let store = ask::Questions::open();
4909 for status in [RunStatus::Blocked, RunStatus::Stalled] {
4915 let mut runner = runner_at(status);
4916 let run = runner.state.id.clone();
4917 let q = ask_open_question(&store, &run);
4918
4919 runner.settle_questions();
4920
4921 let back = store.get(&q.id).unwrap();
4922 assert!(
4923 back.status.open(),
4924 "{status:?} is still alive; the question must still be waiting"
4925 );
4926 }
4927 }
4928
4929 #[test]
4930 fn settle_questions_never_touches_an_already_answered_question() {
4931 ask_test_home();
4932 let store = ask::Questions::open();
4933 let mut runner = runner_at(RunStatus::Failed);
4934 let run = runner.state.id.clone();
4935 let mut q = ask_open_question(&store, &run);
4936 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
4937 .unwrap();
4938 store.put(&mut q).unwrap();
4939
4940 runner.settle_questions();
4945 runner.settle_questions();
4946
4947 let back = store.get(&q.id).unwrap();
4948 assert_eq!(
4949 back.status,
4950 ask::QuestionStatus::Answered,
4951 "a real answer is a decision on record, never overwritten by a sweep"
4952 );
4953 }
4954
4955 #[tokio::test]
4964 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
4965 let tmp = tempfile::tempdir().expect("tempdir");
4966 let repo = tmp.path().join("repo");
4967 std::fs::create_dir_all(&repo).unwrap();
4968 init_repo(&repo);
4969
4970 let mut config = Config::default();
4971 config.merge.mode = MergeMode::Local;
4972
4973 let mut state = RunState::new(
4974 repo.clone(),
4975 "main".to_owned(),
4976 "deadbeef".to_owned(),
4977 "task".to_owned(),
4978 config,
4979 );
4980 state.candidates = vec![Candidate {
4981 index: 0,
4982 label: 'A',
4983 agent: "alpha".to_owned(),
4984 branch: "does-not-exist".to_owned(),
4985 worktree: repo.clone(),
4986 summary: String::new(),
4987 stat: String::new(),
4988 files: 0,
4989 commits: 0,
4990 empty: false,
4991 failed: None,
4992 duration_ms: 0,
4993 folded: false,
4994 }];
4995 state.tally = Some(Tally {
4996 first_choice: BTreeMap::from([('A', 1)]),
4997 borda: BTreeMap::new(),
4998 winner: 'A',
4999 rankings: 1,
5000 unanimous_initial: true,
5001 deliberated: false,
5002 changed_votes: 0,
5003 unanimous_final: true,
5004 tie_break: None,
5005 judges: 0,
5006 present: 0,
5007 quorum: 0,
5008 met_quorum: true,
5009 uncontested: Some("only candidate A produced a change".to_owned()),
5010 });
5011 state.reviews = vec![ReviewRound {
5012 round: 1,
5013 head: "deadbeef".to_owned(),
5014 verified_head: None,
5015 reviews: Vec::new(),
5016 e2e: Vec::new(),
5017 fix: None,
5018 blocking: 0,
5019 answered: 0,
5020 expected: 0,
5021 clean: true,
5022 verify_retried: false,
5023 e2e_deferred: false,
5024 e2e_defer_reason: None,
5025 progressed: false,
5026 vote_split: false,
5027 reconsideration: Vec::new(),
5028 verdict: None,
5029 }];
5030 state.gate = vec![CommandOutcome {
5031 command: "test".to_owned(),
5032 code: Some(0),
5033 output_tail: String::new(),
5034 duration_ms: 0,
5035 }];
5036 state.status = RunStatus::Ready;
5041 state.merge = Some(MergeOutcome {
5042 mode: MergeMode::Local,
5043 ok: false,
5044 detail: "already concluded".to_owned(),
5045 });
5046
5047 let mut runner = Runner {
5048 state,
5049 roles: ResolvedRoles {
5050 implementers: Vec::new(),
5051 judges: Vec::new(),
5052 reviewers: Vec::new(),
5053 fixer: None,
5054 conductor: conductor(),
5055 },
5056 sem: Arc::new(Semaphore::new(1)),
5057 pause: Pause::new(),
5058 interrupt: Pause::new(),
5059 };
5060
5061 runner.merge().await.expect("merge");
5062
5063 assert_eq!(
5064 runner.state.status,
5065 RunStatus::Ready,
5066 "a concluded run's status must not change on reentry"
5067 );
5068 assert_eq!(
5069 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
5070 Some("already concluded"),
5071 "merge must not run again once the node already recorded an outcome"
5072 );
5073 }
5074
5075 #[tokio::test]
5076 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
5077 crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
5078 let tmp = tempfile::tempdir().expect("tempdir");
5079 let repo = tmp.path().join("repo");
5080 std::fs::create_dir_all(&repo).unwrap();
5081 init_repo(&repo);
5082
5083 let mut config = Config::default();
5084 config.merge.mode = MergeMode::Pr;
5085 config.graph.land = true;
5086 config.graph.land_approval = false;
5087
5088 let mut state = RunState::new(
5089 repo.clone(),
5090 "main".to_owned(),
5091 "deadbeef".to_owned(),
5092 "task".to_owned(),
5093 config,
5094 );
5095 state.candidates = vec![Candidate {
5096 index: 0,
5097 label: 'A',
5098 agent: "alpha".to_owned(),
5099 branch: "does-not-exist".to_owned(),
5100 worktree: repo.clone(),
5101 summary: String::new(),
5102 stat: String::new(),
5103 files: 0,
5104 commits: 0,
5105 empty: false,
5106 failed: None,
5107 duration_ms: 0,
5108 folded: false,
5109 }];
5110 state.tally = Some(Tally {
5111 first_choice: BTreeMap::from([('A', 1)]),
5112 borda: BTreeMap::new(),
5113 winner: 'A',
5114 rankings: 1,
5115 unanimous_initial: true,
5116 deliberated: false,
5117 changed_votes: 0,
5118 unanimous_final: true,
5119 tie_break: None,
5120 judges: 0,
5121 present: 0,
5122 quorum: 0,
5123 met_quorum: true,
5124 uncontested: Some("only candidate A produced a change".to_owned()),
5125 });
5126 state.reviews = vec![ReviewRound {
5127 round: 1,
5128 head: "deadbeef".to_owned(),
5129 verified_head: None,
5130 reviews: Vec::new(),
5131 e2e: Vec::new(),
5132 fix: None,
5133 blocking: 0,
5134 answered: 0,
5135 expected: 0,
5136 clean: true,
5137 verify_retried: false,
5138 e2e_deferred: false,
5139 e2e_defer_reason: None,
5140 progressed: false,
5141 vote_split: false,
5142 reconsideration: Vec::new(),
5143 verdict: None,
5144 }];
5145 state.gate = vec![CommandOutcome {
5146 command: "test".to_owned(),
5147 code: Some(0),
5148 output_tail: String::new(),
5149 duration_ms: 0,
5150 }];
5151 state.status = RunStatus::Landing;
5155 state.merge = Some(MergeOutcome {
5156 mode: MergeMode::Pr,
5157 ok: true,
5158 detail: "https://example.invalid/x/y/pull/1".to_owned(),
5159 });
5160
5161 ask_test_home();
5165 let store = ask::Questions::open();
5166 let q = ask_open_question(&store, &state.id);
5167
5168 let mut runner = Runner {
5169 state,
5170 roles: ResolvedRoles {
5171 implementers: Vec::new(),
5172 judges: Vec::new(),
5173 reviewers: Vec::new(),
5174 fixer: None,
5175 conductor: conductor(),
5176 },
5177 sem: Arc::new(Semaphore::new(1)),
5178 pause: Pause::new(),
5179 interrupt: Pause::new(),
5180 };
5181
5182 runner.execute().await.expect("execute");
5187
5188 assert_eq!(
5189 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
5190 Some("https://example.invalid/x/y/pull/1"),
5191 "reentry must not push again or open a second pull request over the \
5192 one `land` is already watching"
5193 );
5194 assert_ne!(
5195 runner.state.status,
5196 RunStatus::Landing,
5197 "land could not actually reach the fake pull request, so it must \
5198 have given up rather than left the run silently parked forever"
5199 );
5200 assert_eq!(runner.state.status, RunStatus::Blocked);
5204 assert!(
5205 store.get(&q.id).unwrap().status.open(),
5206 "Blocked is still alive; settle_questions must have been a no-op here"
5207 );
5208 }
5209
5210 fn state_with_round(round: ReviewRound) -> RunState {
5211 let mut s = RunState::new(
5212 PathBuf::from("/repo"),
5213 "main".to_owned(),
5214 "abc1234".to_owned(),
5215 "add retries".to_owned(),
5216 Config::default(),
5217 );
5218 s.reviews = vec![round];
5219 s
5220 }
5221
5222 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
5223 crate::verdict::Finding {
5224 id: id.to_owned(),
5225 severity,
5226 file: None,
5227 line: None,
5228 title: title.to_owned(),
5229 detail: String::new(),
5230 }
5231 }
5232
5233 #[test]
5234 fn pr_body_names_open_findings_and_declined_ones() {
5235 let round = ReviewRound {
5236 round: 2,
5237 head: "deadbee".to_owned(),
5238 verified_head: None,
5239 reviews: vec![ReviewRecord {
5240 reviewer: 1,
5241 agent: "alpha".to_owned(),
5242 summary: String::new(),
5243 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
5244 vote: None,
5245 failed: None,
5246 duration_ms: 0,
5247 }],
5248 e2e: vec![CommandOutcome {
5249 command: "cargo test".to_owned(),
5250 code: Some(0),
5251 output_tail: String::new(),
5252 duration_ms: 0,
5253 }],
5254 verify_retried: false,
5255 e2e_deferred: false,
5256 e2e_defer_reason: None,
5257 fix: Some(FixRecord {
5258 agent: "alpha".to_owned(),
5259 addressed: Vec::new(),
5260 rejected: vec![crate::verdict::Rejection {
5261 id: "R1-1-1".to_owned(),
5262 why: "not reachable from any caller".to_owned(),
5263 }],
5264 notes: String::new(),
5265 committed: true,
5266 failed: None,
5267 duration_ms: 0,
5268 }),
5269 blocking: 0,
5270 answered: 1,
5271 expected: 1,
5272 clean: false,
5273 progressed: true,
5274 vote_split: false,
5275 reconsideration: Vec::new(),
5276 verdict: None,
5277 };
5278 let state = state_with_round(round);
5279 let body = pr_body(&state, 'A');
5280
5281 assert!(body.contains("add retries"), "the task must still be there");
5282 assert!(body.contains("R2-1-1"), "{body}");
5283 assert!(body.contains("unused import"), "{body}");
5284 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
5285 assert!(
5286 body.contains("not reachable from any caller"),
5287 "the reason it was declined: {body}"
5288 );
5289 }
5290
5291 #[test]
5292 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
5293 let round = ReviewRound {
5294 round: 1,
5295 head: "deadbee".to_owned(),
5296 verified_head: None,
5297 reviews: vec![ReviewRecord {
5298 reviewer: 1,
5299 agent: "alpha".to_owned(),
5300 summary: String::new(),
5301 findings: Vec::new(),
5302 vote: None,
5303 failed: None,
5304 duration_ms: 0,
5305 }],
5306 e2e: Vec::new(),
5307 verify_retried: false,
5308 e2e_deferred: false,
5309 e2e_defer_reason: None,
5310 fix: None,
5311 blocking: 0,
5312 answered: 1,
5313 expected: 1,
5314 clean: true,
5315 progressed: false,
5316 vote_split: false,
5317 reconsideration: Vec::new(),
5318 verdict: None,
5319 };
5320 let state = state_with_round(round);
5321 let body = pr_body(&state, 'A');
5322 assert!(!body.contains("Open review findings"), "{body}");
5323 assert!(!body.contains("Declined"), "{body}");
5324 }
5325
5326 #[test]
5327 fn pr_body_titles_itself_from_the_task_not_run_or_candidate() {
5328 let state = RunState::new(
5329 PathBuf::from("/repo"),
5330 "main".to_owned(),
5331 "abc1234".to_owned(),
5332 "add retries".to_owned(),
5333 Config::default(),
5334 );
5335 let body = pr_body(&state, 'A');
5336 let title = body.lines().next().unwrap();
5337
5338 assert_eq!(
5339 title, "add retries",
5340 "the title must be the task, not run/candidate bookkeeping: {body}"
5341 );
5342 assert!(
5343 body.contains(&format!("magi:run/{}", state.id)),
5344 "the run id must still be recoverable from the footer: {body}"
5345 );
5346 assert!(
5347 body.contains("magi:candidate-a"),
5348 "the candidate must still be recoverable from the footer: {body}"
5349 );
5350 }
5351
5352 #[test]
5353 fn pr_body_never_titles_itself_off_a_blank_first_line() {
5354 let leading_blank = RunState::new(
5355 PathBuf::from("/repo"),
5356 "main".to_owned(),
5357 "abc1234".to_owned(),
5358 "\n\n \nadd retries\n\ndetails".to_owned(),
5359 Config::default(),
5360 );
5361 let body = pr_body(&leading_blank, 'A');
5362 assert_eq!(
5363 body.lines().next(),
5364 Some("add retries"),
5365 "a leading blank line must not become an empty title: {body}"
5366 );
5367
5368 let whitespace_only = RunState::new(
5369 PathBuf::from("/repo"),
5370 "main".to_owned(),
5371 "abc1234".to_owned(),
5372 " \n \n".to_owned(),
5373 Config::default(),
5374 );
5375 let body = pr_body(&whitespace_only, 'A');
5376 let title = body.lines().next().unwrap_or_default();
5377 assert!(
5378 !title.is_empty(),
5379 "a whitespace-only instruction must still fall back to a non-empty title: {body}"
5380 );
5381 }
5382
5383 #[test]
5384 fn manual_merge_command_matches_the_configured_style() {
5385 let repo = Path::new("/repo");
5386 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
5387
5388 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
5389 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
5390
5391 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
5392 assert_eq!(
5393 squash,
5394 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
5395 \"Merge magi run 0832 (candidate A)\""
5396 );
5397
5398 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
5399 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
5400 }
5401
5402 #[test]
5403 fn a_nudge_gets_a_quarter_of_the_budget() {
5404 assert_eq!(retry_budget(secs(1200), true), secs(300));
5406 assert_eq!(retry_budget(secs(3600), true), secs(900));
5407 }
5408
5409 #[test]
5410 fn a_resent_prompt_keeps_the_whole_budget() {
5411 assert_eq!(retry_budget(secs(1200), false), secs(1200));
5414 assert_eq!(retry_budget(secs(60), false), secs(60));
5415 }
5416
5417 #[test]
5418 fn the_floor_never_exceeds_the_original_budget() {
5419 assert_eq!(retry_budget(secs(60), true), secs(60));
5423 assert_eq!(retry_budget(secs(480), true), secs(120));
5424 assert_eq!(retry_budget(secs(0), true), secs(0));
5425 }
5426}