1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::agent::{self, AgentOutput, Invocation, SeatState};
30use crate::blind;
31use crate::config::{
32 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, Prompts, ResolvedRoles,
33};
34use crate::git;
35use crate::land;
36use crate::proc::Quiet as _;
37use crate::prompt::{self, CandidateView, Turn};
38use crate::run::{
39 BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
40 MergeOutcome, QuotaLoss, ReviewRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord,
41 tail, write_artifact,
42};
43use crate::verdict::{self, FinalVote, FixReport, Position, Ranking, Review, Severity};
44
45const OUTPUT_TAIL: usize = 8_000;
47
48const BASE_SYNC_ROUNDS: usize = 4;
61
62#[derive(Clone)]
68struct SeatJob {
69 spec: AgentSpec,
70 seat: SeatState,
71 cwd: PathBuf,
72 prompt: String,
73 timeout: Duration,
74 allow_write: bool,
75 sessions: bool,
76 artifacts: PathBuf,
77 stem: String,
78}
79
80enum AgentOutcome {
92 Ok(AgentOutput),
94 Quota(AgentOutput),
96 Dropped(AgentOutput),
99 Failed(String),
101}
102
103#[derive(Debug, Clone, Default)]
120pub struct Pause(Arc<AtomicBool>);
121
122impl Pause {
123 #[must_use]
125 pub fn new() -> Self {
126 Self::default()
127 }
128
129 pub fn park(&self) {
131 self.0.store(true, Ordering::SeqCst);
132 }
133
134 #[must_use]
136 pub fn parked(&self) -> bool {
137 self.0.load(Ordering::SeqCst)
138 }
139}
140
141pub struct Runner {
143 pub state: RunState,
145 roles: ResolvedRoles,
146 sem: Arc<Semaphore>,
147 pause: Pause,
149}
150
151async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
171 let tracking = format!("{remote}/{base_branch}");
172 let fetched = git::fetch(repo, remote, base_branch).await;
173 if let Ok(out) = &fetched
174 && out.ok()
175 && git::rev_exists(repo, &tracking).await
176 {
177 return git::rev_parse(repo, &tracking).await;
178 }
179 let why = match &fetched {
180 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
181 Ok(_) => format!("{remote} has no {base_branch}"),
182 Err(e) => e.to_string(),
183 };
184 tracing::warn!(
185 "could not read {tracking} ({why}); branching off the local \
186 {base_branch} instead, which may be behind"
187 );
188 git::rev_parse(repo, base_branch).await.with_context(|| {
189 format!(
190 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
191 branch that exists"
192 )
193 })
194}
195
196impl Runner {
197 pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
199 let repo = git::toplevel(repo).await?;
200 let missing = agent::missing_programs(&config.agents);
201 if !missing.is_empty() {
202 bail!(
203 "these agent programs are not on PATH: {}. Fix the roster in \
204 magi.toml or install them.",
205 missing.join(", ")
206 );
207 }
208 let base_branch = match config.merge.base.clone() {
209 Some(b) => b,
210 None => git::current_branch(&repo)
211 .await?
212 .context("HEAD is detached; set [merge] base in magi.toml")?,
213 };
214 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
215 if !git::is_clean(&repo).await? {
219 tracing::warn!(
220 "{} has uncommitted changes; they are not part of this run, \
221 which branches off {base_branch} ({})",
222 repo.display(),
223 &base_commit[..base_commit.len().min(8)]
224 );
225 }
226 let roles = config.resolve_roles()?;
227 let max_parallel = config.graph.max_parallel.max(1);
228 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
229 state.event("start", format!("run {} created", state.id));
230 state.save()?;
231 Ok(Self {
232 state,
233 roles,
234 sem: Arc::new(Semaphore::new(max_parallel)),
235 pause: Pause::new(),
236 })
237 }
238
239 pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
253 let repo = git::toplevel(repo).await?;
254 let missing = agent::missing_programs(&config.agents);
255 if !missing.is_empty() {
256 bail!(
257 "these agent programs are not on PATH: {}. Fix the roster in \
258 magi.toml or install them.",
259 missing.join(", ")
260 );
261 }
262 if !git::branch_exists(&repo, branch).await? {
263 bail!("no branch `{branch}` in {}", repo.display());
264 }
265 let base_branch = match config.merge.base.clone() {
266 Some(b) => b,
267 None => git::current_branch(&repo)
268 .await?
269 .context("HEAD is detached; set [merge] base in magi.toml")?,
270 };
271 if base_branch == branch {
272 bail!("`{branch}` is the base branch; there is nothing to review against");
273 }
274 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
275
276 let roles = config.resolve_roles()?;
277 let max_parallel = config.graph.max_parallel.max(1);
278 let log = git::log_oneline(&repo, &base_commit, branch)
281 .await
282 .unwrap_or_default();
283 let instruction = format!(
284 "Review the work already on branch `{branch}`. There is no task \
285 statement: what the change claims to do is whatever its commits \
286 say.\n\n{}",
287 if log.trim().is_empty() {
288 "(no commit messages)"
289 } else {
290 log.trim()
291 }
292 );
293 let mut state = RunState::new(
294 repo.clone(),
295 base_branch,
296 base_commit.clone(),
297 instruction,
298 config,
299 );
300
301 let worktree = state.worktree_root().join("under-review");
304 if let Some(parent) = worktree.parent() {
305 tokio::fs::create_dir_all(parent).await.ok();
306 }
307 let path = worktree.to_string_lossy().to_string();
308 git::git(&repo, &["worktree", "add", &path, branch])
309 .await
310 .with_context(|| {
311 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
312 })?;
313
314 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
315 .await
316 .unwrap_or(0);
317 if commits == 0 {
318 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
319 }
320 let files = git::changed_files(&worktree, &base_commit, "HEAD")
321 .await
322 .map(|f| f.len())
323 .unwrap_or(0);
324 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
325 .await
326 .unwrap_or_default();
327
328 state.candidates.push(Candidate {
329 index: 0,
330 label: 'A',
331 agent: "(existing branch)".to_owned(),
334 branch: branch.to_owned(),
335 worktree,
336 summary: String::new(),
337 stat,
338 files,
339 commits,
340 empty: false,
341 failed: None,
342 duration_ms: 0,
343 folded: false,
344 });
345 state.tally = Some(Tally {
346 first_choice: BTreeMap::from([('A', 0)]),
347 borda: BTreeMap::new(),
348 winner: 'A',
349 rankings: 0,
350 unanimous_initial: false,
351 deliberated: false,
352 changed_votes: 0,
353 unanimous_final: false,
354 tie_break: None,
355 judges: 0,
359 present: 0,
360 quorum: 0,
361 met_quorum: true,
362 uncontested: Some("review-only run: nothing competed".to_owned()),
363 });
364 state.status = RunStatus::Reviewing;
365 state.event(
366 "start",
367 format!(
368 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
369 state.id
370 ),
371 );
372 state.save()?;
373 Ok(Self {
374 state,
375 roles,
376 sem: Arc::new(Semaphore::new(max_parallel)),
377 pause: Pause::new(),
378 })
379 }
380
381 pub fn resume(id: &str) -> Result<Self> {
383 let state = RunState::load(id)?;
384 let roles = state.config.resolve_roles()?;
385 let max_parallel = state.config.graph.max_parallel.max(1);
386 Ok(Self {
387 state,
388 roles,
389 sem: Arc::new(Semaphore::new(max_parallel)),
390 pause: Pause::new(),
391 })
392 }
393
394 pub async fn execute(&mut self) -> Result<()> {
396 self.state.parked = false;
401 if self.state.clear_active() {
408 self.state.save()?;
409 }
410 if self.state.status == RunStatus::Stalled {
423 if self.recover_stall().await? {
424 self.finish_after_tally().await?;
425 } else {
426 self.state.save()?;
428 }
429 return Ok(());
430 }
431 self.prep().await?;
432 if self.park_here()? {
433 return Ok(());
434 }
435 self.implement().await?;
436 if self.park_here()? {
437 return Ok(());
438 }
439 self.judge().await?;
440 if self.park_here()? {
441 return Ok(());
442 }
443 self.deliberate().await?;
444 if self.park_here()? {
445 return Ok(());
446 }
447 self.vote().await?;
448 if self.park_here()? {
449 return Ok(());
450 }
451 self.tally()?;
452 if self.state.status == RunStatus::Stalled {
457 self.state.save()?;
461 return Ok(());
462 }
463 self.finish_after_tally().await?;
464 Ok(())
465 }
466
467 fn park_here(&mut self) -> Result<bool> {
474 if !self.pause.parked() {
475 return Ok(false);
476 }
477 self.state.event(
478 "park",
479 format!(
480 "parked after `{}` — resume to carry on from here",
481 self.state.status.as_str()
482 ),
483 );
484 self.state.parked = true;
485 self.state.save()?;
486 Ok(true)
487 }
488
489 pub fn on_pause(&mut self, pause: Pause) {
491 self.pause = pause;
492 }
493
494 async fn finish_after_tally(&mut self) -> Result<()> {
497 self.fold_losers().await?;
498 self.sync_to_base().await?;
503 self.review_loop().await?;
504 self.sync_to_base().await?;
505 self.gate().await?;
506 self.merge().await?;
507 self.state.save()?;
508 Ok(())
509 }
510
511 async fn prep(&mut self) -> Result<()> {
514 if !self.state.candidates.is_empty() {
515 return Ok(());
516 }
517 self.state.status = RunStatus::Prep;
518 let repo = self.state.repo.clone();
519 let base = self.state.base_commit.clone();
520 let root = self.state.worktree_root();
521 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
522
523 let hooks_dir = self.state.dir().join("hooks");
526 if self.state.config.blind.commit_msg_hook {
527 std::fs::create_dir_all(&hooks_dir)
528 .with_context(|| format!("create {}", hooks_dir.display()))?;
529 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
530 let path = hooks_dir.join("commit-msg");
531 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
532 make_executable(&path)?;
533 if git::enable_worktree_config(&repo).await? {
534 self.state.enabled_worktree_config = true;
535 }
536 }
537
538 for (index, (spec, label)) in self
539 .roles
540 .implementers
541 .clone()
542 .into_iter()
543 .zip(labels)
544 .enumerate()
545 {
546 let branch = self.state.branch_for(label);
547 let worktree = root.join(format!("cand-{label}"));
548 git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
549 if self.state.config.blind.commit_msg_hook {
550 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
551 }
552 git::local_exclude(&worktree, "/.magi/").await?;
553 self.state.candidates.push(Candidate {
554 index,
555 label,
556 agent: spec.id.clone(),
557 branch,
558 worktree,
559 summary: String::new(),
560 stat: String::new(),
561 files: 0,
562 commits: 0,
563 empty: false,
564 failed: None,
565 duration_ms: 0,
566 folded: false,
567 });
568 }
569
570 for j in 1..=self.roles.judges.len() {
571 let wt = root.join(format!("judge-{j}"));
572 if !wt.exists() {
573 git::worktree_add_detached(&repo, &wt, &base).await?;
574 }
575 }
576
577 let authors: Vec<&str> = self
582 .roles
583 .implementers
584 .iter()
585 .map(|a| a.id.as_str())
586 .collect();
587 let overlap: Vec<String> = self
588 .roles
589 .judges
590 .iter()
591 .enumerate()
592 .filter(|(_, j)| authors.contains(&j.id.as_str()))
593 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
594 .collect();
595 if !overlap.is_empty() {
596 let note = format!(
597 "{} also authored a candidate; blind, but the panel is less \
598 independent than {} distinct agents would be",
599 overlap.join(", "),
600 self.roles.judges.len()
601 );
602 self.state.event("prep", note);
603 }
604
605 self.state.event(
606 "prep",
607 format!(
608 "{} candidates, {} judges, base {} ({})",
609 self.state.candidates.len(),
610 self.roles.judges.len(),
611 &self.state.base_commit[..7.min(self.state.base_commit.len())],
612 self.state.base_branch
613 ),
614 );
615 self.state.status = RunStatus::Implementing;
616 self.state.save()?;
617 Ok(())
618 }
619
620 async fn implement(&mut self) -> Result<()> {
623 let run_id = self.state.id.clone();
628 let prompts = self.state.config.prompts.clone();
629 let todo: Vec<usize> = self
630 .state
631 .candidates
632 .iter()
633 .enumerate()
634 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
635 .map(|(i, _)| i)
636 .collect();
637 if todo.is_empty() {
638 return self.after_implement();
639 }
640 self.state.status = RunStatus::Implementing;
641
642 let language = self.state.config.graph.language.clone();
643 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
644 let sessions = self.state.config.graph.sessions;
645 let artifacts = agent::artifacts_dir(&self.state.dir());
646
647 let mut jobs = Vec::new();
648 for &i in &todo {
649 let (index, label, worktree) = {
650 let c = &self.state.candidates[i];
651 (c.index, c.label, c.worktree.clone())
652 };
653 let spec = self.roles.implementers[index].clone();
654 let seat_key = format!("impl-{label}");
655 let seat = self.seat(&seat_key, &spec.id);
656 let instruction = self.state.instruction.clone();
657 jobs.push(SeatJob {
658 spec,
659 seat,
660 prompt: prompt::implement(&instruction, &worktree.to_string_lossy(), &language),
661 cwd: worktree,
662 timeout,
663 allow_write: true,
664 sessions,
665 artifacts: artifacts.clone(),
666 stem: format!("impl-{label}"),
667 });
668 }
669
670 self.state.event(
671 "implement",
672 format!("{} candidates in parallel", jobs.len()),
673 );
674 let sent = jobs.clone();
677 let cache = self.state.config.cache_dir();
678 let ctx = WaveCtx {
679 run: &run_id,
680 node: "implement",
681 prompts: &prompts,
682 cache: cache.as_deref(),
683 };
684 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
685 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
686 .await;
687
688 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
689 let seat_key = seat.key.clone();
690 self.state.seats.insert(seat.key.clone(), seat);
691 let label = self.state.candidates[i].label;
692 let worktree = self.state.candidates[i].worktree.clone();
693 let base = self.state.base_commit.clone();
694
695 let (summary, duration, failed) = match out {
696 AgentOutcome::Ok(o) => {
697 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
698 let failed = (!o.usable()).then(|| {
699 if o.timed_out {
700 "agent timed out".to_owned()
701 } else {
702 format!("agent exited with {:?}", o.exit_code)
703 }
704 });
705 (text, o.duration_ms, failed)
706 }
707 AgentOutcome::Dropped(o) => {
713 let why = o
714 .dropped
715 .as_ref()
716 .map(|d| d.why.as_str())
717 .unwrap_or("the CLI ended the stream without delivering its answer");
718 (
719 String::new(),
720 o.duration_ms,
721 Some(format!("the CLI dropped the stream ({why})")),
722 )
723 }
724 AgentOutcome::Quota(o) => {
725 self.state.quota.push(QuotaLoss {
726 seat: seat_key,
727 node: "implement".to_owned(),
728 at: Timestamp::now(),
729 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
730 });
731 (
732 String::new(),
733 o.duration_ms,
734 Some("rate limited (quota); produced no change".to_owned()),
735 )
736 }
737 AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
738 };
739
740 let rescued = git::commit_all(
743 &worktree,
744 &format!("magi: candidate {label} (uncommitted work)"),
745 )
746 .await
747 .unwrap_or(false);
748 let commits = git::commits_ahead(&worktree, &base, "HEAD")
749 .await
750 .unwrap_or(0);
751 let patch = git::diff(&worktree, &base, "HEAD")
752 .await
753 .unwrap_or_default();
754 let stat = git::diff_stat(&worktree, &base, "HEAD")
755 .await
756 .unwrap_or_default();
757 let files = git::changed_files(&worktree, &base, "HEAD")
758 .await
759 .map(|f| f.len())
760 .unwrap_or(0);
761 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
762
763 let c = &mut self.state.candidates[i];
764 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
765 c.stat = stat;
766 c.files = files;
767 c.commits = commits;
768 c.duration_ms = duration;
769 c.empty = commits == 0 || patch.trim().is_empty();
770 c.failed = match failed {
773 Some(_) if c.empty => failed,
774 _ => None,
775 };
776 let note = match (&c.failed, c.empty, rescued) {
777 (Some(e), _, _) => format!("candidate {label}: {e}"),
778 (None, true, _) => format!("candidate {label}: no change produced"),
779 (None, false, true) => {
780 format!(
781 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
782 )
783 }
784 (None, false, false) => {
785 format!("candidate {label}: {files} files, {commits} commits")
786 }
787 };
788 self.state.event("implement", note);
789 self.state.save()?;
790 }
791
792 self.after_implement()
793 }
794
795 async fn resume_undelivered(
823 &mut self,
824 results: &mut [(usize, SeatState, AgentOutcome)],
825 sent: &[SeatJob],
826 prompts: &Prompts,
827 run_id: &str,
828 ) {
829 for (wi, seat, out) in results.iter_mut() {
830 let Some(dropped) = (match &*out {
831 AgentOutcome::Dropped(o) => o.dropped.clone(),
832 _ => None,
833 }) else {
834 continue;
835 };
836 let Some(job) = sent.get(*wi) else { continue };
837 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
839 self.state.event(
840 "implement",
841 format!(
842 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
843 work is in the tree",
844 seat.key, dropped.output_tokens, dropped.why
845 ),
846 );
847 continue;
848 }
849 if !has_context(&job.spec, seat, job.sessions) {
857 self.state.event(
858 "implement",
859 format!(
860 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
861 is no session left to resume",
862 seat.key, dropped.output_tokens, dropped.why
863 ),
864 );
865 continue;
866 }
867 self.state.event(
868 "implement",
869 format!(
870 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
871 conversation",
872 seat.key, dropped.output_tokens, dropped.why
873 ),
874 );
875 let mut retry = job.clone();
876 retry.seat = seat.clone();
877 retry.prompt = prompt::resume_after_drop(&dropped.why);
878 retry.timeout = retry_budget(job.timeout, true);
879 retry.stem = format!("{}-resume", job.stem);
880 let cache = self.state.config.cache_dir();
881 let ctx = WaveCtx {
882 run: run_id,
883 node: "implement",
884 prompts,
885 cache: cache.as_deref(),
886 };
887 let (resumed_seat, resumed) =
888 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
889 *seat = resumed_seat;
890 *out = resumed;
891 }
892 }
893
894 fn after_implement(&mut self) -> Result<()> {
895 if self.state.leaks.is_empty() {
897 let cfg = self.state.config.blind.clone();
898 let mut leaks = Vec::new();
899 for c in &self.state.candidates {
900 let Some(patch) =
901 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
902 else {
903 continue;
904 };
905 leaks.extend(blind::scan(
906 &format!("candidate {} patch", c.label),
907 &patch,
908 &cfg.vendor_tokens,
909 ));
910 }
911 if !leaks.is_empty() {
912 let summary = leaks
913 .iter()
914 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
915 .collect::<Vec<_>>()
916 .join(", ");
917 match cfg.on_leak {
918 LeakPolicy::Fail => {
919 self.state.status = RunStatus::Failed;
920 self.state
921 .event("blind", format!("vendor text in a patch: {summary}"));
922 self.state.leaks = leaks;
923 self.state.save()?;
924 bail!(
925 "blind.on_leak = \"fail\" and vendor text reached a \
926 judged patch: {summary}"
927 );
928 }
929 LeakPolicy::Redact => self.state.event(
930 "blind",
931 format!("redacting vendor text for judging: {summary}"),
932 ),
933 LeakPolicy::Warn => self.state.event(
934 "blind",
935 format!("vendor text present in a judged patch (shown as-is): {summary}"),
936 ),
937 }
938 self.state.leaks = leaks;
939 }
940 }
941
942 if self.state.viable().is_empty() {
943 self.state.status = RunStatus::Failed;
944 self.state.save()?;
945 bail!("no candidate produced a change; nothing to judge");
946 }
947 self.state.status = RunStatus::Judging;
948 self.state.save()?;
949 Ok(())
950 }
951
952 async fn judge(&mut self) -> Result<()> {
955 let run_id = self.state.id.clone();
960 let prompts = self.state.config.prompts.clone();
961 if !self.state.judgements.is_empty() || self.state.judge_skipped {
962 return Ok(());
963 }
964 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
965 if viable.len() == 1 {
966 self.state.judge_skipped = true;
973 self.state.event(
974 "judge",
975 format!(
976 "only candidate {} produced a change; judging skipped",
977 viable[0].label
978 ),
979 );
980 self.state.save()?;
981 return Ok(());
982 }
983 self.state.status = RunStatus::Judging;
984
985 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
986 let language = self.state.config.graph.language.clone();
987 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
988 let sessions = self.state.config.graph.sessions;
989 let artifacts = agent::artifacts_dir(&self.state.dir());
990 let root = self.state.worktree_root();
991 let base_short = short(&self.state.base_commit);
992
993 let mut jobs = Vec::new();
994 let mut orders = Vec::new();
995 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
996 let order = blind::presentation_order(viable.len(), j, self.state.seed);
997 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
998 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
999 let seat_key = format!("judge-{}", j + 1);
1000 let seat = self.seat(&seat_key, &spec.id);
1001 jobs.push(SeatJob {
1002 prompt: prompt::judge(
1003 &self.state.instruction,
1004 &views,
1005 self.roles.judges.len(),
1006 &base_short,
1007 &language,
1008 ),
1009 spec,
1010 seat,
1011 cwd: root.join(format!("judge-{}", j + 1)),
1012 timeout,
1013 allow_write: false,
1014 sessions,
1015 artifacts: artifacts.clone(),
1016 stem: format!("judge-{}", j + 1),
1017 });
1018 }
1019
1020 self.state.event(
1021 "judge",
1022 format!(
1023 "{} judges ranking {} candidates blind",
1024 jobs.len(),
1025 viable.len()
1026 ),
1027 );
1028 let labels_for_check = labels.clone();
1029 let mut quota_losses = Vec::new();
1030 let cache = self.state.config.cache_dir();
1031 let ctx = WaveCtx {
1032 run: &run_id,
1033 node: "judge",
1034 prompts: &prompts,
1035 cache: cache.as_deref(),
1036 };
1037 let results = ask_json_wave::<Ranking>(
1038 jobs,
1039 Arc::clone(&self.sem),
1040 self.state.config.graph.retries,
1041 &ctx,
1042 &mut quota_losses,
1043 &mut self.state,
1044 &move |r: &Ranking| r.validate(&labels_for_check),
1045 )
1046 .await;
1047 self.state.quota.extend(quota_losses);
1048
1049 for (j, (seat, res)) in results.into_iter().enumerate() {
1050 let agent_id = seat.agent.clone();
1051 self.state.seats.insert(seat.key.clone(), seat);
1052 let mut record = Judgement {
1053 judge: j + 1,
1054 seat: format!("judge-{}", j + 1),
1055 agent: agent_id,
1056 ranking: Vec::new(),
1057 reasons: BTreeMap::new(),
1058 confidence: None,
1059 order: orders[j].clone(),
1060 failed: None,
1061 duration_ms: 0,
1062 };
1063 match res {
1064 Ok((ranking, out)) => {
1065 record.ranking = ranking.normalized();
1066 record.reasons = ranking.reasons;
1067 record.confidence = ranking.confidence;
1068 record.duration_ms = out.duration_ms;
1069 self.state.event(
1070 "judge",
1071 format!(
1072 "judge {} ranked {}",
1073 j + 1,
1074 record.ranking.iter().collect::<String>()
1075 ),
1076 );
1077 }
1078 Err(e) => {
1079 record.failed = Some(e.to_string());
1080 self.state
1081 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1082 }
1083 }
1084 self.state.judgements.push(record);
1085 self.state.save()?;
1086 }
1087 Ok(())
1088 }
1089
1090 async fn deliberate(&mut self) -> Result<()> {
1093 let run_id = self.state.id.clone();
1098 let prompts = self.state.config.prompts.clone();
1099 if !self.state.deliberation.is_empty() {
1100 return Ok(());
1101 }
1102 let tops: Vec<char> = self
1103 .state
1104 .judgements
1105 .iter()
1106 .filter_map(|j| j.ranking.first().copied())
1107 .collect();
1108 let rounds = self.state.config.graph.deliberate_rounds;
1109 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1110 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1111 self.state.event(
1112 "deliberate",
1113 format!("judges agreed on {} outright; no deliberation", tops[0]),
1114 );
1115 }
1116 self.state.status = RunStatus::Voting;
1117 self.state.save()?;
1118 return Ok(());
1119 }
1120
1121 self.state.status = RunStatus::Deliberating;
1122 self.state.event(
1123 "deliberate",
1124 format!(
1125 "split: first choices were {} — opening {rounds} round(s)",
1126 tops.iter().collect::<String>()
1127 ),
1128 );
1129
1130 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1131 let language = self.state.config.graph.language.clone();
1132 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1133 let sessions = self.state.config.graph.sessions;
1134 let artifacts = agent::artifacts_dir(&self.state.dir());
1135 let root = self.state.worktree_root();
1136 let base_short = short(&self.state.base_commit);
1137
1138 for round in 1..=rounds {
1142 let mut turns: Vec<DeliberationTurn> = Vec::new();
1143 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1144 if self.state.judgements[j].failed.is_some() {
1145 continue;
1146 }
1147 let seat_key = format!("judge-{}", j + 1);
1148 let mut seat = self.seat(&seat_key, &spec.id);
1149 let transcript = self.transcript(&turns, j);
1150 let context = if has_context(&spec, &seat, sessions) {
1151 None
1152 } else {
1153 Some(self.candidate_block(&viable, &base_short))
1154 };
1155 let text = prompt::deliberate(
1156 &self.state.instruction,
1157 context.as_deref(),
1158 &transcript,
1159 round,
1160 rounds,
1161 &language,
1162 );
1163 let job = SeatJob {
1164 spec,
1165 seat: seat.clone(),
1166 prompt: text,
1167 cwd: root.join(format!("judge-{}", j + 1)),
1168 timeout,
1169 allow_write: false,
1170 sessions,
1171 artifacts: artifacts.clone(),
1172 stem: format!("delib-{round}-judge-{}", j + 1),
1173 };
1174 let cache = self.state.config.cache_dir();
1175 let ctx = WaveCtx {
1176 run: &run_id,
1177 node: "deliberate",
1178 prompts: &prompts,
1179 cache: cache.as_deref(),
1180 };
1181 let (updated, out) =
1182 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1183 seat = updated;
1184 let agent_id = seat.agent.clone();
1185 let seat_key = seat.key.clone();
1186 self.state.seats.insert(seat.key.clone(), seat);
1187 let body = match out {
1188 AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1189 AgentOutcome::Dropped(o) => {
1193 let why =
1194 o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1195 "the CLI ended the stream without delivering its answer",
1196 );
1197 self.state.event(
1198 "deliberate",
1199 format!(
1200 "judge {} skipped: the CLI dropped the stream ({why})",
1201 j + 1
1202 ),
1203 );
1204 continue;
1205 }
1206 AgentOutcome::Quota(o) => {
1207 self.state.quota.push(QuotaLoss {
1208 seat: seat_key,
1209 node: "deliberate".to_owned(),
1210 at: Timestamp::now(),
1211 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1212 });
1213 self.state.event(
1214 "deliberate",
1215 format!("judge {} skipped: rate limited (quota)", j + 1),
1216 );
1217 continue;
1218 }
1219 AgentOutcome::Failed(e) => {
1220 self.state
1221 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1222 continue;
1223 }
1224 };
1225 let tentative = verdict::extract_json::<Position>(&body)
1226 .ok()
1227 .and_then(|p| p.tentative)
1228 .and_then(|s| s.trim().chars().next())
1229 .map(|c| c.to_ascii_uppercase());
1230 self.state.event(
1231 "deliberate",
1232 format!(
1233 "round {round}: judge {} now favours {}",
1234 j + 1,
1235 tentative.map_or("—".to_owned(), |c| c.to_string())
1236 ),
1237 );
1238 turns.push(DeliberationTurn {
1239 judge: j + 1,
1240 agent: agent_id,
1241 body: blind::sanitize_prose(&body, &self.state.config.blind),
1242 tentative,
1243 });
1244 }
1245 self.state
1246 .deliberation
1247 .push(DeliberationRound { round, turns });
1248 self.state.save()?;
1249 }
1250
1251 self.state.status = RunStatus::Voting;
1252 self.state.save()?;
1253 Ok(())
1254 }
1255
1256 async fn vote(&mut self) -> Result<()> {
1259 let run_id = self.state.id.clone();
1264 let prompts = self.state.config.prompts.clone();
1265 if !self.state.votes.is_empty() {
1266 return Ok(());
1267 }
1268 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1269 if viable.len() == 1 {
1270 return Ok(());
1271 }
1272 self.state.status = RunStatus::Voting;
1273
1274 let language = self.state.config.graph.language.clone();
1275 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1276 let sessions = self.state.config.graph.sessions;
1277 let artifacts = agent::artifacts_dir(&self.state.dir());
1278 let root = self.state.worktree_root();
1279 let base_short = short(&self.state.base_commit);
1280 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1281
1282 let mut jobs = Vec::new();
1283 let mut seats_at = Vec::new();
1284 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1285 if self
1286 .state
1287 .judgements
1288 .get(j)
1289 .is_some_and(|r| r.failed.is_some())
1290 {
1291 continue;
1292 }
1293 let seat_key = format!("judge-{}", j + 1);
1294 let seat = self.seat(&seat_key, &spec.id);
1295 let mut text = prompt::final_vote(&viable, &language);
1296 if !has_context(&spec, &seat, sessions) {
1297 text = format!(
1298 "{}\n\n# Candidates\n\n{}",
1299 text,
1300 self.candidate_block(&candidates, &base_short)
1301 );
1302 }
1303 jobs.push(SeatJob {
1304 spec,
1305 seat,
1306 prompt: text,
1307 cwd: root.join(format!("judge-{}", j + 1)),
1308 timeout,
1309 allow_write: false,
1310 sessions,
1311 artifacts: artifacts.clone(),
1312 stem: format!("vote-judge-{}", j + 1),
1313 });
1314 seats_at.push(j);
1315 }
1316
1317 self.state.event(
1318 "vote",
1319 format!(
1320 "collecting {} final votes one by one, privately",
1321 jobs.len()
1322 ),
1323 );
1324 let allowed = viable.clone();
1325 let mut quota_losses = Vec::new();
1326 let cache = self.state.config.cache_dir();
1327 let ctx = WaveCtx {
1328 run: &run_id,
1329 node: "vote",
1330 prompts: &prompts,
1331 cache: cache.as_deref(),
1332 };
1333 let results = ask_json_wave::<FinalVote>(
1334 jobs,
1335 Arc::clone(&self.sem),
1336 self.state.config.graph.retries,
1337 &ctx,
1338 &mut quota_losses,
1339 &mut self.state,
1340 &move |v: &FinalVote| match v.label() {
1341 Some(c) if allowed.contains(&c) => Ok(()),
1342 other => bail!("vote {other:?} is not one of {allowed:?}"),
1343 },
1344 )
1345 .await;
1346 self.state.quota.extend(quota_losses);
1347
1348 for (&j, (seat, res)) in seats_at.iter().zip(results) {
1349 let agent_id = seat.agent.clone();
1350 self.state.seats.insert(seat.key.clone(), seat);
1351 let initial = self
1352 .state
1353 .judgements
1354 .get(j)
1355 .and_then(|r| r.ranking.first().copied());
1356 let mut record = VoteRecord {
1357 judge: j + 1,
1358 agent: agent_id,
1359 vote: None,
1360 reason: String::new(),
1361 changed: false,
1362 };
1363 match res {
1364 Ok((v, _)) => {
1365 record.vote = v.label();
1366 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1367 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1368 self.state.event(
1369 "vote",
1370 format!(
1371 "judge {} voted {}{}",
1372 j + 1,
1373 record.vote.unwrap_or('?'),
1374 if record.changed { " (changed)" } else { "" }
1375 ),
1376 );
1377 }
1378 Err(e) => {
1379 self.state
1380 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1381 }
1382 }
1383 self.state.votes.push(record);
1384 self.state.save()?;
1385 }
1386 Ok(())
1387 }
1388
1389 fn tally(&mut self) -> Result<()> {
1392 if self.state.tally.is_some() {
1393 return Ok(());
1394 }
1395 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1396 let tops: Vec<char> = self
1397 .state
1398 .judgements
1399 .iter()
1400 .filter_map(|j| j.ranking.first().copied())
1401 .collect();
1402 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1403
1404 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1407 let mut cast: Vec<char> = Vec::new();
1408 for (i, j) in self.state.judgements.iter().enumerate() {
1409 let vote = self
1410 .state
1411 .votes
1412 .iter()
1413 .find(|v| v.judge == i + 1)
1414 .and_then(|v| v.vote)
1415 .or_else(|| j.ranking.first().copied());
1416 if let Some(v) = vote {
1417 *first_choice.entry(v).or_insert(0) += 1;
1418 cast.push(v);
1419 }
1420 }
1421
1422 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1423 for j in &self.state.judgements {
1424 let n = j.ranking.len();
1425 for (pos, label) in j.ranking.iter().enumerate() {
1426 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1427 }
1428 }
1429
1430 let best = first_choice.values().copied().max().unwrap_or(0);
1431 let mut leaders: Vec<char> = first_choice
1432 .iter()
1433 .filter(|(_, v)| **v == best)
1434 .map(|(k, _)| *k)
1435 .collect();
1436 let mut tie_break = None;
1437 if leaders.len() > 1 {
1438 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1439 let borda_leaders: Vec<char> = leaders
1440 .iter()
1441 .copied()
1442 .filter(|l| borda[l] == top_borda)
1443 .collect();
1444 tie_break = Some(if borda_leaders.len() == 1 {
1445 format!(
1446 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1447 leaders.len()
1448 )
1449 } else {
1450 format!(
1451 "{} way tie on both first-choice votes and Borda points, broken by label order",
1452 leaders.len()
1453 )
1454 });
1455 leaders = borda_leaders;
1456 leaders.sort_unstable();
1457 }
1458 let winner = *leaders
1459 .first()
1460 .or(viable.first())
1461 .context("no candidate to declare a winner from")?;
1462
1463 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1464 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1465 let deliberated = !self.state.deliberation.is_empty();
1466
1467 let quota_seats: std::collections::BTreeSet<&str> =
1471 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1472 let mut present = 0usize;
1473 for (i, j) in self.state.judgements.iter().enumerate() {
1474 if quota_seats.contains(j.seat.as_str()) {
1475 continue;
1476 }
1477 let ranked = !j.ranking.is_empty() && j.failed.is_none();
1478 let voted = self
1479 .state
1480 .votes
1481 .iter()
1482 .any(|v| v.judge == i + 1 && v.vote.is_some());
1483 if ranked || voted {
1484 present += 1;
1485 }
1486 }
1487 let needs_quorum = viable.len() > 1;
1493 let judges_total = if needs_quorum {
1494 self.roles.judges.len()
1495 } else {
1496 0
1497 };
1498 let quorum = if needs_quorum {
1499 judges_total / 2 + 1
1500 } else {
1501 0
1502 };
1503 let met_quorum = !needs_quorum || present >= quorum;
1504 let uncontested = (!needs_quorum).then(|| {
1505 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1506 });
1507
1508 self.state.event(
1509 "tally",
1510 match &uncontested {
1511 Some(reason) => format!("winner {winner} — {reason}"),
1512 None => format!(
1513 "winner {winner} — votes {} | initial {} | {} changed | \
1514 {present}/{judges_total} judges{}",
1515 first_choice
1516 .iter()
1517 .map(|(k, v)| format!("{k}:{v}"))
1518 .collect::<Vec<_>>()
1519 .join(" "),
1520 if unanimous_initial {
1521 "unanimous"
1522 } else {
1523 "split"
1524 },
1525 changed_votes,
1526 if met_quorum {
1527 String::new()
1528 } else {
1529 format!(" — below quorum ({quorum} required)")
1530 },
1531 ),
1532 },
1533 );
1534 if !met_quorum {
1535 self.state.event(
1536 "stall",
1537 format!(
1538 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1539 the run stops here, resumable"
1540 ),
1541 );
1542 }
1543 self.state.tally = Some(Tally {
1544 first_choice,
1545 borda,
1546 winner,
1547 rankings: tops.len(),
1548 unanimous_initial,
1549 deliberated,
1550 changed_votes,
1551 unanimous_final,
1552 tie_break,
1553 judges: judges_total,
1554 present,
1555 quorum,
1556 met_quorum,
1557 uncontested,
1558 });
1559 self.state.status = if met_quorum {
1560 RunStatus::Reviewing
1561 } else {
1562 RunStatus::Stalled
1563 };
1564 self.state.save()?;
1565 Ok(())
1566 }
1567
1568 #[allow(clippy::too_many_lines)]
1589 async fn recover_stall(&mut self) -> Result<bool> {
1590 let run_id = self.state.id.clone();
1595 let prompts = self.state.config.prompts.clone();
1596 let quota_seats: BTreeSet<&str> =
1601 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1602 let absent: Vec<String> = self
1603 .state
1604 .judgements
1605 .iter()
1606 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
1607 .map(|j| j.seat.clone())
1608 .collect();
1609 if absent.is_empty() {
1610 return Ok(false);
1611 }
1612 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1613 if viable.len() <= 1 {
1614 return Ok(false);
1615 }
1616 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1617 let language = self.state.config.graph.language.clone();
1618 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1619 let sessions = self.state.config.graph.sessions;
1620 let artifacts = agent::artifacts_dir(&self.state.dir());
1621 let root = self.state.worktree_root();
1622 let base_short = short(&self.state.base_commit);
1623 let candidates: Vec<Candidate> = viable.clone();
1624
1625 let mut positions: Vec<usize> = absent
1627 .iter()
1628 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
1629 .collect();
1630 if positions.is_empty() {
1631 return Ok(false);
1632 }
1633 positions.sort_unstable();
1634 positions.dedup();
1635
1636 let mut judge_jobs = Vec::new();
1638 for &j in &positions {
1639 let order = blind::presentation_order(viable.len(), j, self.state.seed);
1640 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1641 let seat_key = format!("judge-{}", j + 1);
1642 let spec = self.roles.judges[j].clone();
1643 let seat = self.seat(&seat_key, &spec.id);
1644 judge_jobs.push(SeatJob {
1645 spec,
1646 seat,
1647 prompt: prompt::judge(
1648 &self.state.instruction,
1649 &views,
1650 self.roles.judges.len(),
1651 &base_short,
1652 &language,
1653 ),
1654 cwd: root.join(seat_key),
1655 timeout,
1656 allow_write: false,
1657 sessions,
1658 artifacts: artifacts.clone(),
1659 stem: format!("judge-{}-recover", j + 1),
1660 });
1661 }
1662
1663 let labels_for_check = labels.clone();
1664 let mut judge_losses = Vec::new();
1665 let retries = self.state.config.graph.retries;
1666 let cache = self.state.config.cache_dir();
1667 let ctx = WaveCtx {
1668 run: &run_id,
1669 node: "judge",
1670 prompts: &prompts,
1671 cache: cache.as_deref(),
1672 };
1673 let results = ask_json_wave::<Ranking>(
1674 judge_jobs,
1675 Arc::clone(&self.sem),
1676 retries,
1677 &ctx,
1678 &mut judge_losses,
1679 &mut self.state,
1680 &move |r: &Ranking| r.validate(&labels_for_check),
1681 )
1682 .await;
1683
1684 let mut recovered: BTreeSet<usize> = BTreeSet::new();
1686 for (&j, (seat, res)) in positions.iter().zip(results) {
1687 self.state.seats.insert(seat.key.clone(), seat);
1688 let record = &mut self.state.judgements[j];
1689 match res {
1690 Ok((ranking, out)) => {
1691 record.ranking = ranking.normalized();
1692 record.reasons = ranking.reasons;
1693 record.confidence = ranking.confidence;
1694 record.failed = None;
1695 record.duration_ms = out.duration_ms;
1696 recovered.insert(j);
1697 self.state.event(
1698 "recover",
1699 format!("judge {} ranked again after the limit", j + 1),
1700 );
1701 }
1702 Err(e) => {
1703 self.state
1704 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
1705 }
1706 }
1707 }
1708
1709 let mut vote_jobs = Vec::new();
1711 let mut vote_pos: Vec<usize> = Vec::new();
1712 for &j in &recovered {
1713 let seat_key = format!("judge-{}", j + 1);
1714 let spec = self.roles.judges[j].clone();
1715 let seat = self.seat(&seat_key, &spec.id);
1716 let mut text = prompt::final_vote(&labels, &language);
1717 if !has_context(&spec, &seat, sessions) {
1718 text = format!(
1719 "{}\n\n# Candidates\n\n{}",
1720 text,
1721 self.candidate_block(&candidates, &base_short)
1722 );
1723 }
1724 vote_jobs.push(SeatJob {
1725 spec,
1726 seat,
1727 prompt: text,
1728 cwd: root.join(seat_key),
1729 timeout,
1730 allow_write: false,
1731 sessions,
1732 artifacts: artifacts.clone(),
1733 stem: format!("vote-judge-{}-recover", j + 1),
1734 });
1735 vote_pos.push(j);
1736 }
1737 let allowed = labels.clone();
1738 let mut vote_losses = Vec::new();
1739 let vote_retries = self.state.config.graph.retries;
1740 let vote_cache = self.state.config.cache_dir();
1741 let ctx = WaveCtx {
1742 run: &run_id,
1743 node: "vote",
1744 prompts: &prompts,
1745 cache: vote_cache.as_deref(),
1746 };
1747 let votes = ask_json_wave::<FinalVote>(
1748 vote_jobs,
1749 Arc::clone(&self.sem),
1750 vote_retries,
1751 &ctx,
1752 &mut vote_losses,
1753 &mut self.state,
1754 &move |v: &FinalVote| match v.label() {
1755 Some(c) if allowed.contains(&c) => Ok(()),
1756 other => bail!("vote {other:?} is not one of {allowed:?}"),
1757 },
1758 )
1759 .await;
1760 for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
1761 let agent_id = seat.agent.clone();
1762 self.state.seats.insert(seat.key.clone(), seat);
1763 match res {
1764 Ok((v, _)) => {
1765 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
1766 rec.vote = v.label();
1767 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1768 } else {
1769 self.state.votes.push(VoteRecord {
1770 judge: j + 1,
1771 agent: agent_id,
1772 vote: v.label(),
1773 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
1774 changed: false,
1775 });
1776 }
1777 self.state.event(
1778 "recover",
1779 format!("judge {} voted again after the limit", j + 1),
1780 );
1781 }
1782 Err(e) => {
1783 self.state
1784 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
1785 }
1786 }
1787 }
1788
1789 if !recovered.is_empty() {
1793 let recovered_keys: BTreeSet<String> = recovered
1794 .iter()
1795 .map(|&j| format!("judge-{}", j + 1))
1796 .collect();
1797 self.state
1798 .quota
1799 .retain(|q| !recovered_keys.contains(&q.seat));
1800 }
1801
1802 self.state.tally = None;
1804 self.tally()?;
1805 Ok(self
1806 .state
1807 .tally
1808 .as_ref()
1809 .map(|t| t.met_quorum)
1810 .unwrap_or(false))
1811 }
1812
1813 async fn fold_losers(&mut self) -> Result<()> {
1816 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
1817 return Ok(());
1818 };
1819 let repo = self.state.repo.clone();
1820 let mut folded = Vec::new();
1821 for i in 0..self.state.candidates.len() {
1822 let c = &self.state.candidates[i];
1823 if c.label == winner || c.folded {
1824 continue;
1825 }
1826 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
1827 git::worktree_remove(&repo, &wt).await.ok();
1828 git::branch_delete(&repo, &branch).await.ok();
1829 self.state.candidates[i].folded = true;
1830 folded.push(label.to_string());
1831 }
1832 let root = self.state.worktree_root();
1834 for j in 1..=self.roles.judges.len() {
1835 let wt = root.join(format!("judge-{j}"));
1836 if wt.exists() {
1837 git::worktree_remove(&repo, &wt).await.ok();
1838 }
1839 }
1840 if !folded.is_empty() {
1841 self.state
1842 .event("fold", format!("folded candidates {}", folded.join(", ")));
1843 self.state.save()?;
1844 }
1845 Ok(())
1846 }
1847
1848 async fn sync_to_base(&mut self) -> Result<()> {
1878 if self
1879 .state
1880 .base_sync
1881 .as_ref()
1882 .is_some_and(|s| s.conflict.is_some())
1883 {
1884 return Ok(());
1885 }
1886 let Some(winner) = self.state.winner().cloned() else {
1887 return Ok(());
1888 };
1889
1890 let repo = self.state.repo.clone();
1891 let remote = self.state.config.merge.remote.clone();
1892 let base_branch = self.state.base_branch.clone();
1893 let tracking = format!("{remote}/{base_branch}");
1894
1895 git::fetch(&repo, &remote, &base_branch).await.ok();
1896 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
1900 return Ok(());
1901 };
1902
1903 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1904 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
1905 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
1906
1907 if behind == 0 {
1908 self.state.base_sync = Some(BaseSync {
1909 tip,
1910 behind: 0,
1911 attempts,
1912 conflict: None,
1913 });
1914 self.state.save()?;
1915 return Ok(());
1916 }
1917
1918 if attempts >= BASE_SYNC_ROUNDS {
1919 let why = format!(
1920 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
1921 rebase(s); rebasing again would only race it",
1922 winner.branch
1923 );
1924 self.state.status = RunStatus::Blocked;
1925 self.state.base_sync = Some(BaseSync {
1926 tip,
1927 behind,
1928 attempts,
1929 conflict: Some(why.clone()),
1930 });
1931 self.state.event("land", why);
1932 self.state.save()?;
1933 return Ok(());
1934 }
1935
1936 self.state.event(
1937 "land",
1938 format!(
1939 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
1940 winner.branch
1941 ),
1942 );
1943 self.state.save()?;
1944
1945 let scratch = self.state.dir().join("base-sync");
1946 let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
1947 let attempts = attempts + 1;
1948 match rebased {
1949 Ok(None) => {
1950 git::sync_to_head(&winner.worktree).await?;
1954 self.state.base_sync = Some(BaseSync {
1955 tip: tip.clone(),
1956 behind: 0,
1957 attempts,
1958 conflict: None,
1959 });
1960 self.state
1961 .event("land", format!("rebased {} onto {tracking}", winner.branch));
1962 }
1963 Ok(Some(conflict)) => {
1964 let why = format!(
1965 "{} conflicts with {tracking} and did not rebase: {}",
1966 winner.branch,
1967 conflict.chars().take(600).collect::<String>()
1968 );
1969 self.state.status = RunStatus::Blocked;
1970 self.state.base_sync = Some(BaseSync {
1971 tip,
1972 behind,
1973 attempts,
1974 conflict: Some(why.clone()),
1975 });
1976 self.state.event("land", why);
1977 }
1978 Err(e) => {
1979 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
1980 self.state.status = RunStatus::Blocked;
1981 self.state.base_sync = Some(BaseSync {
1982 tip,
1983 behind,
1984 attempts,
1985 conflict: Some(why.clone()),
1986 });
1987 self.state.event("land", why);
1988 }
1989 }
1990 self.state.save()?;
1991 Ok(())
1992 }
1993
1994 fn landing_base(&self) -> String {
2004 self.state
2005 .base_sync
2006 .as_ref()
2007 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2008 }
2009
2010 async fn review_loop(&mut self) -> Result<()> {
2013 if self
2018 .state
2019 .base_sync
2020 .as_ref()
2021 .is_some_and(|s| s.conflict.is_some())
2022 {
2023 return Ok(());
2024 }
2025 let run_id = self.state.id.clone();
2030 let prompts = self.state.config.prompts.clone();
2031 let Some(winner) = self.state.winner().cloned() else {
2032 return Ok(());
2033 };
2034 let max_rounds = self.state.config.graph.review_rounds;
2035 if max_rounds == 0 || self.state.reviews.iter().any(|r| r.clean) {
2036 self.state.status = RunStatus::Gating;
2037 self.state.save()?;
2038 return Ok(());
2039 }
2040 if self.state.reviews.len() >= max_rounds {
2041 self.state.status = RunStatus::Blocked;
2050 self.state.save()?;
2051 return Ok(());
2052 }
2053 self.state.status = RunStatus::Reviewing;
2054
2055 let repo = self.state.repo.clone();
2056 let root = self.state.worktree_root();
2057 let language = self.state.config.graph.language.clone();
2058 let sessions = self.state.config.graph.sessions;
2059 let artifacts = agent::artifacts_dir(&self.state.dir());
2060 let base = self.landing_base();
2061 let base_short = short(&base);
2062 let reviewers = self.roles.reviewers.clone();
2063 let shell = self.state.config.shell();
2064
2065 let mut prev_e2e: Option<String> = None;
2066 for round in (self.state.reviews.len() + 1)..=max_rounds {
2067 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2068 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2069 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2070
2071 let mut jobs = Vec::new();
2075 for (r, spec) in reviewers.iter().cloned().enumerate() {
2076 let wt = root.join(format!("review-{}", r + 1));
2077 if wt.exists() {
2078 git::reset_detached(&wt, &head).await?;
2079 } else {
2080 git::worktree_add_detached(&repo, &wt, &head).await?;
2081 }
2082 let seat_key = format!("review-{}", r + 1);
2083 let seat = self.seat(&seat_key, &spec.id);
2084 jobs.push(SeatJob {
2085 prompt: prompt::review(&prompt::ReviewCtx {
2086 instruction: &self.state.instruction,
2087 branch: &winner.branch,
2088 base_short: &base_short,
2089 stat: &stat,
2090 patch: &patch,
2091 e2e: prev_e2e.as_deref(),
2092 reviewers: reviewers.len(),
2093 round,
2094 rounds: max_rounds,
2095 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2098 language: &language,
2099 }),
2100 spec,
2101 seat,
2102 cwd: wt,
2103 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2104 allow_write: false,
2105 sessions,
2106 artifacts: artifacts.clone(),
2107 stem: format!("review-{round}-{}", r + 1),
2108 });
2109 }
2110
2111 self.state.event(
2112 "review",
2113 format!(
2114 "round {round}: {} reviewers on {}",
2115 jobs.len(),
2116 short(&head)
2117 ),
2118 );
2119 let mut quota_losses = Vec::new();
2120 let review_retries = self.state.config.graph.retries;
2121 let review_cache = self.state.config.cache_dir();
2122 let ctx = WaveCtx {
2123 run: &run_id,
2124 node: "review",
2125 prompts: &prompts,
2126 cache: review_cache.as_deref(),
2127 };
2128 let results = ask_json_wave::<Review>(
2129 jobs,
2130 Arc::clone(&self.sem),
2131 review_retries,
2132 &ctx,
2133 &mut quota_losses,
2134 &mut self.state,
2135 &|_: &Review| Ok(()),
2136 )
2137 .await;
2138 self.state.quota.extend(quota_losses);
2139
2140 let mut records = Vec::new();
2141 let mut all_findings = Vec::new();
2142 for (r, (seat, res)) in results.into_iter().enumerate() {
2143 let agent_id = seat.agent.clone();
2144 self.state.seats.insert(seat.key.clone(), seat);
2145 let mut record = ReviewRecord {
2146 reviewer: r + 1,
2147 agent: agent_id,
2148 summary: String::new(),
2149 findings: Vec::new(),
2150 failed: None,
2151 duration_ms: 0,
2152 };
2153 match res {
2154 Ok((review, out)) => {
2155 record.summary = review.summary;
2156 record.duration_ms = out.duration_ms;
2157 for (n, mut f) in review.findings.into_iter().enumerate() {
2158 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2161 all_findings.push(f.clone());
2162 record.findings.push(f);
2163 }
2164 self.state.event(
2165 "review",
2166 format!(
2167 "round {round}: reviewer {} raised {} finding(s)",
2168 r + 1,
2169 record.findings.len()
2170 ),
2171 );
2172 }
2173 Err(e) => {
2174 record.failed = Some(e.to_string());
2175 self.state.event(
2176 "review",
2177 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2178 );
2179 }
2180 }
2181 records.push(record);
2182 }
2183
2184 let mut e2e = run_commands(
2185 &shell,
2186 &self.state.config.verify.e2e,
2187 &winner.worktree,
2188 Duration::from_secs(self.state.config.graph.timeout_review),
2189 )
2190 .await;
2191 for o in &e2e {
2192 self.state.event(
2193 "verify",
2194 format!("round {round}: `{}` -> {}", o.command, e2e_outcome_label(o)),
2195 );
2196 }
2197
2198 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
2203 if verify_retried {
2204 self.state.event(
2205 "verify",
2206 format!(
2207 "round {round}: verify could not build/link, not a test result — \
2208 retrying once before concluding"
2209 ),
2210 );
2211 e2e = run_commands(
2212 &shell,
2213 &self.state.config.verify.e2e,
2214 &winner.worktree,
2215 Duration::from_secs(self.state.config.graph.timeout_review),
2216 )
2217 .await;
2218 for o in &e2e {
2219 self.state.event(
2220 "verify",
2221 format!(
2222 "round {round}: retry `{}` -> {}",
2223 o.command,
2224 e2e_outcome_label(o)
2225 ),
2226 );
2227 }
2228 }
2229
2230 let e2e_failures: String = e2e
2231 .iter()
2232 .filter(|o| !o.ok())
2233 .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2234 .collect();
2235
2236 let expected = records.len();
2237 let answered = records.iter().filter(|r| r.failed.is_none()).count();
2238 let incomplete = answered < expected;
2239 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2240 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
2241 let policy = self.state.config.graph.incomplete_review;
2242 let clean = round_is_clean(blocking, e2e_ok, answered, expected, policy);
2243
2244 let mut round_record = ReviewRound {
2245 round,
2246 head: head.clone(),
2247 reviews: records,
2248 e2e,
2249 verify_retried,
2250 fix: None,
2251 blocking,
2252 answered,
2253 expected,
2254 clean,
2255 };
2256
2257 if incomplete {
2258 let missing: Vec<String> = round_record
2259 .reviews
2260 .iter()
2261 .filter(|r| r.failed.is_some())
2262 .map(|r| format!("review-{}", r.reviewer))
2263 .collect();
2264 self.state.event(
2265 "review",
2266 format!(
2267 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
2268 missing.join(", ")
2269 ),
2270 );
2271 }
2272
2273 if clean {
2274 self.state.event(
2275 "review",
2276 if incomplete {
2277 format!(
2278 "round {round}: clean (warn policy, incomplete panel) — no \
2279 blocking findings from the seats that answered, verification green"
2280 )
2281 } else {
2282 format!("round {round}: clean — no blocking findings, verification green")
2283 },
2284 );
2285 self.state.reviews.push(round_record);
2286 self.state.status = RunStatus::Gating;
2287 self.state.save()?;
2288 return Ok(());
2289 }
2290
2291 if incomplete && blocking == 0 && e2e_ok {
2295 self.state.reviews.push(round_record);
2296 self.state.save()?;
2297 if round == max_rounds {
2298 self.state.status = RunStatus::Blocked;
2299 self.state.event(
2300 "review",
2301 format!(
2302 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
2303 refusing to call it clean",
2304 expected - answered
2305 ),
2306 );
2307 return Ok(());
2308 }
2309 prev_e2e = None;
2310 continue;
2311 }
2312
2313 if round == max_rounds {
2314 self.state.reviews.push(round_record);
2315 self.state.status = RunStatus::Blocked;
2316 self.state.event(
2317 "review",
2318 format!("{blocking} blocking finding(s) still open after {max_rounds} rounds"),
2319 );
2320 self.state.save()?;
2321 return Ok(());
2322 }
2323
2324 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2327 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2328 _ => (
2329 self.state
2330 .config
2331 .agent(&winner.agent)
2332 .cloned()
2333 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2334 format!("impl-{}", winner.label),
2335 ),
2336 };
2337 let seat = self.seat(&fix_seat_key, &fix_spec.id);
2338 let blocking_findings: Vec<_> = all_findings
2339 .iter()
2340 .filter(|f| f.severity.blocks())
2341 .cloned()
2342 .collect();
2343 let job = SeatJob {
2344 prompt: prompt::fix(
2345 &self.state.instruction,
2346 &blocking_findings,
2347 (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2348 round,
2349 max_rounds,
2350 &language,
2351 ),
2352 spec: fix_spec.clone(),
2353 seat,
2354 cwd: winner.worktree.clone(),
2355 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2356 allow_write: true,
2357 sessions,
2358 artifacts: artifacts.clone(),
2359 stem: format!("fix-{round}"),
2360 };
2361 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2362 let cache = self.state.config.cache_dir();
2363 let ctx = WaveCtx {
2364 run: &run_id,
2365 node: "fix",
2366 prompts: &prompts,
2367 cache: cache.as_deref(),
2368 };
2369 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2370 let agent_id = seat.agent.clone();
2371 let seat_key = seat.key.clone();
2372 self.state.seats.insert(seat.key.clone(), seat);
2373
2374 let mut fix = FixRecord {
2375 agent: agent_id,
2376 addressed: Vec::new(),
2377 rejected: Vec::new(),
2378 notes: String::new(),
2379 committed: false,
2380 failed: None,
2381 duration_ms: 0,
2382 };
2383 match out {
2384 AgentOutcome::Ok(o) => {
2385 fix.duration_ms = o.duration_ms;
2386 match verdict::extract_json::<FixReport>(&o.text) {
2387 Ok(report) => {
2388 fix.addressed = report.addressed;
2389 fix.rejected = report.rejected;
2390 fix.notes =
2391 blind::sanitize_prose(&report.notes, &self.state.config.blind);
2392 }
2393 Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
2394 }
2395 }
2396 AgentOutcome::Dropped(o) => {
2398 fix.duration_ms = o.duration_ms;
2399 let why = o
2400 .dropped
2401 .as_ref()
2402 .map(|d| d.why.as_str())
2403 .unwrap_or("the CLI ended the stream without delivering its answer");
2404 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
2405 }
2406 AgentOutcome::Quota(o) => {
2407 self.state.quota.push(QuotaLoss {
2408 seat: seat_key,
2409 node: "fix".to_owned(),
2410 at: Timestamp::now(),
2411 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2412 });
2413 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
2414 }
2415 AgentOutcome::Failed(e) => fix.failed = Some(e),
2416 }
2417 git::commit_all(
2418 &winner.worktree,
2419 &format!("magi: review round {round} fixes (uncommitted work)"),
2420 )
2421 .await
2422 .ok();
2423 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
2424 fix.committed = after != before;
2425 let commit_note = if fix.committed {
2426 "committed"
2427 } else {
2428 "NO new commit"
2429 };
2430 self.state.event(
2431 "fix",
2432 match &fix.failed {
2433 Some(reason) => {
2439 format!("round {round}: fixer's adoption report was lost ({reason}); {commit_note}")
2440 }
2441 None => format!(
2442 "round {round}: {} addressed, {} rejected, {commit_note}",
2443 fix.addressed.len(),
2444 fix.rejected.len(),
2445 ),
2446 },
2447 );
2448 let stalled = !fix.committed;
2449 round_record.fix = Some(fix);
2450 self.state.reviews.push(round_record);
2451 self.state.save()?;
2452
2453 prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
2454
2455 if stalled {
2456 self.state.status = RunStatus::Blocked;
2457 self.state.event(
2458 "review",
2459 "the fixer produced no commit; stopping instead of looping on an unchanged tree"
2460 .to_owned(),
2461 );
2462 self.state.save()?;
2463 return Ok(());
2464 }
2465 }
2466 Ok(())
2467 }
2468
2469 async fn gate(&mut self) -> Result<()> {
2472 if self.state.status == RunStatus::Failed
2481 || self
2482 .state
2483 .base_sync
2484 .as_ref()
2485 .is_some_and(|s| s.conflict.is_some())
2486 || self.state.reviews.last().is_some_and(|r| !r.clean)
2487 {
2488 return Ok(());
2489 }
2490 if !self.state.gate.is_empty() {
2491 return Ok(());
2492 }
2493 let Some(winner) = self.state.winner().cloned() else {
2494 return Ok(());
2495 };
2496 self.state.status = RunStatus::Gating;
2497 let shell = self.state.config.shell();
2498 let outcomes = run_commands(
2499 &shell,
2500 &self.state.config.verify.gate,
2501 &winner.worktree,
2502 Duration::from_secs(self.state.config.graph.timeout_review),
2503 )
2504 .await;
2505 for o in &outcomes {
2506 self.state.event(
2507 "gate",
2508 format!(
2509 "`{}` -> {}",
2510 o.command,
2511 if o.ok() {
2512 "pass".to_owned()
2513 } else {
2514 format!("FAIL ({:?})", o.code)
2515 }
2516 ),
2517 );
2518 }
2519 let passed = outcomes.iter().all(CommandOutcome::ok);
2520 self.state.gate = outcomes;
2521 if !passed {
2522 self.state.status = RunStatus::Blocked;
2523 self.state.event("gate", "gate failed; not merging");
2524 }
2525 self.state.save()?;
2526 Ok(())
2527 }
2528
2529 async fn merge(&mut self) -> Result<()> {
2532 if self
2537 .state
2538 .base_sync
2539 .as_ref()
2540 .is_some_and(|s| s.conflict.is_some())
2541 || self.state.reviews.last().is_some_and(|r| !r.clean)
2542 || self.state.gate.iter().any(|o| !o.ok())
2543 {
2544 return Ok(());
2545 }
2546 if self.state.merge.is_some() {
2555 return Ok(());
2556 }
2557 let Some(winner) = self.state.winner().cloned() else {
2558 return Ok(());
2559 };
2560 let repo = self.state.repo.clone();
2561 let base = self.state.base_branch.clone();
2562 let mode = self.state.config.merge.mode;
2563 let message = format!(
2564 "Merge magi run {} (candidate {})\n\n{}",
2565 self.state.id, winner.label, self.state.instruction
2566 );
2567
2568 let outcome = match mode {
2569 MergeMode::None => MergeOutcome {
2570 mode,
2571 ok: true,
2572 detail: format!("git -C {} merge --no-ff {}", repo.display(), winner.branch),
2573 },
2574 MergeMode::Local => {
2575 let on = git::current_branch(&repo).await?;
2576 if on.as_deref() != Some(base.as_str()) {
2577 MergeOutcome {
2578 mode,
2579 ok: false,
2580 detail: format!(
2581 "{} has {} checked out, not the base branch {base}",
2582 repo.display(),
2583 on.unwrap_or_else(|| "a detached HEAD".to_owned())
2584 ),
2585 }
2586 } else if !git::is_clean(&repo).await? {
2587 MergeOutcome {
2588 mode,
2589 ok: false,
2590 detail: format!("{} is dirty; refusing to merge", repo.display()),
2591 }
2592 } else {
2593 let out = git::merge_no_ff(&repo, &winner.branch, &message).await?;
2594 MergeOutcome {
2595 mode,
2596 ok: out.ok(),
2597 detail: if out.ok() { out.stdout } else { out.stderr },
2598 }
2599 }
2600 }
2601 MergeMode::Pr => {
2602 let remote = self.state.config.merge.remote.clone();
2603 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
2604 if !pushed.ok() {
2605 MergeOutcome {
2606 mode,
2607 ok: false,
2608 detail: pushed.stderr,
2609 }
2610 } else {
2611 let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
2612 match out {
2613 Ok(url) => MergeOutcome {
2614 mode,
2615 ok: true,
2616 detail: url,
2617 },
2618 Err(e) => MergeOutcome {
2619 mode,
2620 ok: false,
2621 detail: e.to_string(),
2622 },
2623 }
2624 }
2625 }
2626 };
2627
2628 self.state.status = match (mode, outcome.ok) {
2629 (MergeMode::None, _) => RunStatus::Ready,
2630 (_, true) => RunStatus::Merged,
2631 (_, false) => RunStatus::Blocked,
2632 };
2633 self.state.event(
2634 "merge",
2635 format!(
2636 "{:?}: {}",
2637 mode,
2638 outcome.detail.lines().next().unwrap_or("")
2639 ),
2640 );
2641 self.state.merge = Some(outcome);
2642 self.state.save()?;
2643
2644 if self.state.config.graph.land
2650 && mode == MergeMode::Pr
2651 && self.state.status == RunStatus::Merged
2652 {
2653 let url = self
2654 .state
2655 .merge
2656 .as_ref()
2657 .map(|m| m.detail.clone())
2658 .unwrap_or_default();
2659 let url = url.lines().next().unwrap_or("").trim().to_owned();
2660 if url.starts_with("http") {
2661 match land::land(&mut self.state, &url).await {
2664 Ok(pr) => {
2665 self.state.status = match pr.state {
2666 land::PrLifecycle::Merged => RunStatus::Merged,
2667 _ => RunStatus::Blocked,
2668 };
2669 }
2670 Err(e) => {
2671 self.state.status = RunStatus::Blocked;
2672 self.state.event("land", format!("gave up: {e}"));
2673 }
2674 }
2675 self.state.save()?;
2676 }
2677 }
2678 Ok(())
2679 }
2680
2681 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
2685 if let Some(existing) = self.state.seats.get(key)
2686 && existing.agent == agent
2687 {
2688 return existing.clone();
2689 }
2690 let fresh = SeatState::new(key, agent, self.state.seed);
2691 self.state.seats.insert(key.to_owned(), fresh.clone());
2692 fresh
2693 }
2694
2695 fn view(&self, c: &Candidate) -> CandidateView {
2697 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2698 .unwrap_or_default();
2699 let (patch, _) = blind::sanitize_patch(
2700 &format!("candidate {} patch", c.label),
2701 &raw,
2702 &self.state.config.blind,
2703 );
2704 CandidateView {
2705 label: c.label,
2706 branch: c.branch.clone(),
2707 summary: c.summary.clone(),
2708 stat: c.stat.clone(),
2709 patch,
2710 }
2711 }
2712
2713 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
2715 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
2716 prompt::judge(
2717 "(see above)",
2718 &views,
2719 self.roles.judges.len(),
2720 base_short,
2721 "en",
2722 )
2723 }
2724
2725 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
2732 let mut turns = Vec::new();
2733 for j in &self.state.judgements {
2734 if j.ranking.is_empty() {
2735 continue;
2736 }
2737 let reasons = j
2738 .reasons
2739 .iter()
2740 .map(|(k, v)| format!("- {k}: {v}"))
2741 .collect::<Vec<_>>()
2742 .join("\n");
2743 turns.push(Turn {
2744 who: format!("Judge {} (opening ranking)", j.judge),
2745 is_self: j.judge == self_idx + 1,
2746 body: format!(
2747 "Ranked {}{}{reasons}",
2748 j.ranking.iter().collect::<String>(),
2749 if reasons.is_empty() {
2750 ""
2751 } else {
2752 ", because:\n"
2753 }
2754 ),
2755 });
2756 }
2757 for t in self
2758 .state
2759 .deliberation
2760 .iter()
2761 .flat_map(|r| r.turns.iter())
2762 .chain(current)
2763 {
2764 turns.push(Turn {
2765 who: format!("Judge {}", t.judge),
2766 is_self: t.judge == self_idx + 1,
2767 body: t.body.clone(),
2768 });
2769 }
2770 turns
2771 }
2772}
2773
2774fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
2776 agent::has_session(spec.kind, seat, sessions)
2777}
2778
2779fn short(commit: &str) -> String {
2780 commit.chars().take(7).collect()
2781}
2782
2783fn make_executable(path: &Path) -> Result<()> {
2784 #[cfg(unix)]
2785 {
2786 use std::os::unix::fs::PermissionsExt as _;
2787 let mut perms = std::fs::metadata(path)?.permissions();
2788 perms.set_mode(0o755);
2789 std::fs::set_permissions(path, perms)?;
2790 }
2791 #[cfg(not(unix))]
2792 {
2793 let _ = path;
2794 }
2795 Ok(())
2796}
2797
2798struct WaveCtx<'a> {
2805 run: &'a str,
2808 node: &'a str,
2810 prompts: &'a Prompts,
2811 cache: Option<&'a Path>,
2813}
2814
2815async fn run_one(
2817 job: SeatJob,
2818 sem: Arc<Semaphore>,
2819 ctx: &WaveCtx<'_>,
2820 state: &mut RunState,
2821 attempt: usize,
2822) -> (SeatState, AgentOutcome) {
2823 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
2824 .await
2825 .pop()
2826 .expect("one job in, one result out");
2827 (seat, out)
2828}
2829
2830async fn wave(
2836 jobs: Vec<SeatJob>,
2837 sem: Arc<Semaphore>,
2838 ctx: &WaveCtx<'_>,
2839 state: &mut RunState,
2840 attempt: usize,
2841) -> Vec<(usize, SeatState, AgentOutcome)> {
2842 let WaveCtx {
2843 run,
2844 node,
2845 prompts,
2846 cache,
2847 } = *ctx;
2848 for job in &jobs {
2849 state.seat_started(node, &job.seat.key, job.timeout, attempt);
2850 }
2851 if let Err(e) = state.save() {
2852 tracing::warn!("could not persist in-progress seats: {e:#}");
2857 }
2858 let mut set = tokio::task::JoinSet::new();
2859 let overlay = prompts.overlay(node);
2860 for (i, mut job) in jobs.into_iter().enumerate() {
2861 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
2862 if cache.is_some() {
2863 job.prompt.push('\n');
2864 job.prompt.push_str(prompt::build_cache_note());
2865 }
2866 let sem = Arc::clone(&sem);
2867 let run = run.to_owned();
2868 let node = node.to_owned();
2869 let cache = cache.map(Path::to_path_buf);
2870 set.spawn(async move {
2871 let _permit = sem.acquire().await;
2872 let mut seat = job.seat;
2873 let out = agent::invoke(
2874 &job.spec,
2875 &mut seat,
2876 &Invocation {
2877 cwd: &job.cwd,
2878 prompt: &job.prompt,
2879 timeout: job.timeout,
2880 allow_write: job.allow_write,
2881 sessions: job.sessions,
2882 artifacts: &job.artifacts,
2883 stem: &job.stem,
2884 run: &run,
2885 node: &node,
2886 cache_dir: cache.as_deref(),
2887 },
2888 )
2889 .await;
2890 let out = match out {
2891 Ok(o) if o.usable() => AgentOutcome::Ok(o),
2892 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
2893 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
2901 Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
2902 Ok(o) => AgentOutcome::Failed(format!(
2903 "exited with {:?} and no usable output",
2904 o.exit_code
2905 )),
2906 Err(e) => AgentOutcome::Failed(e.to_string()),
2907 };
2908 (i, seat, out)
2909 });
2910 }
2911 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
2912 while let Some(joined) = set.join_next().await {
2913 let (i, seat, out) = match joined {
2914 Ok(v) => v,
2915 Err(e) => {
2919 tracing::error!("agent task panicked: {e}");
2920 continue;
2921 }
2922 };
2923 state.seat_finished(&seat.key);
2924 if let Err(e) = state.save() {
2925 tracing::warn!("could not persist a seat's completion: {e:#}");
2926 }
2927 if collected.len() <= i {
2928 collected.resize_with(i + 1, || None);
2929 }
2930 collected[i] = Some((i, seat, out));
2931 }
2932 if state
2938 .active
2939 .values()
2940 .any(|a| a.node == node && a.attempt == attempt)
2941 {
2942 state
2943 .active
2944 .retain(|_, a| !(a.node == node && a.attempt == attempt));
2945 if let Err(e) = state.save() {
2946 tracing::warn!("could not persist the end of a wave: {e:#}");
2947 }
2948 }
2949 collected.into_iter().flatten().collect()
2950}
2951
2952fn round_is_clean(
2962 blocking: usize,
2963 e2e_ok: bool,
2964 answered: usize,
2965 expected: usize,
2966 policy: IncompleteReviewPolicy,
2967) -> bool {
2968 blocking == 0 && e2e_ok && (answered == expected || policy == IncompleteReviewPolicy::Warn)
2969}
2970
2971fn retry_budget(full: Duration, nudged: bool) -> Duration {
2986 if nudged {
2987 (full / 4).max(Duration::from_secs(120)).min(full)
2988 } else {
2989 full
2990 }
2991}
2992
2993#[allow(clippy::too_many_arguments)]
3006async fn ask_json_wave<T>(
3007 jobs: Vec<SeatJob>,
3008 sem: Arc<Semaphore>,
3009 retries: usize,
3010 ctx: &WaveCtx<'_>,
3011 losses: &mut Vec<QuotaLoss>,
3012 state: &mut RunState,
3013 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
3014) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
3015where
3016 T: serde::de::DeserializeOwned + Send + 'static,
3017{
3018 let n = jobs.len();
3019 let originals: Vec<SeatJob> = jobs;
3020 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
3021 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
3022 let mut pending: Vec<usize> = (0..n).collect();
3023
3024 for attempt in 0..=retries {
3025 if pending.is_empty() {
3026 break;
3027 }
3028 let mut batch = Vec::with_capacity(pending.len());
3029 for &i in &pending {
3030 let src = &originals[i];
3031 let (prompt, timeout) = if attempt == 0 {
3034 (src.prompt.clone(), src.timeout)
3035 } else {
3036 let why = done[i]
3037 .as_ref()
3038 .and_then(|r| r.as_ref().err().map(ToString::to_string))
3039 .unwrap_or_else(|| "no parsable answer".to_owned());
3040 let nudge = prompt::nudge(&why);
3041 let nudged = has_context(&src.spec, &seats[i], src.sessions);
3042 let prompt = if nudged {
3043 nudge
3044 } else {
3045 format!("{}\n\n---\n\n{}", src.prompt, nudge)
3046 };
3047 (prompt, retry_budget(src.timeout, nudged))
3048 };
3049 batch.push(SeatJob {
3050 spec: src.spec.clone(),
3051 seat: seats[i].clone(),
3052 cwd: src.cwd.clone(),
3053 prompt,
3054 timeout,
3055 allow_write: src.allow_write,
3056 sessions: src.sessions,
3057 artifacts: src.artifacts.clone(),
3058 stem: if attempt == 0 {
3059 src.stem.clone()
3060 } else {
3061 format!("{}-retry{attempt}", src.stem)
3062 },
3063 });
3064 }
3065
3066 if attempt > 0 {
3067 let seats_out: Vec<&str> = pending
3068 .iter()
3069 .map(|&i| originals[i].seat.key.as_str())
3070 .collect();
3071 state.event(
3072 ctx.node,
3073 format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3074 );
3075 }
3076 let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3077 let mut still = Vec::new();
3078 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3079 seats[i] = seat;
3080 let (parsed, quota) = match out {
3081 AgentOutcome::Ok(o) => (
3082 match verdict::extract_json::<T>(&o.text) {
3083 Ok(v) => match validate(&v) {
3084 Ok(()) => Ok((v, o)),
3085 Err(e) => Err(e),
3086 },
3087 Err(e) => Err(e),
3088 },
3089 false,
3090 ),
3091 AgentOutcome::Quota(o) => {
3092 losses.push(QuotaLoss {
3093 seat: originals[i].seat.key.clone(),
3094 node: ctx.node.to_owned(),
3095 at: Timestamp::now(),
3096 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3097 });
3098 (
3099 Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3100 true,
3101 )
3102 }
3103 AgentOutcome::Dropped(o) => {
3108 let why = o
3109 .dropped
3110 .as_ref()
3111 .map(|d| d.why.as_str())
3112 .unwrap_or("the CLI ended the stream without delivering its answer");
3113 (
3114 Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3115 false,
3116 )
3117 }
3118 AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3119 };
3120 let failed = parsed.is_err();
3121 done[i] = Some(parsed);
3122 if failed && !quota {
3125 still.push(i);
3126 }
3127 }
3128 pending = still;
3129 }
3130
3131 seats
3132 .into_iter()
3133 .zip(done)
3134 .map(|(seat, res)| {
3135 (
3136 seat,
3137 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3138 )
3139 })
3140 .collect()
3141}
3142
3143fn e2e_outcome_label(o: &CommandOutcome) -> String {
3147 if o.ok() {
3148 "pass".to_owned()
3149 } else if o.build_failed() {
3150 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
3151 } else {
3152 format!("FAIL ({:?})", o.code)
3153 }
3154}
3155
3156async fn run_commands(
3158 shell: &[String],
3159 commands: &[String],
3160 cwd: &Path,
3161 timeout: Duration,
3162) -> Vec<CommandOutcome> {
3163 let mut out = Vec::new();
3164 for command in commands {
3165 let started = Instant::now();
3166 let mut cmd = tokio::process::Command::new(&shell[0]);
3167 cmd.quiet();
3168 cmd.args(&shell[1..])
3169 .arg(command)
3170 .current_dir(cwd)
3171 .stdin(std::process::Stdio::null())
3172 .stdout(std::process::Stdio::piped())
3173 .stderr(std::process::Stdio::piped())
3174 .kill_on_drop(true);
3175 let spawned = cmd.spawn();
3176 let (code, body) = match spawned {
3177 Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
3178 Ok(Ok(o)) => {
3179 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
3180 body.push_str(&String::from_utf8_lossy(&o.stderr));
3181 (o.status.code(), body)
3182 }
3183 Ok(Err(e)) => (None, format!("failed to run: {e}")),
3184 Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
3185 },
3186 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
3187 };
3188 out.push(CommandOutcome {
3189 command: command.clone(),
3190 code,
3191 output_tail: tail(&body, OUTPUT_TAIL),
3192 duration_ms: started.elapsed().as_millis() as u64,
3193 });
3194 }
3195 out
3196}
3197
3198async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
3200 let title = body.lines().next().unwrap_or("magi run").to_owned();
3201 let out = tokio::process::Command::new("gh")
3202 .args([
3203 "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
3204 ])
3205 .current_dir(cwd)
3206 .stdin(std::process::Stdio::null())
3207 .output()
3208 .await
3209 .context("spawn gh")?;
3210 if out.status.success() {
3211 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
3212 } else {
3213 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
3214 }
3215}
3216
3217pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
3219 let repo = state.repo.clone();
3220 let root = state.worktree_root();
3221 let winner = state.tally.as_ref().map(|t| t.winner);
3222 let mut removed = Vec::new();
3223
3224 for i in 0..state.candidates.len() {
3225 let c = state.candidates[i].clone();
3226 let is_winner = Some(c.label) == winner;
3227 if is_winner && !drop_winner {
3228 continue;
3229 }
3230 if c.worktree.exists() {
3231 git::worktree_remove(&repo, &c.worktree).await.ok();
3232 removed.push(c.worktree.to_string_lossy().into_owned());
3233 }
3234 if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
3235 git::branch_delete(&repo, &c.branch).await.ok();
3236 removed.push(c.branch.clone());
3237 }
3238 state.candidates[i].folded = true;
3239 }
3240
3241 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
3242 let path = name.path();
3243 let keep = !drop_winner
3244 && winner.is_some_and(|w| {
3245 path.file_name()
3246 .is_some_and(|n| n == format!("cand-{w}").as_str())
3247 });
3248 if keep {
3249 continue;
3250 }
3251 git::worktree_remove(&repo, &path).await.ok();
3252 removed.push(path.to_string_lossy().into_owned());
3253 }
3254
3255 if state.enabled_worktree_config && drop_winner {
3256 git::disable_worktree_config(&repo).await.ok();
3257 state.enabled_worktree_config = false;
3258 }
3259 state.save()?;
3260 Ok(removed)
3261}
3262
3263pub fn worst_open(state: &RunState) -> Option<Severity> {
3265 state
3266 .reviews
3267 .last()?
3268 .reviews
3269 .iter()
3270 .flat_map(|r| r.findings.iter())
3271 .map(|f| f.severity)
3272 .max()
3273}
3274
3275#[cfg(test)]
3276mod tests {
3277 use super::*;
3278 use std::time::Duration;
3279
3280 #[test]
3289 fn a_full_panel_that_found_nothing_is_clean() {
3290 assert!(round_is_clean(0, true, 2, 2, IncompleteReviewPolicy::Block));
3291 }
3292
3293 #[test]
3294 fn a_missing_seat_is_never_clean_under_the_default_policy() {
3295 assert!(!round_is_clean(
3296 0,
3297 true,
3298 1,
3299 2,
3300 IncompleteReviewPolicy::Block
3301 ));
3302 }
3303
3304 #[test]
3305 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
3306 assert!(!round_is_clean(1, true, 1, 2, IncompleteReviewPolicy::Warn));
3307 }
3308
3309 #[test]
3310 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
3311 assert!(round_is_clean(0, true, 1, 2, IncompleteReviewPolicy::Warn));
3312 }
3313
3314 #[test]
3315 fn a_full_panel_with_an_open_finding_is_not_clean() {
3316 assert!(!round_is_clean(
3317 1,
3318 true,
3319 2,
3320 2,
3321 IncompleteReviewPolicy::Block
3322 ));
3323 }
3324
3325 #[test]
3326 fn a_full_panel_with_a_red_e2e_is_not_clean() {
3327 assert!(!round_is_clean(
3328 0,
3329 false,
3330 2,
3331 2,
3332 IncompleteReviewPolicy::Block
3333 ));
3334 }
3335
3336 fn secs(n: u64) -> Duration {
3337 Duration::from_secs(n)
3338 }
3339
3340 fn init_repo(dir: &Path) {
3343 let run = |args: &[&str]| {
3344 let out = std::process::Command::new("git")
3345 .args(args)
3346 .current_dir(dir)
3347 .output()
3348 .expect("spawn git");
3349 assert!(
3350 out.status.success(),
3351 "git {args:?} failed: {}",
3352 String::from_utf8_lossy(&out.stderr)
3353 );
3354 };
3355 run(&["init", "-b", "main"]);
3356 run(&["config", "user.name", "magi test"]);
3357 run(&["config", "user.email", "magi@example.com"]);
3358 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
3359 run(&["add", "-A"]);
3360 run(&["commit", "-m", "init"]);
3361 }
3362
3363 #[tokio::test]
3372 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
3373 let tmp = tempfile::tempdir().expect("tempdir");
3374 let repo = tmp.path().join("repo");
3375 std::fs::create_dir_all(&repo).unwrap();
3376 init_repo(&repo);
3377
3378 let mut config = Config::default();
3379 config.merge.mode = MergeMode::Local;
3380
3381 let mut state = RunState::new(
3382 repo.clone(),
3383 "main".to_owned(),
3384 "deadbeef".to_owned(),
3385 "task".to_owned(),
3386 config,
3387 );
3388 state.candidates = vec![Candidate {
3389 index: 0,
3390 label: 'A',
3391 agent: "alpha".to_owned(),
3392 branch: "does-not-exist".to_owned(),
3393 worktree: repo.clone(),
3394 summary: String::new(),
3395 stat: String::new(),
3396 files: 0,
3397 commits: 0,
3398 empty: false,
3399 failed: None,
3400 duration_ms: 0,
3401 folded: false,
3402 }];
3403 state.tally = Some(Tally {
3404 first_choice: BTreeMap::from([('A', 1)]),
3405 borda: BTreeMap::new(),
3406 winner: 'A',
3407 rankings: 1,
3408 unanimous_initial: true,
3409 deliberated: false,
3410 changed_votes: 0,
3411 unanimous_final: true,
3412 tie_break: None,
3413 judges: 0,
3414 present: 0,
3415 quorum: 0,
3416 met_quorum: true,
3417 uncontested: Some("only candidate A produced a change".to_owned()),
3418 });
3419 state.reviews = vec![ReviewRound {
3420 round: 1,
3421 head: "deadbeef".to_owned(),
3422 reviews: Vec::new(),
3423 e2e: Vec::new(),
3424 fix: None,
3425 blocking: 0,
3426 answered: 0,
3427 expected: 0,
3428 clean: true,
3429 verify_retried: false,
3430 }];
3431 state.gate = vec![CommandOutcome {
3432 command: "test".to_owned(),
3433 code: Some(0),
3434 output_tail: String::new(),
3435 duration_ms: 0,
3436 }];
3437 state.status = RunStatus::Ready;
3442 state.merge = Some(MergeOutcome {
3443 mode: MergeMode::Local,
3444 ok: false,
3445 detail: "already concluded".to_owned(),
3446 });
3447
3448 let mut runner = Runner {
3449 state,
3450 roles: ResolvedRoles {
3451 implementers: Vec::new(),
3452 judges: Vec::new(),
3453 reviewers: Vec::new(),
3454 fixer: None,
3455 },
3456 sem: Arc::new(Semaphore::new(1)),
3457 pause: Pause::new(),
3458 };
3459
3460 runner.merge().await.expect("merge");
3461
3462 assert_eq!(
3463 runner.state.status,
3464 RunStatus::Ready,
3465 "a concluded run's status must not change on reentry"
3466 );
3467 assert_eq!(
3468 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
3469 Some("already concluded"),
3470 "merge must not run again once the node already recorded an outcome"
3471 );
3472 }
3473
3474 #[test]
3475 fn a_nudge_gets_a_quarter_of_the_budget() {
3476 assert_eq!(retry_budget(secs(1200), true), secs(300));
3478 assert_eq!(retry_budget(secs(3600), true), secs(900));
3479 }
3480
3481 #[test]
3482 fn a_resent_prompt_keeps_the_whole_budget() {
3483 assert_eq!(retry_budget(secs(1200), false), secs(1200));
3486 assert_eq!(retry_budget(secs(60), false), secs(60));
3487 }
3488
3489 #[test]
3490 fn the_floor_never_exceeds_the_original_budget() {
3491 assert_eq!(retry_budget(secs(60), true), secs(60));
3495 assert_eq!(retry_budget(secs(480), true), secs(120));
3496 assert_eq!(retry_budget(secs(0), true), secs(0));
3497 }
3498}