1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::agent::{self, AgentOutput, Invocation, SeatState};
30use crate::blind;
31use crate::config::{AgentSpec, Config, LeakPolicy, MergeMode, Prompts, ResolvedRoles};
32use crate::git;
33use crate::land;
34use crate::proc::Quiet as _;
35use crate::prompt::{self, CandidateView, Turn};
36use crate::run::{
37 BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
38 MergeOutcome, QuotaLoss, ReviewRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord,
39 tail, write_artifact,
40};
41use crate::verdict::{self, FinalVote, FixReport, Position, Ranking, Review, Severity};
42
43const OUTPUT_TAIL: usize = 8_000;
45
46const BASE_SYNC_ROUNDS: usize = 4;
59
60#[derive(Clone)]
66struct SeatJob {
67 spec: AgentSpec,
68 seat: SeatState,
69 cwd: PathBuf,
70 prompt: String,
71 timeout: Duration,
72 allow_write: bool,
73 sessions: bool,
74 artifacts: PathBuf,
75 stem: String,
76}
77
78enum AgentOutcome {
90 Ok(AgentOutput),
92 Quota(AgentOutput),
94 Dropped(AgentOutput),
97 Failed(String),
99}
100
101#[derive(Debug, Clone, Default)]
118pub struct Pause(Arc<AtomicBool>);
119
120impl Pause {
121 #[must_use]
123 pub fn new() -> Self {
124 Self::default()
125 }
126
127 pub fn park(&self) {
129 self.0.store(true, Ordering::SeqCst);
130 }
131
132 #[must_use]
134 pub fn parked(&self) -> bool {
135 self.0.load(Ordering::SeqCst)
136 }
137}
138
139pub struct Runner {
141 pub state: RunState,
143 roles: ResolvedRoles,
144 sem: Arc<Semaphore>,
145 pause: Pause,
147}
148
149async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
169 let tracking = format!("{remote}/{base_branch}");
170 let fetched = git::fetch(repo, remote, base_branch).await;
171 if let Ok(out) = &fetched
172 && out.ok()
173 && git::rev_exists(repo, &tracking).await
174 {
175 return git::rev_parse(repo, &tracking).await;
176 }
177 let why = match &fetched {
178 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
179 Ok(_) => format!("{remote} has no {base_branch}"),
180 Err(e) => e.to_string(),
181 };
182 tracing::warn!(
183 "could not read {tracking} ({why}); branching off the local \
184 {base_branch} instead, which may be behind"
185 );
186 git::rev_parse(repo, base_branch).await.with_context(|| {
187 format!(
188 "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
189 branch that exists"
190 )
191 })
192}
193
194impl Runner {
195 pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
197 let repo = git::toplevel(repo).await?;
198 let missing = agent::missing_programs(&config.agents);
199 if !missing.is_empty() {
200 bail!(
201 "these agent programs are not on PATH: {}. Fix the roster in \
202 magi.toml or install them.",
203 missing.join(", ")
204 );
205 }
206 let base_branch = match config.merge.base.clone() {
207 Some(b) => b,
208 None => git::current_branch(&repo)
209 .await?
210 .context("HEAD is detached; set [merge] base in magi.toml")?,
211 };
212 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
213 if !git::is_clean(&repo).await? {
217 tracing::warn!(
218 "{} has uncommitted changes; they are not part of this run, \
219 which branches off {base_branch} ({})",
220 repo.display(),
221 &base_commit[..base_commit.len().min(8)]
222 );
223 }
224 let roles = config.resolve_roles()?;
225 let max_parallel = config.graph.max_parallel.max(1);
226 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
227 state.event("start", format!("run {} created", state.id));
228 state.save()?;
229 Ok(Self {
230 state,
231 roles,
232 sem: Arc::new(Semaphore::new(max_parallel)),
233 pause: Pause::new(),
234 })
235 }
236
237 pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
251 let repo = git::toplevel(repo).await?;
252 let missing = agent::missing_programs(&config.agents);
253 if !missing.is_empty() {
254 bail!(
255 "these agent programs are not on PATH: {}. Fix the roster in \
256 magi.toml or install them.",
257 missing.join(", ")
258 );
259 }
260 if !git::branch_exists(&repo, branch).await? {
261 bail!("no branch `{branch}` in {}", repo.display());
262 }
263 let base_branch = match config.merge.base.clone() {
264 Some(b) => b,
265 None => git::current_branch(&repo)
266 .await?
267 .context("HEAD is detached; set [merge] base in magi.toml")?,
268 };
269 if base_branch == branch {
270 bail!("`{branch}` is the base branch; there is nothing to review against");
271 }
272 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
273
274 let roles = config.resolve_roles()?;
275 let max_parallel = config.graph.max_parallel.max(1);
276 let log = git::log_oneline(&repo, &base_commit, branch)
279 .await
280 .unwrap_or_default();
281 let instruction = format!(
282 "Review the work already on branch `{branch}`. There is no task \
283 statement: what the change claims to do is whatever its commits \
284 say.\n\n{}",
285 if log.trim().is_empty() {
286 "(no commit messages)"
287 } else {
288 log.trim()
289 }
290 );
291 let mut state = RunState::new(
292 repo.clone(),
293 base_branch,
294 base_commit.clone(),
295 instruction,
296 config,
297 );
298
299 let worktree = state.worktree_root().join("under-review");
302 if let Some(parent) = worktree.parent() {
303 tokio::fs::create_dir_all(parent).await.ok();
304 }
305 let path = worktree.to_string_lossy().to_string();
306 git::git(&repo, &["worktree", "add", &path, branch])
307 .await
308 .with_context(|| {
309 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
310 })?;
311
312 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
313 .await
314 .unwrap_or(0);
315 if commits == 0 {
316 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
317 }
318 let files = git::changed_files(&worktree, &base_commit, "HEAD")
319 .await
320 .map(|f| f.len())
321 .unwrap_or(0);
322 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
323 .await
324 .unwrap_or_default();
325
326 state.candidates.push(Candidate {
327 index: 0,
328 label: 'A',
329 agent: "(existing branch)".to_owned(),
332 branch: branch.to_owned(),
333 worktree,
334 summary: String::new(),
335 stat,
336 files,
337 commits,
338 empty: false,
339 failed: None,
340 duration_ms: 0,
341 folded: false,
342 });
343 state.tally = Some(Tally {
344 first_choice: BTreeMap::from([('A', 0)]),
345 borda: BTreeMap::new(),
346 winner: 'A',
347 rankings: 0,
348 unanimous_initial: false,
349 deliberated: false,
350 changed_votes: 0,
351 unanimous_final: false,
352 tie_break: None,
353 judges: 0,
357 present: 0,
358 quorum: 0,
359 met_quorum: true,
360 uncontested: Some("review-only run: nothing competed".to_owned()),
361 });
362 state.status = RunStatus::Reviewing;
363 state.event(
364 "start",
365 format!(
366 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
367 state.id
368 ),
369 );
370 state.save()?;
371 Ok(Self {
372 state,
373 roles,
374 sem: Arc::new(Semaphore::new(max_parallel)),
375 pause: Pause::new(),
376 })
377 }
378
379 pub fn resume(id: &str) -> Result<Self> {
381 let state = RunState::load(id)?;
382 let roles = state.config.resolve_roles()?;
383 let max_parallel = state.config.graph.max_parallel.max(1);
384 Ok(Self {
385 state,
386 roles,
387 sem: Arc::new(Semaphore::new(max_parallel)),
388 pause: Pause::new(),
389 })
390 }
391
392 pub async fn execute(&mut self) -> Result<()> {
394 self.state.parked = false;
399 if self.state.clear_active() {
406 self.state.save()?;
407 }
408 if self.state.status == RunStatus::Stalled {
421 if self.recover_stall().await? {
422 self.finish_after_tally().await?;
423 } else {
424 self.state.save()?;
426 }
427 return Ok(());
428 }
429 self.prep().await?;
430 if self.park_here()? {
431 return Ok(());
432 }
433 self.implement().await?;
434 if self.park_here()? {
435 return Ok(());
436 }
437 self.judge().await?;
438 if self.park_here()? {
439 return Ok(());
440 }
441 self.deliberate().await?;
442 if self.park_here()? {
443 return Ok(());
444 }
445 self.vote().await?;
446 if self.park_here()? {
447 return Ok(());
448 }
449 self.tally()?;
450 if self.state.status == RunStatus::Stalled {
455 self.state.save()?;
459 return Ok(());
460 }
461 self.finish_after_tally().await?;
462 Ok(())
463 }
464
465 fn park_here(&mut self) -> Result<bool> {
472 if !self.pause.parked() {
473 return Ok(false);
474 }
475 self.state.event(
476 "park",
477 format!(
478 "parked after `{}` — resume to carry on from here",
479 self.state.status.as_str()
480 ),
481 );
482 self.state.parked = true;
483 self.state.save()?;
484 Ok(true)
485 }
486
487 pub fn on_pause(&mut self, pause: Pause) {
489 self.pause = pause;
490 }
491
492 async fn finish_after_tally(&mut self) -> Result<()> {
495 self.fold_losers().await?;
496 self.sync_to_base().await?;
501 self.review_loop().await?;
502 self.sync_to_base().await?;
503 self.gate().await?;
504 self.merge().await?;
505 self.state.save()?;
506 Ok(())
507 }
508
509 async fn prep(&mut self) -> Result<()> {
512 if !self.state.candidates.is_empty() {
513 return Ok(());
514 }
515 self.state.status = RunStatus::Prep;
516 let repo = self.state.repo.clone();
517 let base = self.state.base_commit.clone();
518 let root = self.state.worktree_root();
519 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
520
521 let hooks_dir = self.state.dir().join("hooks");
524 if self.state.config.blind.commit_msg_hook {
525 std::fs::create_dir_all(&hooks_dir)
526 .with_context(|| format!("create {}", hooks_dir.display()))?;
527 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
528 let path = hooks_dir.join("commit-msg");
529 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
530 make_executable(&path)?;
531 if git::enable_worktree_config(&repo).await? {
532 self.state.enabled_worktree_config = true;
533 }
534 }
535
536 for (index, (spec, label)) in self
537 .roles
538 .implementers
539 .clone()
540 .into_iter()
541 .zip(labels)
542 .enumerate()
543 {
544 let branch = self.state.branch_for(label);
545 let worktree = root.join(format!("cand-{label}"));
546 git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
547 if self.state.config.blind.commit_msg_hook {
548 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
549 }
550 git::local_exclude(&worktree, "/.magi/").await?;
551 self.state.candidates.push(Candidate {
552 index,
553 label,
554 agent: spec.id.clone(),
555 branch,
556 worktree,
557 summary: String::new(),
558 stat: String::new(),
559 files: 0,
560 commits: 0,
561 empty: false,
562 failed: None,
563 duration_ms: 0,
564 folded: false,
565 });
566 }
567
568 for j in 1..=self.roles.judges.len() {
569 let wt = root.join(format!("judge-{j}"));
570 if !wt.exists() {
571 git::worktree_add_detached(&repo, &wt, &base).await?;
572 }
573 }
574
575 let authors: Vec<&str> = self
580 .roles
581 .implementers
582 .iter()
583 .map(|a| a.id.as_str())
584 .collect();
585 let overlap: Vec<String> = self
586 .roles
587 .judges
588 .iter()
589 .enumerate()
590 .filter(|(_, j)| authors.contains(&j.id.as_str()))
591 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
592 .collect();
593 if !overlap.is_empty() {
594 let note = format!(
595 "{} also authored a candidate; blind, but the panel is less \
596 independent than {} distinct agents would be",
597 overlap.join(", "),
598 self.roles.judges.len()
599 );
600 self.state.event("prep", note);
601 }
602
603 self.state.event(
604 "prep",
605 format!(
606 "{} candidates, {} judges, base {} ({})",
607 self.state.candidates.len(),
608 self.roles.judges.len(),
609 &self.state.base_commit[..7.min(self.state.base_commit.len())],
610 self.state.base_branch
611 ),
612 );
613 self.state.status = RunStatus::Implementing;
614 self.state.save()?;
615 Ok(())
616 }
617
618 async fn implement(&mut self) -> Result<()> {
621 let run_id = self.state.id.clone();
626 let prompts = self.state.config.prompts.clone();
627 let todo: Vec<usize> = self
628 .state
629 .candidates
630 .iter()
631 .enumerate()
632 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
633 .map(|(i, _)| i)
634 .collect();
635 if todo.is_empty() {
636 return self.after_implement();
637 }
638 self.state.status = RunStatus::Implementing;
639
640 let language = self.state.config.graph.language.clone();
641 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
642 let sessions = self.state.config.graph.sessions;
643 let artifacts = agent::artifacts_dir(&self.state.dir());
644
645 let mut jobs = Vec::new();
646 for &i in &todo {
647 let (index, label, worktree) = {
648 let c = &self.state.candidates[i];
649 (c.index, c.label, c.worktree.clone())
650 };
651 let spec = self.roles.implementers[index].clone();
652 let seat_key = format!("impl-{label}");
653 let seat = self.seat(&seat_key, &spec.id);
654 let instruction = self.state.instruction.clone();
655 jobs.push(SeatJob {
656 spec,
657 seat,
658 prompt: prompt::implement(&instruction, &worktree.to_string_lossy(), &language),
659 cwd: worktree,
660 timeout,
661 allow_write: true,
662 sessions,
663 artifacts: artifacts.clone(),
664 stem: format!("impl-{label}"),
665 });
666 }
667
668 self.state.event(
669 "implement",
670 format!("{} candidates in parallel", jobs.len()),
671 );
672 let sent = jobs.clone();
675 let cache = self.state.config.cache_dir();
676 let ctx = WaveCtx {
677 run: &run_id,
678 node: "implement",
679 prompts: &prompts,
680 cache: cache.as_deref(),
681 };
682 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
683 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
684 .await;
685
686 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
687 let seat_key = seat.key.clone();
688 self.state.seats.insert(seat.key.clone(), seat);
689 let label = self.state.candidates[i].label;
690 let worktree = self.state.candidates[i].worktree.clone();
691 let base = self.state.base_commit.clone();
692
693 let (summary, duration, failed) = match out {
694 AgentOutcome::Ok(o) => {
695 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
696 let failed = (!o.usable()).then(|| {
697 if o.timed_out {
698 "agent timed out".to_owned()
699 } else {
700 format!("agent exited with {:?}", o.exit_code)
701 }
702 });
703 (text, o.duration_ms, failed)
704 }
705 AgentOutcome::Dropped(o) => {
711 let why = o
712 .dropped
713 .as_ref()
714 .map(|d| d.why.as_str())
715 .unwrap_or("the CLI ended the stream without delivering its answer");
716 (
717 String::new(),
718 o.duration_ms,
719 Some(format!("the CLI dropped the stream ({why})")),
720 )
721 }
722 AgentOutcome::Quota(o) => {
723 self.state.quota.push(QuotaLoss {
724 seat: seat_key,
725 node: "implement".to_owned(),
726 at: Timestamp::now(),
727 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
728 });
729 (
730 String::new(),
731 o.duration_ms,
732 Some("rate limited (quota); produced no change".to_owned()),
733 )
734 }
735 AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
736 };
737
738 let rescued = git::commit_all(
741 &worktree,
742 &format!("magi: candidate {label} (uncommitted work)"),
743 )
744 .await
745 .unwrap_or(false);
746 let commits = git::commits_ahead(&worktree, &base, "HEAD")
747 .await
748 .unwrap_or(0);
749 let patch = git::diff(&worktree, &base, "HEAD")
750 .await
751 .unwrap_or_default();
752 let stat = git::diff_stat(&worktree, &base, "HEAD")
753 .await
754 .unwrap_or_default();
755 let files = git::changed_files(&worktree, &base, "HEAD")
756 .await
757 .map(|f| f.len())
758 .unwrap_or(0);
759 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
760
761 let c = &mut self.state.candidates[i];
762 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
763 c.stat = stat;
764 c.files = files;
765 c.commits = commits;
766 c.duration_ms = duration;
767 c.empty = commits == 0 || patch.trim().is_empty();
768 c.failed = match failed {
771 Some(_) if c.empty => failed,
772 _ => None,
773 };
774 let note = match (&c.failed, c.empty, rescued) {
775 (Some(e), _, _) => format!("candidate {label}: {e}"),
776 (None, true, _) => format!("candidate {label}: no change produced"),
777 (None, false, true) => {
778 format!(
779 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
780 )
781 }
782 (None, false, false) => {
783 format!("candidate {label}: {files} files, {commits} commits")
784 }
785 };
786 self.state.event("implement", note);
787 self.state.save()?;
788 }
789
790 self.after_implement()
791 }
792
793 async fn resume_undelivered(
821 &mut self,
822 results: &mut [(usize, SeatState, AgentOutcome)],
823 sent: &[SeatJob],
824 prompts: &Prompts,
825 run_id: &str,
826 ) {
827 for (wi, seat, out) in results.iter_mut() {
828 let Some(dropped) = (match &*out {
829 AgentOutcome::Dropped(o) => o.dropped.clone(),
830 _ => None,
831 }) else {
832 continue;
833 };
834 let Some(job) = sent.get(*wi) else { continue };
835 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
837 self.state.event(
838 "implement",
839 format!(
840 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
841 work is in the tree",
842 seat.key, dropped.output_tokens, dropped.why
843 ),
844 );
845 continue;
846 }
847 if !has_context(&job.spec, seat, job.sessions) {
855 self.state.event(
856 "implement",
857 format!(
858 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
859 is no session left to resume",
860 seat.key, dropped.output_tokens, dropped.why
861 ),
862 );
863 continue;
864 }
865 self.state.event(
866 "implement",
867 format!(
868 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
869 conversation",
870 seat.key, dropped.output_tokens, dropped.why
871 ),
872 );
873 let mut retry = job.clone();
874 retry.seat = seat.clone();
875 retry.prompt = prompt::resume_after_drop(&dropped.why);
876 retry.timeout = retry_budget(job.timeout, true);
877 retry.stem = format!("{}-resume", job.stem);
878 let cache = self.state.config.cache_dir();
879 let ctx = WaveCtx {
880 run: run_id,
881 node: "implement",
882 prompts,
883 cache: cache.as_deref(),
884 };
885 let (resumed_seat, resumed) =
886 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
887 *seat = resumed_seat;
888 *out = resumed;
889 }
890 }
891
892 fn after_implement(&mut self) -> Result<()> {
893 if self.state.leaks.is_empty() {
895 let cfg = self.state.config.blind.clone();
896 let mut leaks = Vec::new();
897 for c in &self.state.candidates {
898 let Some(patch) =
899 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
900 else {
901 continue;
902 };
903 leaks.extend(blind::scan(
904 &format!("candidate {} patch", c.label),
905 &patch,
906 &cfg.vendor_tokens,
907 ));
908 }
909 if !leaks.is_empty() {
910 let summary = leaks
911 .iter()
912 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
913 .collect::<Vec<_>>()
914 .join(", ");
915 match cfg.on_leak {
916 LeakPolicy::Fail => {
917 self.state.status = RunStatus::Failed;
918 self.state
919 .event("blind", format!("vendor text in a patch: {summary}"));
920 self.state.leaks = leaks;
921 self.state.save()?;
922 bail!(
923 "blind.on_leak = \"fail\" and vendor text reached a \
924 judged patch: {summary}"
925 );
926 }
927 LeakPolicy::Redact => self.state.event(
928 "blind",
929 format!("redacting vendor text for judging: {summary}"),
930 ),
931 LeakPolicy::Warn => self.state.event(
932 "blind",
933 format!("vendor text present in a judged patch (shown as-is): {summary}"),
934 ),
935 }
936 self.state.leaks = leaks;
937 }
938 }
939
940 if self.state.viable().is_empty() {
941 self.state.status = RunStatus::Failed;
942 self.state.save()?;
943 bail!("no candidate produced a change; nothing to judge");
944 }
945 self.state.status = RunStatus::Judging;
946 self.state.save()?;
947 Ok(())
948 }
949
950 async fn judge(&mut self) -> Result<()> {
953 let run_id = self.state.id.clone();
958 let prompts = self.state.config.prompts.clone();
959 if !self.state.judgements.is_empty() || self.state.judge_skipped {
960 return Ok(());
961 }
962 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
963 if viable.len() == 1 {
964 self.state.judge_skipped = true;
971 self.state.event(
972 "judge",
973 format!(
974 "only candidate {} produced a change; judging skipped",
975 viable[0].label
976 ),
977 );
978 self.state.save()?;
979 return Ok(());
980 }
981 self.state.status = RunStatus::Judging;
982
983 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
984 let language = self.state.config.graph.language.clone();
985 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
986 let sessions = self.state.config.graph.sessions;
987 let artifacts = agent::artifacts_dir(&self.state.dir());
988 let root = self.state.worktree_root();
989 let base_short = short(&self.state.base_commit);
990
991 let mut jobs = Vec::new();
992 let mut orders = Vec::new();
993 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
994 let order = blind::presentation_order(viable.len(), j, self.state.seed);
995 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
996 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
997 let seat_key = format!("judge-{}", j + 1);
998 let seat = self.seat(&seat_key, &spec.id);
999 jobs.push(SeatJob {
1000 prompt: prompt::judge(
1001 &self.state.instruction,
1002 &views,
1003 self.roles.judges.len(),
1004 &base_short,
1005 &language,
1006 ),
1007 spec,
1008 seat,
1009 cwd: root.join(format!("judge-{}", j + 1)),
1010 timeout,
1011 allow_write: false,
1012 sessions,
1013 artifacts: artifacts.clone(),
1014 stem: format!("judge-{}", j + 1),
1015 });
1016 }
1017
1018 self.state.event(
1019 "judge",
1020 format!(
1021 "{} judges ranking {} candidates blind",
1022 jobs.len(),
1023 viable.len()
1024 ),
1025 );
1026 let labels_for_check = labels.clone();
1027 let mut quota_losses = Vec::new();
1028 let cache = self.state.config.cache_dir();
1029 let ctx = WaveCtx {
1030 run: &run_id,
1031 node: "judge",
1032 prompts: &prompts,
1033 cache: cache.as_deref(),
1034 };
1035 let results = ask_json_wave::<Ranking>(
1036 jobs,
1037 Arc::clone(&self.sem),
1038 self.state.config.graph.retries,
1039 &ctx,
1040 &mut quota_losses,
1041 &mut self.state,
1042 &move |r: &Ranking| r.validate(&labels_for_check),
1043 )
1044 .await;
1045 self.state.quota.extend(quota_losses);
1046
1047 for (j, (seat, res)) in results.into_iter().enumerate() {
1048 let agent_id = seat.agent.clone();
1049 self.state.seats.insert(seat.key.clone(), seat);
1050 let mut record = Judgement {
1051 judge: j + 1,
1052 seat: format!("judge-{}", j + 1),
1053 agent: agent_id,
1054 ranking: Vec::new(),
1055 reasons: BTreeMap::new(),
1056 confidence: None,
1057 order: orders[j].clone(),
1058 failed: None,
1059 duration_ms: 0,
1060 };
1061 match res {
1062 Ok((ranking, out)) => {
1063 record.ranking = ranking.normalized();
1064 record.reasons = ranking.reasons;
1065 record.confidence = ranking.confidence;
1066 record.duration_ms = out.duration_ms;
1067 self.state.event(
1068 "judge",
1069 format!(
1070 "judge {} ranked {}",
1071 j + 1,
1072 record.ranking.iter().collect::<String>()
1073 ),
1074 );
1075 }
1076 Err(e) => {
1077 record.failed = Some(e.to_string());
1078 self.state
1079 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1080 }
1081 }
1082 self.state.judgements.push(record);
1083 self.state.save()?;
1084 }
1085 Ok(())
1086 }
1087
1088 async fn deliberate(&mut self) -> Result<()> {
1091 let run_id = self.state.id.clone();
1096 let prompts = self.state.config.prompts.clone();
1097 if !self.state.deliberation.is_empty() {
1098 return Ok(());
1099 }
1100 let tops: Vec<char> = self
1101 .state
1102 .judgements
1103 .iter()
1104 .filter_map(|j| j.ranking.first().copied())
1105 .collect();
1106 let rounds = self.state.config.graph.deliberate_rounds;
1107 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1108 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1109 self.state.event(
1110 "deliberate",
1111 format!("judges agreed on {} outright; no deliberation", tops[0]),
1112 );
1113 }
1114 self.state.status = RunStatus::Voting;
1115 self.state.save()?;
1116 return Ok(());
1117 }
1118
1119 self.state.status = RunStatus::Deliberating;
1120 self.state.event(
1121 "deliberate",
1122 format!(
1123 "split: first choices were {} — opening {rounds} round(s)",
1124 tops.iter().collect::<String>()
1125 ),
1126 );
1127
1128 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1129 let language = self.state.config.graph.language.clone();
1130 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1131 let sessions = self.state.config.graph.sessions;
1132 let artifacts = agent::artifacts_dir(&self.state.dir());
1133 let root = self.state.worktree_root();
1134 let base_short = short(&self.state.base_commit);
1135
1136 for round in 1..=rounds {
1140 let mut turns: Vec<DeliberationTurn> = Vec::new();
1141 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1142 if self.state.judgements[j].failed.is_some() {
1143 continue;
1144 }
1145 let seat_key = format!("judge-{}", j + 1);
1146 let mut seat = self.seat(&seat_key, &spec.id);
1147 let transcript = self.transcript(&turns, j);
1148 let context = if has_context(&spec, &seat, sessions) {
1149 None
1150 } else {
1151 Some(self.candidate_block(&viable, &base_short))
1152 };
1153 let text = prompt::deliberate(
1154 &self.state.instruction,
1155 context.as_deref(),
1156 &transcript,
1157 round,
1158 rounds,
1159 &language,
1160 );
1161 let job = SeatJob {
1162 spec,
1163 seat: seat.clone(),
1164 prompt: text,
1165 cwd: root.join(format!("judge-{}", j + 1)),
1166 timeout,
1167 allow_write: false,
1168 sessions,
1169 artifacts: artifacts.clone(),
1170 stem: format!("delib-{round}-judge-{}", j + 1),
1171 };
1172 let cache = self.state.config.cache_dir();
1173 let ctx = WaveCtx {
1174 run: &run_id,
1175 node: "deliberate",
1176 prompts: &prompts,
1177 cache: cache.as_deref(),
1178 };
1179 let (updated, out) =
1180 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1181 seat = updated;
1182 let agent_id = seat.agent.clone();
1183 let seat_key = seat.key.clone();
1184 self.state.seats.insert(seat.key.clone(), seat);
1185 let body = match out {
1186 AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1187 AgentOutcome::Dropped(o) => {
1191 let why =
1192 o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1193 "the CLI ended the stream without delivering its answer",
1194 );
1195 self.state.event(
1196 "deliberate",
1197 format!(
1198 "judge {} skipped: the CLI dropped the stream ({why})",
1199 j + 1
1200 ),
1201 );
1202 continue;
1203 }
1204 AgentOutcome::Quota(o) => {
1205 self.state.quota.push(QuotaLoss {
1206 seat: seat_key,
1207 node: "deliberate".to_owned(),
1208 at: Timestamp::now(),
1209 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1210 });
1211 self.state.event(
1212 "deliberate",
1213 format!("judge {} skipped: rate limited (quota)", j + 1),
1214 );
1215 continue;
1216 }
1217 AgentOutcome::Failed(e) => {
1218 self.state
1219 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1220 continue;
1221 }
1222 };
1223 let tentative = verdict::extract_json::<Position>(&body)
1224 .ok()
1225 .and_then(|p| p.tentative)
1226 .and_then(|s| s.trim().chars().next())
1227 .map(|c| c.to_ascii_uppercase());
1228 self.state.event(
1229 "deliberate",
1230 format!(
1231 "round {round}: judge {} now favours {}",
1232 j + 1,
1233 tentative.map_or("—".to_owned(), |c| c.to_string())
1234 ),
1235 );
1236 turns.push(DeliberationTurn {
1237 judge: j + 1,
1238 agent: agent_id,
1239 body: blind::sanitize_prose(&body, &self.state.config.blind),
1240 tentative,
1241 });
1242 }
1243 self.state
1244 .deliberation
1245 .push(DeliberationRound { round, turns });
1246 self.state.save()?;
1247 }
1248
1249 self.state.status = RunStatus::Voting;
1250 self.state.save()?;
1251 Ok(())
1252 }
1253
1254 async fn vote(&mut self) -> Result<()> {
1257 let run_id = self.state.id.clone();
1262 let prompts = self.state.config.prompts.clone();
1263 if !self.state.votes.is_empty() {
1264 return Ok(());
1265 }
1266 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1267 if viable.len() == 1 {
1268 return Ok(());
1269 }
1270 self.state.status = RunStatus::Voting;
1271
1272 let language = self.state.config.graph.language.clone();
1273 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1274 let sessions = self.state.config.graph.sessions;
1275 let artifacts = agent::artifacts_dir(&self.state.dir());
1276 let root = self.state.worktree_root();
1277 let base_short = short(&self.state.base_commit);
1278 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1279
1280 let mut jobs = Vec::new();
1281 let mut seats_at = Vec::new();
1282 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1283 if self
1284 .state
1285 .judgements
1286 .get(j)
1287 .is_some_and(|r| r.failed.is_some())
1288 {
1289 continue;
1290 }
1291 let seat_key = format!("judge-{}", j + 1);
1292 let seat = self.seat(&seat_key, &spec.id);
1293 let mut text = prompt::final_vote(&viable, &language);
1294 if !has_context(&spec, &seat, sessions) {
1295 text = format!(
1296 "{}\n\n# Candidates\n\n{}",
1297 text,
1298 self.candidate_block(&candidates, &base_short)
1299 );
1300 }
1301 jobs.push(SeatJob {
1302 spec,
1303 seat,
1304 prompt: text,
1305 cwd: root.join(format!("judge-{}", j + 1)),
1306 timeout,
1307 allow_write: false,
1308 sessions,
1309 artifacts: artifacts.clone(),
1310 stem: format!("vote-judge-{}", j + 1),
1311 });
1312 seats_at.push(j);
1313 }
1314
1315 self.state.event(
1316 "vote",
1317 format!(
1318 "collecting {} final votes one by one, privately",
1319 jobs.len()
1320 ),
1321 );
1322 let allowed = viable.clone();
1323 let mut quota_losses = Vec::new();
1324 let cache = self.state.config.cache_dir();
1325 let ctx = WaveCtx {
1326 run: &run_id,
1327 node: "vote",
1328 prompts: &prompts,
1329 cache: cache.as_deref(),
1330 };
1331 let results = ask_json_wave::<FinalVote>(
1332 jobs,
1333 Arc::clone(&self.sem),
1334 self.state.config.graph.retries,
1335 &ctx,
1336 &mut quota_losses,
1337 &mut self.state,
1338 &move |v: &FinalVote| match v.label() {
1339 Some(c) if allowed.contains(&c) => Ok(()),
1340 other => bail!("vote {other:?} is not one of {allowed:?}"),
1341 },
1342 )
1343 .await;
1344 self.state.quota.extend(quota_losses);
1345
1346 for (&j, (seat, res)) in seats_at.iter().zip(results) {
1347 let agent_id = seat.agent.clone();
1348 self.state.seats.insert(seat.key.clone(), seat);
1349 let initial = self
1350 .state
1351 .judgements
1352 .get(j)
1353 .and_then(|r| r.ranking.first().copied());
1354 let mut record = VoteRecord {
1355 judge: j + 1,
1356 agent: agent_id,
1357 vote: None,
1358 reason: String::new(),
1359 changed: false,
1360 };
1361 match res {
1362 Ok((v, _)) => {
1363 record.vote = v.label();
1364 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1365 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1366 self.state.event(
1367 "vote",
1368 format!(
1369 "judge {} voted {}{}",
1370 j + 1,
1371 record.vote.unwrap_or('?'),
1372 if record.changed { " (changed)" } else { "" }
1373 ),
1374 );
1375 }
1376 Err(e) => {
1377 self.state
1378 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1379 }
1380 }
1381 self.state.votes.push(record);
1382 self.state.save()?;
1383 }
1384 Ok(())
1385 }
1386
1387 fn tally(&mut self) -> Result<()> {
1390 if self.state.tally.is_some() {
1391 return Ok(());
1392 }
1393 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1394 let tops: Vec<char> = self
1395 .state
1396 .judgements
1397 .iter()
1398 .filter_map(|j| j.ranking.first().copied())
1399 .collect();
1400 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1401
1402 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1405 let mut cast: Vec<char> = Vec::new();
1406 for (i, j) in self.state.judgements.iter().enumerate() {
1407 let vote = self
1408 .state
1409 .votes
1410 .iter()
1411 .find(|v| v.judge == i + 1)
1412 .and_then(|v| v.vote)
1413 .or_else(|| j.ranking.first().copied());
1414 if let Some(v) = vote {
1415 *first_choice.entry(v).or_insert(0) += 1;
1416 cast.push(v);
1417 }
1418 }
1419
1420 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1421 for j in &self.state.judgements {
1422 let n = j.ranking.len();
1423 for (pos, label) in j.ranking.iter().enumerate() {
1424 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1425 }
1426 }
1427
1428 let best = first_choice.values().copied().max().unwrap_or(0);
1429 let mut leaders: Vec<char> = first_choice
1430 .iter()
1431 .filter(|(_, v)| **v == best)
1432 .map(|(k, _)| *k)
1433 .collect();
1434 let mut tie_break = None;
1435 if leaders.len() > 1 {
1436 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1437 let borda_leaders: Vec<char> = leaders
1438 .iter()
1439 .copied()
1440 .filter(|l| borda[l] == top_borda)
1441 .collect();
1442 tie_break = Some(if borda_leaders.len() == 1 {
1443 format!(
1444 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1445 leaders.len()
1446 )
1447 } else {
1448 format!(
1449 "{} way tie on both first-choice votes and Borda points, broken by label order",
1450 leaders.len()
1451 )
1452 });
1453 leaders = borda_leaders;
1454 leaders.sort_unstable();
1455 }
1456 let winner = *leaders
1457 .first()
1458 .or(viable.first())
1459 .context("no candidate to declare a winner from")?;
1460
1461 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1462 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1463 let deliberated = !self.state.deliberation.is_empty();
1464
1465 let quota_seats: std::collections::BTreeSet<&str> =
1469 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1470 let mut present = 0usize;
1471 for (i, j) in self.state.judgements.iter().enumerate() {
1472 if quota_seats.contains(j.seat.as_str()) {
1473 continue;
1474 }
1475 let ranked = !j.ranking.is_empty() && j.failed.is_none();
1476 let voted = self
1477 .state
1478 .votes
1479 .iter()
1480 .any(|v| v.judge == i + 1 && v.vote.is_some());
1481 if ranked || voted {
1482 present += 1;
1483 }
1484 }
1485 let needs_quorum = viable.len() > 1;
1491 let judges_total = if needs_quorum {
1492 self.roles.judges.len()
1493 } else {
1494 0
1495 };
1496 let quorum = if needs_quorum {
1497 judges_total / 2 + 1
1498 } else {
1499 0
1500 };
1501 let met_quorum = !needs_quorum || present >= quorum;
1502 let uncontested = (!needs_quorum).then(|| {
1503 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1504 });
1505
1506 self.state.event(
1507 "tally",
1508 match &uncontested {
1509 Some(reason) => format!("winner {winner} — {reason}"),
1510 None => format!(
1511 "winner {winner} — votes {} | initial {} | {} changed | \
1512 {present}/{judges_total} judges{}",
1513 first_choice
1514 .iter()
1515 .map(|(k, v)| format!("{k}:{v}"))
1516 .collect::<Vec<_>>()
1517 .join(" "),
1518 if unanimous_initial {
1519 "unanimous"
1520 } else {
1521 "split"
1522 },
1523 changed_votes,
1524 if met_quorum {
1525 String::new()
1526 } else {
1527 format!(" — below quorum ({quorum} required)")
1528 },
1529 ),
1530 },
1531 );
1532 if !met_quorum {
1533 self.state.event(
1534 "stall",
1535 format!(
1536 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1537 the run stops here, resumable"
1538 ),
1539 );
1540 }
1541 self.state.tally = Some(Tally {
1542 first_choice,
1543 borda,
1544 winner,
1545 rankings: tops.len(),
1546 unanimous_initial,
1547 deliberated,
1548 changed_votes,
1549 unanimous_final,
1550 tie_break,
1551 judges: judges_total,
1552 present,
1553 quorum,
1554 met_quorum,
1555 uncontested,
1556 });
1557 self.state.status = if met_quorum {
1558 RunStatus::Reviewing
1559 } else {
1560 RunStatus::Stalled
1561 };
1562 self.state.save()?;
1563 Ok(())
1564 }
1565
1566 #[allow(clippy::too_many_lines)]
1587 async fn recover_stall(&mut self) -> Result<bool> {
1588 let run_id = self.state.id.clone();
1593 let prompts = self.state.config.prompts.clone();
1594 let quota_seats: BTreeSet<&str> =
1599 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1600 let absent: Vec<String> = self
1601 .state
1602 .judgements
1603 .iter()
1604 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
1605 .map(|j| j.seat.clone())
1606 .collect();
1607 if absent.is_empty() {
1608 return Ok(false);
1609 }
1610 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1611 if viable.len() <= 1 {
1612 return Ok(false);
1613 }
1614 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1615 let language = self.state.config.graph.language.clone();
1616 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1617 let sessions = self.state.config.graph.sessions;
1618 let artifacts = agent::artifacts_dir(&self.state.dir());
1619 let root = self.state.worktree_root();
1620 let base_short = short(&self.state.base_commit);
1621 let candidates: Vec<Candidate> = viable.clone();
1622
1623 let mut positions: Vec<usize> = absent
1625 .iter()
1626 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
1627 .collect();
1628 if positions.is_empty() {
1629 return Ok(false);
1630 }
1631 positions.sort_unstable();
1632 positions.dedup();
1633
1634 let mut judge_jobs = Vec::new();
1636 for &j in &positions {
1637 let order = blind::presentation_order(viable.len(), j, self.state.seed);
1638 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1639 let seat_key = format!("judge-{}", j + 1);
1640 let spec = self.roles.judges[j].clone();
1641 let seat = self.seat(&seat_key, &spec.id);
1642 judge_jobs.push(SeatJob {
1643 spec,
1644 seat,
1645 prompt: prompt::judge(
1646 &self.state.instruction,
1647 &views,
1648 self.roles.judges.len(),
1649 &base_short,
1650 &language,
1651 ),
1652 cwd: root.join(seat_key),
1653 timeout,
1654 allow_write: false,
1655 sessions,
1656 artifacts: artifacts.clone(),
1657 stem: format!("judge-{}-recover", j + 1),
1658 });
1659 }
1660
1661 let labels_for_check = labels.clone();
1662 let mut judge_losses = Vec::new();
1663 let retries = self.state.config.graph.retries;
1664 let cache = self.state.config.cache_dir();
1665 let ctx = WaveCtx {
1666 run: &run_id,
1667 node: "judge",
1668 prompts: &prompts,
1669 cache: cache.as_deref(),
1670 };
1671 let results = ask_json_wave::<Ranking>(
1672 judge_jobs,
1673 Arc::clone(&self.sem),
1674 retries,
1675 &ctx,
1676 &mut judge_losses,
1677 &mut self.state,
1678 &move |r: &Ranking| r.validate(&labels_for_check),
1679 )
1680 .await;
1681
1682 let mut recovered: BTreeSet<usize> = BTreeSet::new();
1684 for (&j, (seat, res)) in positions.iter().zip(results) {
1685 self.state.seats.insert(seat.key.clone(), seat);
1686 let record = &mut self.state.judgements[j];
1687 match res {
1688 Ok((ranking, out)) => {
1689 record.ranking = ranking.normalized();
1690 record.reasons = ranking.reasons;
1691 record.confidence = ranking.confidence;
1692 record.failed = None;
1693 record.duration_ms = out.duration_ms;
1694 recovered.insert(j);
1695 self.state.event(
1696 "recover",
1697 format!("judge {} ranked again after the limit", j + 1),
1698 );
1699 }
1700 Err(e) => {
1701 self.state
1702 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
1703 }
1704 }
1705 }
1706
1707 let mut vote_jobs = Vec::new();
1709 let mut vote_pos: Vec<usize> = Vec::new();
1710 for &j in &recovered {
1711 let seat_key = format!("judge-{}", j + 1);
1712 let spec = self.roles.judges[j].clone();
1713 let seat = self.seat(&seat_key, &spec.id);
1714 let mut text = prompt::final_vote(&labels, &language);
1715 if !has_context(&spec, &seat, sessions) {
1716 text = format!(
1717 "{}\n\n# Candidates\n\n{}",
1718 text,
1719 self.candidate_block(&candidates, &base_short)
1720 );
1721 }
1722 vote_jobs.push(SeatJob {
1723 spec,
1724 seat,
1725 prompt: text,
1726 cwd: root.join(seat_key),
1727 timeout,
1728 allow_write: false,
1729 sessions,
1730 artifacts: artifacts.clone(),
1731 stem: format!("vote-judge-{}-recover", j + 1),
1732 });
1733 vote_pos.push(j);
1734 }
1735 let allowed = labels.clone();
1736 let mut vote_losses = Vec::new();
1737 let vote_retries = self.state.config.graph.retries;
1738 let vote_cache = self.state.config.cache_dir();
1739 let ctx = WaveCtx {
1740 run: &run_id,
1741 node: "vote",
1742 prompts: &prompts,
1743 cache: vote_cache.as_deref(),
1744 };
1745 let votes = ask_json_wave::<FinalVote>(
1746 vote_jobs,
1747 Arc::clone(&self.sem),
1748 vote_retries,
1749 &ctx,
1750 &mut vote_losses,
1751 &mut self.state,
1752 &move |v: &FinalVote| match v.label() {
1753 Some(c) if allowed.contains(&c) => Ok(()),
1754 other => bail!("vote {other:?} is not one of {allowed:?}"),
1755 },
1756 )
1757 .await;
1758 for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
1759 let agent_id = seat.agent.clone();
1760 self.state.seats.insert(seat.key.clone(), seat);
1761 match res {
1762 Ok((v, _)) => {
1763 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
1764 rec.vote = v.label();
1765 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1766 } else {
1767 self.state.votes.push(VoteRecord {
1768 judge: j + 1,
1769 agent: agent_id,
1770 vote: v.label(),
1771 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
1772 changed: false,
1773 });
1774 }
1775 self.state.event(
1776 "recover",
1777 format!("judge {} voted again after the limit", j + 1),
1778 );
1779 }
1780 Err(e) => {
1781 self.state
1782 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
1783 }
1784 }
1785 }
1786
1787 if !recovered.is_empty() {
1791 let recovered_keys: BTreeSet<String> = recovered
1792 .iter()
1793 .map(|&j| format!("judge-{}", j + 1))
1794 .collect();
1795 self.state
1796 .quota
1797 .retain(|q| !recovered_keys.contains(&q.seat));
1798 }
1799
1800 self.state.tally = None;
1802 self.tally()?;
1803 Ok(self
1804 .state
1805 .tally
1806 .as_ref()
1807 .map(|t| t.met_quorum)
1808 .unwrap_or(false))
1809 }
1810
1811 async fn fold_losers(&mut self) -> Result<()> {
1814 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
1815 return Ok(());
1816 };
1817 let repo = self.state.repo.clone();
1818 let mut folded = Vec::new();
1819 for i in 0..self.state.candidates.len() {
1820 let c = &self.state.candidates[i];
1821 if c.label == winner || c.folded {
1822 continue;
1823 }
1824 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
1825 git::worktree_remove(&repo, &wt).await.ok();
1826 git::branch_delete(&repo, &branch).await.ok();
1827 self.state.candidates[i].folded = true;
1828 folded.push(label.to_string());
1829 }
1830 let root = self.state.worktree_root();
1832 for j in 1..=self.roles.judges.len() {
1833 let wt = root.join(format!("judge-{j}"));
1834 if wt.exists() {
1835 git::worktree_remove(&repo, &wt).await.ok();
1836 }
1837 }
1838 if !folded.is_empty() {
1839 self.state
1840 .event("fold", format!("folded candidates {}", folded.join(", ")));
1841 self.state.save()?;
1842 }
1843 Ok(())
1844 }
1845
1846 async fn sync_to_base(&mut self) -> Result<()> {
1876 if self
1877 .state
1878 .base_sync
1879 .as_ref()
1880 .is_some_and(|s| s.conflict.is_some())
1881 {
1882 return Ok(());
1883 }
1884 let Some(winner) = self.state.winner().cloned() else {
1885 return Ok(());
1886 };
1887
1888 let repo = self.state.repo.clone();
1889 let remote = self.state.config.merge.remote.clone();
1890 let base_branch = self.state.base_branch.clone();
1891 let tracking = format!("{remote}/{base_branch}");
1892
1893 git::fetch(&repo, &remote, &base_branch).await.ok();
1894 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
1898 return Ok(());
1899 };
1900
1901 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1902 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
1903 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
1904
1905 if behind == 0 {
1906 self.state.base_sync = Some(BaseSync {
1907 tip,
1908 behind: 0,
1909 attempts,
1910 conflict: None,
1911 });
1912 self.state.save()?;
1913 return Ok(());
1914 }
1915
1916 if attempts >= BASE_SYNC_ROUNDS {
1917 let why = format!(
1918 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
1919 rebase(s); rebasing again would only race it",
1920 winner.branch
1921 );
1922 self.state.status = RunStatus::Blocked;
1923 self.state.base_sync = Some(BaseSync {
1924 tip,
1925 behind,
1926 attempts,
1927 conflict: Some(why.clone()),
1928 });
1929 self.state.event("land", why);
1930 self.state.save()?;
1931 return Ok(());
1932 }
1933
1934 self.state.event(
1935 "land",
1936 format!(
1937 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
1938 winner.branch
1939 ),
1940 );
1941 self.state.save()?;
1942
1943 let scratch = self.state.dir().join("base-sync");
1944 let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
1945 let attempts = attempts + 1;
1946 match rebased {
1947 Ok(None) => {
1948 git::sync_to_head(&winner.worktree).await?;
1952 self.state.base_sync = Some(BaseSync {
1953 tip: tip.clone(),
1954 behind: 0,
1955 attempts,
1956 conflict: None,
1957 });
1958 self.state
1959 .event("land", format!("rebased {} onto {tracking}", winner.branch));
1960 }
1961 Ok(Some(conflict)) => {
1962 let why = format!(
1963 "{} conflicts with {tracking} and did not rebase: {}",
1964 winner.branch,
1965 conflict.chars().take(600).collect::<String>()
1966 );
1967 self.state.status = RunStatus::Blocked;
1968 self.state.base_sync = Some(BaseSync {
1969 tip,
1970 behind,
1971 attempts,
1972 conflict: Some(why.clone()),
1973 });
1974 self.state.event("land", why);
1975 }
1976 Err(e) => {
1977 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
1978 self.state.status = RunStatus::Blocked;
1979 self.state.base_sync = Some(BaseSync {
1980 tip,
1981 behind,
1982 attempts,
1983 conflict: Some(why.clone()),
1984 });
1985 self.state.event("land", why);
1986 }
1987 }
1988 self.state.save()?;
1989 Ok(())
1990 }
1991
1992 fn landing_base(&self) -> String {
2002 self.state
2003 .base_sync
2004 .as_ref()
2005 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2006 }
2007
2008 async fn review_loop(&mut self) -> Result<()> {
2011 if self
2016 .state
2017 .base_sync
2018 .as_ref()
2019 .is_some_and(|s| s.conflict.is_some())
2020 {
2021 return Ok(());
2022 }
2023 let run_id = self.state.id.clone();
2028 let prompts = self.state.config.prompts.clone();
2029 let Some(winner) = self.state.winner().cloned() else {
2030 return Ok(());
2031 };
2032 let max_rounds = self.state.config.graph.review_rounds;
2033 if max_rounds == 0 || self.state.reviews.iter().any(|r| r.clean) {
2034 self.state.status = RunStatus::Gating;
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 if self.state.reviews.len() >= max_rounds {
2039 self.state.status = RunStatus::Blocked;
2048 self.state.save()?;
2049 return Ok(());
2050 }
2051 self.state.status = RunStatus::Reviewing;
2052
2053 let repo = self.state.repo.clone();
2054 let root = self.state.worktree_root();
2055 let language = self.state.config.graph.language.clone();
2056 let sessions = self.state.config.graph.sessions;
2057 let artifacts = agent::artifacts_dir(&self.state.dir());
2058 let base = self.landing_base();
2059 let base_short = short(&base);
2060 let reviewers = self.roles.reviewers.clone();
2061 let shell = self.state.config.shell();
2062
2063 let mut prev_e2e: Option<String> = None;
2064 for round in (self.state.reviews.len() + 1)..=max_rounds {
2065 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2066 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2067 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2068
2069 let mut jobs = Vec::new();
2073 for (r, spec) in reviewers.iter().cloned().enumerate() {
2074 let wt = root.join(format!("review-{}", r + 1));
2075 if wt.exists() {
2076 git::reset_detached(&wt, &head).await?;
2077 } else {
2078 git::worktree_add_detached(&repo, &wt, &head).await?;
2079 }
2080 let seat_key = format!("review-{}", r + 1);
2081 let seat = self.seat(&seat_key, &spec.id);
2082 jobs.push(SeatJob {
2083 prompt: prompt::review(&prompt::ReviewCtx {
2084 instruction: &self.state.instruction,
2085 branch: &winner.branch,
2086 base_short: &base_short,
2087 stat: &stat,
2088 patch: &patch,
2089 e2e: prev_e2e.as_deref(),
2090 reviewers: reviewers.len(),
2091 round,
2092 rounds: max_rounds,
2093 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2096 language: &language,
2097 }),
2098 spec,
2099 seat,
2100 cwd: wt,
2101 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2102 allow_write: false,
2103 sessions,
2104 artifacts: artifacts.clone(),
2105 stem: format!("review-{round}-{}", r + 1),
2106 });
2107 }
2108
2109 self.state.event(
2110 "review",
2111 format!(
2112 "round {round}: {} reviewers on {}",
2113 jobs.len(),
2114 short(&head)
2115 ),
2116 );
2117 let mut quota_losses = Vec::new();
2118 let review_retries = self.state.config.graph.retries;
2119 let review_cache = self.state.config.cache_dir();
2120 let ctx = WaveCtx {
2121 run: &run_id,
2122 node: "review",
2123 prompts: &prompts,
2124 cache: review_cache.as_deref(),
2125 };
2126 let results = ask_json_wave::<Review>(
2127 jobs,
2128 Arc::clone(&self.sem),
2129 review_retries,
2130 &ctx,
2131 &mut quota_losses,
2132 &mut self.state,
2133 &|_: &Review| Ok(()),
2134 )
2135 .await;
2136 self.state.quota.extend(quota_losses);
2137
2138 let mut records = Vec::new();
2139 let mut all_findings = Vec::new();
2140 for (r, (seat, res)) in results.into_iter().enumerate() {
2141 let agent_id = seat.agent.clone();
2142 self.state.seats.insert(seat.key.clone(), seat);
2143 let mut record = ReviewRecord {
2144 reviewer: r + 1,
2145 agent: agent_id,
2146 summary: String::new(),
2147 findings: Vec::new(),
2148 failed: None,
2149 duration_ms: 0,
2150 };
2151 match res {
2152 Ok((review, out)) => {
2153 record.summary = review.summary;
2154 record.duration_ms = out.duration_ms;
2155 for (n, mut f) in review.findings.into_iter().enumerate() {
2156 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2159 all_findings.push(f.clone());
2160 record.findings.push(f);
2161 }
2162 self.state.event(
2163 "review",
2164 format!(
2165 "round {round}: reviewer {} raised {} finding(s)",
2166 r + 1,
2167 record.findings.len()
2168 ),
2169 );
2170 }
2171 Err(e) => {
2172 record.failed = Some(e.to_string());
2173 self.state.event(
2174 "review",
2175 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2176 );
2177 }
2178 }
2179 records.push(record);
2180 }
2181
2182 let mut e2e = run_commands(
2183 &shell,
2184 &self.state.config.verify.e2e,
2185 &winner.worktree,
2186 Duration::from_secs(self.state.config.graph.timeout_review),
2187 )
2188 .await;
2189 for o in &e2e {
2190 self.state.event(
2191 "verify",
2192 format!("round {round}: `{}` -> {}", o.command, e2e_outcome_label(o)),
2193 );
2194 }
2195
2196 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
2201 if verify_retried {
2202 self.state.event(
2203 "verify",
2204 format!(
2205 "round {round}: verify could not build/link, not a test result — \
2206 retrying once before concluding"
2207 ),
2208 );
2209 e2e = run_commands(
2210 &shell,
2211 &self.state.config.verify.e2e,
2212 &winner.worktree,
2213 Duration::from_secs(self.state.config.graph.timeout_review),
2214 )
2215 .await;
2216 for o in &e2e {
2217 self.state.event(
2218 "verify",
2219 format!(
2220 "round {round}: retry `{}` -> {}",
2221 o.command,
2222 e2e_outcome_label(o)
2223 ),
2224 );
2225 }
2226 }
2227
2228 let e2e_failures: String = e2e
2229 .iter()
2230 .filter(|o| !o.ok())
2231 .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2232 .collect();
2233
2234 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2235 let clean = blocking == 0 && e2e.iter().all(CommandOutcome::ok);
2236
2237 let mut round_record = ReviewRound {
2238 round,
2239 head: head.clone(),
2240 reviews: records,
2241 e2e,
2242 verify_retried,
2243 fix: None,
2244 blocking,
2245 clean,
2246 };
2247
2248 if clean {
2249 self.state.event(
2250 "review",
2251 format!("round {round}: clean — no blocking findings, verification green"),
2252 );
2253 self.state.reviews.push(round_record);
2254 self.state.status = RunStatus::Gating;
2255 self.state.save()?;
2256 return Ok(());
2257 }
2258
2259 if round == max_rounds {
2260 self.state.reviews.push(round_record);
2261 self.state.status = RunStatus::Blocked;
2262 self.state.event(
2263 "review",
2264 format!("{blocking} blocking finding(s) still open after {max_rounds} rounds"),
2265 );
2266 self.state.save()?;
2267 return Ok(());
2268 }
2269
2270 let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2273 Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2274 _ => (
2275 self.state
2276 .config
2277 .agent(&winner.agent)
2278 .cloned()
2279 .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2280 format!("impl-{}", winner.label),
2281 ),
2282 };
2283 let seat = self.seat(&fix_seat_key, &fix_spec.id);
2284 let blocking_findings: Vec<_> = all_findings
2285 .iter()
2286 .filter(|f| f.severity.blocks())
2287 .cloned()
2288 .collect();
2289 let job = SeatJob {
2290 prompt: prompt::fix(
2291 &self.state.instruction,
2292 &blocking_findings,
2293 (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2294 round,
2295 max_rounds,
2296 &language,
2297 ),
2298 spec: fix_spec.clone(),
2299 seat,
2300 cwd: winner.worktree.clone(),
2301 timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2302 allow_write: true,
2303 sessions,
2304 artifacts: artifacts.clone(),
2305 stem: format!("fix-{round}"),
2306 };
2307 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2308 let cache = self.state.config.cache_dir();
2309 let ctx = WaveCtx {
2310 run: &run_id,
2311 node: "fix",
2312 prompts: &prompts,
2313 cache: cache.as_deref(),
2314 };
2315 let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2316 let agent_id = seat.agent.clone();
2317 let seat_key = seat.key.clone();
2318 self.state.seats.insert(seat.key.clone(), seat);
2319
2320 let mut fix = FixRecord {
2321 agent: agent_id,
2322 addressed: Vec::new(),
2323 rejected: Vec::new(),
2324 notes: String::new(),
2325 committed: false,
2326 failed: None,
2327 duration_ms: 0,
2328 };
2329 match out {
2330 AgentOutcome::Ok(o) => {
2331 fix.duration_ms = o.duration_ms;
2332 match verdict::extract_json::<FixReport>(&o.text) {
2333 Ok(report) => {
2334 fix.addressed = report.addressed;
2335 fix.rejected = report.rejected;
2336 fix.notes =
2337 blind::sanitize_prose(&report.notes, &self.state.config.blind);
2338 }
2339 Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
2340 }
2341 }
2342 AgentOutcome::Dropped(o) => {
2344 fix.duration_ms = o.duration_ms;
2345 let why = o
2346 .dropped
2347 .as_ref()
2348 .map(|d| d.why.as_str())
2349 .unwrap_or("the CLI ended the stream without delivering its answer");
2350 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
2351 }
2352 AgentOutcome::Quota(o) => {
2353 self.state.quota.push(QuotaLoss {
2354 seat: seat_key,
2355 node: "fix".to_owned(),
2356 at: Timestamp::now(),
2357 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2358 });
2359 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
2360 }
2361 AgentOutcome::Failed(e) => fix.failed = Some(e),
2362 }
2363 git::commit_all(
2364 &winner.worktree,
2365 &format!("magi: review round {round} fixes (uncommitted work)"),
2366 )
2367 .await
2368 .ok();
2369 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
2370 fix.committed = after != before;
2371 let commit_note = if fix.committed {
2372 "committed"
2373 } else {
2374 "NO new commit"
2375 };
2376 self.state.event(
2377 "fix",
2378 match &fix.failed {
2379 Some(reason) => {
2385 format!("round {round}: fixer's adoption report was lost ({reason}); {commit_note}")
2386 }
2387 None => format!(
2388 "round {round}: {} addressed, {} rejected, {commit_note}",
2389 fix.addressed.len(),
2390 fix.rejected.len(),
2391 ),
2392 },
2393 );
2394 let stalled = !fix.committed;
2395 round_record.fix = Some(fix);
2396 self.state.reviews.push(round_record);
2397 self.state.save()?;
2398
2399 prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
2400
2401 if stalled {
2402 self.state.status = RunStatus::Blocked;
2403 self.state.event(
2404 "review",
2405 "the fixer produced no commit; stopping instead of looping on an unchanged tree"
2406 .to_owned(),
2407 );
2408 self.state.save()?;
2409 return Ok(());
2410 }
2411 }
2412 Ok(())
2413 }
2414
2415 async fn gate(&mut self) -> Result<()> {
2418 if self.state.status == RunStatus::Failed
2427 || self
2428 .state
2429 .base_sync
2430 .as_ref()
2431 .is_some_and(|s| s.conflict.is_some())
2432 || self.state.reviews.last().is_some_and(|r| !r.clean)
2433 {
2434 return Ok(());
2435 }
2436 if !self.state.gate.is_empty() {
2437 return Ok(());
2438 }
2439 let Some(winner) = self.state.winner().cloned() else {
2440 return Ok(());
2441 };
2442 self.state.status = RunStatus::Gating;
2443 let shell = self.state.config.shell();
2444 let outcomes = run_commands(
2445 &shell,
2446 &self.state.config.verify.gate,
2447 &winner.worktree,
2448 Duration::from_secs(self.state.config.graph.timeout_review),
2449 )
2450 .await;
2451 for o in &outcomes {
2452 self.state.event(
2453 "gate",
2454 format!(
2455 "`{}` -> {}",
2456 o.command,
2457 if o.ok() {
2458 "pass".to_owned()
2459 } else {
2460 format!("FAIL ({:?})", o.code)
2461 }
2462 ),
2463 );
2464 }
2465 let passed = outcomes.iter().all(CommandOutcome::ok);
2466 self.state.gate = outcomes;
2467 if !passed {
2468 self.state.status = RunStatus::Blocked;
2469 self.state.event("gate", "gate failed; not merging");
2470 }
2471 self.state.save()?;
2472 Ok(())
2473 }
2474
2475 async fn merge(&mut self) -> Result<()> {
2478 if self
2483 .state
2484 .base_sync
2485 .as_ref()
2486 .is_some_and(|s| s.conflict.is_some())
2487 || self.state.reviews.last().is_some_and(|r| !r.clean)
2488 || self.state.gate.iter().any(|o| !o.ok())
2489 {
2490 return Ok(());
2491 }
2492 if self.state.merge.is_some() {
2501 return Ok(());
2502 }
2503 let Some(winner) = self.state.winner().cloned() else {
2504 return Ok(());
2505 };
2506 let repo = self.state.repo.clone();
2507 let base = self.state.base_branch.clone();
2508 let mode = self.state.config.merge.mode;
2509 let message = format!(
2510 "Merge magi run {} (candidate {})\n\n{}",
2511 self.state.id, winner.label, self.state.instruction
2512 );
2513
2514 let outcome = match mode {
2515 MergeMode::None => MergeOutcome {
2516 mode,
2517 ok: true,
2518 detail: format!("git -C {} merge --no-ff {}", repo.display(), winner.branch),
2519 },
2520 MergeMode::Local => {
2521 let on = git::current_branch(&repo).await?;
2522 if on.as_deref() != Some(base.as_str()) {
2523 MergeOutcome {
2524 mode,
2525 ok: false,
2526 detail: format!(
2527 "{} has {} checked out, not the base branch {base}",
2528 repo.display(),
2529 on.unwrap_or_else(|| "a detached HEAD".to_owned())
2530 ),
2531 }
2532 } else if !git::is_clean(&repo).await? {
2533 MergeOutcome {
2534 mode,
2535 ok: false,
2536 detail: format!("{} is dirty; refusing to merge", repo.display()),
2537 }
2538 } else {
2539 let out = git::merge_no_ff(&repo, &winner.branch, &message).await?;
2540 MergeOutcome {
2541 mode,
2542 ok: out.ok(),
2543 detail: if out.ok() { out.stdout } else { out.stderr },
2544 }
2545 }
2546 }
2547 MergeMode::Pr => {
2548 let remote = self.state.config.merge.remote.clone();
2549 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
2550 if !pushed.ok() {
2551 MergeOutcome {
2552 mode,
2553 ok: false,
2554 detail: pushed.stderr,
2555 }
2556 } else {
2557 let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
2558 match out {
2559 Ok(url) => MergeOutcome {
2560 mode,
2561 ok: true,
2562 detail: url,
2563 },
2564 Err(e) => MergeOutcome {
2565 mode,
2566 ok: false,
2567 detail: e.to_string(),
2568 },
2569 }
2570 }
2571 }
2572 };
2573
2574 self.state.status = match (mode, outcome.ok) {
2575 (MergeMode::None, _) => RunStatus::Ready,
2576 (_, true) => RunStatus::Merged,
2577 (_, false) => RunStatus::Blocked,
2578 };
2579 self.state.event(
2580 "merge",
2581 format!(
2582 "{:?}: {}",
2583 mode,
2584 outcome.detail.lines().next().unwrap_or("")
2585 ),
2586 );
2587 self.state.merge = Some(outcome);
2588 self.state.save()?;
2589
2590 if self.state.config.graph.land
2596 && mode == MergeMode::Pr
2597 && self.state.status == RunStatus::Merged
2598 {
2599 let url = self
2600 .state
2601 .merge
2602 .as_ref()
2603 .map(|m| m.detail.clone())
2604 .unwrap_or_default();
2605 let url = url.lines().next().unwrap_or("").trim().to_owned();
2606 if url.starts_with("http") {
2607 match land::land(&mut self.state, &url).await {
2610 Ok(pr) => {
2611 self.state.status = match pr.state {
2612 land::PrLifecycle::Merged => RunStatus::Merged,
2613 _ => RunStatus::Blocked,
2614 };
2615 }
2616 Err(e) => {
2617 self.state.status = RunStatus::Blocked;
2618 self.state.event("land", format!("gave up: {e}"));
2619 }
2620 }
2621 self.state.save()?;
2622 }
2623 }
2624 Ok(())
2625 }
2626
2627 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
2631 if let Some(existing) = self.state.seats.get(key)
2632 && existing.agent == agent
2633 {
2634 return existing.clone();
2635 }
2636 let fresh = SeatState::new(key, agent, self.state.seed);
2637 self.state.seats.insert(key.to_owned(), fresh.clone());
2638 fresh
2639 }
2640
2641 fn view(&self, c: &Candidate) -> CandidateView {
2643 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2644 .unwrap_or_default();
2645 let (patch, _) = blind::sanitize_patch(
2646 &format!("candidate {} patch", c.label),
2647 &raw,
2648 &self.state.config.blind,
2649 );
2650 CandidateView {
2651 label: c.label,
2652 branch: c.branch.clone(),
2653 summary: c.summary.clone(),
2654 stat: c.stat.clone(),
2655 patch,
2656 }
2657 }
2658
2659 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
2661 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
2662 prompt::judge(
2663 "(see above)",
2664 &views,
2665 self.roles.judges.len(),
2666 base_short,
2667 "en",
2668 )
2669 }
2670
2671 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
2678 let mut turns = Vec::new();
2679 for j in &self.state.judgements {
2680 if j.ranking.is_empty() {
2681 continue;
2682 }
2683 let reasons = j
2684 .reasons
2685 .iter()
2686 .map(|(k, v)| format!("- {k}: {v}"))
2687 .collect::<Vec<_>>()
2688 .join("\n");
2689 turns.push(Turn {
2690 who: format!("Judge {} (opening ranking)", j.judge),
2691 is_self: j.judge == self_idx + 1,
2692 body: format!(
2693 "Ranked {}{}{reasons}",
2694 j.ranking.iter().collect::<String>(),
2695 if reasons.is_empty() {
2696 ""
2697 } else {
2698 ", because:\n"
2699 }
2700 ),
2701 });
2702 }
2703 for t in self
2704 .state
2705 .deliberation
2706 .iter()
2707 .flat_map(|r| r.turns.iter())
2708 .chain(current)
2709 {
2710 turns.push(Turn {
2711 who: format!("Judge {}", t.judge),
2712 is_self: t.judge == self_idx + 1,
2713 body: t.body.clone(),
2714 });
2715 }
2716 turns
2717 }
2718}
2719
2720fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
2722 agent::has_session(spec.kind, seat, sessions)
2723}
2724
2725fn short(commit: &str) -> String {
2726 commit.chars().take(7).collect()
2727}
2728
2729fn make_executable(path: &Path) -> Result<()> {
2730 #[cfg(unix)]
2731 {
2732 use std::os::unix::fs::PermissionsExt as _;
2733 let mut perms = std::fs::metadata(path)?.permissions();
2734 perms.set_mode(0o755);
2735 std::fs::set_permissions(path, perms)?;
2736 }
2737 #[cfg(not(unix))]
2738 {
2739 let _ = path;
2740 }
2741 Ok(())
2742}
2743
2744struct WaveCtx<'a> {
2751 run: &'a str,
2754 node: &'a str,
2756 prompts: &'a Prompts,
2757 cache: Option<&'a Path>,
2759}
2760
2761async fn run_one(
2763 job: SeatJob,
2764 sem: Arc<Semaphore>,
2765 ctx: &WaveCtx<'_>,
2766 state: &mut RunState,
2767 attempt: usize,
2768) -> (SeatState, AgentOutcome) {
2769 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
2770 .await
2771 .pop()
2772 .expect("one job in, one result out");
2773 (seat, out)
2774}
2775
2776async fn wave(
2782 jobs: Vec<SeatJob>,
2783 sem: Arc<Semaphore>,
2784 ctx: &WaveCtx<'_>,
2785 state: &mut RunState,
2786 attempt: usize,
2787) -> Vec<(usize, SeatState, AgentOutcome)> {
2788 let WaveCtx {
2789 run,
2790 node,
2791 prompts,
2792 cache,
2793 } = *ctx;
2794 for job in &jobs {
2795 state.seat_started(node, &job.seat.key, job.timeout, attempt);
2796 }
2797 if let Err(e) = state.save() {
2798 tracing::warn!("could not persist in-progress seats: {e:#}");
2803 }
2804 let mut set = tokio::task::JoinSet::new();
2805 let overlay = prompts.overlay(node);
2806 for (i, mut job) in jobs.into_iter().enumerate() {
2807 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
2808 if cache.is_some() {
2809 job.prompt.push('\n');
2810 job.prompt.push_str(prompt::build_cache_note());
2811 }
2812 let sem = Arc::clone(&sem);
2813 let run = run.to_owned();
2814 let node = node.to_owned();
2815 let cache = cache.map(Path::to_path_buf);
2816 set.spawn(async move {
2817 let _permit = sem.acquire().await;
2818 let mut seat = job.seat;
2819 let out = agent::invoke(
2820 &job.spec,
2821 &mut seat,
2822 &Invocation {
2823 cwd: &job.cwd,
2824 prompt: &job.prompt,
2825 timeout: job.timeout,
2826 allow_write: job.allow_write,
2827 sessions: job.sessions,
2828 artifacts: &job.artifacts,
2829 stem: &job.stem,
2830 run: &run,
2831 node: &node,
2832 cache_dir: cache.as_deref(),
2833 },
2834 )
2835 .await;
2836 let out = match out {
2837 Ok(o) if o.usable() => AgentOutcome::Ok(o),
2838 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
2839 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
2847 Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
2848 Ok(o) => AgentOutcome::Failed(format!(
2849 "exited with {:?} and no usable output",
2850 o.exit_code
2851 )),
2852 Err(e) => AgentOutcome::Failed(e.to_string()),
2853 };
2854 (i, seat, out)
2855 });
2856 }
2857 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
2858 while let Some(joined) = set.join_next().await {
2859 let (i, seat, out) = match joined {
2860 Ok(v) => v,
2861 Err(e) => {
2865 tracing::error!("agent task panicked: {e}");
2866 continue;
2867 }
2868 };
2869 state.seat_finished(&seat.key);
2870 if let Err(e) = state.save() {
2871 tracing::warn!("could not persist a seat's completion: {e:#}");
2872 }
2873 if collected.len() <= i {
2874 collected.resize_with(i + 1, || None);
2875 }
2876 collected[i] = Some((i, seat, out));
2877 }
2878 if state
2884 .active
2885 .values()
2886 .any(|a| a.node == node && a.attempt == attempt)
2887 {
2888 state
2889 .active
2890 .retain(|_, a| !(a.node == node && a.attempt == attempt));
2891 if let Err(e) = state.save() {
2892 tracing::warn!("could not persist the end of a wave: {e:#}");
2893 }
2894 }
2895 collected.into_iter().flatten().collect()
2896}
2897
2898fn retry_budget(full: Duration, nudged: bool) -> Duration {
2913 if nudged {
2914 (full / 4).max(Duration::from_secs(120)).min(full)
2915 } else {
2916 full
2917 }
2918}
2919
2920#[allow(clippy::too_many_arguments)]
2933async fn ask_json_wave<T>(
2934 jobs: Vec<SeatJob>,
2935 sem: Arc<Semaphore>,
2936 retries: usize,
2937 ctx: &WaveCtx<'_>,
2938 losses: &mut Vec<QuotaLoss>,
2939 state: &mut RunState,
2940 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
2941) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
2942where
2943 T: serde::de::DeserializeOwned + Send + 'static,
2944{
2945 let n = jobs.len();
2946 let originals: Vec<SeatJob> = jobs;
2947 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
2948 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
2949 let mut pending: Vec<usize> = (0..n).collect();
2950
2951 for attempt in 0..=retries {
2952 if pending.is_empty() {
2953 break;
2954 }
2955 let mut batch = Vec::with_capacity(pending.len());
2956 for &i in &pending {
2957 let src = &originals[i];
2958 let (prompt, timeout) = if attempt == 0 {
2961 (src.prompt.clone(), src.timeout)
2962 } else {
2963 let why = done[i]
2964 .as_ref()
2965 .and_then(|r| r.as_ref().err().map(ToString::to_string))
2966 .unwrap_or_else(|| "no parsable answer".to_owned());
2967 let nudge = prompt::nudge(&why);
2968 let nudged = has_context(&src.spec, &seats[i], src.sessions);
2969 let prompt = if nudged {
2970 nudge
2971 } else {
2972 format!("{}\n\n---\n\n{}", src.prompt, nudge)
2973 };
2974 (prompt, retry_budget(src.timeout, nudged))
2975 };
2976 batch.push(SeatJob {
2977 spec: src.spec.clone(),
2978 seat: seats[i].clone(),
2979 cwd: src.cwd.clone(),
2980 prompt,
2981 timeout,
2982 allow_write: src.allow_write,
2983 sessions: src.sessions,
2984 artifacts: src.artifacts.clone(),
2985 stem: if attempt == 0 {
2986 src.stem.clone()
2987 } else {
2988 format!("{}-retry{attempt}", src.stem)
2989 },
2990 });
2991 }
2992
2993 if attempt > 0 {
2994 let seats_out: Vec<&str> = pending
2995 .iter()
2996 .map(|&i| originals[i].seat.key.as_str())
2997 .collect();
2998 state.event(
2999 ctx.node,
3000 format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3001 );
3002 }
3003 let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3004 let mut still = Vec::new();
3005 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3006 seats[i] = seat;
3007 let (parsed, quota) = match out {
3008 AgentOutcome::Ok(o) => (
3009 match verdict::extract_json::<T>(&o.text) {
3010 Ok(v) => match validate(&v) {
3011 Ok(()) => Ok((v, o)),
3012 Err(e) => Err(e),
3013 },
3014 Err(e) => Err(e),
3015 },
3016 false,
3017 ),
3018 AgentOutcome::Quota(o) => {
3019 losses.push(QuotaLoss {
3020 seat: originals[i].seat.key.clone(),
3021 node: ctx.node.to_owned(),
3022 at: Timestamp::now(),
3023 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3024 });
3025 (
3026 Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3027 true,
3028 )
3029 }
3030 AgentOutcome::Dropped(o) => {
3035 let why = o
3036 .dropped
3037 .as_ref()
3038 .map(|d| d.why.as_str())
3039 .unwrap_or("the CLI ended the stream without delivering its answer");
3040 (
3041 Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3042 false,
3043 )
3044 }
3045 AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3046 };
3047 let failed = parsed.is_err();
3048 done[i] = Some(parsed);
3049 if failed && !quota {
3052 still.push(i);
3053 }
3054 }
3055 pending = still;
3056 }
3057
3058 seats
3059 .into_iter()
3060 .zip(done)
3061 .map(|(seat, res)| {
3062 (
3063 seat,
3064 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3065 )
3066 })
3067 .collect()
3068}
3069
3070fn e2e_outcome_label(o: &CommandOutcome) -> String {
3074 if o.ok() {
3075 "pass".to_owned()
3076 } else if o.build_failed() {
3077 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
3078 } else {
3079 format!("FAIL ({:?})", o.code)
3080 }
3081}
3082
3083async fn run_commands(
3085 shell: &[String],
3086 commands: &[String],
3087 cwd: &Path,
3088 timeout: Duration,
3089) -> Vec<CommandOutcome> {
3090 let mut out = Vec::new();
3091 for command in commands {
3092 let started = Instant::now();
3093 let mut cmd = tokio::process::Command::new(&shell[0]);
3094 cmd.quiet();
3095 cmd.args(&shell[1..])
3096 .arg(command)
3097 .current_dir(cwd)
3098 .stdin(std::process::Stdio::null())
3099 .stdout(std::process::Stdio::piped())
3100 .stderr(std::process::Stdio::piped())
3101 .kill_on_drop(true);
3102 let spawned = cmd.spawn();
3103 let (code, body) = match spawned {
3104 Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
3105 Ok(Ok(o)) => {
3106 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
3107 body.push_str(&String::from_utf8_lossy(&o.stderr));
3108 (o.status.code(), body)
3109 }
3110 Ok(Err(e)) => (None, format!("failed to run: {e}")),
3111 Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
3112 },
3113 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
3114 };
3115 out.push(CommandOutcome {
3116 command: command.clone(),
3117 code,
3118 output_tail: tail(&body, OUTPUT_TAIL),
3119 duration_ms: started.elapsed().as_millis() as u64,
3120 });
3121 }
3122 out
3123}
3124
3125async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
3127 let title = body.lines().next().unwrap_or("magi run").to_owned();
3128 let out = tokio::process::Command::new("gh")
3129 .args([
3130 "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
3131 ])
3132 .current_dir(cwd)
3133 .stdin(std::process::Stdio::null())
3134 .output()
3135 .await
3136 .context("spawn gh")?;
3137 if out.status.success() {
3138 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
3139 } else {
3140 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
3141 }
3142}
3143
3144pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
3146 let repo = state.repo.clone();
3147 let root = state.worktree_root();
3148 let winner = state.tally.as_ref().map(|t| t.winner);
3149 let mut removed = Vec::new();
3150
3151 for i in 0..state.candidates.len() {
3152 let c = state.candidates[i].clone();
3153 let is_winner = Some(c.label) == winner;
3154 if is_winner && !drop_winner {
3155 continue;
3156 }
3157 if c.worktree.exists() {
3158 git::worktree_remove(&repo, &c.worktree).await.ok();
3159 removed.push(c.worktree.to_string_lossy().into_owned());
3160 }
3161 if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
3162 git::branch_delete(&repo, &c.branch).await.ok();
3163 removed.push(c.branch.clone());
3164 }
3165 state.candidates[i].folded = true;
3166 }
3167
3168 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
3169 let path = name.path();
3170 let keep = !drop_winner
3171 && winner.is_some_and(|w| {
3172 path.file_name()
3173 .is_some_and(|n| n == format!("cand-{w}").as_str())
3174 });
3175 if keep {
3176 continue;
3177 }
3178 git::worktree_remove(&repo, &path).await.ok();
3179 removed.push(path.to_string_lossy().into_owned());
3180 }
3181
3182 if state.enabled_worktree_config && drop_winner {
3183 git::disable_worktree_config(&repo).await.ok();
3184 state.enabled_worktree_config = false;
3185 }
3186 state.save()?;
3187 Ok(removed)
3188}
3189
3190pub fn worst_open(state: &RunState) -> Option<Severity> {
3192 state
3193 .reviews
3194 .last()?
3195 .reviews
3196 .iter()
3197 .flat_map(|r| r.findings.iter())
3198 .map(|f| f.severity)
3199 .max()
3200}
3201
3202#[cfg(test)]
3203mod tests {
3204 use super::*;
3205 use std::time::Duration;
3206
3207 fn secs(n: u64) -> Duration {
3208 Duration::from_secs(n)
3209 }
3210
3211 fn init_repo(dir: &Path) {
3214 let run = |args: &[&str]| {
3215 let out = std::process::Command::new("git")
3216 .args(args)
3217 .current_dir(dir)
3218 .output()
3219 .expect("spawn git");
3220 assert!(
3221 out.status.success(),
3222 "git {args:?} failed: {}",
3223 String::from_utf8_lossy(&out.stderr)
3224 );
3225 };
3226 run(&["init", "-b", "main"]);
3227 run(&["config", "user.name", "magi test"]);
3228 run(&["config", "user.email", "magi@example.com"]);
3229 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
3230 run(&["add", "-A"]);
3231 run(&["commit", "-m", "init"]);
3232 }
3233
3234 #[tokio::test]
3243 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
3244 let tmp = tempfile::tempdir().expect("tempdir");
3245 let repo = tmp.path().join("repo");
3246 std::fs::create_dir_all(&repo).unwrap();
3247 init_repo(&repo);
3248
3249 let mut config = Config::default();
3250 config.merge.mode = MergeMode::Local;
3251
3252 let mut state = RunState::new(
3253 repo.clone(),
3254 "main".to_owned(),
3255 "deadbeef".to_owned(),
3256 "task".to_owned(),
3257 config,
3258 );
3259 state.candidates = vec![Candidate {
3260 index: 0,
3261 label: 'A',
3262 agent: "alpha".to_owned(),
3263 branch: "does-not-exist".to_owned(),
3264 worktree: repo.clone(),
3265 summary: String::new(),
3266 stat: String::new(),
3267 files: 0,
3268 commits: 0,
3269 empty: false,
3270 failed: None,
3271 duration_ms: 0,
3272 folded: false,
3273 }];
3274 state.tally = Some(Tally {
3275 first_choice: BTreeMap::from([('A', 1)]),
3276 borda: BTreeMap::new(),
3277 winner: 'A',
3278 rankings: 1,
3279 unanimous_initial: true,
3280 deliberated: false,
3281 changed_votes: 0,
3282 unanimous_final: true,
3283 tie_break: None,
3284 judges: 0,
3285 present: 0,
3286 quorum: 0,
3287 met_quorum: true,
3288 uncontested: Some("only candidate A produced a change".to_owned()),
3289 });
3290 state.reviews = vec![ReviewRound {
3291 round: 1,
3292 head: "deadbeef".to_owned(),
3293 reviews: Vec::new(),
3294 e2e: Vec::new(),
3295 fix: None,
3296 blocking: 0,
3297 clean: true,
3298 verify_retried: false,
3299 }];
3300 state.gate = vec![CommandOutcome {
3301 command: "test".to_owned(),
3302 code: Some(0),
3303 output_tail: String::new(),
3304 duration_ms: 0,
3305 }];
3306 state.status = RunStatus::Ready;
3311 state.merge = Some(MergeOutcome {
3312 mode: MergeMode::Local,
3313 ok: false,
3314 detail: "already concluded".to_owned(),
3315 });
3316
3317 let mut runner = Runner {
3318 state,
3319 roles: ResolvedRoles {
3320 implementers: Vec::new(),
3321 judges: Vec::new(),
3322 reviewers: Vec::new(),
3323 fixer: None,
3324 },
3325 sem: Arc::new(Semaphore::new(1)),
3326 pause: Pause::new(),
3327 };
3328
3329 runner.merge().await.expect("merge");
3330
3331 assert_eq!(
3332 runner.state.status,
3333 RunStatus::Ready,
3334 "a concluded run's status must not change on reentry"
3335 );
3336 assert_eq!(
3337 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
3338 Some("already concluded"),
3339 "merge must not run again once the node already recorded an outcome"
3340 );
3341 }
3342
3343 #[test]
3344 fn a_nudge_gets_a_quarter_of_the_budget() {
3345 assert_eq!(retry_budget(secs(1200), true), secs(300));
3347 assert_eq!(retry_budget(secs(3600), true), secs(900));
3348 }
3349
3350 #[test]
3351 fn a_resent_prompt_keeps_the_whole_budget() {
3352 assert_eq!(retry_budget(secs(1200), false), secs(1200));
3355 assert_eq!(retry_budget(secs(60), false), secs(60));
3356 }
3357
3358 #[test]
3359 fn the_floor_never_exceeds_the_original_budget() {
3360 assert_eq!(retry_budget(secs(60), true), secs(60));
3364 assert_eq!(retry_budget(secs(480), true), secs(120));
3365 assert_eq!(retry_budget(secs(0), true), secs(0));
3366 }
3367}