1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
488 let tracking = format!("{remote}/{base_branch}");
489 let fetched = git::fetch(repo, remote, base_branch).await;
490 if let Ok(out) = &fetched
491 && out.ok()
492 && git::rev_exists(repo, &tracking).await
493 {
494 return git::rev_parse(repo, &tracking).await;
495 }
496 let why = match &fetched {
497 Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
498 Ok(_) => format!("{remote} has no {base_branch}"),
499 Err(e) => e.to_string(),
500 };
501 bail!(
504 "cannot read {tracking} ({why}); refusing to branch off the local \
505 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
506 base / remote in magi.toml"
507 )
508}
509
510struct FixClaim {
527 path: PathBuf,
528}
529
530impl FixClaim {
531 fn acquire(dir: &Path) -> Result<Self> {
532 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
533 let path = dir.join("fix.lock");
534 match Self::create(&path) {
535 Ok(claim) => Ok(claim),
536 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
537 if Self::reclaim_if_dead(&path) {
538 Self::create(&path).with_context(|| format!("lock {}", path.display()))
539 } else {
540 bail!(
541 "another `magi fix` is already running for this run ({} exists)",
542 path.display()
543 )
544 }
545 }
546 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
547 }
548 }
549
550 fn create(path: &Path) -> std::io::Result<Self> {
551 let mut f = std::fs::OpenOptions::new()
552 .write(true)
553 .create_new(true)
554 .open(path)?;
555 use std::io::Write as _;
556 writeln!(f, "{}", std::process::id())?;
558 Ok(Self {
559 path: path.to_owned(),
560 })
561 }
562
563 fn reclaim_if_dead(path: &Path) -> bool {
567 let dead = std::fs::read_to_string(path)
568 .ok()
569 .and_then(|body| body.trim().parse::<u32>().ok())
570 .is_some_and(|pid| !crate::proc::pid_alive(pid));
571 dead && std::fs::remove_file(path).is_ok()
572 }
573}
574
575impl Drop for FixClaim {
576 fn drop(&mut self) {
577 let _ = std::fs::remove_file(&self.path);
578 }
579}
580
581impl Runner {
582 pub async fn start(
584 repo: &Path,
585 instruction: String,
586 config: Config,
587 origin: Origin,
588 ) -> Result<Self> {
589 Self::start_naming(repo, instruction, "", config, origin).await
590 }
591
592 pub async fn start_naming(
596 repo: &Path,
597 instruction: String,
598 also_scan: &str,
599 config: Config,
600 origin: Origin,
601 ) -> Result<Self> {
602 let repo = git::toplevel(repo).await?;
603 let missing = agent::missing_programs(&config.agents);
604 if !missing.is_empty() {
605 bail!(
606 "these agent programs are not on PATH: {}. Fix the roster in \
607 magi.toml or install them.",
608 missing.join(", ")
609 );
610 }
611 let base_branch =
612 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
613 .await?;
614 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
615 let roles = config.resolve_roles()?;
616 let max_parallel = config.graph.max_parallel.max(1);
617 let seeds = refs::resolve(
620 &repo,
621 &base_commit,
622 &config.merge.remote,
623 &format!("{also_scan}\n{instruction}"),
624 )
625 .await;
626 refs::plan(&repo, &seeds).await?;
627 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
628 state.origin = Some(origin);
632 for seed in &seeds {
633 state.event(
634 "seed",
635 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
636 );
637 }
638 state.seeds = seeds;
639 state.event("start", format!("run {} created", state.id));
640 state.save()?;
641 Ok(Self {
642 state,
643 roles,
644 sem: Arc::new(Semaphore::new(max_parallel)),
645 pause: Pause::new(),
646 interrupt: Pause::new(),
647 })
648 }
649
650 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
664 Self::review_taking_over(repo, branch, config, None, origin).await
665 }
666
667 pub async fn review_taking_over(
674 repo: &Path,
675 branch: &str,
676 config: Config,
677 takeover: Option<crate::handover::Takeover>,
678 origin: Origin,
679 ) -> Result<Self> {
680 let repo = git::toplevel(repo).await?;
681 let missing = agent::missing_programs(&config.agents);
682 if !missing.is_empty() {
683 bail!(
684 "these agent programs are not on PATH: {}. Fix the roster in \
685 magi.toml or install them.",
686 missing.join(", ")
687 );
688 }
689 let base_branch =
690 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
691 .await?;
692 if base_branch == branch {
693 bail!("`{branch}` is the base branch; there is nothing to review against");
694 }
695 let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
696
697 let roles = config.resolve_roles()?;
698 let max_parallel = config.graph.max_parallel.max(1);
699 let mut state = RunState::new(
700 repo.clone(),
701 base_branch,
702 base_commit.clone(),
703 String::new(),
704 config,
705 );
706 state.origin = Some(origin);
707
708 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
715 earlier: Vec::new(),
716 home: crate::run::home(),
717 choice: None,
718 task_instruction: None,
719 });
720 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
721 if let Some(released) = &released {
722 state.event(
723 "release",
724 format!(
725 "took `{branch}` over from run {}: its worktree was released: {}",
726 crate::run::short_of(&released.old_id),
727 released.audit
728 ),
729 );
730 }
731 if let Some(choice) = takeover.choice.as_ref()
735 && let Err(e) =
736 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
737 .await
738 {
739 if let Some(released) = &released {
740 released.restore(&repo, branch).await;
741 }
742 return Err(e.context("applying the owner's answer about the diverged branch"));
743 }
744 for id in takeover.earlier.iter().rev() {
750 if let Ok(old) = RunState::load_under(id, &takeover.home)
751 && let Some(head) = old.pr_head.clone()
752 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
753 {
754 state.pr_head = Some(head);
755 break;
756 }
757 }
758 let opened = Self::open_review(
759 &repo,
760 branch,
761 state,
762 roles,
763 max_parallel,
764 base_commit,
765 &takeover,
766 )
767 .await;
768 if opened.is_err()
769 && let Some(released) = &released
770 {
771 released.restore(&repo, branch).await;
772 }
773 opened
774 }
775
776 async fn inherit(
781 repo: &Path,
782 branch: &str,
783 base_commit: &str,
784 takeover: &crate::handover::Takeover,
785 ) -> Option<crate::recovery::RecoveryRecord> {
786 use crate::recovery::{Relation, assess};
787 let newest = takeover.earlier.last()?;
788 let mut chosen = None;
789 for id in takeover.earlier.iter().rev() {
790 if let Ok(old) = RunState::load_under(id, &takeover.home)
791 && !old.reviews.is_empty()
792 && (old.winner().is_some_and(|w| w.branch == branch)
793 || old.pr_head.as_deref() == Some(branch))
794 {
795 chosen = Some(old);
796 break;
797 }
798 }
799 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
800 .await
801 .unwrap_or_default();
802 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
803 Some(last) if last.head == tip => Relation::Same,
804 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
805 _ => Relation::Diverged,
806 };
807 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
808 Some(assess(from, chosen.as_ref(), base_commit, relation))
809 }
810
811 async fn open_review(
814 repo: &Path,
815 branch: &str,
816 mut state: RunState,
817 roles: ResolvedRoles,
818 max_parallel: usize,
819 base_commit: String,
820 takeover: &crate::handover::Takeover,
821 ) -> Result<Self> {
822 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
823 let start = review_base(
826 repo,
827 &state.config.merge.remote,
828 &state.base_branch,
829 &base_commit,
830 branch,
831 )
832 .await;
833 let log = git::log_oneline(repo, &start, branch)
834 .await
835 .unwrap_or_default();
836 let log_text = if log.trim().is_empty() {
837 "(no commit messages)"
838 } else {
839 log.trim()
840 };
841 let mut instruction = format!(
842 "Review the work already on branch `{branch}`. There is no task \
843 statement: what the change claims to do is whatever its commits \
844 say.\n\n{log_text}"
845 );
846 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
850 if let Some(task) = takeover
851 .task_instruction
852 .as_deref()
853 .filter(|t| !t.trim().is_empty())
854 && rec.reviewed_head.is_some()
855 {
856 instruction = format!(
857 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
858 task.trim()
859 );
860 rec.criteria_inherited = true;
861 }
862 state.event("recovery", rec.summary());
863 state.recovery = Some(rec);
864 }
865 state.instruction = instruction;
866 state.reviewed_commits = Some(
867 git::subjects(repo, &start, branch)
868 .await
869 .unwrap_or_default(),
870 );
871
872 let worktree = state.worktree_root().join("under-review");
875 if let Some(parent) = worktree.parent() {
876 tokio::fs::create_dir_all(parent).await.ok();
877 }
878 let path = worktree.to_string_lossy().to_string();
879 git::git(repo, &["worktree", "add", &path, branch])
880 .await
881 .with_context(|| {
882 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
883 })?;
884
885 let setup = match ensure_setup_config(&mut state, repo).await {
886 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
887 Err(e) => Err(e),
888 };
889 if let Err(e) = setup {
890 git::worktree_remove(repo, &worktree).await.ok();
891 return Err(e);
892 }
893 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
894 .await
895 .unwrap_or(0);
896 if commits == 0 {
897 git::worktree_remove(repo, &worktree).await.ok();
898 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
899 }
900 let files = git::changed_files(&worktree, &base_commit, "HEAD")
901 .await
902 .map(|f| f.len())
903 .unwrap_or(0);
904 if files == 0
905 && let (Ok(head_tree), Ok(base_tree)) = (
906 git::tree_of(&worktree, "HEAD").await,
907 git::tree_of(&worktree, &base_commit).await,
908 )
909 && head_tree == base_tree
910 {
911 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
912 git::worktree_remove(repo, &worktree).await.ok();
913 bail!(
914 "`{branch}` at {} has a tree identical to base {}; this usually means \
915 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
916 against `{}/{branch}`) rather than an empty change",
917 short(&head),
918 short(&base_commit),
919 state.config.merge.remote
920 );
921 }
922 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
923 .await
924 .unwrap_or_default();
925
926 state.candidates.push(Candidate {
927 index: 0,
928 label: 'A',
929 agent: EXISTING_BRANCH.to_owned(),
932 branch: branch.to_owned(),
933 worktree,
934 summary: String::new(),
935 stat,
936 files,
937 commits,
938 empty: false,
939 failed: None,
940 verified_noop: None,
941 duration_ms: 0,
942 folded: false,
943 });
944 state.tally = Some(Tally {
945 first_choice: BTreeMap::from([('A', 0)]),
946 borda: BTreeMap::new(),
947 winner: 'A',
948 rankings: 0,
949 unanimous_initial: false,
950 deliberated: false,
951 changed_votes: 0,
952 unanimous_final: false,
953 tie_break: None,
954 judges: 0,
958 present: 0,
959 quorum: 0,
960 met_quorum: true,
961 uncontested: Some("review-only run: nothing competed".to_owned()),
962 });
963 state.status = RunStatus::Reviewing;
964 state.event(
965 "start",
966 format!(
967 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
968 state.id
969 ),
970 );
971 state.save()?;
972 Ok(Self {
973 state,
974 roles,
975 sem: Arc::new(Semaphore::new(max_parallel)),
976 pause: Pause::new(),
977 interrupt: Pause::new(),
978 })
979 }
980
981 pub fn resume(id: &str) -> Result<Self> {
983 let state = RunState::load(id)?;
984 if let Some(to) = &state.released_to {
985 bail!(
986 "run {} cannot be resumed: its worktree was released to run {}",
987 state.short(),
988 crate::run::short_of(to)
989 );
990 }
991 let roles = state.config.resolve_roles()?;
992 let max_parallel = state.config.graph.max_parallel.max(1);
993 Ok(Self {
994 state,
995 roles,
996 sem: Arc::new(Semaphore::new(max_parallel)),
997 pause: Pause::new(),
998 interrupt: Pause::new(),
999 })
1000 }
1001
1002 pub async fn execute(&mut self) -> Result<()> {
1009 let result = self.execute_graph().await;
1010 self.mark_driver_exited();
1011 let ended = if result.is_err() {
1012 Some(crate::notices::run_stopped(&self.state.id, &self.state))
1013 } else {
1014 crate::notices::run_ended(&self.state)
1015 };
1016 if let Some(notice) = ended {
1017 crate::notices::raise(notice);
1018 }
1019 result
1020 }
1021
1022 fn mark_driver_exited(&mut self) {
1031 self.state.driver_exited = true;
1032 let pid = std::process::id();
1033 let Ok(_lock) = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) else {
1034 return;
1035 };
1036 let Ok(mut disk) = RunState::load(&self.state.id) else {
1037 return;
1038 };
1039 if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
1040 return;
1041 }
1042 disk.driver_exited = true;
1043 if let Err(e) = disk.save() {
1044 tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
1045 }
1046 }
1047
1048 fn claim_driver(&mut self) -> Result<()> {
1050 let _lock = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1051 let disk = RunState::load(&self.state.id)?;
1052 crate::pr_head::ensure_owner(&disk)?;
1053 self.state.parked = false;
1058 self.state.resource_wait = None;
1059 self.state.resource_block = None;
1060 self.state.clear_active();
1067 if let Some(to) = &disk.released_to {
1085 bail!(
1086 "run {} cannot continue: its worktree was released to run {}",
1087 self.state.short(),
1088 crate::run::short_of(to)
1089 );
1090 }
1091 let pid = std::process::id();
1092 self.state.driver_pid = Some(pid);
1093 self.state.driver_started_at = crate::proc::process_started_at(pid);
1094 self.state.driver_exited = false;
1095 self.state.begin_pass();
1096 self.state.save()?;
1097 Ok(())
1098 }
1099
1100 async fn execute_graph(&mut self) -> Result<()> {
1101 self.claim_driver()?;
1102 if self.state.pr_publication_pending {
1105 self.merge().await?;
1106 self.state.save()?;
1107 self.settle_questions();
1108 return Ok(());
1109 }
1110 if self.state.status == RunStatus::Stalled {
1123 if self.recover_stall().await? {
1124 self.finish_after_tally().await?;
1125 } else {
1126 self.state.save()?;
1128 }
1129 return Ok(());
1130 }
1131 if self.state.status == RunStatus::Landing {
1141 self.run_land().await?;
1142 self.settle_questions();
1147 return Ok(());
1148 }
1149 if self.state.github_text.is_some()
1154 && self.state.merge.is_none()
1155 && self.state.status == RunStatus::Gating
1156 {
1157 self.merge().await?;
1158 self.state.save()?;
1159 return Ok(());
1160 }
1161 self.prep().await?;
1162 if self.park_here()? {
1163 return Ok(());
1164 }
1165 self.advise().await?;
1166 if self.park_here()? {
1167 return Ok(());
1168 }
1169 self.implement().await?;
1170 if self.park_here()? {
1171 return Ok(());
1172 }
1173 if self.state.status == RunStatus::VerifiedNoop {
1177 return Ok(());
1178 }
1179 self.judge().await?;
1180 if self.park_here()? {
1181 return Ok(());
1182 }
1183 self.deliberate().await?;
1184 if self.park_here()? {
1185 return Ok(());
1186 }
1187 self.vote().await?;
1188 if self.park_here()? {
1189 return Ok(());
1190 }
1191 self.tally()?;
1192 if self.state.status == RunStatus::Stalled {
1197 self.state.save()?;
1201 return Ok(());
1202 }
1203 self.finish_after_tally().await?;
1204 Ok(())
1205 }
1206
1207 fn stop_for_resource(&mut self) -> Result<()> {
1212 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1214 {
1215 self.state.status = RunStatus::Blocked;
1216 }
1217 self.state.save()
1218 }
1219
1220 fn resource_halt(&mut self) -> Result<bool> {
1222 if self.state.resource_block.is_none() {
1223 return Ok(false);
1224 }
1225 self.stop_for_resource()?;
1226 Ok(true)
1227 }
1228
1229 fn park_here(&mut self) -> Result<bool> {
1236 let parking = self.pause.parked() || self.interrupt.parked();
1237 if !parking && self.state.resource_block.is_some() {
1238 self.stop_for_resource()?;
1239 return Ok(true);
1240 }
1241 if !self.pause.parked() && !self.interrupt.parked() {
1246 return Ok(false);
1247 }
1248 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1249 let why = match cause.as_ref().map(ParkCause::label) {
1250 Some(reason) => format!(
1251 "parked after `{}` ({reason}) — resume to carry on from here",
1252 self.state.status.as_str()
1253 ),
1254 None => format!(
1255 "parked after `{}` — resume to carry on from here",
1256 self.state.status.as_str()
1257 ),
1258 };
1259 self.state.event("park", why);
1260 self.state
1261 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1262 self.state.parked = true;
1263 self.state.save()?;
1264 Ok(true)
1265 }
1266
1267 pub fn on_pause(&mut self, pause: Pause) {
1269 self.pause = pause;
1270 }
1271
1272 pub fn watch_interrupt(&mut self, pause: Pause) {
1278 self.interrupt = pause;
1279 }
1280
1281 fn settle_questions(&mut self) {
1301 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1302 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1303 }
1304 }
1305
1306 async fn finish_after_tally(&mut self) -> Result<()> {
1309 self.graft().await?;
1312 if self.resource_halt()? {
1313 return Ok(());
1314 }
1315 self.fold_losers().await?;
1316 self.sync_to_base().await?;
1321 if self.state.status == RunStatus::AlreadyInBase {
1322 return Ok(());
1323 }
1324 self.review_loop().await?;
1325 if self.resource_halt()? {
1326 return Ok(());
1327 }
1328 self.sync_to_base().await?;
1329 if self.state.status == RunStatus::AlreadyInBase {
1330 return Ok(());
1331 }
1332 self.gate().await?;
1333 self.merge().await?;
1334 self.state.save()?;
1335 Ok(())
1336 }
1337
1338 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1345 let result = match ensure_setup_config(&mut self.state, repo).await {
1346 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1347 Err(e) => Err(e),
1348 };
1349 if let Err(e) = result {
1350 git::worktree_remove(repo, wt).await.ok();
1351 if in_prep {
1352 self.rollback_prep(repo).await;
1353 }
1354 self.state.status = RunStatus::Blocked;
1355 self.state.event(
1356 "setup",
1357 format!("worktree setup failed in {}: {e:#}", wt.display()),
1358 );
1359 self.state.save()?;
1360 return Err(e);
1361 }
1362 Ok(())
1363 }
1364
1365 async fn rollback_prep(&mut self, repo: &Path) {
1367 let root = self.state.worktree_root();
1368 for c in std::mem::take(&mut self.state.candidates) {
1369 git::worktree_remove(repo, &c.worktree).await.ok();
1370 git::branch_delete(repo, &c.branch).await.ok();
1371 }
1372 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1373 for j in 1..=seats {
1374 git::worktree_remove(repo, &root.join(format!("judge-{j}")))
1375 .await
1376 .ok();
1377 }
1378 for k in 1..=self.state.config.graph.advisors {
1379 git::worktree_remove(repo, &root.join(format!("advisor-{k}")))
1380 .await
1381 .ok();
1382 }
1383 }
1384
1385 async fn prep(&mut self) -> Result<()> {
1386 if !self.state.candidates.is_empty() {
1387 return Ok(());
1388 }
1389 self.state.status = RunStatus::Prep;
1390 let repo = self.state.repo.clone();
1391 let base = self.state.base_commit.clone();
1392 let plan = refs::plan(&repo, &self.state.seeds).await?;
1393 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1394 let root = self.state.worktree_root();
1395 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1396
1397 let hooks_dir = self.state.dir().join("hooks");
1400 if self.state.config.blind.commit_msg_hook {
1401 std::fs::create_dir_all(&hooks_dir)
1402 .with_context(|| format!("create {}", hooks_dir.display()))?;
1403 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1404 let path = hooks_dir.join("commit-msg");
1405 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1406 make_executable(&path)?;
1407 git::acquire_worktree_config(&repo).await?;
1415 self.state.enabled_worktree_config = true;
1416 }
1417
1418 ensure_setup_config(&mut self.state, &repo).await?;
1421
1422 for (index, (spec, label)) in self
1423 .roles
1424 .implementers
1425 .clone()
1426 .into_iter()
1427 .zip(labels)
1428 .enumerate()
1429 {
1430 let branch = self.state.branch_for(label);
1431 let worktree = root.join(format!("cand-{label}"));
1432 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1433 if self.state.config.blind.commit_msg_hook {
1434 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1435 }
1436 git::local_exclude(&worktree, "/.magi/").await?;
1437 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1438 git::worktree_remove(&repo, &worktree).await.ok();
1441 git::branch_delete(&repo, &branch).await.ok();
1442 self.rollback_prep(&repo).await;
1443 self.state.status = RunStatus::Blocked;
1444 self.state
1445 .event("prep", format!("worktree setup failed: {e:#}"));
1446 self.state.save()?;
1447 return Err(e);
1448 }
1449 for pick in &plan.picks {
1450 if let Err(e) = git::cherry_pick(&worktree, pick).await {
1451 self.state.status = RunStatus::Blocked;
1452 self.state
1453 .event("prep", format!("cannot apply referenced commit: {e}"));
1454 self.state.save()?;
1455 return Err(e);
1456 }
1457 }
1458 self.state.candidates.push(Candidate {
1459 index,
1460 label,
1461 agent: spec.id.clone(),
1462 branch,
1463 worktree,
1464 summary: String::new(),
1465 stat: String::new(),
1466 files: 0,
1467 commits: 0,
1468 empty: false,
1469 failed: None,
1470 verified_noop: None,
1471 duration_ms: 0,
1472 folded: false,
1473 });
1474 }
1475
1476 for j in 1..=self.roles.judges.len() {
1477 let wt = root.join(format!("judge-{j}"));
1478 if !wt.exists() {
1479 git::worktree_add_detached(&repo, &wt, &base).await?;
1480 self.setup_seat_worktree(&repo, &wt, true).await?;
1481 }
1482 }
1483
1484 if self.state.config.graph.advise {
1492 for k in 1..=self.state.config.graph.advisors {
1493 let wt = root.join(format!("advisor-{k}"));
1494 if !wt.exists() {
1495 git::worktree_add_detached(&repo, &wt, &base).await?;
1496 self.setup_seat_worktree(&repo, &wt, true).await?;
1497 }
1498 }
1499 }
1500
1501 let authors: Vec<&str> = self
1506 .roles
1507 .implementers
1508 .iter()
1509 .map(|a| a.id.as_str())
1510 .collect();
1511 let overlap: Vec<String> = self
1512 .roles
1513 .judges
1514 .iter()
1515 .enumerate()
1516 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1517 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1518 .collect();
1519 if !overlap.is_empty() {
1520 let note = format!(
1521 "{} also authored a candidate; blind, but the panel is less \
1522 independent than {} distinct agents would be",
1523 overlap.join(", "),
1524 self.roles.judges.len()
1525 );
1526 self.state.event("prep", note);
1527 }
1528
1529 self.state.event(
1530 "prep",
1531 format!(
1532 "{} candidates, {} judges, base {} ({})",
1533 self.state.candidates.len(),
1534 self.roles.judges.len(),
1535 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1536 self.state.base_branch
1537 ),
1538 );
1539 self.state.status = RunStatus::Implementing;
1540 self.state.save()?;
1541 Ok(())
1542 }
1543
1544 async fn advise(&mut self) -> Result<()> {
1577 let implement_untouched = self
1578 .state
1579 .candidates
1580 .iter()
1581 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1582 if !self.state.config.graph.advise || self.state.advise_attempted {
1583 return Ok(());
1584 }
1585 if !implement_untouched {
1586 self.state.event(
1587 "advise",
1588 "skipping the design-deliberation stage: at least one \
1589 candidate already shows implementation progress, so this \
1590 run is past the point the stage exists to run before"
1591 .to_owned(),
1592 );
1593 self.state.advise_attempted = true;
1594 self.state.save()?;
1595 return Ok(());
1596 }
1597 let run_id = self.state.id.clone();
1598 let prompts = self.state.config.prompts.clone();
1599 let instruction = self.state.instruction.clone();
1600 let language = self.state.config.graph.language.clone();
1601 let root = self.state.worktree_root();
1602 let n = self.state.config.graph.advisors;
1603 let where_recorded = self.state.dir().join("run.json");
1604
1605 let seats = match self.state.config.advisors() {
1606 Ok(seats) if !seats.is_empty() => seats,
1607 Ok(_) => {
1608 self.state.event(
1609 "advise",
1610 format!(
1611 "[graph] advisors is 0; skipping the design-deliberation \
1612 stage and continuing without a synthesis brief (see {})",
1613 where_recorded.display()
1614 ),
1615 );
1616 self.state.advise_attempted = true;
1617 self.state.save()?;
1618 return Ok(());
1619 }
1620 Err(e) => {
1621 self.state.event(
1622 "advise",
1623 format!(
1624 "could not resolve advisor seats ({e:#}); continuing \
1625 without a design-deliberation brief (see {})",
1626 where_recorded.display()
1627 ),
1628 );
1629 self.state.advise_attempted = true;
1630 self.state.save()?;
1631 return Ok(());
1632 }
1633 };
1634
1635 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1636 let artifacts = agent::artifacts_dir(&self.state.dir());
1637 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1638
1639 let mut jobs = Vec::new();
1640 for (i, spec) in seats.iter().cloned().enumerate() {
1641 let seat_key = format!("advisor-{}", i + 1);
1642 let seat = self.seat(&seat_key, &spec.id);
1643 jobs.push(SeatJob {
1644 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1645 spec,
1646 seat,
1647 cwd: worktrees[i % worktrees.len()].clone(),
1648 timeout,
1649 allow_write: false,
1650 sessions: false,
1651 artifacts: artifacts.clone(),
1652 stem: seat_key,
1653 handover: None,
1654 });
1655 }
1656
1657 self.state.event(
1658 "advise",
1659 format!(
1660 "{} advisor seat(s) sketching a design in parallel",
1661 jobs.len()
1662 ),
1663 );
1664 let mut quota_losses = Vec::new();
1665 let cache = self.state.config.cache_dir();
1666 let ctx = WaveCtx {
1667 carry_seats: false,
1668 pauses: Some((&self.pause, &self.interrupt)),
1669 run: &run_id,
1670 node: "advise",
1671 prompts: &prompts,
1672 cache: cache.as_deref(),
1673 round: None,
1674 };
1675 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1676 let results = ask_json_wave::<Proposal>(
1677 jobs,
1678 Arc::clone(&self.sem),
1679 self.state.config.graph.retries,
1680 &advisor_roster,
1681 &ctx,
1682 &mut quota_losses,
1683 &mut self.state,
1684 &|p: &Proposal| p.validate(),
1685 )
1686 .await;
1687 if self.state.resource_block.is_some() {
1688 return self.stop_for_resource();
1689 }
1690 self.state.quota.extend(quota_losses);
1691
1692 let mut records = Vec::with_capacity(results.len());
1693 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1694 let agent_id = seat.agent.clone();
1695 self.state.seats.insert(seat.key.clone(), seat);
1696 match res {
1697 Ok((proposal, out)) => {
1698 self.state
1699 .event("advise", format!("advisor-{} proposed a design", i + 1));
1700 records.push(advise::AdvisorRecord::proposed(
1701 i + 1,
1702 agent_id,
1703 proposal,
1704 out.duration_ms,
1705 ));
1706 }
1707 Err(e) => {
1708 self.state.event(
1709 "advise",
1710 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1711 );
1712 records.push(advise::AdvisorRecord::failed(
1713 i + 1,
1714 agent_id,
1715 e.to_string(),
1716 ));
1717 }
1718 }
1719 }
1720
1721 let mut advice = advise::Advice {
1722 records,
1723 synthesis: None,
1724 };
1725 if advice.proposals().is_empty() {
1726 self.state.event(
1727 "advise",
1728 "no advisor produced a usable proposal; continuing without a \
1729 synthesis brief"
1730 .to_owned(),
1731 );
1732 } else {
1733 match self
1734 .synthesize_brief(
1735 &advice,
1736 &instruction,
1737 &language,
1738 &worktrees[0],
1739 &artifacts,
1740 &run_id,
1741 &prompts,
1742 cache.as_deref(),
1743 )
1744 .await
1745 {
1746 Ok(Some(text)) => {
1747 self.state.event(
1748 "advise",
1749 "synthesized a design brief for the implementer".to_owned(),
1750 );
1751 advice.synthesis = Some(text);
1752 }
1753 Ok(None) => {
1754 self.state.event(
1755 "advise",
1756 "the synthesis seat produced nothing usable; continuing \
1757 without a design brief"
1758 .to_owned(),
1759 );
1760 }
1761 Err(e) => {
1762 self.state.event(
1763 "advise",
1764 format!("could not synthesize a design brief: {e:#}"),
1765 );
1766 }
1767 }
1768 }
1769 advise::apply_reflection(&mut advice);
1770
1771 self.state.advice = Some(advice);
1772 self.state.advise_attempted = true;
1773 self.state.save()?;
1774 Ok(())
1775 }
1776
1777 #[allow(clippy::too_many_arguments)]
1789 async fn synthesize_brief(
1790 &mut self,
1791 advice: &advise::Advice,
1792 instruction: &str,
1793 language: &str,
1794 cwd: &Path,
1795 artifacts: &Path,
1796 run_id: &str,
1797 prompts: &Prompts,
1798 cache: Option<&Path>,
1799 ) -> Result<Option<String>> {
1800 let chain = agent::pick_chain(
1801 &self.state.config.agents,
1802 self.state.config.roles.synthesizer.as_ref(),
1803 &agent::installed,
1804 "synthesizer",
1805 )?;
1806 let proposals = advice.proposals();
1807 let mut prompt = prompt::with_overlay(
1808 prompt::synthesize_brief(instruction, &proposals, language),
1809 prompts.overlay("advise"),
1810 );
1811 if cache.is_some() {
1812 prompt.push('\n');
1817 prompt.push_str(&prompt::build_cache_note("advise", false));
1818 }
1819 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1820 let mut last = None;
1825 for (n, spec) in chain.iter().enumerate() {
1826 if n > 0 {
1827 self.state
1828 .event("advise", format!("synthesis falling back to {}", spec.id));
1829 }
1830 let mut seat = self.seat("advise-synthesis", &spec.id);
1831 let outcome = agent::invoke(
1832 spec,
1833 &mut seat,
1834 &Invocation {
1835 cwd,
1836 prompt: &prompt,
1837 timeout,
1838 allow_write: false,
1839 unsandboxed: false,
1840 sessions: false,
1841 artifacts,
1842 stem: &if n == 0 {
1843 "advise-synthesis".to_owned()
1844 } else {
1845 format!("advise-synthesis-{}", spec.id)
1846 },
1847 run: run_id,
1848 node: "advise",
1849 cache_dir: None,
1850 attachments: &[],
1851 writable: &[],
1852 },
1853 )
1854 .await;
1855 if outcome.is_ok() {
1856 self.state.seats.insert(seat.key.clone(), seat);
1857 }
1858 let advance = agent::chain_advances(&outcome);
1859 last = Some(outcome);
1860 if !advance {
1861 break;
1862 }
1863 }
1864 let out = last.expect("a chain holds at least one agent")?;
1867 if !out.usable() {
1868 return Ok(None);
1869 }
1870 let text =
1871 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1872 Ok((!text.trim().is_empty()).then_some(text))
1873 }
1874
1875 async fn implement(&mut self) -> Result<()> {
1878 let run_id = self.state.id.clone();
1883 let prompts = self.state.config.prompts.clone();
1884 let todo: Vec<usize> = self
1885 .state
1886 .candidates
1887 .iter()
1888 .enumerate()
1889 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1890 .map(|(i, _)| i)
1891 .collect();
1892 if todo.is_empty() {
1893 return self.after_implement();
1894 }
1895 self.state.status = RunStatus::Implementing;
1896
1897 let language = self.state.config.graph.language.clone();
1898 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1899 let sessions = self.state.config.graph.sessions;
1900 let artifacts = agent::artifacts_dir(&self.state.dir());
1901 let brief = self
1905 .state
1906 .advice
1907 .as_ref()
1908 .and_then(|a| a.synthesis.as_deref())
1909 .map(str::to_owned);
1910 let attachments = self.state.attachments.clone();
1911
1912 let mut jobs = Vec::new();
1913 for &i in &todo {
1914 let (index, label, worktree) = {
1915 let c = &self.state.candidates[i];
1916 (c.index, c.label, c.worktree.clone())
1917 };
1918 let spec = self.roles.implementers[index].clone();
1919 let seat_key = format!("impl-{label}");
1920 let seat = self.seat(&seat_key, &spec.id);
1921 let instruction = seeded_instruction(&self.state);
1922 jobs.push(SeatJob {
1923 spec,
1924 seat,
1925 prompt: prompt::implement(
1926 &instruction,
1927 &worktree.to_string_lossy(),
1928 &language,
1929 brief.as_deref(),
1930 &attachments,
1931 ),
1932 cwd: worktree,
1933 timeout,
1934 allow_write: true,
1935 sessions,
1936 artifacts: artifacts.clone(),
1937 stem: format!("impl-{label}"),
1938 handover: None,
1939 });
1940 }
1941
1942 self.state.event(
1943 "implement",
1944 format!("{} candidates in parallel", jobs.len()),
1945 );
1946 let mut sent = jobs.clone();
1952 let cache = self.state.config.cache_dir();
1953 let ctx = WaveCtx {
1954 carry_seats: false,
1955 pauses: Some((&self.pause, &self.interrupt)),
1956 run: &run_id,
1957 node: "implement",
1958 prompts: &prompts,
1959 cache: cache.as_deref(),
1960 round: None,
1961 };
1962 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1963 if wave_blocked(&results) {
1964 return self.stop_for_resource();
1965 }
1966 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1967 .await;
1968 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1969 .await;
1970 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1971 .await;
1972 if self.state.resource_block.is_some() {
1973 return self.stop_for_resource();
1974 }
1975
1976 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1977 let seat_key = seat.key.clone();
1978 let agent = seat.agent.clone();
1987 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1988 self.state.seats.insert(seat.key.clone(), seat);
1989 let label = self.state.candidates[i].label;
1990 let worktree = self.state.candidates[i].worktree.clone();
1991 let base = self.state.base_commit.clone();
1992
1993 let (summary, duration, failed, verified_claim) = match out {
1994 AgentOutcome::Ok(o) => {
1995 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1996 let failed = (!o.usable()).then(|| {
1997 if o.timed_out {
1998 "agent timed out".to_owned()
1999 } else {
2000 format!("agent exited with {:?}", o.exit_code)
2001 }
2002 });
2003 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2004 (text, o.duration_ms, failed, verified_claim)
2005 }
2006 AgentOutcome::Dropped(o) => {
2012 let why = o
2013 .dropped
2014 .as_ref()
2015 .map(|d| d.why.as_str())
2016 .unwrap_or("the CLI ended the stream without delivering its answer");
2017 (
2018 String::new(),
2019 o.duration_ms,
2020 Some(format!("the CLI dropped the stream ({why})")),
2021 None,
2022 )
2023 }
2024 AgentOutcome::Quota(o) => {
2025 self.state.quota.push(QuotaLoss {
2026 seat: seat_key,
2027 node: "implement".to_owned(),
2028 at: Timestamp::now(),
2029 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2030 });
2031 (
2032 String::new(),
2033 o.duration_ms,
2034 Some("rate limited (quota); produced no change".to_owned()),
2035 None,
2036 )
2037 }
2038 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2039 (String::new(), 0, Some(e), None)
2040 }
2041 };
2042
2043 let rescued = match git::rescue_commit(
2046 &worktree,
2047 &format!("magi: candidate {label} (uncommitted work)"),
2048 )
2049 .await
2050 {
2051 Ok(r) => {
2052 self.state.note_withheld("implement", &r.withheld);
2053 r.committed
2054 }
2055 Err(_) => false,
2056 };
2057 let commits = git::commits_ahead(&worktree, &base, "HEAD")
2058 .await
2059 .unwrap_or(0);
2060 let patch = git::diff(&worktree, &base, "HEAD")
2061 .await
2062 .unwrap_or_default();
2063 let stat = git::diff_stat(&worktree, &base, "HEAD")
2064 .await
2065 .unwrap_or_default();
2066 let files = git::changed_files(&worktree, &base, "HEAD")
2067 .await
2068 .map(|f| f.len())
2069 .unwrap_or(0);
2070 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2071
2072 let c = &mut self.state.candidates[i];
2073 if !exhausted_the_fallback_chain {
2074 c.agent = agent;
2075 }
2076 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2077 c.stat = stat;
2078 c.files = files;
2079 c.commits = commits;
2080 c.duration_ms = duration;
2081 c.empty = commits == 0 || patch.trim().is_empty();
2082 c.failed = match failed {
2085 Some(_) if c.empty => failed,
2086 _ => None,
2087 };
2088 c.verified_noop = if c.empty { verified_claim } else { None };
2093 let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
2094 (Some(e), _, _, _) => format!("candidate {label}: {e}"),
2095 (None, true, Some(_), _) => {
2096 format!("candidate {label}: no change produced (agent-verified no-op)")
2097 }
2098 (None, true, None, _) => format!("candidate {label}: no change produced"),
2099 (None, false, _, true) => {
2100 format!(
2101 "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
2102 )
2103 }
2104 (None, false, _, false) => {
2105 format!("candidate {label}: {files} files, {commits} commits")
2106 }
2107 };
2108 self.state.event("implement", note);
2109 self.state.save()?;
2110 }
2111
2112 self.after_implement()
2113 }
2114
2115 async fn resume_undelivered(
2143 &mut self,
2144 results: &mut [(usize, SeatState, AgentOutcome)],
2145 sent: &[SeatJob],
2146 prompts: &Prompts,
2147 run_id: &str,
2148 ) {
2149 for (wi, seat, out) in results.iter_mut() {
2150 let Some(dropped) = (match &*out {
2151 AgentOutcome::Dropped(o) => o.dropped.clone(),
2152 _ => None,
2153 }) else {
2154 continue;
2155 };
2156 let Some(job) = sent.get(*wi) else { continue };
2157 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2159 self.state.event(
2160 "implement",
2161 format!(
2162 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2163 work is in the tree",
2164 seat.key, dropped.output_tokens, dropped.why
2165 ),
2166 );
2167 continue;
2168 }
2169 if !has_context(&job.spec, seat, job.sessions) {
2177 self.state.event(
2178 "implement",
2179 format!(
2180 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2181 is no session left to resume",
2182 seat.key, dropped.output_tokens, dropped.why
2183 ),
2184 );
2185 continue;
2186 }
2187 self.state.event(
2188 "implement",
2189 format!(
2190 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2191 conversation",
2192 seat.key, dropped.output_tokens, dropped.why
2193 ),
2194 );
2195 let mut retry = job.clone();
2196 retry.seat = seat.clone();
2197 retry.prompt = prompt::resume_after_drop(&dropped.why);
2198 retry.timeout = retry_budget(job.timeout, true);
2199 retry.stem = format!("{}-resume", job.stem);
2200 let cache = self.state.config.cache_dir();
2201 let ctx = WaveCtx {
2202 carry_seats: false,
2203 pauses: Some((&self.pause, &self.interrupt)),
2204 run: run_id,
2205 node: "implement",
2206 prompts,
2207 cache: cache.as_deref(),
2208 round: None,
2209 };
2210 let (resumed_seat, resumed) =
2211 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2212 *seat = resumed_seat;
2213 *out = resumed;
2214 }
2215 }
2216
2217 async fn resume_seat_handovers(
2282 &mut self,
2283 results: &mut [(usize, SeatState, AgentOutcome)],
2284 sent: &mut [SeatJob],
2285 prompts: &Prompts,
2286 run_id: &str,
2287 ) {
2288 let instruction = seeded_instruction(&self.state);
2289 let language = self.state.config.graph.language.clone();
2290 let brief = self
2291 .state
2292 .advice
2293 .as_ref()
2294 .and_then(|a| a.synthesis.as_deref())
2295 .map(str::to_owned);
2296 let attachments = self.state.attachments.clone();
2297 for (wi, seat, out) in results.iter_mut() {
2298 let others: BTreeSet<String> = sent
2301 .iter()
2302 .enumerate()
2303 .filter(|(j, _)| j != wi)
2304 .map(|(_, j)| j.spec.id.clone())
2305 .collect();
2306 let Some(job) = sent.get_mut(*wi) else {
2307 continue;
2308 };
2309 let start = self
2314 .roles
2315 .implementer_roster
2316 .iter()
2317 .position(|s| s.id == job.spec.id)
2318 .unwrap_or(0);
2319 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2320 let mut fallback_attempt = 0usize;
2321 let mut prev: Option<FailClass> = None;
2322 while let Some(cur) = FailClass::of(&*out) {
2323 if !should_hand_over(prev.as_ref(), &cur) {
2324 break;
2325 }
2326 let Some(next) =
2327 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2328 .cloned()
2329 else {
2330 break;
2331 };
2332 tried.insert(next.id.clone());
2333 fallback_attempt += 1;
2334
2335 if let Ok(r) = git::rescue_commit(
2336 &job.cwd,
2337 &format!(
2338 "magi: candidate {} (uncommitted work before {} fallback)",
2339 seat.key,
2340 if cur == FailClass::Quota {
2341 "quota"
2342 } else {
2343 "handover"
2344 }
2345 ),
2346 )
2347 .await
2348 {
2349 self.state.note_withheld("implement", &r.withheld);
2350 }
2351
2352 record_handover(
2353 &mut self.state,
2354 "implement",
2355 &seat.key,
2356 &seat.agent,
2357 &next.id,
2358 &cur,
2359 &fail_reason(&*out),
2360 );
2361 prev = Some(cur.clone());
2362
2363 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2364 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2365 job.spec = next.clone();
2373 let mut retry = job.clone();
2374 retry.seat = new_seat;
2375 retry.prompt = prompt::implement(
2376 &instruction,
2377 &job.cwd.to_string_lossy(),
2378 &language,
2379 brief.as_deref(),
2380 &attachments,
2381 );
2382 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2383 let cache = self.state.config.cache_dir();
2384 let ctx = WaveCtx {
2385 carry_seats: false,
2386 pauses: Some((&self.pause, &self.interrupt)),
2387 run: run_id,
2388 node: "implement",
2389 prompts,
2390 cache: cache.as_deref(),
2391 round: None,
2392 };
2393 let (fallback_seat, fallback_out) = run_one(
2394 retry,
2395 Arc::clone(&self.sem),
2396 &ctx,
2397 &mut self.state,
2398 fallback_attempt,
2399 )
2400 .await;
2401 *seat = fallback_seat;
2402 *out = fallback_out;
2403 }
2404 }
2405 }
2406
2407 async fn resume_unconfirmed_commands(
2431 &mut self,
2432 results: &mut [(usize, SeatState, AgentOutcome)],
2433 sent: &[SeatJob],
2434 prompts: &Prompts,
2435 run_id: &str,
2436 ) {
2437 for (wi, seat, out) in results.iter_mut() {
2438 let AgentOutcome::Ok(o) = &*out else {
2439 continue;
2440 };
2441 if !has_unconfirmed_command(&o.commands) {
2442 continue;
2443 }
2444 let Some(job) = sent.get(*wi) else { continue };
2445 if !has_context(&job.spec, seat, job.sessions) {
2446 self.state.event(
2447 "implement",
2448 format!(
2449 "{}: the reply named a command whose own CLI never confirmed the exit \
2450 status of, but there is no session left to resume",
2451 seat.key
2452 ),
2453 );
2454 continue;
2455 }
2456 self.state.event(
2457 "implement",
2458 format!(
2459 "{}: the reply named a command whose own CLI never confirmed the exit \
2460 status of; resuming the conversation",
2461 seat.key
2462 ),
2463 );
2464 let mut retry = job.clone();
2465 retry.seat = seat.clone();
2466 retry.prompt = prompt::resume_incomplete(
2467 "a command in your last reply had no confirmed exit status",
2468 );
2469 retry.timeout = retry_budget(job.timeout, true);
2470 retry.stem = format!("{}-confirm", job.stem);
2471 let cache = self.state.config.cache_dir();
2472 let ctx = WaveCtx {
2473 carry_seats: false,
2474 pauses: Some((&self.pause, &self.interrupt)),
2475 run: run_id,
2476 node: "implement",
2477 prompts,
2478 cache: cache.as_deref(),
2479 round: None,
2480 };
2481 let (resumed_seat, resumed) =
2482 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2483 *seat = resumed_seat;
2484 *out = resumed;
2485 }
2486 }
2487
2488 async fn continue_fix_report(
2509 &mut self,
2510 mut seat: SeatState,
2511 parse_err: String,
2512 job: &SeatJob,
2513 prompts: &Prompts,
2514 run_id: &str,
2515 round: usize,
2516 ) -> (
2517 SeatState,
2518 Option<FixReport>,
2519 Option<String>,
2520 ContinuationRecord,
2521 ) {
2522 let mut last_err = parse_err;
2523 let mut cumulative_wait_ms = 0u64;
2524 let mut attempts = 0usize;
2525 loop {
2526 if !has_context(&job.spec, &seat, job.sessions) {
2527 self.state.event(
2528 "fix",
2529 format!(
2530 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2531 session left to resume into"
2532 ),
2533 );
2534 let outcome = if attempts == 0 {
2535 ContinuationOutcome::NoSession
2536 } else {
2537 ContinuationOutcome::Exhausted
2538 };
2539 return (
2540 seat,
2541 None,
2542 Some(format!("unparsable fix report: {last_err}")),
2543 ContinuationRecord {
2544 attempts,
2545 cumulative_wait_ms,
2546 outcome,
2547 },
2548 );
2549 }
2550 if attempts >= MAX_FIX_CONTINUATIONS {
2551 self.state.event(
2552 "fix",
2553 format!(
2554 "round {round}: fixer's reply still had no adoption report after \
2555 {attempts} continuation(s) ({last_err}); giving up"
2556 ),
2557 );
2558 return (
2559 seat,
2560 None,
2561 Some(format!(
2562 "unparsable fix report after {attempts} continuation(s): {last_err}"
2563 )),
2564 ContinuationRecord {
2565 attempts,
2566 cumulative_wait_ms,
2567 outcome: ContinuationOutcome::Exhausted,
2568 },
2569 );
2570 }
2571 attempts += 1;
2572 self.state.event(
2573 "fix",
2574 format!(
2575 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2576 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2577 ),
2578 );
2579 let mut retry = job.clone();
2580 retry.seat = seat.clone();
2581 retry.prompt = prompt::resume_incomplete(&last_err);
2582 retry.timeout = retry_budget(job.timeout, true);
2583 retry.stem = format!("{}-continue{attempts}", job.stem);
2584 let cache = self.state.config.cache_dir();
2585 let ctx = WaveCtx {
2586 carry_seats: false,
2587 pauses: Some((&self.pause, &self.interrupt)),
2588 run: run_id,
2589 node: "fix",
2590 prompts,
2591 cache: cache.as_deref(),
2592 round: Some(round),
2593 };
2594 let (resumed_seat, resumed_out) = run_one(
2595 retry,
2596 Arc::clone(&self.sem),
2597 &ctx,
2598 &mut self.state,
2599 attempts,
2600 )
2601 .await;
2602 seat = resumed_seat;
2603 match resumed_out {
2604 AgentOutcome::Ok(o) => {
2605 cumulative_wait_ms += o.duration_ms;
2606 match verdict::extract_json::<FixReport>(&o.text) {
2607 Ok(report) if !has_unconfirmed_command(&o.commands) => {
2608 self.state.event(
2609 "fix",
2610 format!(
2611 "round {round}: fixer's adoption report recovered after \
2612 {attempts} continuation(s)"
2613 ),
2614 );
2615 return (
2616 seat,
2617 Some(report),
2618 None,
2619 ContinuationRecord {
2620 attempts,
2621 cumulative_wait_ms,
2622 outcome: ContinuationOutcome::Resumed,
2623 },
2624 );
2625 }
2626 Ok(_) => {
2634 last_err = "the reply parsed, but it reported a command whose own CLI \
2635 never confirmed an exit status"
2636 .to_owned();
2637 }
2638 Err(e) => last_err = e.to_string(),
2639 }
2640 }
2641 AgentOutcome::Quota(o) => {
2642 cumulative_wait_ms += o.duration_ms;
2643 self.state.quota.push(QuotaLoss {
2644 seat: seat.key.clone(),
2645 node: "fix".to_owned(),
2646 at: Timestamp::now(),
2647 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2648 });
2649 self.state.event(
2650 "fix",
2651 format!(
2652 "round {round}: continuation rate limited (quota); not retrying now"
2653 ),
2654 );
2655 return (
2656 seat,
2657 None,
2658 Some("rate limited (quota) while recovering the fix report".to_owned()),
2659 ContinuationRecord {
2660 attempts,
2661 cumulative_wait_ms,
2662 outcome: ContinuationOutcome::QuotaLost,
2663 },
2664 );
2665 }
2666 AgentOutcome::Dropped(o) => {
2667 cumulative_wait_ms += o.duration_ms;
2668 let why = o
2669 .dropped
2670 .as_ref()
2671 .map(|d| d.why.as_str())
2672 .unwrap_or("the CLI ended the stream without delivering its answer");
2673 last_err = format!("the CLI dropped the stream ({why})");
2674 }
2675 AgentOutcome::Failed(e) => last_err = e,
2676 AgentOutcome::ResourceBlocked(e) => {
2679 return (
2680 seat,
2681 None,
2682 Some(format!("the continuation was not started: {e}")),
2683 ContinuationRecord {
2684 attempts: attempts.saturating_sub(1),
2685 cumulative_wait_ms,
2686 outcome: ContinuationOutcome::NoSession,
2687 },
2688 );
2689 }
2690 }
2691 }
2692 }
2693
2694 fn after_implement(&mut self) -> Result<()> {
2695 if self.state.leaks.is_empty() {
2697 let cfg = self.state.config.blind.clone();
2698 let mut leaks = Vec::new();
2699 for c in &self.state.candidates {
2700 let Some(patch) =
2701 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2702 else {
2703 continue;
2704 };
2705 leaks.extend(blind::scan(
2706 &format!("candidate {} patch", c.label),
2707 &patch,
2708 &cfg.vendor_tokens,
2709 ));
2710 }
2711 if !leaks.is_empty() {
2712 let summary = leaks
2713 .iter()
2714 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2715 .collect::<Vec<_>>()
2716 .join(", ");
2717 match cfg.on_leak {
2718 LeakPolicy::Fail => {
2719 self.state.status = RunStatus::Failed;
2720 self.state
2721 .event("blind", format!("vendor text in a patch: {summary}"));
2722 self.state.leaks = leaks;
2723 self.state.save()?;
2724 self.settle_questions();
2725 bail!(
2726 "blind.on_leak = \"fail\" and vendor text reached a \
2727 judged patch: {summary}"
2728 );
2729 }
2730 LeakPolicy::Redact => self.state.event(
2731 "blind",
2732 format!("redacting vendor text for judging: {summary}"),
2733 ),
2734 LeakPolicy::Warn => self.state.event(
2735 "blind",
2736 format!("vendor text present in a judged patch (shown as-is): {summary}"),
2737 ),
2738 }
2739 self.state.leaks = leaks;
2740 }
2741 }
2742
2743 if self.state.viable().is_empty() {
2744 if self.state.all_candidates_verified_noop() {
2745 self.state.status = RunStatus::VerifiedNoop;
2756 self.state.save()?;
2757 self.settle_questions();
2758 return Ok(());
2759 }
2760 self.state.status = RunStatus::Failed;
2761 self.state.save()?;
2762 self.settle_questions();
2763 bail!("no candidate produced a change; nothing to judge");
2764 }
2765 self.state.status = RunStatus::Judging;
2766 self.state.save()?;
2767 Ok(())
2768 }
2769
2770 async fn judge(&mut self) -> Result<()> {
2773 let run_id = self.state.id.clone();
2778 let prompts = self.state.config.prompts.clone();
2779 if !self.state.judgements.is_empty() || self.state.judge_skipped {
2780 return Ok(());
2781 }
2782 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2783 if viable.len() == 1 {
2784 self.state.judge_skipped = true;
2791 self.state.event(
2792 "judge",
2793 format!(
2794 "only candidate {} produced a change; judging skipped",
2795 viable[0].label
2796 ),
2797 );
2798 self.state.save()?;
2799 return Ok(());
2800 }
2801 self.state.status = RunStatus::Judging;
2802
2803 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2804 let language = self.state.config.graph.language.clone();
2805 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2806 let sessions = self.state.config.graph.sessions;
2807 let artifacts = agent::artifacts_dir(&self.state.dir());
2808 let root = self.state.worktree_root();
2809 let base_short = short(&self.state.base_commit);
2810
2811 let mut jobs = Vec::new();
2812 let mut orders = Vec::new();
2813 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2814 let order = blind::presentation_order(viable.len(), j, self.state.seed);
2815 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2816 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2817 let seat_key = format!("judge-{}", j + 1);
2818 let seat = self.seat(&seat_key, &spec.id);
2819 jobs.push(SeatJob {
2820 prompt: prompt::judge(
2821 &self.state.instruction,
2822 &views,
2823 self.roles.judges.len(),
2824 &base_short,
2825 &language,
2826 ),
2827 spec,
2828 seat,
2829 cwd: root.join(format!("judge-{}", j + 1)),
2830 timeout,
2831 allow_write: false,
2832 sessions,
2833 artifacts: artifacts.clone(),
2834 stem: format!("judge-{}", j + 1),
2835 handover: None,
2836 });
2837 }
2838
2839 self.state.event(
2840 "judge",
2841 format!(
2842 "{} judges ranking {} candidates blind",
2843 jobs.len(),
2844 viable.len()
2845 ),
2846 );
2847 let labels_for_check = labels.clone();
2848 let mut quota_losses = Vec::new();
2849 let cache = self.state.config.cache_dir();
2850 let ctx = WaveCtx {
2851 carry_seats: false,
2852 pauses: Some((&self.pause, &self.interrupt)),
2853 run: &run_id,
2854 node: "judge",
2855 prompts: &prompts,
2856 cache: cache.as_deref(),
2857 round: None,
2858 };
2859 let results = ask_json_wave::<Ranking>(
2860 jobs,
2861 Arc::clone(&self.sem),
2862 self.state.config.graph.retries,
2863 &self.roles.judge_roster,
2864 &ctx,
2865 &mut quota_losses,
2866 &mut self.state,
2867 &move |r: &Ranking| r.validate(&labels_for_check),
2868 )
2869 .await;
2870 if self.state.resource_block.is_some() {
2871 return self.stop_for_resource();
2872 }
2873 self.state.quota.extend(quota_losses);
2874
2875 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2876 let agent_id = seat.agent.clone();
2877 self.state.seats.insert(seat.key.clone(), seat);
2878 let mut record = Judgement {
2879 judge: j + 1,
2880 seat: format!("judge-{}", j + 1),
2881 agent: agent_id,
2882 ranking: Vec::new(),
2883 reasons: BTreeMap::new(),
2884 confidence: None,
2885 strengths: BTreeMap::new(),
2886 order: orders[j].clone(),
2887 failed: None,
2888 duration_ms: 0,
2889 };
2890 match res {
2891 Ok((ranking, out)) => {
2892 record.ranking = ranking.normalized();
2893 record.reasons = ranking.reasons;
2894 record.strengths = ranking.strengths;
2895 record.confidence = ranking.confidence;
2896 record.duration_ms = out.duration_ms;
2897 self.state.event(
2898 "judge",
2899 format!(
2900 "judge {} ranked {}",
2901 j + 1,
2902 record.ranking.iter().collect::<String>()
2903 ),
2904 );
2905 }
2906 Err(e) => {
2907 record.failed = Some(e.to_string());
2908 self.state
2909 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2910 }
2911 }
2912 self.state.judgements.push(record);
2913 self.state.save()?;
2914 }
2915 Ok(())
2916 }
2917
2918 async fn deliberate(&mut self) -> Result<()> {
2921 let run_id = self.state.id.clone();
2926 let prompts = self.state.config.prompts.clone();
2927 if !self.state.deliberation.is_empty() {
2928 return Ok(());
2929 }
2930 let tops: Vec<char> = self
2931 .state
2932 .judgements
2933 .iter()
2934 .filter_map(|j| j.ranking.first().copied())
2935 .collect();
2936 let rounds = self.state.config.graph.deliberate_rounds;
2937 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2938 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2939 self.state.event(
2940 "deliberate",
2941 format!("judges agreed on {} outright; no deliberation", tops[0]),
2942 );
2943 }
2944 self.state.status = RunStatus::Voting;
2945 self.state.save()?;
2946 return Ok(());
2947 }
2948
2949 self.state.status = RunStatus::Deliberating;
2950 self.state.event(
2951 "deliberate",
2952 format!(
2953 "split: first choices were {} — opening {rounds} round(s)",
2954 tops.iter().collect::<String>()
2955 ),
2956 );
2957
2958 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2959 let language = self.state.config.graph.language.clone();
2960 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2961 let sessions = self.state.config.graph.sessions;
2962 let artifacts = agent::artifacts_dir(&self.state.dir());
2963 let root = self.state.worktree_root();
2964 let base_short = short(&self.state.base_commit);
2965
2966 for round in 1..=rounds {
2970 let mut turns: Vec<DeliberationTurn> = Vec::new();
2971 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2972 if self.state.judgements[j].failed.is_some() {
2973 continue;
2974 }
2975 let seat_key = format!("judge-{}", j + 1);
2976 let spec = self.occupant(&seat_key, spec);
2977 let mut seat = self.seat(&seat_key, &spec.id);
2978 let transcript = self.transcript(&turns, j);
2979 let build = |context: Option<&str>| {
2980 prompt::deliberate(
2981 &self.state.instruction,
2982 context,
2983 &transcript,
2984 round,
2985 rounds,
2986 &language,
2987 )
2988 };
2989 let block = self.candidate_block(&viable, &base_short);
2990 let full = build(Some(&block));
2991 let text = if has_context(&spec, &seat, sessions) {
2992 build(None)
2993 } else {
2994 full.clone()
2995 };
2996 let job = SeatJob {
2997 spec,
2998 seat: seat.clone(),
2999 prompt: text,
3000 cwd: root.join(format!("judge-{}", j + 1)),
3001 timeout,
3002 allow_write: false,
3003 sessions,
3004 artifacts: artifacts.clone(),
3005 stem: format!("delib-{round}-judge-{}", j + 1),
3006 handover: Some(full),
3007 };
3008 let cache = self.state.config.cache_dir();
3009 let ctx = WaveCtx {
3010 carry_seats: false,
3011 pauses: Some((&self.pause, &self.interrupt)),
3012 run: &run_id,
3013 node: "deliberate",
3014 prompts: &prompts,
3015 cache: cache.as_deref(),
3016 round: None,
3017 };
3018 let mut losses = Vec::new();
3022 let mut results = ask_wave_with::<String>(
3023 vec![job],
3024 Arc::clone(&self.sem),
3025 0,
3026 &self.roles.judge_roster,
3027 &ctx,
3028 &mut losses,
3029 &mut self.state,
3030 &|text: &str| {
3031 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3032 },
3033 )
3034 .await;
3035 self.state.quota.extend(losses);
3036 let (updated, res, _) = results.pop().expect("one job in, one result out");
3037 seat = updated;
3038 let agent_id = seat.agent.clone();
3039 self.state.seats.insert(seat.key.clone(), seat);
3040 let body = match res {
3041 Ok((body, _)) => body,
3042 Err(e) => {
3045 self.state
3046 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3047 continue;
3048 }
3049 };
3050 let tentative = verdict::extract_json::<Position>(&body)
3051 .ok()
3052 .and_then(|p| p.tentative)
3053 .and_then(|s| s.trim().chars().next())
3054 .map(|c| c.to_ascii_uppercase());
3055 self.state.event(
3056 "deliberate",
3057 format!(
3058 "round {round}: judge {} now favours {}",
3059 j + 1,
3060 tentative.map_or("—".to_owned(), |c| c.to_string())
3061 ),
3062 );
3063 turns.push(DeliberationTurn {
3064 judge: j + 1,
3065 agent: agent_id,
3066 body: blind::sanitize_prose(&body, &self.state.config.blind),
3067 tentative,
3068 });
3069 }
3070 self.state
3071 .deliberation
3072 .push(DeliberationRound { round, turns });
3073 self.state.save()?;
3074 }
3075
3076 self.state.status = RunStatus::Voting;
3077 self.state.save()?;
3078 Ok(())
3079 }
3080
3081 async fn vote(&mut self) -> Result<()> {
3084 let run_id = self.state.id.clone();
3089 let prompts = self.state.config.prompts.clone();
3090 if !self.state.votes.is_empty() {
3091 return Ok(());
3092 }
3093 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3094 if viable.len() == 1 {
3095 return Ok(());
3096 }
3097 self.state.status = RunStatus::Voting;
3098
3099 let language = self.state.config.graph.language.clone();
3100 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3101 let sessions = self.state.config.graph.sessions;
3102 let artifacts = agent::artifacts_dir(&self.state.dir());
3103 let root = self.state.worktree_root();
3104 let base_short = short(&self.state.base_commit);
3105 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3106
3107 let mut jobs = Vec::new();
3108 let mut seats_at = Vec::new();
3109 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3110 if self
3111 .state
3112 .judgements
3113 .get(j)
3114 .is_some_and(|r| r.failed.is_some())
3115 {
3116 continue;
3117 }
3118 let seat_key = format!("judge-{}", j + 1);
3119 let spec = self.occupant(&seat_key, spec);
3120 let seat = self.seat(&seat_key, &spec.id);
3121 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3122 let text = if has_context(&spec, &seat, sessions) {
3123 prompt::final_vote(&viable, &language)
3124 } else {
3125 full.clone()
3126 };
3127 jobs.push(SeatJob {
3128 spec,
3129 seat,
3130 prompt: text,
3131 cwd: root.join(format!("judge-{}", j + 1)),
3132 timeout,
3133 allow_write: false,
3134 sessions,
3135 artifacts: artifacts.clone(),
3136 stem: format!("vote-judge-{}", j + 1),
3137 handover: Some(full),
3138 });
3139 seats_at.push(j);
3140 }
3141
3142 self.state.event(
3143 "vote",
3144 format!(
3145 "collecting {} final votes one by one, privately",
3146 jobs.len()
3147 ),
3148 );
3149 let allowed = viable.clone();
3150 let mut quota_losses = Vec::new();
3151 let cache = self.state.config.cache_dir();
3152 let ctx = WaveCtx {
3153 carry_seats: false,
3154 pauses: Some((&self.pause, &self.interrupt)),
3155 run: &run_id,
3156 node: "vote",
3157 prompts: &prompts,
3158 cache: cache.as_deref(),
3159 round: None,
3160 };
3161 let results = ask_json_wave::<FinalVote>(
3162 jobs,
3163 Arc::clone(&self.sem),
3164 self.state.config.graph.retries,
3165 &self.roles.judge_roster,
3166 &ctx,
3167 &mut quota_losses,
3168 &mut self.state,
3169 &move |v: &FinalVote| match v.label() {
3170 Some(c) if allowed.contains(&c) => Ok(()),
3171 other => bail!("vote {other:?} is not one of {allowed:?}"),
3172 },
3173 )
3174 .await;
3175 if self.state.resource_block.is_some() {
3176 return self.stop_for_resource();
3177 }
3178 self.state.quota.extend(quota_losses);
3179
3180 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3181 let agent_id = seat.agent.clone();
3182 self.state.seats.insert(seat.key.clone(), seat);
3183 let initial = self
3184 .state
3185 .judgements
3186 .get(j)
3187 .and_then(|r| r.ranking.first().copied());
3188 let mut record = VoteRecord {
3189 judge: j + 1,
3190 agent: agent_id,
3191 vote: None,
3192 reason: String::new(),
3193 changed: false,
3194 };
3195 match res {
3196 Ok((v, _)) => {
3197 record.vote = v.label();
3198 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3199 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3200 self.state.event(
3201 "vote",
3202 format!(
3203 "judge {} voted {}{}",
3204 j + 1,
3205 record.vote.unwrap_or('?'),
3206 if record.changed { " (changed)" } else { "" }
3207 ),
3208 );
3209 }
3210 Err(e) => {
3211 self.state
3212 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3213 }
3214 }
3215 self.state.votes.push(record);
3216 self.state.save()?;
3217 }
3218 Ok(())
3219 }
3220
3221 fn tally(&mut self) -> Result<()> {
3224 if self.state.tally.is_some() {
3225 return Ok(());
3226 }
3227 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3228 let tops: Vec<char> = self
3229 .state
3230 .judgements
3231 .iter()
3232 .filter_map(|j| j.ranking.first().copied())
3233 .collect();
3234 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3235
3236 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3239 let mut cast: Vec<char> = Vec::new();
3240 for (i, j) in self.state.judgements.iter().enumerate() {
3241 let vote = self
3242 .state
3243 .votes
3244 .iter()
3245 .find(|v| v.judge == i + 1)
3246 .and_then(|v| v.vote)
3247 .or_else(|| j.ranking.first().copied());
3248 if let Some(v) = vote {
3249 *first_choice.entry(v).or_insert(0) += 1;
3250 cast.push(v);
3251 }
3252 }
3253
3254 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3255 for j in &self.state.judgements {
3256 let n = j.ranking.len();
3257 for (pos, label) in j.ranking.iter().enumerate() {
3258 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3259 }
3260 }
3261
3262 let best = first_choice.values().copied().max().unwrap_or(0);
3263 let mut leaders: Vec<char> = first_choice
3264 .iter()
3265 .filter(|(_, v)| **v == best)
3266 .map(|(k, _)| *k)
3267 .collect();
3268 let mut tie_break = None;
3269 if leaders.len() > 1 {
3270 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3271 let borda_leaders: Vec<char> = leaders
3272 .iter()
3273 .copied()
3274 .filter(|l| borda[l] == top_borda)
3275 .collect();
3276 tie_break = Some(if borda_leaders.len() == 1 {
3277 format!(
3278 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3279 leaders.len()
3280 )
3281 } else {
3282 format!(
3283 "{} way tie on both first-choice votes and Borda points, broken by label order",
3284 leaders.len()
3285 )
3286 });
3287 leaders = borda_leaders;
3288 leaders.sort_unstable();
3289 }
3290 let winner = *leaders
3291 .first()
3292 .or(viable.first())
3293 .context("no candidate to declare a winner from")?;
3294
3295 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3296 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3297 let deliberated = !self.state.deliberation.is_empty();
3298
3299 let quota_seats: std::collections::BTreeSet<&str> =
3303 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3304 let mut present = 0usize;
3305 for (i, j) in self.state.judgements.iter().enumerate() {
3306 if quota_seats.contains(j.seat.as_str()) {
3307 continue;
3308 }
3309 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3310 let voted = self
3311 .state
3312 .votes
3313 .iter()
3314 .any(|v| v.judge == i + 1 && v.vote.is_some());
3315 if ranked || voted {
3316 present += 1;
3317 }
3318 }
3319 let needs_quorum = viable.len() > 1;
3325 let judges_total = if needs_quorum {
3326 self.roles.judges.len()
3327 } else {
3328 0
3329 };
3330 let quorum = if needs_quorum {
3331 judges_total / 2 + 1
3332 } else {
3333 0
3334 };
3335 let met_quorum = !needs_quorum || present >= quorum;
3336 let uncontested = (!needs_quorum).then(|| {
3337 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3338 });
3339
3340 self.state.event(
3341 "tally",
3342 match &uncontested {
3343 Some(reason) => format!("winner {winner} — {reason}"),
3344 None => format!(
3345 "winner {winner} — votes {} | initial {} | {} changed | \
3346 {present}/{judges_total} judges{}",
3347 first_choice
3348 .iter()
3349 .map(|(k, v)| format!("{k}:{v}"))
3350 .collect::<Vec<_>>()
3351 .join(" "),
3352 if unanimous_initial {
3353 "unanimous"
3354 } else {
3355 "split"
3356 },
3357 changed_votes,
3358 if met_quorum {
3359 String::new()
3360 } else {
3361 format!(" — below quorum ({quorum} required)")
3362 },
3363 ),
3364 },
3365 );
3366 if !met_quorum {
3367 self.state.event(
3368 "stall",
3369 format!(
3370 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3371 the run stops here, resumable"
3372 ),
3373 );
3374 }
3375 self.state.tally = Some(Tally {
3376 first_choice,
3377 borda,
3378 winner,
3379 rankings: tops.len(),
3380 unanimous_initial,
3381 deliberated,
3382 changed_votes,
3383 unanimous_final,
3384 tie_break,
3385 judges: judges_total,
3386 present,
3387 quorum,
3388 met_quorum,
3389 uncontested,
3390 });
3391 self.state.status = if met_quorum {
3392 RunStatus::Reviewing
3393 } else {
3394 RunStatus::Stalled
3395 };
3396 self.state.save()?;
3397 Ok(())
3398 }
3399
3400 #[allow(clippy::too_many_lines)]
3421 async fn recover_stall(&mut self) -> Result<bool> {
3422 let run_id = self.state.id.clone();
3427 let prompts = self.state.config.prompts.clone();
3428 let quota_seats: BTreeSet<&str> =
3433 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3434 let absent: Vec<String> = self
3435 .state
3436 .judgements
3437 .iter()
3438 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3439 .map(|j| j.seat.clone())
3440 .collect();
3441 if absent.is_empty() {
3442 return Ok(false);
3443 }
3444 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3445 if viable.len() <= 1 {
3446 return Ok(false);
3447 }
3448 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3449 let language = self.state.config.graph.language.clone();
3450 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3451 let sessions = self.state.config.graph.sessions;
3452 let artifacts = agent::artifacts_dir(&self.state.dir());
3453 let root = self.state.worktree_root();
3454 let base_short = short(&self.state.base_commit);
3455 let candidates: Vec<Candidate> = viable.clone();
3456
3457 let mut positions: Vec<usize> = absent
3459 .iter()
3460 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3461 .collect();
3462 if positions.is_empty() {
3463 return Ok(false);
3464 }
3465 positions.sort_unstable();
3466 positions.dedup();
3467
3468 let mut judge_jobs = Vec::new();
3470 for &j in &positions {
3471 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3472 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3473 let seat_key = format!("judge-{}", j + 1);
3474 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3475 let seat = self.seat(&seat_key, &spec.id);
3476 judge_jobs.push(SeatJob {
3477 spec,
3478 seat,
3479 prompt: prompt::judge(
3480 &self.state.instruction,
3481 &views,
3482 self.roles.judges.len(),
3483 &base_short,
3484 &language,
3485 ),
3486 cwd: root.join(seat_key),
3487 timeout,
3488 allow_write: false,
3489 sessions,
3490 artifacts: artifacts.clone(),
3491 stem: format!("judge-{}-recover", j + 1),
3492 handover: None,
3493 });
3494 }
3495
3496 let labels_for_check = labels.clone();
3497 let mut judge_losses = Vec::new();
3498 let retries = self.state.config.graph.retries;
3499 let cache = self.state.config.cache_dir();
3500 let ctx = WaveCtx {
3501 carry_seats: false,
3502 pauses: Some((&self.pause, &self.interrupt)),
3503 run: &run_id,
3504 node: "judge",
3505 prompts: &prompts,
3506 cache: cache.as_deref(),
3507 round: None,
3508 };
3509 let results = ask_json_wave::<Ranking>(
3510 judge_jobs,
3511 Arc::clone(&self.sem),
3512 retries,
3513 &self.roles.judge_roster,
3514 &ctx,
3515 &mut judge_losses,
3516 &mut self.state,
3517 &move |r: &Ranking| r.validate(&labels_for_check),
3518 )
3519 .await;
3520 if self.state.resource_block.is_some() {
3521 self.stop_for_resource()?;
3522 return Ok(false);
3523 }
3524
3525 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3527 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3528 let agent_id = seat.agent.clone();
3529 self.state.seats.insert(seat.key.clone(), seat);
3530 let record = &mut self.state.judgements[j];
3531 match res {
3532 Ok((ranking, out)) => {
3533 record.agent = agent_id;
3534 record.ranking = ranking.normalized();
3535 record.reasons = ranking.reasons;
3536 record.strengths = ranking.strengths;
3537 record.confidence = ranking.confidence;
3538 record.failed = None;
3539 record.duration_ms = out.duration_ms;
3540 recovered.insert(j);
3541 self.state.event(
3542 "recover",
3543 format!("judge {} ranked again after the limit", j + 1),
3544 );
3545 }
3546 Err(e) => {
3547 self.state
3548 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3549 }
3550 }
3551 }
3552
3553 let mut vote_jobs = Vec::new();
3555 let mut vote_pos: Vec<usize> = Vec::new();
3556 for &j in &recovered {
3557 let seat_key = format!("judge-{}", j + 1);
3558 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3559 let seat = self.seat(&seat_key, &spec.id);
3560 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3561 let text = if has_context(&spec, &seat, sessions) {
3562 prompt::final_vote(&labels, &language)
3563 } else {
3564 full.clone()
3565 };
3566 vote_jobs.push(SeatJob {
3567 spec,
3568 seat,
3569 prompt: text,
3570 cwd: root.join(seat_key),
3571 timeout,
3572 allow_write: false,
3573 sessions,
3574 artifacts: artifacts.clone(),
3575 stem: format!("vote-judge-{}-recover", j + 1),
3576 handover: Some(full),
3577 });
3578 vote_pos.push(j);
3579 }
3580 let allowed = labels.clone();
3581 let mut vote_losses = Vec::new();
3582 let vote_retries = self.state.config.graph.retries;
3583 let vote_cache = self.state.config.cache_dir();
3584 let ctx = WaveCtx {
3585 carry_seats: false,
3586 pauses: Some((&self.pause, &self.interrupt)),
3587 run: &run_id,
3588 node: "vote",
3589 prompts: &prompts,
3590 cache: vote_cache.as_deref(),
3591 round: None,
3592 };
3593 let votes = ask_json_wave::<FinalVote>(
3594 vote_jobs,
3595 Arc::clone(&self.sem),
3596 vote_retries,
3597 &self.roles.judge_roster,
3598 &ctx,
3599 &mut vote_losses,
3600 &mut self.state,
3601 &move |v: &FinalVote| match v.label() {
3602 Some(c) if allowed.contains(&c) => Ok(()),
3603 other => bail!("vote {other:?} is not one of {allowed:?}"),
3604 },
3605 )
3606 .await;
3607 if self.state.resource_block.is_some() {
3608 self.stop_for_resource()?;
3609 return Ok(false);
3610 }
3611 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3612 let agent_id = seat.agent.clone();
3613 self.state.seats.insert(seat.key.clone(), seat);
3614 match res {
3615 Ok((v, _)) => {
3616 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3617 rec.vote = v.label();
3618 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3619 } else {
3620 self.state.votes.push(VoteRecord {
3621 judge: j + 1,
3622 agent: agent_id,
3623 vote: v.label(),
3624 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3625 changed: false,
3626 });
3627 }
3628 self.state.event(
3629 "recover",
3630 format!("judge {} voted again after the limit", j + 1),
3631 );
3632 }
3633 Err(e) => {
3634 self.state
3635 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3636 }
3637 }
3638 }
3639
3640 let recovered_keys: BTreeSet<String> = recovered
3644 .iter()
3645 .map(|&j| format!("judge-{}", j + 1))
3646 .collect();
3647 self.state
3648 .quota
3649 .retain(|q| !recovered_keys.contains(&q.seat));
3650 for loss in judge_losses.into_iter().chain(vote_losses) {
3654 if recovered_keys.contains(&loss.seat) {
3655 continue;
3656 }
3657 self.state.quota.retain(|q| q.seat != loss.seat);
3658 self.state.quota.push(loss);
3659 }
3660
3661 self.state.tally = None;
3663 self.tally()?;
3664 Ok(self
3665 .state
3666 .tally
3667 .as_ref()
3668 .map(|t| t.met_quorum)
3669 .unwrap_or(false))
3670 }
3671
3672 async fn graft(&mut self) -> Result<()> {
3689 let Err(e) = self.graft_inner().await else {
3690 return Ok(());
3691 };
3692 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
3693 self.state.status = RunStatus::Blocked;
3694 self.state.event("graft", format!("blocked: {e:#}"));
3695 self.state.save()?;
3696 return Err(e);
3697 }
3698 if let Some(rec) = self.state.graft.clone()
3703 && !rec.outcome.settled()
3704 {
3705 if let Some(winner) = self.state.winner().cloned() {
3706 if let Err(re) = Self::graft_restore(&winner.worktree, &rec.pre).await {
3707 self.state.status = RunStatus::Blocked;
3708 self.state.event("graft", format!("blocked: {re:#}"));
3709 self.state.save()?;
3710 return Err(re);
3711 }
3712 }
3713 if let Some(g) = self.state.graft.as_mut() {
3714 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
3715 }
3716 }
3717 self.state.event("graft", format!("skipped: {e:#}"));
3718 if let Err(e) = self.state.save() {
3719 tracing::warn!("save after graft: {e:#}");
3720 }
3721 Ok(())
3722 }
3723
3724 async fn graft_restore(wt: &Path, pre: &str) -> Result<()> {
3727 let restored = git::restore_to(wt, pre).await;
3728 let tip = git::rev_parse(wt, "HEAD").await;
3729 match (restored, tip) {
3730 (Ok(()), Ok(tip)) if tip == pre => Ok(()),
3731 (Err(e), _) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
3732 (Ok(()), Ok(tip)) => {
3733 Err(GraftRestoreFailed(format!("the winner is at {tip}, not {pre}")).into())
3734 }
3735 (Ok(()), Err(e)) => Err(GraftRestoreFailed(format!("{e:#}")).into()),
3736 }
3737 }
3738
3739 fn graft_skipped(&mut self, why: &str) {
3741 let message = format!("skipped: {why}");
3742 if !self
3743 .state
3744 .events
3745 .iter()
3746 .any(|e| e.node == "graft" && e.message == message)
3747 {
3748 self.state.event("graft", message);
3749 }
3750 }
3751
3752 async fn graft_inner(&mut self) -> Result<()> {
3753 if let Some(rec) = self.state.graft.clone()
3756 && !rec.outcome.settled()
3757 {
3758 if self.state.reviews.is_empty()
3762 && let Some(winner) = self.state.winner().cloned()
3763 {
3764 Self::graft_restore(&winner.worktree, &rec.pre).await?;
3765 }
3766 if let Some(g) = self.state.graft.as_mut() {
3767 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
3768 }
3769 self.state.event(
3770 "graft",
3771 "rolled back: the previous round was interrupted; the winner is untouched",
3772 );
3773 self.state.save()?;
3774 return Ok(());
3775 }
3776 let plan = match graft::plan(&self.state) {
3777 Ok(plan) => plan,
3778 Err(skip) => {
3779 if skip != graft::Skip::AlreadyDone {
3780 self.graft_skipped(skip.reason());
3781 self.state.save()?;
3782 }
3783 return Ok(());
3784 }
3785 };
3786 let Some(winner) = self.state.winner().cloned() else {
3787 return Ok(());
3788 };
3789 let wt = winner.worktree.clone();
3790 if !git::is_clean(&wt).await? {
3791 self.graft_skipped("the winner's worktree has uncommitted changes");
3792 self.state.save()?;
3793 return Ok(());
3794 }
3795 let pre = git::rev_parse(&wt, "HEAD").await?;
3796 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
3797 else {
3798 self.graft_skipped("the winner's diff could not be measured");
3799 self.state.save()?;
3800 return Ok(());
3801 };
3802 let limit = graft::limit(winner_lines);
3803 self.state.graft = Some(GraftRecord {
3806 pre: pre.clone(),
3807 post: None,
3808 lines: None,
3809 limit: Some(limit),
3810 agent: None,
3811 outcome: GraftOutcome::Running,
3812 requested: graft::requested(&plan),
3813 adoption: None,
3814 adoption_error: None,
3815 });
3816 self.state.event(
3817 "graft",
3818 format!(
3819 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
3820 plan.strengths
3821 .iter()
3822 .map(|(l, _, _)| l.to_string())
3823 .collect::<Vec<_>>()
3824 .join(", "),
3825 plan.winner
3826 ),
3827 );
3828 self.state.save()?;
3829
3830 let fix_prompt = prompt::graft(
3831 &self.state.instruction,
3832 plan.winner,
3833 &plan.strengths,
3834 limit,
3835 &self.state.config.graph.language,
3836 );
3837 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
3838 let sessions = self.state.config.graph.sessions;
3839 let artifacts = agent::artifacts_dir(&self.state.dir());
3840 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
3841 let (_, seat, out) = self
3842 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
3843 SeatJob {
3844 prompt: fix_prompt.clone(),
3845 spec,
3846 seat,
3847 cwd: wt.clone(),
3848 timeout,
3849 allow_write: true,
3850 sessions,
3851 artifacts: artifacts.clone(),
3852 stem: "graft".to_owned(),
3853 handover: None,
3854 }
3855 })
3856 .await;
3857 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3862 Self::graft_restore(&wt, &pre).await?;
3863 self.state.graft = None;
3864 self.state.save()?;
3865 return self.stop_for_resource();
3866 }
3867 let mut why: Option<String> = None;
3868 match &out {
3869 AgentOutcome::Ok(o) if agent::output_advances(o) => {
3870 why = Some("the fixer produced nothing usable".to_owned());
3871 }
3872 AgentOutcome::Ok(_) => {}
3873 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
3874 AgentOutcome::Quota(o) => {
3875 self.state.quota.push(QuotaLoss {
3876 seat: seat.key.clone(),
3877 node: "graft".to_owned(),
3878 at: Timestamp::now(),
3879 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3880 });
3881 why = Some("rate limited (quota)".to_owned());
3882 }
3883 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3884 why = Some(format!(
3885 "the fixer failed: {}",
3886 e.lines()
3887 .next()
3888 .unwrap_or("")
3889 .chars()
3890 .take(160)
3891 .collect::<String>()
3892 ));
3893 }
3894 }
3895 let agent_id = seat.agent.clone();
3896 self.state.seats.insert(seat.key.clone(), seat);
3897
3898 let mut post = None;
3899 let mut lines = None;
3900 if why.is_none() {
3901 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
3902 if let Ok(r) = &rescued {
3903 self.state.note_withheld("graft", &r.withheld);
3904 }
3905 let head = git::rev_parse(&wt, "HEAD").await?;
3906 let dirty = match &rescued {
3910 Ok(r) => git::status_porcelain(&wt)
3911 .await
3912 .map(|st| {
3913 st.lines().any(|l| {
3914 !r.withheld
3915 .iter()
3916 .any(|w| l.trim_end().ends_with(w.path.as_str()))
3917 })
3918 })
3919 .unwrap_or(true),
3920 Err(_) => true,
3921 };
3922 if dirty {
3923 why = Some("the fixer left changes that could not be committed".to_owned());
3924 } else if head == pre {
3925 why = Some("the fixer changed nothing".to_owned());
3926 } else {
3927 lines = git::changed_lines(&wt, &pre, &head).await;
3928 if graft::within_budget(lines, limit) {
3929 post = Some(head);
3930 } else {
3931 why = Some(format!(
3932 "{} changed lines against a limit of {limit}",
3933 lines.map_or_else(
3934 || "an unmeasurable number of".to_owned(),
3935 |l| l.to_string()
3936 )
3937 ));
3938 }
3939 }
3940 }
3941 let (adoption, adoption_error) = match &out {
3942 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
3943 _ => (None, None),
3944 };
3945 let requested = graft::requested(&plan);
3946 let outcome = match why {
3947 None => {
3948 self.state.event(
3949 "graft",
3950 format!(
3951 "applied: {} changed lines (limit {limit})",
3952 lines.unwrap_or(0)
3953 ),
3954 );
3955 GraftOutcome::Applied
3956 }
3957 Some(why) => {
3958 Self::graft_restore(&wt, &pre).await?;
3961 self.state.event(
3962 "graft",
3963 format!("rolled back: {why}; the winner is untouched"),
3964 );
3965 GraftOutcome::RolledBack(why)
3966 }
3967 };
3968 self.state.graft = Some(GraftRecord {
3969 pre,
3970 post,
3971 lines,
3972 limit: Some(limit),
3973 agent: Some(agent_id),
3974 outcome,
3975 requested,
3976 adoption,
3977 adoption_error,
3978 });
3979 self.state.save()?;
3980 Ok(())
3981 }
3982
3983 async fn fold_losers(&mut self) -> Result<()> {
3986 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3987 return Ok(());
3988 };
3989 let repo = self.state.repo.clone();
3990 let mut folded = Vec::new();
3991 for i in 0..self.state.candidates.len() {
3992 let c = &self.state.candidates[i];
3993 if c.label == winner || c.folded {
3994 continue;
3995 }
3996 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3997 git::worktree_remove(&repo, &wt).await.ok();
3998 git::branch_delete(&repo, &branch).await.ok();
3999 self.state.candidates[i].folded = true;
4000 folded.push(label.to_string());
4001 }
4002 let root = self.state.worktree_root();
4004 for j in 1..=self.roles.judges.len() {
4005 let wt = root.join(format!("judge-{j}"));
4006 if wt.exists() {
4007 git::worktree_remove(&repo, &wt).await.ok();
4008 }
4009 }
4010 if self.state.config.graph.advise {
4013 for k in 1..=self.state.config.graph.advisors {
4014 let wt = root.join(format!("advisor-{k}"));
4015 if wt.exists() {
4016 git::worktree_remove(&repo, &wt).await.ok();
4017 }
4018 }
4019 }
4020 if !folded.is_empty() {
4021 self.state
4022 .event("fold", format!("folded candidates {}", folded.join(", ")));
4023 self.state.save()?;
4024 }
4025 Ok(())
4026 }
4027
4028 async fn sync_to_base(&mut self) -> Result<()> {
4068 if self.state.status == RunStatus::AlreadyInBase {
4069 return Ok(());
4070 }
4071 let conflicted = self
4072 .state
4073 .base_sync
4074 .as_ref()
4075 .is_some_and(|s| s.conflict.is_some());
4076 let Some(winner) = self.state.winner().cloned() else {
4077 return Ok(());
4078 };
4079
4080 let repo = self.state.repo.clone();
4081 let remote = self.state.config.merge.remote.clone();
4082 let base_branch = self.state.base_branch.clone();
4083 let tracking = format!("{remote}/{base_branch}");
4084
4085 git::fetch(&repo, &remote, &base_branch).await.ok();
4086 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4091 return Ok(());
4092 };
4093
4094 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4095 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4096 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4097
4098 if (behind > 0 || conflicted || head == tip)
4107 && self
4108 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4109 .await?
4110 {
4111 return Ok(());
4112 }
4113 if conflicted {
4114 return Ok(());
4115 }
4116
4117 if behind == 0 {
4118 if let Some(from) = self
4127 .state
4128 .rebase_fixes
4129 .iter()
4130 .rev()
4131 .find_map(|r| r.from.clone())
4132 && from != head
4133 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4134 .await
4135 .is_ok_and(|o| o.ok())
4136 {
4137 git::sync_to_head(&winner.worktree).await?;
4138 }
4139 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4145 if let Some(why) = &conflict {
4146 self.state.status = RunStatus::Blocked;
4147 self.state.event("land", why.clone());
4148 }
4149 self.state.base_sync = Some(BaseSync {
4150 tip,
4151 behind: 0,
4152 attempts,
4153 conflict,
4154 already_in: None,
4155 });
4156 self.state.save()?;
4157 return Ok(());
4158 }
4159
4160 if attempts >= BASE_SYNC_ROUNDS {
4161 let why = format!(
4162 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4163 rebase(s); rebasing again would only race it",
4164 winner.branch
4165 );
4166 self.state.status = RunStatus::Blocked;
4167 self.state.base_sync = Some(BaseSync {
4168 tip,
4169 behind,
4170 attempts,
4171 conflict: Some(why.clone()),
4172 already_in: None,
4173 });
4174 self.state.event("land", why);
4175 self.state.save()?;
4176 return Ok(());
4177 }
4178
4179 self.state.event(
4180 "land",
4181 format!(
4182 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4183 winner.branch
4184 ),
4185 );
4186 self.state.save()?;
4187
4188 let branch_tracking = format!("{remote}/{}", winner.branch);
4194 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4195 let remote_tip = if self.state.task_pr_branch().is_none()
4196 && matches!(&fetched_branch, Ok(o) if o.ok())
4197 {
4198 git::rev_parse(&repo, &branch_tracking).await.ok()
4199 } else {
4200 None
4201 };
4202 if let Some(theirs) = &remote_tip
4206 && !git::is_ancestor(&repo, theirs, &head).await
4207 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4208 .await
4209 .is_ok_and(|missing| missing.is_empty())
4210 {
4211 let why = format!(
4212 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4213 them",
4214 short(theirs),
4215 winner.branch
4216 );
4217 self.state.status = RunStatus::Blocked;
4218 self.state.base_sync = Some(BaseSync {
4219 tip,
4220 behind,
4221 attempts,
4222 conflict: Some(why.clone()),
4223 already_in: None,
4224 });
4225 self.state.event("land", why);
4226 self.state.save()?;
4227 return Ok(());
4228 }
4229
4230 let scratch = self.state.dir().join("base-sync");
4231 let rebased = match crate::rebase::rebase_with_fixer(
4232 &mut self.state,
4233 &scratch,
4234 &winner.branch,
4235 &tracking,
4236 )
4237 .await
4238 {
4239 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4240 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4241 Err(e) => Err(e),
4242 };
4243 let attempts = attempts + 1;
4244 match rebased {
4245 Ok(None) => {
4246 git::sync_to_head(&winner.worktree).await?;
4250 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4251 let mut conflict = None;
4252 if let Some(pinned) = &remote_tip {
4253 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4254 match pushed {
4255 Ok(o) if o.ok() => self.state.event(
4256 "land",
4257 format!("pushed rebased {} to {remote}", winner.branch),
4258 ),
4259 Ok(o) => {
4260 conflict = Some(format!(
4261 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4262 winner.branch,
4263 short(pinned),
4264 o.stderr.chars().take(600).collect::<String>()
4265 ));
4266 }
4267 Err(e) => {
4268 conflict = Some(format!(
4269 "rebased {} locally but could not push it: {e:#}",
4270 winner.branch
4271 ));
4272 }
4273 }
4274 }
4275 if let Some(why) = &conflict {
4276 self.state.status = RunStatus::Blocked;
4277 self.state.event("land", why.clone());
4278 }
4279 self.state.base_sync = Some(BaseSync {
4280 tip: tip.clone(),
4281 behind: 0,
4282 attempts,
4283 conflict,
4284 already_in: None,
4285 });
4286 self.state
4287 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4288 }
4289 Ok(Some(conflict)) => {
4290 let why = format!(
4291 "{} conflicts with {tracking} and did not rebase: {}",
4292 winner.branch,
4293 conflict.chars().take(600).collect::<String>()
4294 );
4295 self.state.status = RunStatus::Blocked;
4296 self.state.base_sync = Some(BaseSync {
4297 tip,
4298 behind,
4299 attempts,
4300 conflict: Some(why.clone()),
4301 already_in: None,
4302 });
4303 self.state.event("land", why);
4304 }
4305 Err(e) => {
4306 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4307 self.state.status = RunStatus::Blocked;
4308 self.state.base_sync = Some(BaseSync {
4309 tip,
4310 behind,
4311 attempts,
4312 conflict: Some(why.clone()),
4313 already_in: None,
4314 });
4315 self.state.event("land", why);
4316 }
4317 }
4318 self.state.save()?;
4319 Ok(())
4320 }
4321
4322 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4328 if self.state.task_pr_branch().is_some() {
4331 return None;
4332 }
4333 let repo = self.state.repo.clone();
4334 let remote = self.state.config.merge.remote.clone();
4335 let fetched = git::fetch(&repo, &remote, branch).await;
4336 if !matches!(&fetched, Ok(o) if o.ok()) {
4337 return None;
4338 }
4339 let branch_tracking = format!("{remote}/{branch}");
4340 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4341 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4342 return None;
4343 }
4344 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4345 .await
4346 .is_ok_and(|missing| missing.is_empty())
4347 {
4348 return Some(format!(
4349 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4350 them",
4351 short(&theirs)
4352 ));
4353 }
4354 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4355 Ok(o) if o.ok() => {
4356 self.state
4357 .event("land", format!("pushed rebased {branch} to {remote}"));
4358 None
4359 }
4360 Ok(o) => Some(format!(
4361 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4362 someone may have pushed): {}",
4363 short(&theirs),
4364 o.stderr.chars().take(600).collect::<String>()
4365 )),
4366 Err(e) => Some(format!(
4367 "{branch} is rebased locally but could not be pushed: {e:#}"
4368 )),
4369 }
4370 }
4371
4372 async fn settle_already_in(
4389 &mut self,
4390 branch: &str,
4391 tip: &str,
4392 head: &str,
4393 attempts: usize,
4394 behind: usize,
4395 ) -> Result<bool> {
4396 let repo = self.state.repo.clone();
4397 let remote = self.state.config.merge.remote.clone();
4398 let start = self.state.base_commit.clone();
4399 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4400 Ok(Some(e)) => e,
4401 Ok(None) => return Ok(false),
4402 Err(e) => {
4403 tracing::warn!("already-in-base check for {branch}: {e:#}");
4404 return Ok(false);
4405 }
4406 };
4407 let mut verified = vec![head.to_owned()];
4408 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4409 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4410 if matches!(&fetched, Ok(o) if o.ok())
4411 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4412 && theirs != head
4413 {
4414 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4415 Ok(Some(_)) => verified.push(theirs),
4416 _ => return Ok(false),
4417 }
4418 }
4419 let base_branch = self.state.base_branch.clone();
4420 let message = format!(
4421 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4422 land",
4423 evidence.names(),
4424 evidence.proof.as_str()
4425 );
4426 let closed =
4427 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4428 .await;
4429 match closed {
4430 Ok(Ok(url)) => self
4431 .state
4432 .event("land", format!("closed {url}: superseded on {base_branch}")),
4433 Ok(Err(why)) => self
4434 .state
4435 .event("land", format!("did not close a pull request: {why}")),
4436 Err(e) => {
4437 let why = format!(
4438 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4439 not be closed ({e:#}); resume to retry"
4440 );
4441 self.state.status = RunStatus::Blocked;
4442 self.state.base_sync = Some(BaseSync {
4443 tip: tip.to_owned(),
4444 behind,
4445 attempts,
4446 conflict: Some(why.clone()),
4447 already_in: None,
4448 });
4449 self.state.event("land", why);
4450 self.state.save()?;
4451 return Ok(true);
4452 }
4453 }
4454 self.state.status = RunStatus::AlreadyInBase;
4455 self.state.base_sync = Some(BaseSync {
4456 tip: tip.to_owned(),
4457 behind,
4458 attempts,
4459 conflict: None,
4460 already_in: Some(evidence),
4461 });
4462 self.state.event("land", message);
4463 self.state.save()?;
4464 self.settle_questions();
4465 Ok(true)
4466 }
4467
4468 fn landing_base(&self) -> String {
4478 self.state
4479 .base_sync
4480 .as_ref()
4481 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4482 }
4483
4484 pub async fn fix_selected(
4517 &mut self,
4518 ids: &[String],
4519 reason: &str,
4520 allow_stale: bool,
4521 ) -> Result<()> {
4522 let reason = reason.trim();
4523 if reason.is_empty() {
4524 bail!("a fix request needs a reason — that is the operator's own record of why");
4525 }
4526 if ids.is_empty() {
4527 bail!("no finding id given");
4528 }
4529 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4530 bail!(
4531 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4532 has already concluded — can be given a targeted fix. A run still \
4533 in progress should simply be resumed; a `merged` run's branch has \
4534 already landed, so its answer is a fresh `magi review <branch>`, \
4535 not reopening this run's own record",
4536 self.state.id,
4537 self.state.status.as_str()
4538 );
4539 }
4540 let Some(winner) = self.state.winner().cloned() else {
4541 bail!("run {} has no winning candidate to fix", self.state.id);
4542 };
4543 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4544 bail!(
4545 "branch `{}` no longer exists; this run cannot be extended",
4546 winner.branch
4547 );
4548 }
4549 let home = crate::run::home();
4550 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
4551 bail!(
4552 "run {} is currently being worked on by another magi process",
4553 self.state.id
4554 );
4555 }
4556 let _claim = FixClaim::acquire(&self.state.dir())?;
4562
4563 let mut seen = BTreeSet::new();
4567 let mut findings = Vec::new();
4568 let mut missing = Vec::new();
4569 for id in ids {
4570 if !seen.insert(id.clone()) {
4571 continue;
4572 }
4573 match self.state.finding(id) {
4574 Some((round, rec, f)) => findings.push(OperatorFixFinding {
4575 id: f.id.clone(),
4576 severity: f.severity,
4577 reviewer_vote: rec.vote,
4578 round: round.round,
4579 round_head: round.head.clone(),
4580 reviewer: rec.reviewer,
4581 agent: rec.agent.clone(),
4582 file: f.file.clone(),
4583 line: f.line,
4584 title: f.title.clone(),
4585 detail: f.detail.clone(),
4586 outcome: OperatorFixOutcome::Pending,
4587 }),
4588 None => missing.push(id.clone()),
4589 }
4590 }
4591 if !missing.is_empty() {
4592 bail!(
4593 "unknown finding id(s): {}; nothing was changed",
4594 missing.join(", ")
4595 );
4596 }
4597
4598 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
4599 let stale_details: Vec<(String, String)> = findings
4600 .iter()
4601 .filter(|f| f.round_head != head_at_request)
4602 .map(|f| (f.id.clone(), f.round_head.clone()))
4603 .collect();
4604 let stale = !stale_details.is_empty();
4605 if stale && !allow_stale {
4606 bail!(
4607 "the branch has moved since some finding(s) were raised — {} — now \
4608 at {}; pass --allow-stale to fix anyway, or re-run review first",
4609 stale_details
4610 .iter()
4611 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
4612 .collect::<Vec<_>>()
4613 .join(", "),
4614 short(&head_at_request)
4615 );
4616 }
4617
4618 let request = OperatorFixRequest {
4619 requested_at: Timestamp::now(),
4620 reason: reason.to_owned(),
4621 findings,
4622 head_at_request: head_at_request.clone(),
4623 allow_stale,
4624 stale,
4625 fix: None,
4626 result_head: None,
4627 follow_up_review_run: None,
4628 };
4629 self.state.event(
4630 "fix",
4631 format!(
4632 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
4633 request.findings.len(),
4634 request
4635 .findings
4636 .iter()
4637 .map(|f| f.id.as_str())
4638 .collect::<Vec<_>>()
4639 .join(", "),
4640 ),
4641 );
4642 self.state.operator_fixes.push(request);
4649 self.state.save()?;
4650 let request_index = self.state.operator_fixes.len() - 1;
4651
4652 if winner.worktree.exists() {
4661 let dirty = git::git(
4664 &winner.worktree,
4665 &["status", "--porcelain", "--untracked-files=all"],
4666 )
4667 .await?;
4668 let only_withheld = dirty.lines().all(|l| {
4669 l.strip_prefix("?? ")
4670 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
4671 });
4672 if !only_withheld {
4673 bail!(
4674 "`{}` has uncommitted changes; refusing to touch it — commit or \
4675 discard them first",
4676 winner.worktree.display()
4677 );
4678 }
4679 git::worktree_remove(&self.state.repo, &winner.worktree)
4680 .await
4681 .ok();
4682 }
4683 let fix_worktree = self.state.worktree_root().join("operator-fix");
4684 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
4685 git::git(
4686 &self.state.repo,
4687 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
4688 )
4689 .await
4690 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
4691 if !git::is_clean(&fix_worktree).await? {
4692 git::worktree_remove(&self.state.repo, &fix_worktree)
4693 .await
4694 .ok();
4695 bail!(
4696 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
4697 winner.branch
4698 );
4699 }
4700 let repo_for_setup = self.state.repo.clone();
4701 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
4702 Ok(()) => {
4703 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
4704 }
4705 Err(e) => Err(e),
4706 };
4707 if let Err(e) = setup {
4708 git::worktree_remove(&self.state.repo, &fix_worktree)
4709 .await
4710 .ok();
4711 return Err(e);
4712 }
4713
4714 let run_id = self.state.id.clone();
4715 let prompts = self.state.config.prompts.clone();
4716 let language = self.state.config.graph.language.clone();
4717 let sessions = self.state.config.graph.sessions;
4718 let artifacts = agent::artifacts_dir(&self.state.dir());
4719 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
4720 .findings
4721 .iter()
4722 .map(|f| Finding {
4723 id: f.id.clone(),
4724 severity: f.severity,
4725 file: f.file.clone(),
4726 line: f.line,
4727 title: f.title.clone(),
4728 detail: f.detail.clone(),
4729 })
4730 .collect();
4731 let fix_prompt = prompt::operator_fix(
4732 &self.state.instruction,
4733 &finding_list,
4734 reason,
4735 &stale_details,
4736 &head_at_request,
4737 &language,
4738 );
4739 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4740 let (job, seat, out) = self
4741 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
4742 prompt: fix_prompt.clone(),
4743 spec,
4744 seat,
4745 cwd: fix_worktree.clone(),
4746 timeout,
4747 allow_write: true,
4748 sessions,
4749 artifacts: artifacts.clone(),
4750 stem: "operator-fix".to_owned(),
4751 handover: None,
4752 })
4753 .await;
4754 let agent_id = seat.agent.clone();
4755
4756 let mut fix = FixRecord {
4757 agent: agent_id,
4758 addressed: Vec::new(),
4759 rejected: Vec::new(),
4760 notes: String::new(),
4761 committed: false,
4762 failed: None,
4763 duration_ms: 0,
4764 continuation: None,
4765 };
4766 let mut final_seat = seat.clone();
4767 match out {
4768 AgentOutcome::Ok(o) => {
4769 fix.duration_ms = o.duration_ms;
4770 let parsed = verdict::extract_json::<FixReport>(&o.text);
4771 let incomplete_reason = match &parsed {
4772 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4773 "the reply parsed, but it reported a command whose own CLI \
4774 never confirmed an exit status"
4775 .to_owned(),
4776 ),
4777 Ok(_) => None,
4778 Err(e) => Some(e.to_string()),
4779 };
4780 match incomplete_reason {
4781 None => {
4782 let report = parsed.expect("checked Ok above");
4783 fix.addressed = report.addressed;
4784 fix.rejected = report.rejected;
4785 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4786 }
4787 Some(reason) => {
4788 let (resumed_seat, resolved, failure, cont) = self
4789 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4790 .await;
4791 fix.duration_ms += cont.cumulative_wait_ms;
4792 fix.continuation = Some(cont);
4793 final_seat = resumed_seat;
4794 match resolved {
4795 Some(report) => {
4796 fix.addressed = report.addressed;
4797 fix.rejected = report.rejected;
4798 fix.notes =
4799 blind::sanitize_prose(&report.notes, &self.state.config.blind);
4800 }
4801 None => fix.failed = failure,
4802 }
4803 }
4804 }
4805 }
4806 AgentOutcome::Dropped(o) => {
4807 fix.duration_ms = o.duration_ms;
4808 let why = o
4809 .dropped
4810 .as_ref()
4811 .map(|d| d.why.as_str())
4812 .unwrap_or("the CLI ended the stream without delivering its answer");
4813 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4814 }
4815 AgentOutcome::Quota(o) => {
4816 self.state.quota.push(QuotaLoss {
4817 seat: final_seat.key.clone(),
4818 node: "fix".to_owned(),
4819 at: Timestamp::now(),
4820 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4821 });
4822 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4823 }
4824 AgentOutcome::Failed(e) => fix.failed = Some(e),
4825 AgentOutcome::ResourceBlocked(e) => {
4826 fix.failed = Some(format!("the fixer was not started: {e}"));
4827 }
4828 }
4829 if fix.continuation.is_none() {
4830 fix.continuation = Some(ContinuationRecord::not_needed());
4831 }
4832 self.state.seats.insert(final_seat.key.clone(), final_seat);
4833
4834 let rescue_message = format!(
4835 "magi: operator-selected fix ({}) (uncommitted work)",
4836 self.state.operator_fixes[request_index]
4837 .findings
4838 .iter()
4839 .map(|f| f.id.as_str())
4840 .collect::<Vec<_>>()
4841 .join(", ")
4842 );
4843 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4844 self.state.note_withheld("fix", &r.withheld);
4845 }
4846 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4847 fix.committed = after != head_at_request;
4848 git::worktree_remove(&self.state.repo, &fix_worktree)
4849 .await
4850 .ok();
4851
4852 self.state.event(
4853 "fix",
4854 match &fix.failed {
4855 Some(reason) => format!(
4856 "operator fix: adoption report was lost ({reason}); {}",
4857 if fix.committed {
4858 "committed"
4859 } else {
4860 "NO new commit"
4861 }
4862 ),
4863 None => format!(
4864 "operator fix: {} addressed, {} rejected, {}",
4865 fix.addressed.len(),
4866 fix.rejected.len(),
4867 if fix.committed {
4868 "committed"
4869 } else {
4870 "NO new commit"
4871 }
4872 ),
4873 },
4874 );
4875
4876 for f in &mut self.state.operator_fixes[request_index].findings {
4883 f.outcome = if fix.failed.is_some() {
4884 OperatorFixOutcome::Unreported
4885 } else if fix.addressed.contains(&f.id) {
4886 OperatorFixOutcome::Addressed
4887 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4888 OperatorFixOutcome::Rejected { why: r.why.clone() }
4889 } else {
4890 OperatorFixOutcome::Unreported
4891 };
4892 }
4893
4894 let committed = fix.committed;
4895 if committed {
4896 self.state.operator_fixes[request_index].result_head = Some(after.clone());
4897 }
4898 self.state.operator_fixes[request_index].fix = Some(fix);
4899 self.state.save()?;
4902
4903 if committed {
4904 self.state.event(
4905 "fix",
4906 format!(
4907 "operator fix committed {}; opening a follow-up review-only run",
4908 short(&after)
4909 ),
4910 );
4911 let origin =
4914 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4915 match Self::review(
4916 &self.state.repo,
4917 &winner.branch,
4918 self.state.config.clone(),
4919 origin,
4920 )
4921 .await
4922 {
4923 Ok(mut follow_up) => {
4924 follow_up.state.event(
4925 "start",
4926 format!(
4927 "requested by an operator fix on run {} for finding(s) {}",
4928 self.state.id,
4929 self.state.operator_fixes[request_index]
4930 .findings
4931 .iter()
4932 .map(|f| f.id.as_str())
4933 .collect::<Vec<_>>()
4934 .join(", "),
4935 ),
4936 );
4937 follow_up.state.save()?;
4938 let follow_up_id = follow_up.state.id.clone();
4939 let adopted = match crate::direct::adopt(
4942 &follow_up.state,
4943 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
4944 ) {
4945 Ok(a) => a,
4946 Err(e) => {
4947 self.state.event(
4950 "fix",
4951 format!(
4952 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
4953 ),
4954 );
4955 self.state.save()?;
4956 return Ok(());
4957 }
4958 };
4959 let executed = follow_up.execute().await;
4960 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
4961 if let Some(a) = adopted {
4962 a.finish(&follow_up.state, executed);
4963 }
4964 if let Some(e) = failure {
4965 self.state.event(
4966 "fix",
4967 format!(
4968 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4969 ),
4970 );
4971 }
4972 self.state.operator_fixes[request_index].follow_up_review_run =
4973 Some(follow_up_id);
4974 }
4975 Err(e) => {
4976 self.state.event(
4977 "fix",
4978 format!("committed the fix but could not open a follow-up review: {e:#}"),
4979 );
4980 }
4981 }
4982 self.state.save()?;
4983 }
4984
4985 Ok(())
4986 }
4987
4988 async fn ask_fixer(
5008 &mut self,
5009 winner: &Candidate,
5010 node: &'static str,
5011 round: Option<usize>,
5012 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5013 ) -> (SeatJob, SeatState, AgentOutcome) {
5014 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5015 self.ask_fixer_among(attempts, node, round, None, build)
5016 .await
5017 }
5018
5019 async fn ask_fixer_among(
5024 &mut self,
5025 attempts: Vec<(AgentSpec, String)>,
5026 node: &'static str,
5027 round: Option<usize>,
5028 restore: Option<(&Path, &str)>,
5029 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5030 ) -> (SeatJob, SeatState, AgentOutcome) {
5031 let run_id = self.state.id.clone();
5032 let prompts = self.state.config.prompts.clone();
5033 let cache = self.state.config.cache_dir();
5034 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5035 let mut last = None;
5036 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5037 let seat = self.seat(&key, &spec.id);
5038 let mut job = build(spec.clone(), seat);
5039 if i > 0 {
5040 job.stem = format!("{}-{}", job.stem, spec.id);
5041 }
5042 let ctx = WaveCtx {
5043 carry_seats: false,
5044 pauses: Some((&self.pause, &self.interrupt)),
5045 run: &run_id,
5046 node,
5047 prompts: &prompts,
5048 cache: cache.as_deref(),
5049 round,
5050 };
5051 let (seat, out) =
5052 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5053 let advances = match &out {
5056 AgentOutcome::Ok(o) => agent::output_advances(o),
5057 AgentOutcome::ResourceBlocked(_) => false,
5058 _ => true,
5059 };
5060 if let Some(next) = ids.get(i + 1)
5061 && advances
5062 {
5063 self.state.seats.insert(seat.key.clone(), seat.clone());
5064 let class =
5065 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5066 let reason = match &out {
5067 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5068 other => fail_reason(other),
5069 };
5070 record_handover(
5071 &mut self.state,
5072 node,
5073 &seat.key,
5074 &spec.id,
5075 next,
5076 &class,
5077 &reason,
5078 );
5079 if let Some((wt, rev)) = restore
5083 && let Err(e) = git::restore_to(wt, rev).await
5084 {
5085 let why =
5086 format!("could not restore the worktree before the next fixer: {e:#}");
5087 tracing::warn!("{why}");
5088 last = Some((job, seat, AgentOutcome::Failed(why)));
5089 break;
5090 }
5091 continue;
5092 }
5093 last = Some((job, seat, out));
5094 break;
5095 }
5096 last.expect("the fixer chain always has an entry")
5097 }
5098
5099 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5110 use crate::recovery::{Mode, Repair};
5111 if !self.state.reviews.is_empty() {
5112 return Ok(());
5113 }
5114 let Some(mut rec) = self.state.recovery.clone() else {
5115 return Ok(());
5116 };
5117 if rec.mode != Mode::VerifyRepair {
5118 return Ok(());
5119 }
5120 let wt = winner.worktree.clone();
5121 let head = git::rev_parse(&wt, "HEAD").await?;
5122 match rec.repair.clone() {
5123 Repair::Done { .. } => return Ok(()),
5124 Repair::Started { before } => {
5125 if let Err(e) = Self::graft_restore(&wt, &before).await {
5129 return self
5130 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5131 .await;
5132 }
5133 rec.repair = Repair::Done {
5134 before: before.clone(),
5135 after: before,
5136 committed: false,
5137 failed: Some("interrupted; rolled back".to_owned()),
5138 };
5139 self.state
5140 .event("recovery", "the repair was interrupted and rolled back");
5141 self.state.recovery = Some(rec);
5142 self.state.save()?;
5143 return Ok(());
5144 }
5145 Repair::NotStarted => {}
5146 }
5147 if crate::recovery::recheck(&mut rec, &head) {
5148 self.state.event("recovery", rec.summary());
5149 self.state.recovery = Some(rec);
5150 self.state.save()?;
5151 return Ok(());
5152 }
5153 rec.repair = Repair::Started {
5154 before: head.clone(),
5155 };
5156 self.state.recovery = Some(rec.clone());
5157 self.state.save()?;
5158
5159 let language = self.state.config.graph.language.clone();
5160 let sessions = self.state.config.graph.sessions;
5161 let artifacts = agent::artifacts_dir(&self.state.dir());
5162 let fix_prompt = prompt::recovery_fix(
5163 &self.state.instruction,
5164 &rec,
5165 self.state.config.graph.review_rounds,
5166 &language,
5167 );
5168 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5169 self.state.event(
5170 "recovery",
5171 format!(
5172 "repair: the fixer starts from the failed verification on {}",
5173 short(&head)
5174 ),
5175 );
5176 let (_job, seat, out) = self
5177 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5178 prompt: fix_prompt.clone(),
5179 spec,
5180 seat,
5181 cwd: wt.clone(),
5182 timeout,
5183 allow_write: true,
5184 sessions,
5185 artifacts: artifacts.clone(),
5186 stem: "fix-recovery".to_owned(),
5187 handover: None,
5188 })
5189 .await;
5190 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5193 if let Err(e) = Self::graft_restore(&wt, &head).await {
5194 return self
5195 .stop_recovery(rec, &head, format!("resource wait; restore failed: {e:#}"))
5196 .await;
5197 }
5198 rec.repair = Repair::NotStarted;
5199 self.state.recovery = Some(rec);
5200 self.state.save()?;
5201 return self.stop_for_resource();
5202 }
5203 let failed = match &out {
5204 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5205 AgentOutcome::Quota(o) => {
5206 self.state.quota.push(QuotaLoss {
5207 seat: seat.key.clone(),
5208 node: "fix".to_owned(),
5209 at: Timestamp::now(),
5210 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5211 });
5212 Some("rate limited (quota); the fixer could not run".to_owned())
5213 }
5214 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5215 AgentOutcome::Failed(e) => Some(e.clone()),
5216 };
5217 self.state.seats.insert(seat.key.clone(), seat);
5218 let failed = match failed {
5219 Some(why) => {
5221 if let Err(e) = git::restore_to(&wt, &head).await {
5222 return self
5223 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5224 .await;
5225 }
5226 Some(why)
5227 }
5228 None => {
5229 if let Ok(r) = git::rescue_commit(
5230 &wt,
5231 "magi: recovery repair of the failed verification (uncommitted work)",
5232 )
5233 .await
5234 {
5235 self.state.note_withheld("fix", &r.withheld);
5236 }
5237 None
5238 }
5239 };
5240 let after = git::rev_parse(&wt, "HEAD").await?;
5241 self.state.event(
5242 "recovery",
5243 match &failed {
5244 Some(why) => format!("repair: nothing kept ({why})"),
5245 None if after != head => {
5246 format!("repair: committed {}", short(&after))
5247 }
5248 None => "repair: the fixer made no commit".to_owned(),
5249 },
5250 );
5251 rec.repair = Repair::Done {
5252 before: head.clone(),
5253 committed: after != head,
5254 after,
5255 failed,
5256 };
5257 self.state.recovery = Some(rec);
5258 self.state.save()?;
5259 Ok(())
5260 }
5261
5262 async fn stop_recovery(
5265 &mut self,
5266 mut rec: crate::recovery::RecoveryRecord,
5267 head: &str,
5268 why: String,
5269 ) -> Result<()> {
5270 rec.repair = crate::recovery::Repair::Started {
5272 before: head.to_owned(),
5273 };
5274 self.state.recovery = Some(rec);
5275 self.state
5276 .event("recovery", format!("repair could not be undone: {why}"));
5277 self.state.status = RunStatus::Blocked;
5278 self.state.save()?;
5279 bail!("recovery repair could not be rolled back: {why}")
5280 }
5281
5282 async fn review_loop(&mut self) -> Result<()> {
5283 if self
5288 .state
5289 .base_sync
5290 .as_ref()
5291 .is_some_and(|s| s.conflict.is_some())
5292 {
5293 return Ok(());
5294 }
5295 let run_id = self.state.id.clone();
5300 let prompts = self.state.config.prompts.clone();
5301 let Some(winner) = self.state.winner().cloned() else {
5302 return Ok(());
5303 };
5304 let max_rounds = self.state.config.graph.review_rounds;
5305 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5315 if status == RunStatus::Gating {
5318 self.record_contested_handoff();
5319 }
5320 self.state.status = status;
5321 self.state.save()?;
5322 return Ok(());
5323 }
5324 self.state.status = RunStatus::Reviewing;
5325 if self
5335 .state
5336 .reviews
5337 .last()
5338 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5339 {
5340 let shell = self.state.config.shell();
5341 return self
5342 .stop_reviewing(
5343 "the last round's own verification never resolved",
5344 &shell,
5345 &winner.worktree,
5346 )
5347 .await;
5348 }
5349
5350 self.recovery_repair(&winner).await?;
5351 if self.resource_halt()? {
5352 return Ok(());
5353 }
5354
5355 let repo = self.state.repo.clone();
5356 let root = self.state.worktree_root();
5357 let language = self.state.config.graph.language.clone();
5358 let sessions = self.state.config.graph.sessions;
5359 let artifacts = agent::artifacts_dir(&self.state.dir());
5360 let base = self.landing_base();
5361 let base_short = short(&base);
5362 let reviewers = self.roles.reviewers.clone();
5363 let shell = self.state.config.shell();
5364
5365 for round in (self.state.reviews.len() + 1)..=max_rounds {
5366 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5367 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5368 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5369 let prev_verification = self
5378 .state
5379 .reviews
5380 .last()
5381 .and_then(|r| r.verification_summary(&head));
5382 let prior_section = match (&self.state.recovery, round) {
5385 (Some(rec), 1) if self.state.reviews.is_empty() => {
5386 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5387 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5388 Some(crate::recovery::head_of(
5389 &git::diff(&winner.worktree, reviewed, "HEAD")
5390 .await
5391 .unwrap_or_default(),
5392 crate::recovery::SINCE_DIFF_MAX,
5393 ))
5394 }
5395 _ => None,
5396 };
5397 Some(prompt::recovery_review(rec, since.as_deref()))
5398 }
5399 _ => None,
5400 };
5401
5402 let mut jobs = Vec::new();
5406 for (r, spec) in reviewers.iter().cloned().enumerate() {
5407 let wt = root.join(format!("review-{}", r + 1));
5408 if wt.exists() {
5411 git::reset_detached(&wt, &head).await?;
5412 } else {
5413 git::worktree_add_detached(&repo, &wt, &head).await?;
5414 }
5415 self.setup_seat_worktree(&repo, &wt, false).await?;
5416 let seat_key = format!("review-{}", r + 1);
5417 let spec = pick_start_spec(
5420 &self.roles.reviewer_roster,
5421 spec,
5422 self.state.seat_history.get(&seat_key),
5423 );
5424 let seat = self.seat(&seat_key, &spec.id);
5425 jobs.push(SeatJob {
5426 prompt: prompt::review(&prompt::ReviewCtx {
5427 instruction: &self.state.instruction,
5428 branch: &winner.branch,
5429 base_short: &base_short,
5430 stat: &stat,
5431 patch: &patch,
5432 verification: prev_verification.as_ref(),
5433 prior: prior_section.as_deref(),
5434 reviewers: reviewers.len(),
5435 round,
5436 rounds: max_rounds,
5437 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5440 lens: Lens::for_seat(r),
5441 language: &language,
5442 }),
5443 spec,
5444 seat,
5445 cwd: wt,
5446 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5447 allow_write: false,
5448 sessions,
5449 artifacts: artifacts.clone(),
5450 stem: format!("review-{round}-{}", r + 1),
5451 handover: None,
5452 });
5453 }
5454
5455 self.state.event(
5456 "review",
5457 format!(
5458 "round {round}: {} reviewers on {}",
5459 jobs.len(),
5460 short(&head)
5461 ),
5462 );
5463 let mut quota_losses = Vec::new();
5464 let review_retries = self.state.config.graph.retries;
5465 let review_cache = self.state.config.cache_dir();
5466 let ctx = WaveCtx {
5467 carry_seats: true,
5468 pauses: Some((&self.pause, &self.interrupt)),
5469 run: &run_id,
5470 node: "review",
5471 prompts: &prompts,
5472 cache: review_cache.as_deref(),
5473 round: Some(round),
5474 };
5475 let results = ask_json_wave::<Review>(
5476 jobs,
5477 Arc::clone(&self.sem),
5478 review_retries,
5479 &self.roles.reviewer_roster,
5480 &ctx,
5481 &mut quota_losses,
5482 &mut self.state,
5483 &|_: &Review| Ok(()),
5484 )
5485 .await;
5486 if self.state.resource_block.is_some() {
5487 return self.stop_for_resource();
5488 }
5489 let round_quota_missing = quota_losses.len();
5493 self.state.quota.extend(quota_losses);
5494
5495 let mut records = Vec::new();
5496 let mut all_findings = Vec::new();
5497 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5498 let agent_id = seat.agent.clone();
5499 self.state.seats.insert(seat.key.clone(), seat);
5500 let mut record = ReviewRecord {
5501 reviewer: r + 1,
5502 agent: agent_id,
5503 summary: String::new(),
5504 findings: Vec::new(),
5505 vote: None,
5506 failed: None,
5507 duration_ms: 0,
5508 attempts,
5514 };
5515 match res {
5516 Ok((review, out)) => {
5517 record.summary =
5525 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5526 record.vote = Some(review.vote);
5527 record.duration_ms = out.duration_ms;
5528 for (n, mut f) in review.findings.into_iter().enumerate() {
5529 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
5532 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
5533 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
5534 f.file = f
5540 .file
5541 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
5542 all_findings.push(f.clone());
5543 record.findings.push(f);
5544 }
5545 self.state.event(
5546 "review",
5547 format!(
5548 "round {round}: reviewer {} voted {} with {} finding(s)",
5549 r + 1,
5550 review.vote.label(),
5551 record.findings.len()
5552 ),
5553 );
5554 }
5555 Err(e) => {
5556 record.failed = Some(e.to_string());
5557 self.state.event(
5558 "review",
5559 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
5560 );
5561 }
5562 }
5563 records.push(record);
5564 }
5565
5566 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
5573 let vote_split =
5574 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
5575 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
5576 if vote_split {
5577 self.state.event(
5578 "review",
5579 format!(
5580 "round {round}: votes split ({}) — one round of reconsideration",
5581 initial_votes
5582 .iter()
5583 .map(|v| v.label())
5584 .collect::<Vec<_>>()
5585 .join(", ")
5586 ),
5587 );
5588 let panel: Vec<ReviewSeatReport<'_>> = records
5591 .iter()
5592 .filter_map(|r| {
5593 r.vote.map(|vote| ReviewSeatReport {
5594 reviewer: r.reviewer,
5595 vote,
5596 summary: &r.summary,
5597 findings: &r.findings,
5598 })
5599 })
5600 .collect();
5601
5602 let mut jobs = Vec::new();
5603 let mut seats_at = Vec::new();
5604 for (r, spec) in reviewers.iter().cloned().enumerate() {
5605 if records[r].vote.is_none() {
5609 continue;
5610 }
5611 let wt = root.join(format!("review-{}", r + 1));
5612 let seat_key = format!("review-{}", r + 1);
5613 let spec = self.occupant(&seat_key, spec);
5614 let seat = self.seat(&seat_key, &spec.id);
5615 let build = |with_patch: bool| {
5623 prompt::review_reconsider(&ReviewReconsiderCtx {
5624 instruction: &self.state.instruction,
5625 reviewer: r + 1,
5626 lens: Lens::for_seat(r),
5627 panel: &panel,
5628 patch: with_patch.then_some(ReviewPatch {
5629 branch: &winner.branch,
5630 base_short: &base_short,
5631 stat: &stat,
5632 patch: &patch,
5633 }),
5634 round,
5635 rounds: max_rounds,
5636 language: &language,
5637 })
5638 };
5639 let full = build(true);
5640 let prompt = if has_context(&spec, &seat, sessions) {
5641 build(false)
5642 } else {
5643 full.clone()
5644 };
5645 jobs.push(SeatJob {
5646 prompt,
5647 spec,
5648 seat,
5649 cwd: wt,
5650 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5651 allow_write: false,
5652 sessions,
5653 artifacts: artifacts.clone(),
5654 stem: format!("review-{round}-reconsider-{}", r + 1),
5655 handover: Some(full),
5656 });
5657 seats_at.push(r);
5658 }
5659
5660 let mut recon_quota_losses = Vec::new();
5661 let recon_cache = self.state.config.cache_dir();
5662 let recon_ctx = WaveCtx {
5663 carry_seats: true,
5664 pauses: Some((&self.pause, &self.interrupt)),
5665 run: &run_id,
5666 node: "review",
5667 prompts: &prompts,
5668 cache: recon_cache.as_deref(),
5669 round: Some(round),
5670 };
5671 let recon_results = ask_json_wave::<ReviewRevote>(
5672 jobs,
5673 Arc::clone(&self.sem),
5674 review_retries,
5675 &self.roles.reviewer_roster,
5676 &recon_ctx,
5677 &mut recon_quota_losses,
5678 &mut self.state,
5679 &|_: &ReviewRevote| Ok(()),
5680 )
5681 .await;
5682 if self.state.resource_block.is_some() {
5683 return self.stop_for_resource();
5684 }
5685 self.state.quota.extend(recon_quota_losses);
5686
5687 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
5688 let agent_id = seat.agent.clone();
5689 self.state.seats.insert(seat.key.clone(), seat);
5690 let mut rec = ReviewRevoteRecord {
5691 reviewer: r + 1,
5692 agent: agent_id,
5693 vote: None,
5694 reason: String::new(),
5695 failed: None,
5696 };
5697 match res {
5698 Ok((rv, _)) => {
5699 rec.vote = Some(rv.vote);
5700 rec.reason =
5701 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
5702 self.state.event(
5703 "review",
5704 format!(
5705 "round {round}: reviewer {} revoted {}",
5706 r + 1,
5707 rv.vote.label()
5708 ),
5709 );
5710 }
5711 Err(e) => {
5712 rec.failed = Some(e.to_string());
5713 self.state.event(
5714 "review",
5715 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
5716 );
5717 }
5718 }
5719 reconsideration.push(rec);
5720 }
5721 } else if initial_votes.len() > 1 {
5722 self.state.event(
5723 "review",
5724 format!(
5725 "round {round}: votes agreed ({}) — no reconsideration",
5726 initial_votes[0].label()
5727 ),
5728 );
5729 }
5730
5731 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
5732 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5733 let defer_e2e =
5744 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
5745 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
5746 let reason =
5747 format!("{blocking} blocking finding(s) already required a fix this round");
5748 self.state.event(
5749 "verify",
5750 format!(
5751 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
5752 {}); it will run once a round has none left",
5753 short(&head)
5754 ),
5755 );
5756 (Vec::new(), false, true, Some(reason))
5757 } else {
5758 let e2e_commands = self.state.config.verify.e2e.clone();
5759 let cache_dir = self.state.config.cache_dir();
5760 let context = format!("round {round}");
5761 let (e2e, verify_retried) = with_cache_lease(
5762 &mut self.state,
5763 cache_dir.as_deref(),
5764 "e2e",
5765 "e2e",
5766 &winner.worktree,
5767 &head,
5768 verify_timeout,
5769 &context,
5770 |state, budget| {
5771 let shell = shell.clone();
5772 let e2e_commands = e2e_commands.clone();
5773 let worktree = winner.worktree.clone();
5774 let context = context.clone();
5775 async move {
5776 run_e2e_with_retry(
5777 state,
5778 &shell,
5779 &e2e_commands,
5780 &worktree,
5781 budget,
5782 &context,
5783 )
5784 .await
5785 }
5786 },
5787 )
5788 .await;
5789 (e2e, verify_retried, false, None)
5790 };
5791
5792 let expected = records.len();
5793 let answered = records.iter().filter(|r| r.failed.is_none()).count();
5794 let incomplete = answered < expected;
5795 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
5796 let policy = self.state.config.graph.incomplete_review;
5797 let clean = round_is_clean(
5798 blocking,
5799 e2e_ok,
5800 answered,
5801 expected,
5802 round_quota_missing,
5803 policy,
5804 );
5805
5806 let mut round_record = ReviewRound {
5807 round,
5808 head: head.clone(),
5809 verified_head: None,
5810 verified_at: None,
5811 reviews: records,
5812 e2e,
5813 verify_retried,
5814 e2e_deferred,
5815 e2e_defer_reason,
5816 fix: None,
5817 blocking,
5818 answered,
5819 expected,
5820 clean,
5821 progressed: false,
5822 vote_split,
5823 reconsideration,
5824 verdict: None,
5825 };
5826 round_record.verdict = ReviewVote::worst(
5830 round_record
5831 .final_votes()
5832 .into_iter()
5833 .map(|(_, _, vote)| vote),
5834 );
5835 if !matches!(
5846 round_record.e2e_status(),
5847 E2eStatus::Deferred | E2eStatus::NotConfigured
5848 ) {
5849 round_record.verified_head = Some(head.clone());
5850 round_record.verified_at = Some(Timestamp::now());
5851 }
5852 let this_round_verification = round_record.verification_summary(&head);
5853
5854 if incomplete {
5855 let missing: Vec<String> = round_record
5856 .reviews
5857 .iter()
5858 .filter(|r| r.failed.is_some())
5859 .map(|r| format!("review-{}", r.reviewer))
5860 .collect();
5861 self.state.event(
5862 "review",
5863 format!(
5864 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
5865 missing.join(", ")
5866 ),
5867 );
5868 }
5869
5870 if clean {
5871 self.state.event(
5872 "review",
5873 if incomplete && policy == IncompleteReviewPolicy::Warn {
5874 format!(
5875 "round {round}: clean (warn policy, incomplete panel) — no \
5876 blocking findings from the seats that answered, verification green"
5877 )
5878 } else if incomplete {
5879 format!(
5880 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
5881 quorum) — no blocking findings from the seats that answered, \
5882 verification green",
5883 expected - answered
5884 )
5885 } else {
5886 format!("round {round}: clean — no blocking findings, verification green")
5887 },
5888 );
5889 self.state.reviews.push(round_record);
5890 self.state.status = RunStatus::Gating;
5891 self.state.save()?;
5892 return Ok(());
5893 }
5894
5895 if incomplete && blocking == 0 && e2e_ok {
5903 self.state.reviews.push(round_record);
5904 self.state.save()?;
5905 if round == max_rounds {
5906 self.state.status = RunStatus::Blocked;
5907 self.state.event(
5908 "review",
5909 format!(
5910 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
5911 refusing to call it clean",
5912 expected - answered
5913 ),
5914 );
5915 return Ok(());
5916 }
5917 continue;
5918 }
5919
5920 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
5932 self.state.reviews.push(round_record);
5933 return self
5934 .stop_reviewing(
5935 "the round's own verification could not run",
5936 &shell,
5937 &winner.worktree,
5938 )
5939 .await;
5940 }
5941
5942 if round == max_rounds {
5943 self.state.reviews.push(round_record);
5944 return self
5945 .stop_reviewing(
5946 &format!(
5947 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
5948 ),
5949 &shell,
5950 &winner.worktree,
5951 )
5952 .await;
5953 }
5954
5955 let blocking_findings: Vec<_> = all_findings
5958 .iter()
5959 .filter(|f| f.severity.blocks())
5960 .cloned()
5961 .collect();
5962 let fix_prompt = prompt::fix(
5963 &self.state.instruction,
5964 &blocking_findings,
5965 this_round_verification.as_ref(),
5966 round,
5967 max_rounds,
5968 &language,
5969 );
5970 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5971 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5972 let (job, seat, out) = self
5973 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
5974 prompt: fix_prompt.clone(),
5975 spec,
5976 seat,
5977 cwd: winner.worktree.clone(),
5978 timeout,
5979 allow_write: true,
5980 sessions,
5981 artifacts: artifacts.clone(),
5982 stem: format!("fix-{round}"),
5983 handover: None,
5984 })
5985 .await;
5986 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5989 return self.stop_for_resource();
5990 }
5991 let agent_id = seat.agent.clone();
5992
5993 let mut fix = FixRecord {
5994 agent: agent_id,
5995 addressed: Vec::new(),
5996 rejected: Vec::new(),
5997 notes: String::new(),
5998 committed: false,
5999 failed: None,
6000 duration_ms: 0,
6001 continuation: None,
6002 };
6003 let mut continuation = ContinuationRecord::not_needed();
6004 let mut final_seat = seat.clone();
6005 match out {
6006 AgentOutcome::Ok(o) => {
6007 fix.duration_ms = o.duration_ms;
6008 let parsed = verdict::extract_json::<FixReport>(&o.text);
6009 let incomplete_reason = match &parsed {
6016 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6017 "the reply parsed, but it reported a command whose own CLI never \
6018 confirmed an exit status"
6019 .to_owned(),
6020 ),
6021 Ok(_) => None,
6022 Err(e) => Some(e.to_string()),
6023 };
6024 match incomplete_reason {
6025 None => {
6026 let report = parsed.expect("checked Ok above");
6027 fix.addressed = report.addressed;
6028 fix.rejected = report.rejected;
6029 fix.notes =
6030 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6031 }
6032 Some(reason) => {
6033 let (resumed_seat, resolved, failure, cont) = self
6034 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6035 .await;
6036 fix.duration_ms += cont.cumulative_wait_ms;
6037 continuation = cont;
6038 final_seat = resumed_seat;
6039 match resolved {
6040 Some(report) => {
6041 fix.addressed = report.addressed;
6042 fix.rejected = report.rejected;
6043 fix.notes = blind::sanitize_prose(
6044 &report.notes,
6045 &self.state.config.blind,
6046 );
6047 }
6048 None => fix.failed = failure,
6049 }
6050 }
6051 }
6052 }
6053 AgentOutcome::Dropped(o) => {
6055 fix.duration_ms = o.duration_ms;
6056 let why = o
6057 .dropped
6058 .as_ref()
6059 .map(|d| d.why.as_str())
6060 .unwrap_or("the CLI ended the stream without delivering its answer");
6061 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6062 }
6063 AgentOutcome::Quota(o) => {
6064 self.state.quota.push(QuotaLoss {
6065 seat: final_seat.key.clone(),
6066 node: "fix".to_owned(),
6067 at: Timestamp::now(),
6068 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6069 });
6070 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6071 }
6072 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6073 }
6074 fix.continuation = Some(continuation);
6075 self.state.seats.insert(final_seat.key.clone(), final_seat);
6076 if let Ok(r) = git::rescue_commit(
6077 &winner.worktree,
6078 &format!("magi: review round {round} fixes (uncommitted work)"),
6079 )
6080 .await
6081 {
6082 self.state.note_withheld("fix", &r.withheld);
6083 }
6084 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6085 fix.committed = after != before;
6086 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6094 let progressed = diff_after != patch;
6095 let commit_note = if fix.committed {
6096 "committed"
6097 } else {
6098 "NO new commit"
6099 };
6100 let tree_note = if progressed {
6101 "changed vs base"
6102 } else {
6103 "unchanged vs base"
6104 };
6105 self.state.event(
6106 "fix",
6107 match &fix.failed {
6108 Some(reason) => {
6114 format!(
6115 "round {round}: fixer's adoption report was lost ({reason}); \
6116 {commit_note}, tree {tree_note}"
6117 )
6118 }
6119 None => format!(
6120 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6121 {tree_note}{}",
6122 fix.addressed.len(),
6123 fix.rejected.len(),
6124 if continuation.outcome == ContinuationOutcome::Resumed {
6125 format!(
6126 " (adoption report recovered after {} continuation(s))",
6127 continuation.attempts
6128 )
6129 } else {
6130 String::new()
6131 },
6132 ),
6133 },
6134 );
6135 round_record.fix = Some(fix);
6136 round_record.progressed = progressed;
6137 self.state.reviews.push(round_record);
6138 self.state.save()?;
6139
6140 if matches!(
6153 continuation.outcome,
6154 ContinuationOutcome::Exhausted
6155 | ContinuationOutcome::QuotaLost
6156 | ContinuationOutcome::NoSession
6157 ) {
6158 return self
6159 .stop_reviewing(
6160 "the fixer's adoption report never came back, even after resuming its \
6161 own seat; refusing to start another round against the same worktree \
6162 while that is unresolved",
6163 &shell,
6164 &winner.worktree,
6165 )
6166 .await;
6167 }
6168
6169 let streak = self
6170 .state
6171 .reviews
6172 .iter()
6173 .rev()
6174 .take_while(|r| !r.progressed)
6175 .count();
6176 if streak >= STAGNANT_LIMIT {
6177 return self
6178 .stop_reviewing(
6179 &format!(
6180 "the tree has not moved against base for {streak} round(s) in a row"
6181 ),
6182 &shell,
6183 &winner.worktree,
6184 )
6185 .await;
6186 }
6187 }
6188 Ok(())
6189 }
6190
6191 fn record_contested_handoff(&mut self) {
6225 if self.state.contested_handoff.is_some() {
6226 return;
6227 }
6228 let Some(contested) = self
6229 .state
6230 .reviews
6231 .last()
6232 .and_then(ReviewRound::contested_handoff)
6233 else {
6234 return;
6235 };
6236 self.state.event(
6237 "review",
6238 format!(
6239 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6240 wait for the owner's approval",
6241 contested.findings.len(),
6242 contested.rejecters.len()
6243 ),
6244 );
6245 self.state.contested_handoff = Some(contested);
6246 }
6247
6248 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6249 let round_idx = self.state.reviews.len() - 1;
6250 let needs_catchup_run = matches!(
6258 self.state.reviews[round_idx].e2e_status(),
6259 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6260 );
6261 if needs_catchup_run {
6262 let round = self.state.reviews[round_idx].round;
6263 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6264 let commands = self.state.config.verify.e2e.clone();
6265 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6266 let cache_dir = self.state.config.cache_dir();
6267 let context = format!(
6268 "round {round}: verification unresolved, catching up before the final decision"
6269 );
6270 let (outcomes, verify_retried) = with_cache_lease(
6271 &mut self.state,
6272 cache_dir.as_deref(),
6273 "e2e",
6274 "e2e",
6275 worktree,
6276 &attempted_head,
6277 timeout,
6278 &context,
6279 |state, budget| {
6280 let shell = shell.to_vec();
6281 let commands = commands.clone();
6282 let context = context.clone();
6283 async move {
6284 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6285 .await
6286 }
6287 },
6288 )
6289 .await;
6290 let last = &mut self.state.reviews[round_idx];
6291 last.e2e = outcomes;
6292 last.verify_retried = verify_retried;
6293 last.verified_head = Some(attempted_head);
6300 last.verified_at = Some(Timestamp::now());
6301 if verify_inconclusive(&last.e2e) {
6302 self.state.save()?;
6309 return Ok(());
6310 }
6311 last.e2e_deferred = false;
6312 }
6313 let last = &self.state.reviews[round_idx];
6314 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6315
6316 match last.e2e_status() {
6317 E2eStatus::Failed => {
6318 let red: Vec<String> = last
6319 .e2e
6320 .iter()
6321 .filter(|o| !o.ok())
6322 .map(|o| {
6323 format!(
6324 "`{}` -> {:?}\n{}",
6325 o.command,
6326 o.code,
6327 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6328 )
6329 })
6330 .collect();
6331 self.state
6332 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6333 self.state.status = RunStatus::Blocked;
6334 }
6335 E2eStatus::ResourceBlocked => {
6340 self.state.event(
6341 "review",
6342 format!(
6343 "{why}; e2e could not run (shared build cache unavailable); not \
6344 deciding yet"
6345 ),
6346 );
6347 }
6348 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6349 self.state.event(
6350 "review",
6351 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6352 );
6353 self.record_contested_handoff();
6354 self.state.status = RunStatus::Gating;
6355 }
6356 }
6357 self.state.save()?;
6358 Ok(())
6359 }
6360
6361 async fn gate(&mut self) -> Result<()> {
6364 if self.state.status == RunStatus::Failed
6376 || self
6377 .state
6378 .base_sync
6379 .as_ref()
6380 .is_some_and(|s| s.conflict.is_some())
6381 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6382 != Some(RunStatus::Gating)
6383 {
6384 return Ok(());
6385 }
6386 if self.state.gate_ran {
6387 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6398 self.state.status = RunStatus::Blocked;
6399 self.state.save()?;
6400 }
6401 return Ok(());
6402 }
6403 let Some(winner) = self.state.winner().cloned() else {
6404 return Ok(());
6405 };
6406 self.state.status = RunStatus::Gating;
6407 let mut outcomes = self.run_gate(&winner).await?;
6408 loop {
6409 if verify_inconclusive(&outcomes) {
6420 self.state.save()?;
6421 return Ok(());
6422 }
6423 if outcomes.iter().all(CommandOutcome::ok) {
6424 break;
6425 }
6426 match self.gate_fix_round(&winner, &outcomes).await? {
6427 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6428 GateFix::Stop => break,
6429 GateFix::Defer => {
6430 self.state.save()?;
6431 return Ok(());
6432 }
6433 }
6434 }
6435 let passed = outcomes.iter().all(CommandOutcome::ok);
6436 self.state.gate = outcomes;
6437 self.state.gate_ran = true;
6438 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6439 if !passed {
6440 self.state.status = RunStatus::Blocked;
6441 let spent = self.state.gate_fixes.len();
6442 self.state.event(
6443 "gate",
6444 if spent == 0 {
6445 "gate failed; not merging".to_owned()
6446 } else {
6447 format!("gate failed after {spent} gate-fix round(s); not merging")
6448 },
6449 );
6450 }
6451 self.state.save()?;
6452 Ok(())
6453 }
6454
6455 async fn run_pre_gate(&mut self, winner: &Candidate) {
6465 let commands = self.state.config.verify.pre_gate.clone();
6466 if commands.is_empty() {
6467 return;
6468 }
6469 let shell = self.state.config.shell();
6470 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6471 let (outcomes, _) = run_commands(
6472 &mut self.state,
6473 "pre_gate",
6474 "pre_gate",
6475 0,
6476 &shell,
6477 &commands,
6478 &winner.worktree,
6479 timeout,
6480 )
6481 .await;
6482 for o in &outcomes {
6483 if !o.ok() {
6484 tracing::warn!(
6485 "pre_gate `{}` failed ({:?}); the gate decides",
6486 o.command,
6487 o.code
6488 );
6489 }
6490 self.state.event(
6491 "pre_gate",
6492 format!(
6493 "`{}` -> {}",
6494 o.command,
6495 if o.ok() {
6496 "pass".to_owned()
6497 } else {
6498 format!(
6499 "FAIL ({:?})\n{}",
6500 o.code,
6501 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6502 )
6503 }
6504 ),
6505 );
6506 }
6507 self.state.pre_gate = outcomes;
6508 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6509 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6510 Ok(head) => {
6511 self.state
6512 .event("pre_gate", format!("committed mechanical fixes ({head})"));
6513 self.state.pre_gate_commit = Some(head);
6514 }
6515 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6516 },
6517 Ok(false) => {}
6518 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6519 }
6520 if let Err(e) = self.state.save() {
6521 tracing::warn!("could not persist the pre_gate record: {e:#}");
6522 }
6523 }
6524
6525 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
6528 self.run_pre_gate(winner).await;
6529 let shell = self.state.config.shell();
6530 let gate_commands = self.state.config.verify.gate.clone();
6531 let outcomes = if gate_commands.is_empty() {
6540 Vec::new()
6541 } else {
6542 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6543 let cache_dir = self.state.config.cache_dir();
6544 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6545 let (outcomes, _) = with_cache_lease(
6546 &mut self.state,
6547 cache_dir.as_deref(),
6548 "gate",
6549 "gate",
6550 &winner.worktree,
6551 &head,
6552 timeout,
6553 "final gate",
6554 |state, budget| {
6555 let shell = shell.clone();
6556 let gate_commands = gate_commands.clone();
6557 let worktree = winner.worktree.clone();
6558 async move {
6559 let (outcomes, timed_out_pids) = run_commands(
6560 state,
6561 "gate",
6562 "gate",
6563 0,
6564 &shell,
6565 &gate_commands,
6566 &worktree,
6567 budget,
6568 )
6569 .await;
6570 (outcomes, false, timed_out_pids)
6571 }
6572 },
6573 )
6574 .await;
6575 outcomes
6576 };
6577 if outcomes.is_empty() {
6578 self.state.event(
6583 "gate",
6584 "no gate commands configured; nothing to check, passing",
6585 );
6586 }
6587 for o in &outcomes {
6588 self.state.event(
6589 "gate",
6590 format!(
6591 "`{}` -> {}",
6592 o.command,
6593 if o.ok() {
6594 "pass".to_owned()
6595 } else {
6596 format!(
6597 "FAIL ({:?})\n{}",
6598 o.code,
6599 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6600 )
6601 }
6602 ),
6603 );
6604 }
6605 Ok(outcomes)
6606 }
6607
6608 async fn gate_fix_round(
6621 &mut self,
6622 winner: &Candidate,
6623 outcomes: &[CommandOutcome],
6624 ) -> Result<GateFix> {
6625 let cap = self.state.config.graph.gate_fix_rounds;
6626 let spent = self.state.gate_fixes.len();
6627 if spent >= cap {
6628 if cap > 0 {
6629 self.state.event(
6630 "gate",
6631 format!("{spent} gate-fix round(s) spent and the gate still fails"),
6632 );
6633 }
6634 return Ok(GateFix::Stop);
6635 }
6636 if !gate_fixable(outcomes) {
6637 self.state.event(
6638 "gate",
6639 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
6640 command or similar); not spending a fix round on it",
6641 );
6642 return Ok(GateFix::Stop);
6643 }
6644 let min_free = self.state.config.disk.min_free_bytes;
6645 if min_free > 0 {
6646 match crate::disk::free_bytes(&winner.worktree) {
6647 Ok(free) if crate::disk::enough_space(free, min_free) => {}
6648 Ok(free) => {
6649 self.state.event(
6650 "gate",
6651 format!(
6652 "only {free} bytes free ({min_free} required by `[disk] \
6653 min_free_bytes`); not spending a fix round on a failure the disk \
6654 may explain"
6655 ),
6656 );
6657 return Ok(GateFix::Stop);
6658 }
6659 Err(e) => {
6660 self.state.event(
6661 "gate",
6662 format!("free disk space could not be measured ({e:#}); no fix round"),
6663 );
6664 return Ok(GateFix::Stop);
6665 }
6666 }
6667 }
6668
6669 let attempt = spent + 1;
6670 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
6671 let base = self.landing_base();
6672 let fix_prompt = prompt::gate_fix(
6673 &self.state.instruction,
6674 &failed,
6675 attempt,
6676 cap,
6677 &self.state.config.graph.language,
6678 );
6679 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6680 let sessions = self.state.config.graph.sessions;
6681 let artifacts = agent::artifacts_dir(&self.state.dir());
6682 self.state.event(
6683 "gate",
6684 format!("gate failed; gate-fix round {attempt} of {cap}"),
6685 );
6686 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6687 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6688 let (_, seat, out) = self
6689 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
6690 prompt: fix_prompt.clone(),
6691 spec,
6692 seat,
6693 cwd: winner.worktree.clone(),
6694 timeout,
6695 allow_write: true,
6696 sessions,
6697 artifacts: artifacts.clone(),
6698 stem: format!("gate-fix-{attempt}"),
6699 handover: None,
6700 })
6701 .await;
6702 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6705 return Ok(GateFix::Defer);
6706 }
6707 let mut record = GateFixRecord {
6708 agent: seat.agent.clone(),
6709 failed,
6710 notes: String::new(),
6711 committed: false,
6712 error: None,
6713 };
6714 match out {
6715 AgentOutcome::Ok(o) => {
6716 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
6719 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6720 }
6721 }
6722 AgentOutcome::Dropped(_) => {
6723 record.error = Some("the CLI dropped the stream".to_owned());
6724 }
6725 AgentOutcome::Quota(o) => {
6726 self.state.quota.push(QuotaLoss {
6727 seat: seat.key.clone(),
6728 node: "gate-fix".to_owned(),
6729 at: Timestamp::now(),
6730 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6731 });
6732 record.error = Some("rate limited (quota); fixer could not run".to_owned());
6733 }
6734 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
6735 }
6736 self.state.seats.insert(seat.key.clone(), seat);
6737 if let Ok(r) = git::rescue_commit(
6738 &winner.worktree,
6739 &format!("magi: gate fix {attempt} (uncommitted work)"),
6740 )
6741 .await
6742 {
6743 self.state.note_withheld("gate-fix", &r.withheld);
6744 }
6745 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6746 record.committed = after != before;
6747 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
6748 let note = record.error.clone();
6749 self.state.gate_fixes.push(record);
6750 self.state.save()?;
6751 if !changed {
6752 self.state.event(
6753 "gate",
6754 match note {
6755 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
6756 None => format!("gate-fix round {attempt}: the tree did not change"),
6757 },
6758 );
6759 return Ok(GateFix::Stop);
6760 }
6761 self.state.event(
6762 "gate",
6763 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
6764 );
6765
6766 let commands = self.state.config.verify.e2e.clone();
6767 if !commands.is_empty() {
6768 let shell = self.state.config.shell();
6769 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6770 let cache_dir = self.state.config.cache_dir();
6771 let context = format!("gate-fix round {attempt}");
6772 let (e2e, _) = with_cache_lease(
6773 &mut self.state,
6774 cache_dir.as_deref(),
6775 "e2e",
6776 "e2e",
6777 &winner.worktree,
6778 &after,
6779 timeout,
6780 &context,
6781 |state, budget| {
6782 let shell = shell.clone();
6783 let commands = commands.clone();
6784 let context = context.clone();
6785 let worktree = winner.worktree.clone();
6786 async move {
6787 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
6788 .await
6789 }
6790 },
6791 )
6792 .await;
6793 if verify_inconclusive(&e2e) {
6794 return Ok(GateFix::Defer);
6795 }
6796 if e2e.iter().any(|o| !o.ok()) {
6797 self.state.event(
6798 "gate",
6799 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
6800 );
6801 return Ok(GateFix::Stop);
6802 }
6803 }
6804 Ok(GateFix::Retry)
6805 }
6806
6807 async fn guarded_pr_message(
6814 &mut self,
6815 winner: &Candidate,
6816 facts: Option<&BranchFacts>,
6817 posting: bool,
6818 ) -> Result<Option<PrMessage>> {
6819 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6820 let head = match final_head.as_deref() {
6821 Some(h) => crate::pr_outcome::FinalHead::At(h),
6822 None => crate::pr_outcome::FinalHead::Unknown,
6823 };
6824 let mut pr = pr_message_raw(&self.state, winner.label, facts, head);
6825 if !posting {
6826 let identity = crate::scrub::Identity::current();
6827 return Ok(Some(PrMessage {
6828 title: crate::scrub::scrub(&pr.title, &identity),
6829 body: crate::scrub::scrub(&pr.body, &identity),
6830 }));
6831 }
6832 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
6835 if let Some(known) = self
6836 .state
6837 .github_text
6838 .clone()
6839 .filter(|g| g.fingerprint == raw_fp)
6840 {
6841 return self.settle_withheld(winner, &pr, known).await;
6842 }
6843 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
6844 let mut violations = crate::github_text::check_with(&pr.title, &pr.body, decision);
6845 let identity = crate::scrub::Identity::current();
6846 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
6847 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
6848 && !violations.contains(&crate::github_text::Violation::SensitiveData)
6849 {
6850 violations.push(crate::github_text::Violation::SensitiveData);
6851 }
6852 let has_language = violations
6856 .iter()
6857 .any(|v| *v != crate::github_text::Violation::SensitiveData);
6858 if self.state.config.graph.github_text_guard && !violations.is_empty() {
6859 self.state.event(
6860 "github-text",
6861 if has_language {
6862 format!("description rejected: {violations:?}; requesting one rewrite")
6863 } else {
6864 format!("description rejected: {violations:?}; asking the owner")
6865 },
6866 );
6867 let mut rewritten = false;
6868 let mut rejected_rewrite = None;
6869 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
6870 {
6871 let output = text;
6872 if let Some(title) = summary_title(&output) {
6873 let raw_body = summary_without_title(&output);
6876 let clean_title = crate::scrub::scrub(&title, &identity);
6877 let mut body = crate::scrub::scrub(&raw_body, &identity);
6878 let redacted = clean_title != title || body != raw_body;
6879 let title = clean_title;
6880 let marker = format!("magi:run/{}", self.state.id);
6881 if !body.trim().is_empty() && !body.contains(&marker) {
6882 body.push_str(&format!("\n\n{marker}"));
6883 }
6884 let rewrite = PrMessage {
6885 title: title.clone(),
6886 body: body.clone(),
6887 };
6888 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
6889 let mut failed = crate::github_text::check_with(&title, &body, redecision);
6890 if (!crate::github_text::shareable(&title)
6891 || !crate::github_text::shareable(&body))
6892 && !failed.contains(&crate::github_text::Violation::SensitiveData)
6893 {
6894 failed.push(crate::github_text::Violation::SensitiveData);
6895 }
6896 if !title.trim().is_empty() && !body.trim().is_empty() && !failed.is_empty() {
6897 rejected_rewrite = Some(crate::run::GithubTextRewrite {
6898 title: title.clone(),
6899 body: body.clone(),
6900 failed: failed
6901 .iter()
6902 .map(|v| crate::github_text::category(*v).to_owned())
6903 .collect(),
6904 });
6905 }
6906 if !title.trim().is_empty() && !body.trim().is_empty() && failed.is_empty() {
6907 decision = redecision;
6908 pr = PrMessage { title, body };
6909 rewritten = true;
6910 self.state
6911 .event("github-text", "description rewrite passed");
6912 if redacted {
6913 self.state.event(
6914 "github-text",
6915 "sensitive data removed from description rewrite",
6916 );
6917 }
6918 }
6919 }
6920 }
6921 if !rewritten {
6922 self.state.event(
6923 "github-text",
6924 "description rewrite unavailable or rejected; asking the owner",
6925 );
6926 if let Some(w) =
6927 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
6928 {
6929 self.retire_github_text_question("the withheld text changed");
6932 let record = crate::run::GithubTextAsk {
6933 rewrite: rejected_rewrite,
6934 fingerprint: raw_fp,
6935 title: w.title,
6936 body: w.body,
6937 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
6938 question: None,
6939 asks: 0,
6940 resolved: false,
6941 chosen_title: None,
6942 choice: None,
6943 };
6944 return self.settle_withheld(winner, &pr, record).await;
6945 }
6946 }
6947 }
6948 let (title, body) =
6949 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
6950 Ok(Some(PrMessage { title, body }))
6951 }
6952
6953 async fn rewrite_pr_text(
6961 &mut self,
6962 winner: &Candidate,
6963 pr: &PrMessage,
6964 violations: &[crate::github_text::Violation],
6965 ) -> Option<String> {
6966 let prompt = format!(
6967 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the complete Markdown body. Preserve the magi run marker.\n\nTITLE: {}\n{}",
6968 pr.title, pr.body
6969 );
6970 let sessions = self.state.config.graph.sessions;
6971 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
6972 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
6974 let key = format!("impl-{}", winner.label);
6975 if let Some(spec) = self
6976 .state
6977 .config
6978 .agents
6979 .iter()
6980 .find(|a| a.id == winner.agent)
6981 .cloned()
6982 && let Some(existing) = self.state.seats.get(&key)
6983 && existing.agent == spec.id
6984 && agent::has_session(spec.kind, existing, sessions)
6985 {
6986 attempts.push((spec, existing.clone(), true));
6987 } else {
6988 let chain = agent::pick_chain(
6989 &self.state.config.agents,
6990 self.state.config.roles.language_judge.as_ref(),
6991 &agent::installed,
6992 "language_judge",
6993 );
6994 match chain {
6995 Ok(chain) => {
6996 for spec in chain {
6997 let seat = crate::fixer::seat_for(
6998 &mut self.state,
6999 "github-text-rewrite",
7000 &spec.id,
7001 );
7002 attempts.push((spec, seat, false));
7003 }
7004 }
7005 Err(e) => {
7006 tracing::warn!("no agent for the description rewrite: {e:#}");
7007 self.state
7008 .event("github-text", "no rewrite agent available");
7009 return None;
7010 }
7011 }
7012 }
7013 let prompts = self.state.config.prompts.clone();
7014 let cache = self.state.config.cache_dir();
7015 let run = self.state.id.clone();
7016 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7017 let job = SeatJob {
7018 spec: spec.clone(),
7019 seat,
7020 cwd: winner.worktree.clone(),
7021 prompt: prompt.clone(),
7022 timeout: retry_budget(timeout_full, resumed),
7023 allow_write: false,
7024 sessions,
7025 artifacts: agent::artifacts_dir(&self.state.dir()),
7026 stem: if i == 0 {
7027 "github-text-rewrite".to_owned()
7028 } else {
7029 format!("github-text-rewrite-{}", spec.id)
7030 },
7031 handover: None,
7032 };
7033 let ctx = WaveCtx {
7034 run: &run,
7035 node: "github-text",
7036 prompts: &prompts,
7037 cache: cache.as_deref(),
7038 round: None,
7039 carry_seats: false,
7040 pauses: Some((&self.pause, &self.interrupt)),
7041 };
7042 let (seat, outcome) =
7043 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7044 self.state.seats.insert(seat.key.clone(), seat);
7045 match outcome {
7046 AgentOutcome::Ok(o)
7047 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7048 {
7049 return Some(o.text);
7050 }
7051 AgentOutcome::ResourceBlocked(_) => return None,
7052 _ => {}
7053 }
7054 }
7055 None
7056 }
7057
7058 fn retire_github_text_question(&mut self, why: &str) {
7060 let Some(id) = self
7061 .state
7062 .github_text
7063 .as_ref()
7064 .and_then(|g| g.question.clone())
7065 else {
7066 return;
7067 };
7068 let _ = ask::Questions::open().update(&id, |q| {
7069 q.abandon(why);
7070 Ok(())
7071 });
7072 }
7073
7074 async fn settle_withheld(
7078 &mut self,
7079 winner: &Candidate,
7080 pr: &PrMessage,
7081 mut g: crate::run::GithubTextAsk,
7082 ) -> Result<Option<PrMessage>> {
7083 use crate::github_text as gt;
7084 let store = ask::Questions::open();
7085 let timeout = self.state.config.graph.answer_timeout;
7086 if !g.resolved {
7087 let mut standing = g
7088 .question
7089 .as_ref()
7090 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7091 if let Some(q) = standing.as_ref()
7092 && q.status.open()
7093 && gt::expired(q, timeout)
7094 {
7095 let why = format!("no answer within {}s of asking", timeout.max(1));
7096 standing = store
7097 .update(&q.id, |q| {
7098 q.abandon(&why);
7099 Ok(())
7100 })
7101 .ok()
7102 .map(|(q, ())| q);
7103 }
7104 let mut ask_again: Option<Option<Vec<String>>> = None;
7105 match standing {
7106 None if g.asks <= 1 => {
7109 g.asks = 0;
7110 ask_again = Some(None);
7111 }
7112 None => {
7113 g.resolved = true;
7114 g.chosen_title = None;
7115 g.choice = Some(GithubTextChoice::Fallback);
7116 }
7117 Some(q) if q.status.open() => {
7118 self.park_on_github_text(&g)?;
7119 return Ok(None);
7120 }
7121 Some(q) => match gt::read_reply(&q) {
7122 gt::Reply::Fallback => {
7123 g.resolved = true;
7124 g.chosen_title = None;
7125 g.choice = Some(GithubTextChoice::Fallback);
7126 }
7127 gt::Reply::Original => {
7128 g.resolved = true;
7129 g.chosen_title = None;
7130 g.choice = Some(GithubTextChoice::Original);
7131 }
7132 gt::Reply::Rewrite => {
7133 g.resolved = true;
7134 g.chosen_title = None;
7135 g.choice = Some(if g.rewrite.is_some() {
7136 GithubTextChoice::Rewrite
7137 } else {
7138 GithubTextChoice::Fallback
7139 });
7140 }
7141 gt::Reply::Title(text) => {
7142 let first = text
7143 .lines()
7144 .map(str::trim)
7145 .find(|l| !l.is_empty())
7146 .unwrap_or("");
7147 let decision = if first.is_empty() {
7148 None
7149 } else {
7150 let candidate = PrMessage {
7151 title: first.to_owned(),
7152 body: String::new(),
7153 };
7154 self.judge_pr_language(&winner.worktree, &candidate).await
7155 };
7156 match gt::vet_title(&text, decision) {
7157 Ok(title) => {
7158 g.resolved = true;
7159 g.chosen_title = Some(title);
7160 g.choice = Some(GithubTextChoice::MyTitle);
7161 }
7162 Err(cats) if g.asks < 2 => {
7163 ask_again = Some(Some(
7164 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7165 ))
7166 }
7167 Err(_) => {
7168 g.resolved = true;
7169 g.chosen_title = None;
7170 g.choice = Some(GithubTextChoice::Fallback);
7171 }
7172 }
7173 }
7174 },
7175 }
7176 if let Some(rejected) = ask_again {
7177 let w = gt::Withheld {
7180 title: g.title,
7181 body: g.body,
7182 categories: g.categories.clone(),
7183 };
7184 let artifact = match crate::run::write_artifact(
7187 &self.state,
7188 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7189 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7190 ) {
7191 Ok(p) => Some(p),
7192 Err(e) => {
7193 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7194 None
7195 }
7196 };
7197 let mut q = ask::Question::new(
7198 self.state.id.clone(),
7199 gt::ASK_NODE.to_owned(),
7200 gt::ASK_SEAT.to_owned(),
7201 gt::question_summary(&self.state.config.graph.language, &w),
7202 gt::question_detail(
7203 &w,
7204 &pr.title,
7205 &pr.body,
7206 &self.state.id,
7207 rejected.as_deref(),
7208 artifact.as_deref(),
7209 g.rewrite.as_ref(),
7210 ),
7211 gt::question_choices(&w, g.rewrite.as_ref()),
7212 );
7213 q.answer_timeout = timeout;
7214 g.question = Some(q.id.clone());
7217 g.asks += 1;
7218 self.state.github_text = Some(g.clone());
7219 self.state.event(
7220 "github-text",
7221 format!("asking the owner about the withheld text ({})", q.short()),
7222 );
7223 self.state.save()?;
7224 store
7225 .put(&mut q)
7226 .context("file the posting-gate question")?;
7227 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7228 tracing::warn!(
7229 "could not notify about posting-gate question {}: {e:#}",
7230 q.short()
7231 );
7232 }
7233 self.park_on_github_text(&g)?;
7234 return Ok(None);
7235 }
7236 }
7237 let resolved = g.chosen_title.clone();
7238 let original = g.choice == Some(GithubTextChoice::Original);
7239 self.state.event(
7241 "github-text",
7242 if original {
7243 format!(
7244 "posting the original text for: {}",
7245 match (g.title, g.body) {
7246 (true, true) => "title, description",
7247 (true, false) => "title",
7248 _ => "description",
7249 }
7250 )
7251 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7252 "posting the saved rewrite title and description".to_owned()
7253 } else if resolved.is_some() {
7254 "posting the owner's replacement title".to_owned()
7255 } else {
7256 "posting the neutral text for the withheld fields".to_owned()
7257 },
7258 );
7259 self.state.github_text = Some(g.clone());
7260 self.state.save()?;
7261 if g.choice == Some(GithubTextChoice::Rewrite)
7262 && let Some(r) = g.rewrite.as_ref()
7263 {
7264 return Ok(Some(PrMessage {
7265 title: r.title.clone(),
7266 body: r.body.clone(),
7267 }));
7268 }
7269 let w = gt::Withheld {
7270 title: g.title,
7271 body: g.body,
7272 categories: g.categories.clone(),
7273 };
7274 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7275 if let Some(t) = resolved {
7276 title = t;
7277 }
7278 let english = gt::LanguageDecision {
7280 title_english: true,
7281 body_english: true,
7282 body_complete: true,
7283 };
7284 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7285 if original {
7286 if g.title {
7289 title = pr.title.clone();
7290 }
7291 if g.body {
7292 body = pr.body.clone();
7293 }
7294 }
7295 Ok(Some(PrMessage { title, body }))
7296 }
7297
7298 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7299 self.state.github_text = Some(g.clone());
7300 self.state.end_pass_parked(ParkCause::Question);
7301 self.state.parked = true;
7302 self.state.event(
7303 "github-text",
7304 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7305 );
7306 self.state.save()
7307 }
7308
7309 async fn judge_pr_language(
7312 &mut self,
7313 cwd: &Path,
7314 pr: &PrMessage,
7315 ) -> Option<crate::github_text::LanguageDecision> {
7316 self.state.config.roles.language_judge.as_ref()?;
7317 let decision =
7318 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
7319 self.state.event(
7320 "github-text",
7321 if decision.is_some() {
7322 "language decided by the language judge"
7323 } else {
7324 "language judge unavailable; using built-in heuristics"
7325 },
7326 );
7327 decision
7328 }
7329
7330 async fn merge(&mut self) -> Result<()> {
7331 if self
7346 .state
7347 .base_sync
7348 .as_ref()
7349 .is_some_and(|s| s.conflict.is_some())
7350 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7351 != Some(RunStatus::Gating)
7352 || !self.state.gate_status().ok()
7361 {
7362 return Ok(());
7363 }
7364 if self.state.merge.is_some() && !self.state.pr_publication_pending {
7373 return Ok(());
7374 }
7375 let Some(winner) = self.state.winner().cloned() else {
7376 return Ok(());
7377 };
7378 let repo = self.state.repo.clone();
7379 let base = self.state.base_branch.clone();
7380 let mode = self.state.config.merge.mode;
7381 let style = self.state.config.merge.style;
7382 let facts = if is_review_run(&self.state) {
7383 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
7384 let start = review_base(
7385 &repo,
7386 &self.state.config.merge.remote,
7387 &base,
7388 &self.state.base_commit,
7389 &winner.branch,
7390 )
7391 .await;
7392 branch_facts(&repo, &start, &winner.branch).await
7393 } else {
7394 None
7395 };
7396 let leaked = if is_review_run(&self.state) {
7399 leaked_subjects(&self.state, &winner.branch).await
7400 } else {
7401 Some(Vec::new())
7402 };
7403 let Some(pr) = self
7407 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
7408 .await?
7409 else {
7410 return Ok(());
7411 };
7412 let message = pr.commit_message();
7413
7414 let outcome = match mode {
7415 MergeMode::None => MergeOutcome {
7416 mode,
7417 ok: true,
7418 detail: manual_merge_command(style, &repo, &winner.branch, &message),
7419 empty: false,
7420 },
7421 MergeMode::Pr | MergeMode::Local
7422 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
7423 {
7424 MergeOutcome {
7425 mode,
7426 ok: false,
7427 detail: empty_candidate_detail(&self.state, &base),
7428 empty: true,
7429 }
7430 }
7431 MergeMode::Local => {
7432 let on = git::current_branch(&repo).await?;
7433 if on.as_deref() != Some(base.as_str()) {
7434 MergeOutcome {
7435 mode,
7436 ok: false,
7437 detail: format!(
7438 "{} has {} checked out, not the base branch {base}",
7439 repo.display(),
7440 on.unwrap_or_else(|| "a detached HEAD".to_owned())
7441 ),
7442 empty: false,
7443 }
7444 } else if !git::is_clean(&repo).await? {
7445 MergeOutcome {
7446 mode,
7447 ok: false,
7448 detail: format!("{} is dirty; refusing to merge", repo.display()),
7449 empty: false,
7450 }
7451 } else {
7452 let out = match style {
7453 MergeStyle::Merge => {
7454 git::merge_no_ff(&repo, &winner.branch, &message).await?
7455 }
7456 MergeStyle::Squash => {
7457 git::merge_squash(&repo, &winner.branch, &message).await?
7458 }
7459 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
7460 };
7461 MergeOutcome {
7462 mode,
7463 ok: out.ok(),
7464 detail: if out.ok() { out.stdout } else { out.stderr },
7465 empty: false,
7466 }
7467 }
7468 }
7469 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
7470 self.state.pr_publication_pending = true;
7471 self.state.save()?;
7472 let out =
7473 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
7474 self.state.pr_publication_pending = out.is_err();
7475 MergeOutcome {
7476 mode,
7477 ok: out.is_ok(),
7478 detail: match out {
7479 Ok(url) => url,
7480 Err(e) => format!("{e:#}"),
7481 },
7482 empty: false,
7483 }
7484 }
7485 MergeMode::Pr => {
7486 let remote = self.state.config.merge.remote.clone();
7487 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
7488 if !pushed.ok() {
7489 MergeOutcome {
7490 mode,
7491 ok: false,
7492 detail: pushed.stderr,
7493 empty: false,
7494 }
7495 } else {
7496 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
7502 let out = match pr_merge_plan(found) {
7503 PrPlan::Create => {
7504 gh_pr_create(
7505 &winner.worktree,
7506 &base,
7507 &winner.branch,
7508 &pr.title,
7509 &pr.body,
7510 )
7511 .await
7512 }
7513 PrPlan::Adopt { url, title } => {
7514 self.state
7515 .event("merge", format!("Pr: adopted open pull request {url}"));
7516 if self.state.github_text.as_ref().is_some_and(|g| {
7517 g.resolved
7518 && g.choice == Some(GithubTextChoice::Rewrite)
7519 && g.rewrite.is_some()
7520 }) {
7521 land::set_pr_text_verbatim(
7524 &winner.worktree,
7525 &url,
7526 &pr.title,
7527 &pr.body,
7528 )
7529 .await
7530 .map(|()| url)
7531 } else {
7532 if title != pr.title
7533 && (!is_review_run(&self.state)
7534 || leaked
7535 .as_deref()
7536 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
7537 && let Err(e) =
7538 if self.state.github_text.as_ref().is_some_and(|g| {
7539 g.resolved
7540 && g.title
7541 && g.choice == Some(GithubTextChoice::Original)
7542 }) {
7543 land::set_pr_title_verbatim(
7545 &winner.worktree,
7546 &url,
7547 &pr.title,
7548 )
7549 .await
7550 } else {
7551 land::set_pr_title(
7552 &mut self.state,
7553 &winner.worktree,
7554 &url,
7555 &pr.title,
7556 )
7557 .await
7558 }
7559 {
7560 tracing::warn!("could not refresh title of {url}: {e:#}");
7561 self.state
7562 .event("merge", format!("Pr: title refresh failed: {e:#}"));
7563 }
7564 Ok(url)
7565 }
7566 }
7567 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
7568 };
7569 match out {
7570 Ok(url) => MergeOutcome {
7571 mode,
7572 ok: true,
7573 detail: url,
7574 empty: false,
7575 },
7576 Err(e) => MergeOutcome {
7577 mode,
7578 ok: false,
7579 detail: e.to_string(),
7580 empty: false,
7581 },
7582 }
7583 }
7584 }
7585 };
7586
7587 self.state.status = match (mode, outcome.ok) {
7588 (MergeMode::None, _) => RunStatus::Ready,
7589 (_, true) => RunStatus::Merged,
7590 (_, false) => RunStatus::Blocked,
7591 };
7592 self.state.event(
7593 "merge",
7594 format!(
7595 "{:?}: {}",
7596 mode,
7597 outcome.detail.lines().next().unwrap_or("")
7598 ),
7599 );
7600 self.state.merge = Some(outcome);
7601 self.state.save()?;
7602
7603 if self.state.config.graph.land
7609 && mode == MergeMode::Pr
7610 && self.state.status == RunStatus::Merged
7611 {
7612 self.run_land().await?;
7613 }
7614 self.settle_questions();
7619 Ok(())
7620 }
7621
7622 async fn run_land(&mut self) -> Result<()> {
7633 crate::pr_head::ensure_owner(&self.state)?;
7634 let url = self
7635 .state
7636 .merge
7637 .as_ref()
7638 .map(|m| m.detail.clone())
7639 .unwrap_or_default();
7640 let url = url.lines().next().unwrap_or("").trim().to_owned();
7641 if !url.starts_with("http") {
7642 return Ok(());
7643 }
7644 match land::land(&mut self.state, &url).await {
7647 Ok(pr) if self.state.parked => {
7648 let _ = pr;
7652 }
7653 Ok(pr) => {
7654 self.state.status = match pr.state {
7655 land::PrLifecycle::Merged => RunStatus::Merged,
7656 _ => RunStatus::Blocked,
7657 };
7658 if bump::should_release_bump(self.state.status)
7665 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
7666 {
7667 self.state
7673 .event("bump", format!("release bump skipped: {e:#}"));
7674 }
7675 if self.state.status == RunStatus::Merged {
7678 crate::followup::after_merge(&mut self.state, &pr.url).await;
7679 }
7680 self.state.save()?;
7681 }
7682 Err(e) => {
7683 self.state.status = RunStatus::Blocked;
7684 self.state.event("land", format!("gave up: {e}"));
7685 self.state.save()?;
7686 }
7687 }
7688 Ok(())
7689 }
7690
7691 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
7695 if let Some(existing) = self.state.seats.get(key)
7696 && existing.agent == agent
7697 {
7698 return existing.clone();
7699 }
7700 let fresh = if self.state.seats.contains_key(key) {
7703 handover_seat(key, agent, self.state.next_seat_seed())
7704 } else {
7705 SeatState::new(key, agent, self.state.seed)
7706 };
7707 self.state.seats.insert(key.to_owned(), fresh.clone());
7708 fresh
7709 }
7710
7711 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
7717 match self.state.seats.get(key) {
7718 Some(s) if s.agent != spec.id => {
7719 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
7720 }
7721 _ => spec,
7722 }
7723 }
7724
7725 fn view(&self, c: &Candidate) -> CandidateView {
7727 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
7728 .unwrap_or_default();
7729 let (patch, _) = blind::sanitize_patch(
7730 &format!("candidate {} patch", c.label),
7731 &raw,
7732 &self.state.config.blind,
7733 );
7734 CandidateView {
7735 label: c.label,
7736 branch: c.branch.clone(),
7737 summary: c.summary.clone(),
7738 stat: c.stat.clone(),
7739 patch,
7740 }
7741 }
7742
7743 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
7745 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
7746 prompt::judge(
7747 "(see above)",
7748 &views,
7749 self.roles.judges.len(),
7750 base_short,
7751 "en",
7752 )
7753 }
7754
7755 fn vote_prompt_full(
7761 &self,
7762 j: usize,
7763 labels: &[char],
7764 language: &str,
7765 candidates: &[Candidate],
7766 base_short: &str,
7767 ) -> String {
7768 let mut text = format!(
7769 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
7770 prompt::final_vote(labels, language),
7771 self.state.instruction,
7772 self.candidate_block(candidates, base_short)
7773 );
7774 if let Some(own) = self
7775 .state
7776 .judgements
7777 .get(j)
7778 .filter(|r| !r.ranking.is_empty())
7779 {
7780 let reasons = own
7781 .reasons
7782 .iter()
7783 .map(|(k, v)| format!("- {k}: {v}"))
7784 .collect::<Vec<_>>()
7785 .join("\n");
7786 text.push_str(&format!(
7787 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
7788 own.ranking.iter().collect::<String>(),
7789 if reasons.is_empty() {
7790 ""
7791 } else {
7792 ", because:\n"
7793 }
7794 ));
7795 }
7796 if !self.state.deliberation.is_empty() {
7797 text.push_str("\n# What was argued before this vote\n");
7798 for t in self.transcript(&[], j) {
7799 text.push_str(&format!(
7800 "\n## {}{}\n\n{}\n",
7801 t.who,
7802 if t.is_self { " (you)" } else { "" },
7803 t.body.trim()
7804 ));
7805 }
7806 }
7807 text
7808 }
7809
7810 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
7817 let mut turns = Vec::new();
7818 for j in &self.state.judgements {
7819 if j.ranking.is_empty() {
7820 continue;
7821 }
7822 let reasons = j
7823 .reasons
7824 .iter()
7825 .map(|(k, v)| format!("- {k}: {v}"))
7826 .collect::<Vec<_>>()
7827 .join("\n");
7828 turns.push(Turn {
7829 who: format!("Judge {} (opening ranking)", j.judge),
7830 is_self: j.judge == self_idx + 1,
7831 body: format!(
7832 "Ranked {}{}{reasons}",
7833 j.ranking.iter().collect::<String>(),
7834 if reasons.is_empty() {
7835 ""
7836 } else {
7837 ", because:\n"
7838 }
7839 ),
7840 });
7841 }
7842 for t in self
7843 .state
7844 .deliberation
7845 .iter()
7846 .flat_map(|r| r.turns.iter())
7847 .chain(current)
7848 {
7849 turns.push(Turn {
7850 who: format!("Judge {}", t.judge),
7851 is_self: t.judge == self_idx + 1,
7852 body: t.body.clone(),
7853 });
7854 }
7855 turns
7856 }
7857}
7858
7859fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
7861 agent::has_session(spec.kind, seat, sessions)
7862}
7863
7864fn next_untried_in_roster<'a>(
7885 roster: &'a [AgentSpec],
7886 start: usize,
7887 tried: &BTreeSet<String>,
7888) -> Option<&'a AgentSpec> {
7889 roster
7890 .get(start + 1..)?
7891 .iter()
7892 .find(|s| !tried.contains(&s.id))
7893}
7894
7895fn pick_successor<'a>(
7916 roster: &'a [AgentSpec],
7917 start: usize,
7918 tried: &BTreeSet<String>,
7919 carried: Option<&BTreeSet<String>>,
7920 others: &BTreeSet<String>,
7921) -> Option<&'a AgentSpec> {
7922 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
7923 roster
7924 .get(start + 1..)
7925 .and_then(|tail| {
7926 tail.iter()
7927 .filter(free)
7928 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
7929 })
7930 .or_else(|| {
7931 carried
7932 .is_some()
7933 .then(|| roster.iter().find(free))
7934 .flatten()
7935 })
7936 .or_else(|| match carried {
7937 Some(c) => next_for_seat(roster, start, tried, c),
7938 None => next_untried_in_roster(roster, start, tried),
7939 })
7940}
7941
7942fn next_for_seat<'a>(
7952 roster: &'a [AgentSpec],
7953 start: usize,
7954 round_tried: &BTreeSet<String>,
7955 carried_failed: &BTreeSet<String>,
7956) -> Option<&'a AgentSpec> {
7957 roster
7958 .get(start + 1..)?
7959 .iter()
7960 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
7961 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
7962}
7963
7964fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
7970 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
7971 return spec;
7972 };
7973 let ok = |id: &str| !h.failed.contains(id);
7974 if let Some(last) = h.last_ok.as_deref()
7975 && ok(last)
7976 && let Some(s) = roster.iter().find(|s| s.id == last)
7977 {
7978 return s.clone();
7979 }
7980 if ok(&spec.id) {
7981 return spec;
7982 }
7983 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
7984 roster
7985 .get(from + 1..)
7986 .into_iter()
7987 .flatten()
7988 .chain(roster.iter())
7989 .find(|s| ok(&s.id))
7990 .cloned()
7991 .unwrap_or(spec)
7992}
7993
7994pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
7998 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
7999}
8000
8001const TIMED_OUT: &str = "timed out";
8004
8005impl FailClass {
8006 fn of(out: &AgentOutcome) -> Option<Self> {
8008 match out {
8009 AgentOutcome::Ok(_) => None,
8010 AgentOutcome::Quota(_) => Some(Self::Quota),
8011 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8012 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8013 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8014 AgentOutcome::ResourceBlocked(_) => None,
8017 }
8018 }
8019
8020 fn stem_word(&self) -> &'static str {
8022 match self {
8023 Self::Quota => "quota",
8024 _ => "handover",
8025 }
8026 }
8027}
8028
8029fn failure_signature(msg: &str) -> String {
8033 let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8034 let mut out = Vec::new();
8035 for word in line.split_whitespace() {
8036 if word.contains('/') || word.contains('\\') {
8037 out.push("<path>".to_owned());
8038 continue;
8039 }
8040 let mut w = String::new();
8041 let mut in_digits = false;
8042 for c in word.chars() {
8043 if c.is_ascii_digit() {
8044 if !in_digits {
8045 w.push('#');
8046 }
8047 in_digits = true;
8048 } else {
8049 in_digits = false;
8050 w.push(c);
8051 }
8052 }
8053 out.push(w);
8054 }
8055 out.join(" ").chars().take(120).collect()
8056}
8057
8058fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8065 match cur {
8066 FailClass::Quota | FailClass::Timeout => true,
8067 FailClass::Other(_) => prev != Some(cur),
8068 }
8069}
8070
8071fn fail_reason(out: &AgentOutcome) -> String {
8073 match out {
8074 AgentOutcome::Ok(_) => String::new(),
8075 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8076 AgentOutcome::Dropped(o) => format!(
8077 "the CLI dropped the stream ({})",
8078 o.dropped
8079 .as_ref()
8080 .map(|d| d.why.as_str())
8081 .unwrap_or("it ended without delivering its answer")
8082 ),
8083 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8084 e.lines().next().unwrap_or("").chars().take(160).collect()
8085 }
8086 }
8087}
8088
8089pub(crate) fn record_handover(
8092 state: &mut RunState,
8093 node: &str,
8094 seat: &str,
8095 from: &str,
8096 to: &str,
8097 class: &FailClass,
8098 reason: &str,
8099) {
8100 let message = if *class == FailClass::Quota {
8101 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
8102 } else {
8103 format!("{seat}: handed over {from} -> {to} ({reason})")
8104 };
8105 state.event(node, message);
8106 state.handovers.push(Handover {
8107 at: Timestamp::now(),
8108 node: node.to_owned(),
8109 seat: seat.to_owned(),
8110 from: from.to_owned(),
8111 to: to.to_owned(),
8112 reason: reason.to_owned(),
8113 });
8114}
8115
8116fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
8130 commands.iter().any(|c| c.exit_code.is_none())
8131}
8132
8133fn verified_noop_claim(
8146 usable: bool,
8147 commands: &[agent::CommandEvidence],
8148 text: &str,
8149) -> Option<String> {
8150 (usable && !has_unconfirmed_command(commands))
8151 .then(|| verdict::verified_noop(text))
8152 .flatten()
8153}
8154
8155fn short(commit: &str) -> String {
8156 commit.chars().take(7).collect()
8157}
8158
8159fn make_executable(path: &Path) -> Result<()> {
8160 #[cfg(unix)]
8161 {
8162 use std::os::unix::fs::PermissionsExt as _;
8163 let mut perms = std::fs::metadata(path)?.permissions();
8164 perms.set_mode(0o755);
8165 std::fs::set_permissions(path, perms)?;
8166 }
8167 #[cfg(not(unix))]
8168 {
8169 let _ = path;
8170 }
8171 Ok(())
8172}
8173
8174struct WaveCtx<'a> {
8181 run: &'a str,
8184 node: &'a str,
8186 prompts: &'a Prompts,
8187 cache: Option<&'a Path>,
8189 round: Option<usize>,
8192 carry_seats: bool,
8196 pauses: Option<(&'a Pause, &'a Pause)>,
8199}
8200
8201async fn run_one(
8203 job: SeatJob,
8204 sem: Arc<Semaphore>,
8205 ctx: &WaveCtx<'_>,
8206 state: &mut RunState,
8207 attempt: usize,
8208) -> (SeatState, AgentOutcome) {
8209 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
8210 .await
8211 .pop()
8212 .expect("one job in, one result out");
8213 (seat, out)
8214}
8215
8216async fn wave(
8222 jobs: Vec<SeatJob>,
8223 sem: Arc<Semaphore>,
8224 ctx: &WaveCtx<'_>,
8225 state: &mut RunState,
8226 attempt: usize,
8227) -> Vec<(usize, SeatState, AgentOutcome)> {
8228 let WaveCtx {
8229 run,
8230 node,
8231 prompts,
8232 cache,
8233 round,
8234 carry_seats: _,
8235 pauses,
8236 } = *ctx;
8237 for job in &jobs {
8238 state.seat_started(node, &job.seat.key, job.timeout, attempt);
8239 }
8240 if let Err(e) = state.save() {
8241 tracing::warn!("could not persist in-progress seats: {e:#}");
8246 }
8247 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
8263 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
8267 let cache_guard = if let Some(cache_dir) = cache {
8268 if jobs_had_a_writer {
8269 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
8270 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
8271 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
8272 {
8273 Ok(Some(g)) => Some(g),
8274 Ok(None) => {
8277 state.resource_block = Some(crate::run::ResourceBlock {
8278 node: node.to_owned(),
8279 kind: "shared build cache".to_owned(),
8280 detail: "the run is parking".to_owned(),
8281 at: Timestamp::now(),
8282 });
8283 return block_wave(
8284 jobs,
8285 state,
8286 node,
8287 attempt,
8288 "parked while waiting for the shared build cache",
8289 );
8290 }
8291 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
8294 Err(e) => {
8295 let why = format!("{e:#}");
8299 state.resource_block = Some(crate::run::ResourceBlock {
8300 node: node.to_owned(),
8301 kind: "shared build cache".to_owned(),
8302 detail: why.clone(),
8303 at: Timestamp::now(),
8304 });
8305 state.event(
8306 "resource",
8307 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
8308 );
8309 return block_wave(jobs, state, node, attempt, &why);
8310 }
8311 }
8312 } else {
8313 None
8314 }
8315 } else {
8316 None
8317 };
8318 state.seats_spawning(node, attempt);
8321 if let Err(e) = state.save() {
8322 tracing::warn!("could not persist the start of a wave: {e:#}");
8323 }
8324 let slot_limit = resource_limit
8328 + jobs
8329 .iter()
8330 .map(|j| j.timeout)
8331 .max()
8332 .unwrap_or_default()
8333 .saturating_mul(jobs.len() as u32);
8334 let mut set = tokio::task::JoinSet::new();
8335 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
8338 let overlay = prompts.overlay(node);
8339 for (i, mut job) in jobs.into_iter().enumerate() {
8340 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
8341 if cache.is_some() {
8342 job.prompt.push('\n');
8343 job.prompt
8344 .push_str(&prompt::build_cache_note(node, job.allow_write));
8345 }
8346 let sem = Arc::clone(&sem);
8347 let run = run.to_owned();
8348 let node = node.to_owned();
8349 let attachments = if node == "implement" {
8352 state.attachments.clone()
8353 } else {
8354 Vec::new()
8355 };
8356 let cache = cache
8367 .filter(|_| job.allow_write && cache_guard.is_some())
8368 .map(Path::to_path_buf);
8369 let spawned_tx = spawned_tx.clone();
8370 set.spawn(async move {
8371 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
8375 Ok(p) => p,
8376 Err(_) => {
8377 return (
8378 i,
8379 job.seat,
8380 AgentOutcome::ResourceBlocked(format!(
8381 "no parallel slot became free within {}s",
8382 slot_limit.as_secs()
8383 )),
8384 );
8385 }
8386 };
8387 spawned_tx.send(job.seat.key.clone()).ok();
8388 let mut seat = job.seat;
8389 let out = agent::invoke(
8390 &job.spec,
8391 &mut seat,
8392 &Invocation {
8393 cwd: &job.cwd,
8394 prompt: &job.prompt,
8395 timeout: job.timeout,
8396 allow_write: job.allow_write,
8397 unsandboxed: false,
8398 sessions: job.sessions,
8399 artifacts: &job.artifacts,
8400 stem: &job.stem,
8401 run: &run,
8402 node: &node,
8403 cache_dir: cache.as_deref(),
8404 attachments: &attachments,
8405 writable: &[],
8406 },
8407 )
8408 .await;
8409 let out = match out {
8410 Ok(o) if o.usable() => AgentOutcome::Ok(o),
8411 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
8412 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
8420 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
8421 Ok(o) => AgentOutcome::Failed(format!(
8422 "exited with {:?} and no usable output",
8423 o.exit_code
8424 )),
8425 Err(e) => AgentOutcome::Failed(e.to_string()),
8426 };
8427 (i, seat, out)
8428 });
8429 }
8430 drop(spawned_tx);
8431 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
8432 loop {
8433 let joined = tokio::select! {
8434 Some(key) = spawned_rx.recv() => {
8435 state.seat_spawned(&key);
8437 if let Err(e) = state.save() {
8438 tracing::warn!("could not persist a seat's start: {e:#}");
8439 }
8440 continue;
8441 }
8442 joined = set.join_next() => joined,
8443 };
8444 let Some(joined) = joined else {
8445 break;
8446 };
8447 let (i, seat, out) = match joined {
8448 Ok(v) => v,
8449 Err(e) => {
8453 tracing::error!("agent task panicked: {e}");
8454 continue;
8455 }
8456 };
8457 state.seat_finished(&seat.key);
8458 if let AgentOutcome::ResourceBlocked(why) = &out
8459 && state.resource_block.is_none()
8460 {
8461 state.resource_block = Some(crate::run::ResourceBlock {
8462 node: node.to_owned(),
8463 kind: "parallel slot".to_owned(),
8464 detail: why.clone(),
8465 at: Timestamp::now(),
8466 });
8467 state.event("resource", format!("{node}: no agent was started - {why}"));
8468 }
8469 record_jobs(state, node, round, &seat.key, &out);
8470 if let Err(e) = state.save() {
8471 tracing::warn!("could not persist a seat's completion: {e:#}");
8472 }
8473 if collected.len() <= i {
8474 collected.resize_with(i + 1, || None);
8475 }
8476 collected[i] = Some((i, seat, out));
8477 }
8478 if state
8484 .active
8485 .values()
8486 .any(|a| a.node == node && a.attempt == attempt)
8487 {
8488 state
8489 .active
8490 .retain(|_, a| !(a.node == node && a.attempt == attempt));
8491 if let Err(e) = state.save() {
8492 tracing::warn!("could not persist the end of a wave: {e:#}");
8493 }
8494 }
8495 if let Some(cache_dir) = cache
8502 && jobs_had_a_writer
8503 {
8504 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
8505 }
8506 if let Some(guard) = cache_guard {
8507 guard.release();
8508 }
8509 collected.into_iter().flatten().collect()
8510}
8511
8512fn record_jobs(
8523 state: &mut RunState,
8524 node: &str,
8525 round: Option<usize>,
8526 seat: &str,
8527 out: &AgentOutcome,
8528) {
8529 let commands: &[agent::CommandEvidence] = match out {
8530 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
8531 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
8532 };
8533 let checked_at = Timestamp::now();
8534 for c in commands {
8535 state.jobs.push(JobRecord {
8536 node: node.to_owned(),
8537 round,
8538 seat: seat.to_owned(),
8539 id: c.id.clone(),
8540 description: c.description.clone(),
8541 checked_at,
8542 status: match c.exit_code {
8543 Some(0) => JobStatus::Completed,
8544 Some(_) => JobStatus::Failed,
8545 None => JobStatus::Unknown,
8546 },
8547 exit_code: c.exit_code,
8548 result_summary: c.result_summary.clone(),
8549 source: c.source.clone(),
8550 });
8551 }
8552}
8553
8554fn round_is_clean(
8575 blocking: usize,
8576 e2e_ok: bool,
8577 answered: usize,
8578 expected: usize,
8579 quota_missing: usize,
8580 policy: IncompleteReviewPolicy,
8581) -> bool {
8582 if blocking != 0 || !e2e_ok {
8583 return false;
8584 }
8585 if answered == expected || policy == IncompleteReviewPolicy::Warn {
8586 return true;
8587 }
8588 answered > 0 && expected - answered <= quota_missing
8589}
8590
8591fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
8615 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
8616 return Some(RunStatus::Gating);
8617 }
8618 let last = reviews.last()?;
8619 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
8620 if reviews.len() < max_rounds && !stagnant {
8621 return None;
8622 }
8623 if last.incomplete() && last.blocking == 0 {
8624 return Some(RunStatus::Blocked);
8625 }
8626 if last.e2e_status() == E2eStatus::ResourceBlocked {
8627 return None;
8628 }
8629 Some(if last.e2e.iter().all(CommandOutcome::ok) {
8630 RunStatus::Gating
8631 } else {
8632 RunStatus::Blocked
8633 })
8634}
8635
8636pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
8651 if nudged {
8652 (full / 4).max(Duration::from_secs(120)).min(full)
8653 } else {
8654 full
8655 }
8656}
8657
8658#[allow(clippy::too_many_arguments)]
8687async fn ask_json_wave<T>(
8688 jobs: Vec<SeatJob>,
8689 sem: Arc<Semaphore>,
8690 retries: usize,
8691 roster: &[AgentSpec],
8692 ctx: &WaveCtx<'_>,
8693 losses: &mut Vec<QuotaLoss>,
8694 state: &mut RunState,
8695 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
8696) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
8697where
8698 T: serde::de::DeserializeOwned + Send + 'static,
8699{
8700 ask_wave_with(
8701 jobs,
8702 sem,
8703 retries,
8704 roster,
8705 ctx,
8706 losses,
8707 state,
8708 &|text: &str| {
8709 let v = verdict::extract_json::<T>(text)?;
8710 validate(&v)?;
8711 Ok(v)
8712 },
8713 )
8714 .await
8715}
8716
8717#[allow(clippy::too_many_arguments)]
8726async fn ask_wave_with<T>(
8727 jobs: Vec<SeatJob>,
8728 sem: Arc<Semaphore>,
8729 retries: usize,
8730 roster: &[AgentSpec],
8731 ctx: &WaveCtx<'_>,
8732 losses: &mut Vec<QuotaLoss>,
8733 state: &mut RunState,
8734 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
8735) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
8736where
8737 T: Send + 'static,
8738{
8739 let n = jobs.len();
8740 let originals: Vec<SeatJob> = jobs;
8741 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
8742 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
8743 let mut nudges: Vec<usize> = vec![0; n];
8750 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
8754 let mut tried: Vec<BTreeSet<String>> = specs
8755 .iter()
8756 .map(|s| BTreeSet::from([s.id.clone()]))
8757 .collect();
8758 let starts: Vec<usize> = specs
8759 .iter()
8760 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
8761 .collect();
8762 let carry = ctx.carry_seats && !roster.is_empty();
8763 let carried: Vec<BTreeSet<String>> = originals
8766 .iter()
8767 .map(|j| {
8768 state
8769 .seat_history
8770 .get(&j.seat.key)
8771 .filter(|_| carry)
8772 .map(|h| h.failed.clone())
8773 .unwrap_or_default()
8774 })
8775 .collect();
8776 let mut prev: Vec<Option<FailClass>> = originals
8777 .iter()
8778 .map(|j| {
8779 state
8780 .seat_history
8781 .get(&j.seat.key)
8782 .filter(|_| carry)
8783 .and_then(|h| h.last_fail.clone())
8784 })
8785 .collect();
8786 let next_agent =
8787 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
8788 let others: BTreeSet<String> = specs
8789 .iter()
8790 .enumerate()
8791 .filter(|(j, _)| *j != i)
8792 .map(|(_, s)| s.id.clone())
8793 .collect();
8794 pick_successor(
8795 roster,
8796 starts[i],
8797 tried,
8798 carry.then(|| &carried[i]),
8799 &others,
8800 )
8801 .cloned()
8802 };
8803 let mut fresh: Vec<Option<String>> = vec![None; n];
8804 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
8805 let mut pending: Vec<usize> = (0..n).collect();
8806
8807 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
8812 for round in 0..max_rounds {
8813 if pending.is_empty() {
8814 break;
8815 }
8816 let mut batch = Vec::with_capacity(pending.len());
8817 let mut renudged: Vec<&str> = Vec::new();
8818 for &i in &pending {
8819 let src = &originals[i];
8820 let full: &str = match &src.handover {
8825 Some(h) if specs[i].id != src.spec.id => h,
8826 _ => &src.prompt,
8827 };
8828 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
8831 (
8832 full.to_owned(),
8833 src.timeout,
8834 format!("{}-{word}-{}", src.stem, specs[i].id),
8835 )
8836 } else if nudges[i] == 0 {
8837 (full.to_owned(), src.timeout, src.stem.clone())
8838 } else {
8839 renudged.push(src.seat.key.as_str());
8840 let why = done[i]
8841 .as_ref()
8842 .and_then(|r| r.as_ref().err().map(ToString::to_string))
8843 .unwrap_or_else(|| "no parsable answer".to_owned());
8844 let nudge = prompt::nudge(&why);
8845 let nudged = has_context(&specs[i], &seats[i], src.sessions);
8846 let prompt = if nudged {
8847 nudge
8848 } else {
8849 format!("{full}\n\n---\n\n{nudge}")
8850 };
8851 (
8852 prompt,
8853 retry_budget(src.timeout, nudged),
8854 format!("{}-retry{}", src.stem, nudges[i]),
8855 )
8856 };
8857 batch.push(SeatJob {
8858 spec: specs[i].clone(),
8859 seat: seats[i].clone(),
8860 cwd: src.cwd.clone(),
8861 prompt,
8862 timeout,
8863 allow_write: src.allow_write,
8864 sessions: src.sessions,
8865 artifacts: src.artifacts.clone(),
8866 stem,
8867 handover: None,
8868 });
8869 }
8870
8871 if !renudged.is_empty() {
8872 state.event(
8873 ctx.node,
8874 format!("retry {round}: re-asking {}", renudged.join(", ")),
8875 );
8876 }
8877 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
8878 let mut still = Vec::new();
8879 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
8880 seats[i] = seat;
8881 if let AgentOutcome::ResourceBlocked(why) = &out {
8884 done[i] = Some(Err(anyhow::anyhow!("{why}")));
8885 continue;
8886 }
8887 let class = FailClass::of(&out);
8888 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
8891 && nudges[i] < retries
8892 && !roster.is_empty();
8893 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
8894 let next = should_hand_over(prev[i].as_ref(), &cur)
8895 .then(|| next_agent(i, &tried[i], &specs))
8896 .flatten();
8897 if carry {
8898 let h = state
8899 .seat_history
8900 .entry(originals[i].seat.key.clone())
8901 .or_default();
8902 h.failed.insert(specs[i].id.clone());
8903 h.last_fail = Some(cur.clone());
8904 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
8905 h.last_ok = None;
8906 }
8907 if let Err(e) = state.save() {
8910 tracing::warn!("could not persist a seat's failure history: {e:#}");
8911 }
8912 }
8913 if let Some(next) = next {
8914 record_handover(
8915 state,
8916 ctx.node,
8917 &originals[i].seat.key,
8918 &specs[i].id,
8919 &next.id,
8920 &cur,
8921 &fail_reason(&out),
8922 );
8923 tried[i].insert(next.id.clone());
8924 prev[i] = Some(cur.clone());
8925 seats[i] =
8926 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
8927 specs[i] = next;
8928 fresh[i] = Some(cur.stem_word().to_owned());
8929 nudges[i] = 0;
8930 done[i] = Some(Err(anyhow::anyhow!(
8931 "handed over after: {}",
8932 fail_reason(&out)
8933 )));
8934 still.push(i);
8935 continue;
8936 }
8937 }
8938 if carry
8939 && !nudge_first
8940 && let Some(cur) = class.clone()
8941 {
8942 let h = state
8944 .seat_history
8945 .entry(originals[i].seat.key.clone())
8946 .or_default();
8947 h.failed.insert(specs[i].id.clone());
8948 h.last_fail = Some(cur);
8949 }
8950 let parsed = match out {
8951 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
8952 AgentOutcome::Quota(o) => {
8953 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
8954 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
8955 }
8956 AgentOutcome::Dropped(o) => {
8960 let why = o
8961 .dropped
8962 .as_ref()
8963 .map(|d| d.why.as_str())
8964 .unwrap_or("the CLI ended the stream without delivering its answer");
8965 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
8966 }
8967 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8968 Err(anyhow::anyhow!(e))
8969 }
8970 };
8971 let quota = class == Some(FailClass::Quota);
8972 let failed = parsed.is_err();
8973 done[i] = Some(parsed);
8974 if carry && !failed {
8975 let h = state
8976 .seat_history
8977 .entry(originals[i].seat.key.clone())
8978 .or_default();
8979 h.failed.remove(&specs[i].id);
8980 h.last_ok = Some(specs[i].id.clone());
8981 h.last_fail = None;
8982 }
8983 let agent_failure = class.is_some()
8991 && !nudge_first
8992 && !roster.is_empty()
8993 && next_agent(i, &tried[i], &specs).is_some();
8994 if failed && !quota && !agent_failure && nudges[i] < retries {
8995 nudges[i] += 1;
8996 still.push(i);
8997 }
8998 }
8999 pending = still;
9000 }
9001
9002 for (i, q) in last_quota.into_iter().enumerate() {
9005 if let Some(reset) = q {
9006 losses.push(QuotaLoss {
9007 seat: originals[i].seat.key.clone(),
9008 node: ctx.node.to_owned(),
9009 at: Timestamp::now(),
9010 reset,
9011 });
9012 }
9013 }
9014
9015 seats
9016 .into_iter()
9017 .zip(done)
9018 .zip(nudges)
9019 .map(|((seat, res), attempts)| {
9020 (
9021 seat,
9022 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
9023 attempts,
9024 )
9025 })
9026 .collect()
9027}
9028
9029fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
9031 results
9032 .iter()
9033 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
9034}
9035
9036fn block_wave(
9040 jobs: Vec<SeatJob>,
9041 state: &mut RunState,
9042 node: &str,
9043 attempt: usize,
9044 why: &str,
9045) -> Vec<(usize, SeatState, AgentOutcome)> {
9046 state.resource_wait = None;
9047 state
9048 .active
9049 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9050 if let Err(e) = state.save() {
9051 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
9052 }
9053 jobs.into_iter()
9054 .enumerate()
9055 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
9056 .collect()
9057}
9058
9059async fn acquire_cache_lease(
9072 state: &mut RunState,
9073 cache_dir: &Path,
9074 owner: &crate::cache::Owner,
9075 budget: Duration,
9076 context: &str,
9077 cancel: &(dyn Fn() -> bool + Sync),
9078) -> Result<Option<crate::cache::Guard>> {
9079 let home = crate::run::home();
9080 let started = Instant::now();
9081 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
9082 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
9083 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
9084 Err(e) => {
9085 state.event(
9086 "verify",
9087 format!("{context}: could not check the shared build cache: {e:#}"),
9088 );
9089 if let Err(e2) = state.save() {
9090 tracing::warn!("could not persist a cache-check failure: {e2:#}");
9091 }
9092 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
9093 }
9094 };
9095 state.event(
9096 "verify",
9097 format!(
9098 "{context}: waiting for the shared build cache at {} ({})",
9099 cache_dir.display(),
9100 busy.describe()
9101 ),
9102 );
9103 state.resource_wait = Some(crate::run::ResourceWait {
9106 node: context.to_owned(),
9107 kind: "shared build cache".to_owned(),
9108 holder: busy.describe(),
9109 since: Timestamp::now(),
9110 limit_secs: budget.as_secs(),
9111 });
9112 if let Err(e) = state.save() {
9113 tracing::warn!("could not persist a cache wait: {e:#}");
9114 }
9115 let remaining = budget.saturating_sub(started.elapsed());
9116 let waited = crate::cache::wait_for_unless(
9117 &home,
9118 cache_dir,
9119 owner,
9120 remaining,
9121 Duration::from_secs(5),
9122 cancel,
9123 )
9124 .await;
9125 state.resource_wait = None;
9126 match waited {
9127 Ok(g) => Ok(g),
9128 Err(e) => {
9129 state.event("verify", format!("{context}: {e:#}"));
9130 if let Err(e2) = state.save() {
9131 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
9132 }
9133 Err(e)
9134 }
9135 }
9136}
9137
9138#[derive(Debug)]
9143struct CacheCheckError(String);
9144
9145impl std::fmt::Display for CacheCheckError {
9146 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
9147 f.write_str(&self.0)
9148 }
9149}
9150
9151impl std::error::Error for CacheCheckError {}
9152
9153#[allow(clippy::too_many_arguments)]
9174async fn with_cache_lease<'s, F, Fut>(
9175 state: &'s mut RunState,
9176 cache_dir: Option<&Path>,
9177 node: &str,
9178 seat: &str,
9179 worktree: &Path,
9180 head: &str,
9181 budget: Duration,
9182 context: &str,
9183 body: F,
9184) -> (Vec<CommandOutcome>, bool)
9185where
9186 F: FnOnce(&'s mut RunState, Duration) -> Fut,
9187 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
9188{
9189 let Some(cache_dir) = cache_dir else {
9190 let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
9191 return (outcomes, retried);
9192 };
9193 let home = crate::run::home();
9194 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
9195 let started = Instant::now();
9196 let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context, &|| false)
9197 .await
9198 .and_then(|g| {
9199 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
9200 }) {
9201 Ok(g) => g,
9202 Err(e) => {
9203 return (
9204 vec![CommandOutcome {
9205 command: "(waiting for the shared build cache)".to_owned(),
9206 code: None,
9207 output_tail: e.to_string(),
9208 duration_ms: started.elapsed().as_millis() as u64,
9209 resource_blocked: true,
9210 }],
9211 false,
9212 );
9213 }
9214 };
9215 let identity = crate::cache::Identity::new(worktree, head);
9216 if let Err(e) = crate::cache::ensure_fresh(
9217 &home,
9218 cache_dir,
9219 &identity,
9220 &state.config.verify.cargo_manifests(),
9221 ) {
9222 state.event(
9230 "verify",
9231 format!(
9232 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
9233 worktree.display(),
9234 short(head)
9235 ),
9236 );
9237 guard.release();
9238 return (
9239 vec![CommandOutcome {
9240 command: "(confirming the shared build cache is fresh)".to_owned(),
9241 code: None,
9242 output_tail: e.to_string(),
9243 duration_ms: started.elapsed().as_millis() as u64,
9244 resource_blocked: true,
9245 }],
9246 false,
9247 );
9248 }
9249 let remaining = budget.saturating_sub(started.elapsed());
9250 let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
9251 if !timed_out_pids.is_empty() {
9256 wait_for_timed_out_children_to_die(&timed_out_pids).await;
9257 }
9258 guard.release();
9259 (outcomes, retried)
9260}
9261
9262async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
9274 wait_for_pids_with(
9275 pids,
9276 crate::proc::pid_alive,
9277 LEASE_RELEASE_POLL,
9278 LEASE_RELEASE_MAX_WAIT,
9279 )
9280 .await;
9281}
9282
9283async fn wait_for_pids_with<F: Fn(u32) -> bool>(
9289 pids: &[u32],
9290 alive: F,
9291 poll: Duration,
9292 max_wait: Duration,
9293) {
9294 let deadline = Instant::now() + max_wait;
9295 loop {
9296 if pids.iter().all(|&pid| !alive(pid)) {
9297 return;
9298 }
9299 if Instant::now() >= deadline {
9300 return;
9301 }
9302 tokio::time::sleep(poll).await;
9303 }
9304}
9305
9306fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
9313 outcomes.iter().any(|o| o.resource_blocked)
9314}
9315
9316enum GateFix {
9318 Retry,
9320 Stop,
9323 Defer,
9326}
9327
9328fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
9336 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
9337 red.peek().is_some()
9338 && red.all(|o| {
9339 !o.resource_blocked
9340 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
9341 && !o.output_tail.trim().is_empty()
9342 })
9343}
9344
9345fn e2e_outcome_label(o: &CommandOutcome) -> String {
9349 if o.ok() {
9350 return "pass".to_owned();
9351 }
9352 let reason = if o.build_failed() {
9353 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
9354 } else {
9355 format!("FAIL ({:?})", o.code)
9356 };
9357 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
9358}
9359
9360async fn run_e2e_with_retry(
9368 state: &mut RunState,
9369 shell: &[String],
9370 commands: &[String],
9371 worktree: &Path,
9372 timeout: Duration,
9373 context: &str,
9374) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
9375 let (mut e2e, mut timed_out_pids) = run_commands(
9376 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
9377 )
9378 .await;
9379 for o in &e2e {
9380 state.event(
9381 "verify",
9382 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
9383 );
9384 }
9385 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
9389 if verify_retried {
9390 state.event(
9391 "verify",
9392 format!(
9393 "{context}: verify could not build/link, not a test result — retrying once \
9394 before concluding"
9395 ),
9396 );
9397 let retried = run_commands(
9398 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
9399 )
9400 .await;
9401 e2e = retried.0;
9402 timed_out_pids.extend(retried.1);
9405 for o in &e2e {
9406 state.event(
9407 "verify",
9408 format!(
9409 "{context}: retry `{}` -> {}",
9410 o.command,
9411 e2e_outcome_label(o)
9412 ),
9413 );
9414 }
9415 }
9416 (e2e, verify_retried, timed_out_pids)
9417}
9418
9419#[allow(clippy::too_many_arguments)]
9435async fn run_commands(
9436 state: &mut RunState,
9437 node: &str,
9438 task: &str,
9439 attempt: usize,
9440 shell: &[String],
9441 commands: &[String],
9442 cwd: &Path,
9443 timeout: Duration,
9444) -> (Vec<CommandOutcome>, Vec<u32>) {
9445 if commands.is_empty() {
9446 return (Vec::new(), Vec::new());
9451 }
9452 let mut out = Vec::new();
9453 let mut timed_out_pids = Vec::new();
9454 let total = commands.len();
9455 for (idx, command) in commands.iter().enumerate() {
9456 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
9457 if let Err(e) = state.save() {
9458 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
9459 }
9460 let started = Instant::now();
9461 let mut cmd = tokio::process::Command::new(&shell[0]);
9462 cmd.quiet();
9463 cmd.args(&shell[1..])
9464 .arg(command)
9465 .current_dir(cwd)
9466 .stdin(std::process::Stdio::null())
9467 .stdout(std::process::Stdio::piped())
9468 .stderr(std::process::Stdio::piped())
9469 .kill_on_drop(true);
9470 let spawned = cmd.spawn();
9471 let (code, body) = match spawned {
9472 Ok(child) => {
9473 let pid = child.id();
9478 match tokio::time::timeout(timeout, child.wait_with_output()).await {
9479 Ok(Ok(o)) => {
9480 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
9481 body.push_str(&String::from_utf8_lossy(&o.stderr));
9482 (o.status.code(), body)
9483 }
9484 Ok(Err(e)) => (None, format!("failed to run: {e}")),
9485 Err(_) => {
9486 if let Some(pid) = pid {
9487 timed_out_pids.push(pid);
9488 }
9489 (None, format!("timed out after {}s", timeout.as_secs()))
9490 }
9491 }
9492 }
9493 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
9494 };
9495 out.push(CommandOutcome {
9496 command: command.clone(),
9497 code,
9498 output_tail: tail(&body, OUTPUT_TAIL),
9499 duration_ms: started.elapsed().as_millis() as u64,
9500 resource_blocked: false,
9501 });
9502 }
9503 state.task_finished(task);
9504 if let Err(e) = state.save() {
9505 tracing::warn!("could not persist the end of {task}: {e:#}");
9506 }
9507 (out, timed_out_pids)
9508}
9509
9510fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
9522 let repo = repo.display();
9523 match style {
9524 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
9525 MergeStyle::Squash => {
9526 let subject = message
9529 .lines()
9530 .next()
9531 .unwrap_or(branch)
9532 .replace(['\\', '"', '$', '`'], "");
9533 format!(
9534 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
9535 )
9536 }
9537 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
9538 }
9539}
9540
9541const PR_TITLE_MAX: usize = 240;
9552
9553fn english_title(text: &str, max: usize) -> Option<String> {
9564 let line = queue::first_line(text)?;
9565 let chars: Vec<char> = line.chars().collect();
9566 let (kept, mark) = if chars.len() <= max {
9567 (line.to_owned(), "")
9568 } else if let Some(end) = sentence_end(&chars, max) {
9569 (chars[..end].iter().collect::<String>(), "")
9570 } else {
9571 let room = max.saturating_sub(3);
9572 let cut = if chars[room].is_whitespace() {
9575 room
9576 } else {
9577 chars[..room]
9578 .iter()
9579 .rposition(|c| c.is_whitespace())
9580 .unwrap_or(room)
9581 };
9582 let head: String = chars[..cut].iter().collect();
9583 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
9584 (head.to_owned(), "...")
9585 };
9586 if kept.is_empty() || !kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic()) {
9587 return None;
9588 }
9589 Some(format!("{kept}{mark}"))
9590}
9591
9592fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
9596 const MIN: usize = 20;
9597 for i in MIN..max.min(chars.len()) {
9598 if !matches!(chars[i], '.' | '!' | '?') {
9599 continue;
9600 }
9601 let Some(&next) = chars.get(i + 1) else {
9602 continue;
9603 };
9604 if !next.is_whitespace() {
9605 continue;
9606 }
9607 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
9608 if after.is_some_and(|c| c.is_ascii_lowercase()) {
9609 continue;
9610 }
9611 let word: String = chars[..i]
9612 .iter()
9613 .rev()
9614 .take_while(|c| !c.is_whitespace())
9615 .collect::<Vec<_>>()
9616 .into_iter()
9617 .rev()
9618 .collect();
9619 let word = word.to_ascii_lowercase();
9620 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
9621 continue;
9622 }
9623 let end = chars[..i]
9624 .iter()
9625 .rposition(|c| !c.is_whitespace())
9626 .map_or(i, |p| p + 1);
9627 return Some(end);
9628 }
9629 None
9630}
9631
9632struct PrMessage {
9637 title: String,
9638 body: String,
9639}
9640
9641impl PrMessage {
9642 fn commit_message(&self) -> String {
9646 format!("{}\n\n{}", self.title, self.body)
9647 }
9648}
9649
9650fn title_marker(line: &str) -> Option<&str> {
9652 let line = line.trim();
9653 let head = line.get(..6)?;
9654 head.eq_ignore_ascii_case("title:")
9655 .then(|| line[6..].trim())
9656}
9657
9658fn summary_title(summary: &str) -> Option<String> {
9663 let first = summary.lines().find(|l| !l.trim().is_empty())?;
9664 let raw = title_marker(first)?;
9665 if raw.is_empty() {
9666 return None;
9667 }
9668 let title = queue::title_from(raw, PR_TITLE_MAX);
9669 let lower = title.to_ascii_lowercase();
9670 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
9671 return None;
9672 }
9673 Some(title)
9674}
9675
9676const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
9678
9679const EXISTING_BRANCH: &str = "(existing branch)";
9681
9682fn is_review_run(state: &RunState) -> bool {
9686 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
9687}
9688
9689fn review_title(state: &RunState) -> Option<String> {
9694 english_subject(state.reviewed_commits.as_ref()?.first()?)
9695}
9696
9697fn english_subject(raw: &str) -> Option<String> {
9700 let raw = raw.trim();
9701 if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
9702 return None;
9703 }
9704 let title = queue::title_from(raw, PR_TITLE_MAX);
9705 let lower = title.to_ascii_lowercase();
9706 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
9707 return None;
9708 }
9709 Some(title)
9710}
9711
9712#[derive(Debug, Clone, PartialEq, Eq)]
9715struct BranchFacts {
9716 commits: Vec<(String, String)>,
9718 stat: String,
9720}
9721
9722const STAT_FILE_LINES: usize = 25;
9724const COMMITS_MAX_CHARS: usize = 20_000;
9726
9727async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
9730 let commits = git::commit_log(repo, base, branch).await.ok()?;
9731 if commits.is_empty() {
9732 return None;
9733 }
9734 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
9735 let lines: Vec<&str> = stat.lines().collect();
9736 let stat = if lines.len() > STAT_FILE_LINES + 1 {
9737 let omitted = lines.len() - 1 - STAT_FILE_LINES;
9738 let more = format!(" ... {omitted} more file(s)");
9739 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
9740 kept.push(&more);
9741 kept.push(lines[lines.len() - 1]);
9742 kept.join("\n")
9743 } else {
9744 lines.join("\n")
9745 };
9746 Some(BranchFacts { commits, stat })
9747}
9748
9749fn markdown_safe(text: &str) -> String {
9752 text.replace("</details>", "</details>")
9753 .replace("\x60\x60\x60", "~~~")
9754}
9755
9756fn neutral_title(state: &RunState, winner: char) -> String {
9757 format!(
9758 "chore: land candidate {} of run {}",
9759 winner.to_ascii_uppercase(),
9760 state.id
9761 )
9762}
9763
9764pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
9770 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
9771 ""
9772 } else {
9773 pr_title
9774 }
9775}
9776
9777pub fn landing_subject_source(state: &RunState) -> String {
9781 if is_review_run(state) {
9782 let winner = state.candidates.first().map_or('A', |c| c.label);
9783 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
9784 }
9785 state.instruction.clone()
9786}
9787
9788fn summary_without_title(summary: &str) -> String {
9791 let mut lines = summary.trim().lines().peekable();
9792 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
9793 lines.next();
9794 }
9795 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
9796}
9797
9798#[cfg(test)]
9812fn pr_message(state: &RunState, winner: char) -> PrMessage {
9813 pr_message_with(state, winner, None)
9814}
9815
9816#[cfg(test)]
9819fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
9820 let raw = pr_message_raw(
9821 state,
9822 winner,
9823 facts,
9824 crate::pr_outcome::FinalHead::Unreported,
9825 );
9826 let id = crate::scrub::Identity::current();
9827 PrMessage {
9828 title: crate::scrub::scrub(&raw.title, &id),
9829 body: crate::scrub::scrub(&raw.body, &id),
9830 }
9831}
9832
9833fn pr_message_raw(
9834 state: &RunState,
9835 winner: char,
9836 facts: Option<&BranchFacts>,
9837 head: crate::pr_outcome::FinalHead<'_>,
9838) -> PrMessage {
9839 let summary = state
9840 .candidates
9841 .iter()
9842 .find(|c| c.label == winner)
9843 .map(|c| c.summary.as_str())
9844 .unwrap_or_default();
9845 let review = is_review_run(state);
9848 let title = if review {
9849 facts
9850 .and_then(|f| english_subject(&f.commits.first()?.0))
9851 .or_else(|| review_title(state))
9852 .or_else(|| {
9853 state
9854 .candidates
9855 .iter()
9856 .find(|c| c.label == winner)
9857 .filter(|c| !c.branch.starts_with("magi/"))
9858 .and_then(|c| english_subject(&c.branch))
9859 })
9860 .unwrap_or_else(|| neutral_title(state, winner))
9861 } else {
9862 summary_title(summary).unwrap_or_else(|| {
9863 english_title(&state.instruction, PR_TITLE_MAX)
9864 .unwrap_or_else(|| neutral_title(state, winner))
9865 })
9866 };
9867
9868 let mut body = String::new();
9869 let what = summary_without_title(summary);
9870 if !what.is_empty() {
9871 body.push_str("## Summary\n\n");
9872 body.push_str(&what);
9873 body.push_str("\n\n");
9874 }
9875
9876 body.push_str(&crate::pr_outcome::review_fixes(state, head).0);
9880 body.push_str(&crate::pr_outcome::final_verification(state, head));
9881 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
9882
9883 let open = state.open_findings();
9884 if !open.is_empty() {
9885 body.push_str("## Open review findings\n\n");
9886 for f in &open {
9887 body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
9888 }
9889 body.push('\n');
9890 }
9891
9892 if let Some(fix) = fix
9893 && !fix.rejected.is_empty()
9894 {
9895 body.push_str("## Declined by the fixer\n\n");
9896 for r in &fix.rejected {
9897 body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
9898 }
9899 body.push('\n');
9900 }
9901
9902 if review {
9903 body.push_str("## Commits under review\n\n");
9905 if let Some(facts) = facts {
9906 let mut left = COMMITS_MAX_CHARS;
9907 for (i, (subject, text)) in facts.commits.iter().enumerate() {
9908 let mut entry = format!("- {}\n", markdown_safe(subject));
9909 for l in markdown_safe(text).lines() {
9910 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
9911 }
9912 if left == 0 {
9913 body.push_str(&format!(
9914 "- ... {} more commit(s)\n",
9915 facts.commits.len() - i
9916 ));
9917 break;
9918 }
9919 if entry.len() > left {
9920 let mut end = left;
9923 while !entry.is_char_boundary(end) {
9924 end -= 1;
9925 }
9926 entry.truncate(end);
9927 entry.push_str("\n ... (truncated)\n");
9928 left = 0;
9929 } else {
9930 left -= entry.len();
9931 }
9932 body.push_str(&entry);
9933 }
9934 if !facts.stat.trim().is_empty() {
9935 body.push_str(&format!(
9936 "\n## Diff stat\n\n```\n{}\n```\n",
9937 markdown_safe(facts.stat.trim())
9938 ));
9939 }
9940 } else {
9941 match &state.reviewed_commits {
9942 Some(subjects) => {
9943 for s in subjects {
9944 body.push_str(&format!("- {}\n", s.trim()));
9945 }
9946 }
9947 None => {
9948 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
9951 body.push_str(rest.trim());
9952 body.push('\n');
9953 }
9954 }
9955 }
9956 } else {
9957 let task = state.instruction.trim();
9958 let task = if task.is_empty() {
9959 "(empty task)"
9960 } else {
9961 task
9962 };
9963 body.push_str(&format!(
9964 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
9965 task.replace("</details>", "</details>")
9966 ));
9967 }
9968
9969 body.push_str(&format!(
9970 "\n---\nmagi:run/{} magi:candidate-{}\n",
9971 state.id,
9972 winner.to_ascii_lowercase()
9973 ));
9974
9975 PrMessage { title, body }
9976}
9977
9978fn seeded_instruction(state: &RunState) -> String {
9982 match refs::describe(&state.seeds) {
9983 Some(facts) => format!(
9984 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
9985 Candidates start from the unmerged branch named above, when there \
9986 is one, and carry any unmerged commit named by sha as a \
9987 cherry-pick. Check that this is what the task meant before \
9988 building on it.",
9989 state.instruction
9990 ),
9991 None => state.instruction.clone(),
9992 }
9993}
9994
9995async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
9999 let base = &state.base_branch;
10000 let mut against = base.clone();
10001 if mode == MergeMode::Pr {
10002 let remote = &state.config.merge.remote;
10006 let tracking = format!("{remote}/{base}");
10007 let fetched = git::fetch(repo, remote, base).await;
10008 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
10009 against = tracking;
10010 } else {
10011 return false;
10012 }
10013 }
10014 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
10015}
10016
10017fn empty_candidate_detail(state: &RunState, base: &str) -> String {
10020 let mut detail = format!(
10021 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
10022 nothing to open a pull request for"
10023 );
10024 match refs::describe(&state.seeds) {
10025 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
10026 None => detail.push_str(
10027 "\nThe task names no existing branch or commit; if it means to land work \
10028 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
10029 ),
10030 }
10031 detail
10032}
10033
10034#[derive(Debug, PartialEq, Eq)]
10037enum PrPlan {
10038 Create,
10039 Adopt { url: String, title: String },
10040 Stop(String),
10041}
10042
10043fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
10046 match found {
10047 Ok(land::OpenPr::None) => PrPlan::Create,
10048 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
10049 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
10050 "several open pull requests exist for this branch, not picking one: {}",
10051 urls.join(" ")
10052 )),
10053 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
10054 }
10055}
10056
10057pub(crate) async fn gh_pr_create(
10059 cwd: &Path,
10060 base: &str,
10061 head: &str,
10062 title: &str,
10063 body: &str,
10064) -> Result<String> {
10065 let out = tokio::process::Command::new("gh")
10066 .args([
10067 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
10068 ])
10069 .current_dir(cwd)
10070 .quiet()
10071 .stdin(std::process::Stdio::null())
10072 .output()
10073 .await
10074 .context("spawn gh")?;
10075 if out.status.success() {
10076 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
10077 } else {
10078 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
10079 }
10080}
10081
10082pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
10091 let repo = state.repo.clone();
10092 let root = state.worktree_root();
10093 let winner = state.tally.as_ref().map(|t| t.winner);
10094 let mut removed = Vec::new();
10095
10096 for i in 0..state.candidates.len() {
10097 let c = state.candidates[i].clone();
10098 let is_winner = Some(c.label) == winner;
10099 if is_winner && !drop_winner {
10100 continue;
10101 }
10102 if c.worktree.exists() {
10103 git::worktree_remove(&repo, &c.worktree).await.ok();
10104 removed.push(c.worktree.to_string_lossy().into_owned());
10105 }
10106 let handed_over = state.released_branches.contains(&c.branch);
10109 if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
10110 git::branch_delete(&repo, &c.branch).await.ok();
10111 removed.push(c.branch.clone());
10112 }
10113 state.candidates[i].folded = true;
10114 }
10115
10116 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
10117 let path = name.path();
10118 let keep = !drop_winner
10119 && winner.is_some_and(|w| {
10120 path.file_name()
10121 .is_some_and(|n| n == format!("cand-{w}").as_str())
10122 });
10123 if keep {
10124 continue;
10125 }
10126 git::worktree_remove(&repo, &path).await.ok();
10127 removed.push(path.to_string_lossy().into_owned());
10128 }
10129
10130 remove_if_empty(&root);
10139
10140 if state.enabled_worktree_config && drop_winner {
10141 git::release_worktree_config(&repo).await.ok();
10145 state.enabled_worktree_config = false;
10146 }
10147 state.save_under(home)?;
10148 Ok(removed)
10149}
10150
10151fn remove_if_empty(dir: &Path) {
10162 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
10163 std::fs::remove_dir(dir).ok();
10164 }
10165}
10166
10167pub fn worst_open(state: &RunState) -> Option<Severity> {
10169 state
10170 .reviews
10171 .last()?
10172 .reviews
10173 .iter()
10174 .flat_map(|r| r.findings.iter())
10175 .map(|f| f.severity)
10176 .max()
10177}
10178
10179async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
10183 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
10184 git::acquire_worktree_config(repo).await?;
10185 state.enabled_worktree_config = true;
10186 }
10187 Ok(())
10188}
10189
10190#[cfg(test)]
10191mod tests {
10192 #[test]
10193 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
10194 let leaked = vec!["chore(deps): update a crate".to_owned()];
10195 let own = "fix(daemon): apply a chosen action";
10196 assert!(should_retitle("", own, &leaked));
10197 assert!(should_retitle(
10198 &format!("{REVIEW_PROMPT_OPENING} `x`"),
10199 own,
10200 &leaked
10201 ));
10202 assert!(should_retitle(
10203 "chore: land candidate A of run 1",
10204 own,
10205 &leaked
10206 ));
10207 assert!(should_retitle(
10208 "magi: candidate A (uncommitted work)",
10209 own,
10210 &leaked
10211 ));
10212 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
10213 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
10214 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
10215 }
10216
10217 #[test]
10218 fn pr_merge_plan_creates_adopts_or_stops() {
10219 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
10220 assert_eq!(
10221 pr_merge_plan(Ok(land::OpenPr::One {
10222 url: "u".into(),
10223 title: "t".into()
10224 })),
10225 PrPlan::Adopt {
10226 url: "u".into(),
10227 title: "t".into()
10228 }
10229 );
10230 let PrPlan::Stop(many) =
10231 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
10232 else {
10233 panic!("many must stop");
10234 };
10235 assert!(many.contains('a') && many.contains('b'));
10236 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
10237 panic!("a failed lookup must stop");
10238 };
10239 assert!(err.contains("bad token"));
10240 }
10241
10242 use super::*;
10243 use crate::run::GateStatus;
10244 use std::collections::BTreeMap;
10245 use std::time::Duration;
10246
10247 fn conductor() -> AgentSpec {
10248 AgentSpec {
10249 id: "conductor".to_owned(),
10250 kind: crate::config::AgentKind::Command,
10251 model: None,
10252 command: vec!["true".to_owned()],
10253 extra_args: Vec::new(),
10254 env: BTreeMap::new(),
10255 prompt_delivery: None,
10256 }
10257 }
10258
10259 fn spec(id: &str) -> AgentSpec {
10260 AgentSpec {
10261 id: id.to_owned(),
10262 kind: crate::config::AgentKind::Command,
10263 model: None,
10264 command: vec!["true".to_owned()],
10265 extra_args: Vec::new(),
10266 env: BTreeMap::new(),
10267 prompt_delivery: None,
10268 }
10269 }
10270
10271 fn ids(xs: &[&str]) -> BTreeSet<String> {
10272 xs.iter().map(|s| (*s).to_owned()).collect()
10273 }
10274
10275 #[test]
10276 fn pick_successor_skips_an_agent_another_seat_holds() {
10277 let roster = [spec("a"), spec("b"), spec("c")];
10279 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
10280 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
10281 }
10282
10283 #[test]
10284 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
10285 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
10287 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
10288 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
10289 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
10290 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
10291 }
10292
10293 #[test]
10294 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
10295 let roster = [spec("a"), spec("b")];
10296 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
10297 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10298 let carried = ids(&["b"]);
10299 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
10300 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10301 }
10302
10303 #[test]
10304 fn pick_successor_returns_none_without_any_untried_successor() {
10305 let roster = [spec("a"), spec("b")];
10306 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
10307 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
10308 let carried = ids(&["a"]);
10309 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
10310 }
10311
10312 #[test]
10313 fn pick_successor_rescue_avoids_another_seats_occupant() {
10314 let roster = [spec("a"), spec("b"), spec("c")];
10316 let carried = ids(&["b", "c"]);
10317 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
10318 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10319 }
10320
10321 #[test]
10322 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
10323 let roster = [spec("a"), spec("b"), spec("c")];
10324 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
10325 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
10326 }
10327
10328 #[test]
10329 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
10330 let roster = [spec("a"), spec("b"), spec("c")];
10331 let failed = ids(&["a", "b", "c"]);
10332 let mut tried = ids(&["b"]);
10334 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
10335 assert_eq!(first.id, "a");
10336 tried.insert(first.id.clone());
10337 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
10338 assert_eq!(second.id, "c");
10339 tried.insert(second.id.clone());
10340 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
10341 }
10342
10343 #[test]
10344 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
10345 let roster = [spec("a"), spec("b")];
10346 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
10347 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10348 }
10349
10350 #[test]
10351 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
10352 let roster = [spec("a"), spec("b"), spec("c")];
10353 let h = SeatHistory {
10354 failed: ids(&["a"]),
10355 last_ok: Some("b".to_owned()),
10356 last_fail: None,
10357 };
10358 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
10359 let gone = SeatHistory {
10361 last_ok: Some("zzz".to_owned()),
10362 ..h.clone()
10363 };
10364 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
10365 let failed = SeatHistory {
10366 failed: ids(&["a", "b"]),
10367 last_ok: Some("b".to_owned()),
10368 last_fail: None,
10369 };
10370 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
10371 }
10372
10373 #[test]
10374 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
10375 let first = SeatState::new("review-1", "alpha", 7);
10376 let next = handover_seat("review-1", "gamma", 7);
10377 assert_ne!(first.claude_session, next.claude_session);
10378 }
10379
10380 #[test]
10381 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
10382 let mut state = state_with_summary("x", "y");
10383 let a = handover_seat("review-1", "beta", state.next_seat_seed());
10384 let b = handover_seat("review-1", "beta", state.next_seat_seed());
10385 assert_ne!(a.claude_session, b.claude_session);
10386 }
10387
10388 #[test]
10389 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
10390 let roster = [spec("a"), spec("b")];
10391 let h = SeatHistory {
10392 failed: ids(&["a", "b"]),
10393 ..SeatHistory::default()
10394 };
10395 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
10396 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
10397 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
10398 }
10399
10400 #[test]
10407 fn failure_signature_ignores_numbers_and_paths() {
10408 assert_eq!(
10409 failure_signature("exited with Some(2) and no usable output"),
10410 failure_signature("exited with Some(137) and no usable output")
10411 );
10412 assert_eq!(
10413 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
10414 failure_signature("cannot open /var/x.txt: denied")
10415 );
10416 assert_ne!(failure_signature("boom"), failure_signature("bang"));
10417 }
10418
10419 #[test]
10420 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
10421 let other = FailClass::Other("x".into());
10422 assert!(should_hand_over(None, &FailClass::Quota));
10423 assert!(should_hand_over(Some(&other), &FailClass::Quota));
10424 assert!(should_hand_over(
10425 Some(&FailClass::Timeout),
10426 &FailClass::Timeout
10427 ));
10428 assert!(should_hand_over(None, &other));
10429 assert!(!should_hand_over(Some(&other), &other));
10430 assert!(should_hand_over(
10431 Some(&other),
10432 &FailClass::Other("y".into())
10433 ));
10434 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
10436 assert!(should_hand_over(Some(&FailClass::Quota), &other));
10437 }
10438
10439 #[test]
10440 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
10441 let a = SeatState::new("judge-1", "alpha", 7);
10442 let b = handover_seat("judge-1", "beta", 7);
10443 assert_ne!(a.claude_session, b.claude_session);
10444 assert_eq!(b.turns, 0);
10445 }
10446
10447 #[test]
10448 fn a_timeout_is_classified_apart_from_other_failures() {
10449 assert_eq!(
10450 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
10451 Some(FailClass::Timeout)
10452 );
10453 assert!(matches!(
10454 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
10455 Some(FailClass::Other(_))
10456 ));
10457 }
10458
10459 #[test]
10460 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
10461 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
10462 let tried = BTreeSet::from(["beta".to_owned()]);
10463 let next = next_untried_in_roster(&roster, 1, &tried);
10466 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
10467 }
10468
10469 #[test]
10470 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
10471 let roster = vec![spec("alpha"), spec("beta")];
10472 let tried = BTreeSet::from(["beta".to_owned()]);
10473 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
10477 }
10478
10479 #[test]
10480 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
10481 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
10482 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
10483 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
10487 }
10488
10489 #[test]
10490 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
10491 let roster = vec![spec("a"), spec("a"), spec("b")];
10492 let tried = BTreeSet::from(["a".to_owned()]);
10493 let next = next_untried_in_roster(&roster, 0, &tried);
10494 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
10495 }
10496
10497 #[test]
10498 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
10499 let roster = vec![spec("a"), spec("b")];
10500 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
10501 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
10502 }
10503
10504 #[test]
10505 fn remove_if_empty_only_ever_takes_a_bare_directory() {
10506 let dir = tempfile::tempdir().unwrap();
10507 let bay = dir.path().join("ffff");
10508
10509 remove_if_empty(&bay);
10511 assert!(!bay.exists());
10512
10513 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
10516 remove_if_empty(&bay);
10517 assert!(bay.exists(), "non-empty directory must survive");
10518
10519 std::fs::remove_dir(bay.join("cand-A")).unwrap();
10521 remove_if_empty(&bay);
10522 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
10523 }
10524
10525 #[test]
10534 fn a_full_panel_that_found_nothing_is_clean() {
10535 assert!(round_is_clean(
10536 0,
10537 true,
10538 2,
10539 2,
10540 0,
10541 IncompleteReviewPolicy::Block
10542 ));
10543 }
10544
10545 #[test]
10546 fn a_missing_seat_is_never_clean_under_the_default_policy() {
10547 assert!(!round_is_clean(
10548 0,
10549 true,
10550 1,
10551 2,
10552 0,
10553 IncompleteReviewPolicy::Block
10554 ));
10555 }
10556
10557 #[test]
10558 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
10559 assert!(!round_is_clean(
10560 1,
10561 true,
10562 1,
10563 2,
10564 0,
10565 IncompleteReviewPolicy::Warn
10566 ));
10567 }
10568
10569 #[test]
10570 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
10571 assert!(round_is_clean(
10572 0,
10573 true,
10574 1,
10575 2,
10576 0,
10577 IncompleteReviewPolicy::Warn
10578 ));
10579 }
10580
10581 #[test]
10582 fn a_full_panel_with_an_open_finding_is_not_clean() {
10583 assert!(!round_is_clean(
10584 1,
10585 true,
10586 2,
10587 2,
10588 0,
10589 IncompleteReviewPolicy::Block
10590 ));
10591 }
10592
10593 #[test]
10594 fn a_full_panel_with_a_red_e2e_is_not_clean() {
10595 assert!(!round_is_clean(
10596 0,
10597 false,
10598 2,
10599 2,
10600 0,
10601 IncompleteReviewPolicy::Block
10602 ));
10603 }
10604
10605 #[test]
10612 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
10613 assert!(round_is_clean(
10616 0,
10617 true,
10618 1,
10619 2,
10620 1,
10621 IncompleteReviewPolicy::Block
10622 ));
10623 }
10624
10625 #[test]
10626 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
10627 assert!(!round_is_clean(
10630 0,
10631 true,
10632 1,
10633 2,
10634 0,
10635 IncompleteReviewPolicy::Block
10636 ));
10637 }
10638
10639 #[test]
10640 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
10641 assert!(!round_is_clean(
10642 1,
10643 true,
10644 1,
10645 2,
10646 1,
10647 IncompleteReviewPolicy::Block
10648 ));
10649 assert!(!round_is_clean(
10650 0,
10651 false,
10652 1,
10653 2,
10654 1,
10655 IncompleteReviewPolicy::Block
10656 ));
10657 }
10658
10659 #[test]
10660 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
10661 assert!(!round_is_clean(
10665 0,
10666 true,
10667 0,
10668 2,
10669 2,
10670 IncompleteReviewPolicy::Block
10671 ));
10672 }
10673
10674 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
10675 CommandOutcome {
10676 command: "test".to_owned(),
10677 code,
10678 output_tail: String::new(),
10679 duration_ms: 0,
10680 resource_blocked,
10681 }
10682 }
10683
10684 #[test]
10685 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
10686 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
10687 assert!(
10688 !verify_inconclusive(&[outcome(Some(1), false)]),
10689 "an ordinary failure is still evidence about the patch"
10690 );
10691 assert!(verify_inconclusive(&[outcome(None, true)]));
10692 assert!(
10693 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
10694 "one inconclusive outcome taints the whole batch"
10695 );
10696 assert!(!verify_inconclusive(&[]));
10697 }
10698
10699 #[tokio::test]
10700 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
10701 let calls = std::sync::atomic::AtomicUsize::new(0);
10705 let started = Instant::now();
10706 wait_for_pids_with(
10707 &[123],
10708 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
10709 Duration::from_millis(5),
10710 Duration::from_secs(5),
10711 )
10712 .await;
10713 assert!(
10714 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
10715 "must keep checking rather than deciding on the first answer"
10716 );
10717 assert!(
10718 started.elapsed() < Duration::from_secs(1),
10719 "must return the moment it is confirmed dead, not wait out the ceiling"
10720 );
10721 }
10722
10723 #[tokio::test]
10724 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
10725 let started = Instant::now();
10726 wait_for_pids_with(
10727 &[123],
10728 |_| true, Duration::from_millis(5),
10730 Duration::from_millis(30),
10731 )
10732 .await;
10733 let elapsed = started.elapsed();
10734 assert!(
10735 elapsed >= Duration::from_millis(30),
10736 "must not give up before its own ceiling: {elapsed:?}"
10737 );
10738 assert!(
10739 elapsed < Duration::from_secs(1),
10740 "must not wait past its own ceiling either: {elapsed:?}"
10741 );
10742 }
10743
10744 #[tokio::test]
10745 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
10746 let started = Instant::now();
10747 wait_for_pids_with(
10748 &[],
10749 |_| true,
10750 Duration::from_secs(5),
10751 Duration::from_secs(5),
10752 )
10753 .await;
10754 assert!(
10755 started.elapsed() < Duration::from_millis(200),
10756 "an empty pid list has nothing to confirm"
10757 );
10758 }
10759
10760 fn review_round(
10766 clean: bool,
10767 blocking: usize,
10768 answered: usize,
10769 expected: usize,
10770 progressed: bool,
10771 e2e_ok: bool,
10772 ) -> ReviewRound {
10773 ReviewRound {
10774 round: 1,
10775 head: "h".to_owned(),
10776 verified_head: None,
10777 verified_at: None,
10778 reviews: Vec::new(),
10779 e2e: vec![CommandOutcome {
10780 command: "test".to_owned(),
10781 code: Some(if e2e_ok { 0 } else { 1 }),
10782 output_tail: String::new(),
10783 duration_ms: 0,
10784 resource_blocked: false,
10785 }],
10786 verify_retried: false,
10787 e2e_deferred: false,
10788 e2e_defer_reason: None,
10789 fix: None,
10790 blocking,
10791 answered,
10792 expected,
10793 clean,
10794 progressed,
10795 vote_split: false,
10796 reconsideration: Vec::new(),
10797 verdict: None,
10798 }
10799 }
10800
10801 #[test]
10802 fn review_conclusion_is_none_when_nothing_has_run() {
10803 assert_eq!(review_conclusion(&[], 3), None);
10804 }
10805
10806 #[test]
10807 fn review_conclusion_is_none_while_rounds_remain() {
10808 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
10809 assert_eq!(review_conclusion(&rounds, 3), None);
10810 }
10811
10812 #[test]
10813 fn review_conclusion_is_gating_once_a_round_is_clean() {
10814 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
10815 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
10816 }
10817
10818 #[test]
10819 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
10820 let rounds = vec![
10821 review_round(false, 1, 2, 2, true, true),
10822 review_round(false, 1, 2, 2, true, true),
10823 ];
10824 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
10825 }
10826
10827 #[test]
10828 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
10829 let rounds = vec![
10830 review_round(false, 1, 2, 2, true, true),
10831 review_round(false, 1, 2, 2, true, false),
10832 ];
10833 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
10834 }
10835
10836 #[test]
10837 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
10838 let mut blocked = review_round(false, 1, 2, 2, true, false);
10845 blocked.e2e[0].resource_blocked = true;
10846 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
10847 assert_eq!(review_conclusion(&rounds, 2), None);
10848 }
10849
10850 #[test]
10851 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
10852 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
10854 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
10855 }
10856
10857 #[test]
10858 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
10859 let rounds = vec![
10860 review_round(false, 1, 2, 2, false, true),
10861 review_round(false, 1, 2, 2, false, true),
10862 ];
10863 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
10864 }
10865
10866 fn secs(n: u64) -> Duration {
10867 Duration::from_secs(n)
10868 }
10869
10870 fn init_repo(dir: &Path) {
10873 let run = |args: &[&str]| {
10874 let out = std::process::Command::new("git")
10875 .args(args)
10876 .current_dir(dir)
10877 .quiet()
10878 .output()
10879 .expect("spawn git");
10880 assert!(
10881 out.status.success(),
10882 "git {args:?} failed: {}",
10883 String::from_utf8_lossy(&out.stderr)
10884 );
10885 };
10886 run(&["init", "-b", "main"]);
10887 run(&["config", "user.name", "magi test"]);
10888 run(&["config", "user.email", "magi@example.com"]);
10889 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
10890 run(&["add", "-A"]);
10891 run(&["commit", "-m", "init"]);
10892 }
10893
10894 fn ask_test_home() {
10902 crate::run::pin_test_home();
10903 }
10904
10905 fn runner_at(status: RunStatus) -> Runner {
10908 let mut state = RunState::new(
10909 PathBuf::from("/nonexistent/repo"),
10910 "main".to_owned(),
10911 "deadbeef".to_owned(),
10912 "task".to_owned(),
10913 Config::default(),
10914 );
10915 state.status = status;
10916 Runner {
10917 state,
10918 roles: ResolvedRoles {
10919 implementers: Vec::new(),
10920 judges: Vec::new(),
10921 reviewers: Vec::new(),
10922 fixer: None,
10923 conductor: conductor(),
10924 implementer_roster: Vec::new(),
10925 judge_roster: Vec::new(),
10926 reviewer_roster: Vec::new(),
10927 },
10928 sem: Arc::new(Semaphore::new(1)),
10929 pause: Pause::new(),
10930 interrupt: Pause::new(),
10931 }
10932 }
10933
10934 #[test]
10938 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
10939 crate::run::pin_test_home();
10940 let mut runner = runner_at(RunStatus::Implementing);
10941 let interrupt = Pause::new();
10942 runner.watch_interrupt(interrupt.clone());
10943
10944 interrupt.park_because("task a1b2 asked to run first");
10945
10946 assert!(runner.park_here().expect("park_here"));
10947 assert!(runner.state.parked);
10948 let last = runner.state.events.last().expect("a park event");
10949 assert_eq!(last.node, "park");
10950 assert!(
10951 last.message.contains("task a1b2 asked to run first"),
10952 "expected the interrupt reason in {:?}",
10953 last.message
10954 );
10955 }
10956
10957 #[test]
10960 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
10961 crate::run::pin_test_home();
10962 let mut runner = runner_at(RunStatus::Implementing);
10963 runner.state.begin_pass();
10964 let shutdown = Pause::new();
10965 runner.on_pause(shutdown.clone());
10966 let upgrade = ParkCause::Upgrade {
10967 from: "0.121.0".to_owned(),
10968 to: Some("0.121.1".to_owned()),
10969 };
10970 shutdown.park_for(upgrade.clone());
10971 shutdown.park_for(ParkCause::Operator);
10972
10973 assert!(runner.park_here().expect("park_here"));
10974 assert_eq!(
10975 runner.state.passes[0].end,
10976 Some(crate::run::PassEnd::Parked { cause: upgrade })
10977 );
10978 let last = runner.state.events.last().expect("a park event");
10979 assert!(last.message.contains("0.121.0"), "{}", last.message);
10980 }
10981
10982 #[test]
10990 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
10991 crate::run::pin_test_home();
10992 let mut runner = runner_at(RunStatus::Implementing);
10993 let shutdown = Pause::new();
10994 runner.on_pause(shutdown.clone());
10995 let interrupt = Pause::new();
10996 runner.watch_interrupt(interrupt.clone());
10997
10998 assert!(!runner.park_here().expect("park_here"));
11000 assert!(!runner.state.parked);
11001
11002 interrupt.park_because("test");
11004 assert!(!shutdown.parked());
11005 assert!(runner.park_here().expect("park_here"));
11006 }
11007
11008 #[tokio::test]
11022 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
11023 crate::run::pin_test_home();
11024 let mut runner = runner_at(RunStatus::Implementing);
11025 let interrupt = Pause::new();
11026 runner.watch_interrupt(interrupt.clone());
11027
11028 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
11029 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
11030
11031 let node = async move {
11035 started_tx.send(()).expect("send started");
11036 finish_rx.await.expect("recv finish");
11037 "node finished"
11038 };
11039
11040 let interrupter = async move {
11041 started_rx.await.expect("recv started");
11042 interrupt.park_because("higher-priority task waiting");
11044 tokio::task::yield_now().await;
11048 finish_tx.send(()).expect("send finish");
11049 };
11050
11051 let (node_result, ()) = tokio::join!(node, interrupter);
11052 assert_eq!(
11053 node_result, "node finished",
11054 "the in-flight call ran to completion"
11055 );
11056
11057 assert!(runner.park_here().expect("park_here"));
11060 assert!(runner.state.parked);
11061 }
11062
11063 #[test]
11069 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
11070 crate::run::pin_test_home();
11071 let mut runner = runner_at(RunStatus::Judging);
11072 runner.state.config.agents = vec![conductor()];
11076 runner.state.candidates = vec![Candidate {
11077 index: 0,
11078 label: 'A',
11079 agent: "alpha".to_owned(),
11080 branch: "magi/x/A".to_owned(),
11081 worktree: PathBuf::from("/nonexistent/worktree"),
11082 summary: "did the thing".to_owned(),
11083 stat: "1 file changed".to_owned(),
11084 files: 1,
11085 commits: 1,
11086 empty: false,
11087 failed: None,
11088 verified_noop: None,
11089 duration_ms: 1234,
11090 folded: false,
11091 }];
11092 let run_id = runner.state.id.clone();
11093
11094 let interrupt = Pause::new();
11095 runner.watch_interrupt(interrupt.clone());
11096 interrupt.park_because("task c3d4 asked to run first");
11097 assert!(runner.park_here().expect("park_here"));
11098
11099 let resumed = Runner::resume(&run_id).expect("resume");
11100 assert_eq!(resumed.state.candidates.len(), 1);
11101 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
11102 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
11103 assert_eq!(resumed.state.status, runner.state.status);
11104 assert!(
11105 resumed.state.parked,
11106 "still parked until `execute` actually walks the graph again"
11107 );
11108 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
11109 }
11110
11111 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
11113 let mut q = ask::Question::new(
11114 run.to_owned(),
11115 "implement".to_owned(),
11116 "impl-A".to_owned(),
11117 "Which storage backend should the cache use?".to_owned(),
11118 String::new(),
11119 vec!["SQLite".to_owned(), "Redis".to_owned()],
11120 );
11121 store.put(&mut q).unwrap();
11122 q
11123 }
11124
11125 #[test]
11126 fn a_failed_runs_open_question_is_abandoned() {
11127 ask_test_home();
11128 let store = ask::Questions::open();
11129 let mut runner = runner_at(RunStatus::Failed);
11130 let run = runner.state.id.clone();
11131 let q = ask_open_question(&store, &run);
11132
11133 runner.settle_questions();
11134
11135 let back = store.get(&q.id).unwrap();
11136 assert!(
11137 !back.status.open(),
11138 "the seat that asked died with the run; nobody is left to read an answer"
11139 );
11140 assert!(
11141 back.detail.contains(&run) && back.detail.contains("failed"),
11142 "the reason names what the run became, not just that it is gone: {}",
11143 back.detail
11144 );
11145 }
11146
11147 #[test]
11148 fn a_merged_runs_open_question_is_abandoned_too() {
11149 ask_test_home();
11150 let store = ask::Questions::open();
11151 for status in [RunStatus::Merged, RunStatus::Ready] {
11154 let mut runner = runner_at(status);
11155 let run = runner.state.id.clone();
11156 let q = ask_open_question(&store, &run);
11157
11158 runner.settle_questions();
11159
11160 let back = store.get(&q.id).unwrap();
11161 assert!(
11162 !back.status.open(),
11163 "{status:?} run's question must not outlive the run"
11164 );
11165 }
11166 }
11167
11168 #[test]
11169 fn a_still_resumable_runs_open_question_is_left_alone() {
11170 ask_test_home();
11171 let store = ask::Questions::open();
11172 for status in [RunStatus::Blocked, RunStatus::Stalled] {
11178 let mut runner = runner_at(status);
11179 let run = runner.state.id.clone();
11180 let q = ask_open_question(&store, &run);
11181
11182 runner.settle_questions();
11183
11184 let back = store.get(&q.id).unwrap();
11185 assert!(
11186 back.status.open(),
11187 "{status:?} is still alive; the question must still be waiting"
11188 );
11189 }
11190 }
11191
11192 #[test]
11193 fn settle_questions_never_touches_an_already_answered_question() {
11194 ask_test_home();
11195 let store = ask::Questions::open();
11196 let mut runner = runner_at(RunStatus::Failed);
11197 let run = runner.state.id.clone();
11198 let mut q = ask_open_question(&store, &run);
11199 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
11200 .unwrap();
11201 store.put(&mut q).unwrap();
11202
11203 runner.settle_questions();
11208 runner.settle_questions();
11209
11210 let back = store.get(&q.id).unwrap();
11211 assert_eq!(
11212 back.status,
11213 ask::QuestionStatus::Answered,
11214 "a real answer is a decision on record, never overwritten by a sweep"
11215 );
11216 }
11217
11218 #[tokio::test]
11229 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
11230 crate::run::pin_test_home();
11231 let tmp = tempfile::tempdir().expect("tempdir");
11232 let repo = tmp.path().join("repo");
11233 std::fs::create_dir_all(&repo).unwrap();
11234 init_repo(&repo);
11235
11236 let mut config = Config::default();
11237 config.graph.worktree_root = Some(tmp.path().join("wt"));
11238
11239 let mut state = RunState::new(
11240 repo.clone(),
11241 "main".to_owned(),
11242 "deadbeef".to_owned(),
11243 "task".to_owned(),
11244 config,
11245 );
11246 let root = state.worktree_root();
11247 let wt_a = root.join("cand-A");
11248 let wt_b = root.join("cand-B");
11249 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
11250 .await
11251 .expect("worktree A");
11252 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
11253 .await
11254 .expect("worktree B");
11255
11256 state.candidates = vec![
11257 Candidate {
11258 index: 0,
11259 label: 'A',
11260 agent: "alpha".to_owned(),
11261 branch: "magi/x/A".to_owned(),
11262 worktree: wt_a.clone(),
11263 summary: String::new(),
11264 stat: String::new(),
11265 files: 0,
11266 commits: 0,
11267 empty: false,
11268 failed: None,
11269 verified_noop: None,
11270 duration_ms: 0,
11271 folded: false,
11272 },
11273 Candidate {
11274 index: 1,
11275 label: 'B',
11276 agent: "beta".to_owned(),
11277 branch: "magi/x/B".to_owned(),
11278 worktree: wt_b.clone(),
11279 summary: String::new(),
11280 stat: String::new(),
11281 files: 0,
11282 commits: 0,
11283 empty: false,
11284 failed: None,
11285 verified_noop: None,
11286 duration_ms: 0,
11287 folded: false,
11288 },
11289 ];
11290 state.tally = Some(Tally {
11291 first_choice: BTreeMap::from([('A', 1)]),
11292 borda: BTreeMap::new(),
11293 winner: 'A',
11294 rankings: 1,
11295 unanimous_initial: true,
11296 deliberated: false,
11297 changed_votes: 0,
11298 unanimous_final: true,
11299 tie_break: None,
11300 judges: 1,
11301 present: 1,
11302 quorum: 1,
11303 met_quorum: true,
11304 uncontested: None,
11305 });
11306 state.status = RunStatus::Ready;
11307
11308 fold_run(&mut state, false, &crate::run::home())
11309 .await
11310 .expect("fold_run");
11311
11312 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
11313 assert!(
11314 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
11315 "the unmerged winner's branch survives"
11316 );
11317 assert!(
11318 !state.candidates[0].folded,
11319 "the winner is not marked folded"
11320 );
11321
11322 assert!(!wt_b.exists(), "the loser's worktree is removed");
11323 assert!(
11324 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
11325 "the loser's branch is removed"
11326 );
11327 assert!(state.candidates[1].folded, "the loser is marked folded");
11328 }
11329
11330 #[tokio::test]
11333 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
11334 let tmp = tempfile::tempdir().expect("tempdir");
11335 let repo = tmp.path().join("repo");
11336 std::fs::create_dir_all(&repo).unwrap();
11337 init_repo(&repo);
11338 let home = tmp.path().join("home");
11339
11340 let mut config = Config::default();
11341 config.graph.worktree_root = Some(tmp.path().join("wt"));
11342 let mut state = RunState::new(
11343 repo.clone(),
11344 "main".to_owned(),
11345 "deadbeef".to_owned(),
11346 "task".to_owned(),
11347 config,
11348 );
11349 git::git(&repo, &["branch", "magi/x/A", "main"])
11351 .await
11352 .expect("branch");
11353 state.candidates = vec![Candidate {
11354 index: 0,
11355 label: 'A',
11356 agent: "alpha".to_owned(),
11357 branch: "magi/x/A".to_owned(),
11358 worktree: state.worktree_root().join("cand-A"),
11359 summary: String::new(),
11360 stat: String::new(),
11361 files: 0,
11362 commits: 0,
11363 empty: false,
11364 failed: None,
11365 verified_noop: None,
11366 duration_ms: 0,
11367 folded: true,
11368 }];
11369 state.released_to = Some("20260901-000000-new1".to_owned());
11370 state.released_branches = vec!["magi/x/A".to_owned()];
11371
11372 fold_run(&mut state, true, &home).await.expect("fold_run");
11373
11374 assert!(
11375 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
11376 "the handed-over branch survives a fold"
11377 );
11378 }
11379
11380 #[tokio::test]
11384 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
11385 let tmp = tempfile::tempdir().expect("tempdir");
11386 let repo = tmp.path().join("repo");
11387 std::fs::create_dir_all(&repo).unwrap();
11388 init_repo(&repo);
11389 let run = |args: &[&str]| {
11390 let out = std::process::Command::new("git")
11391 .quiet()
11392 .args(args)
11393 .current_dir(&repo)
11394 .output()
11395 .expect("spawn git");
11396 assert!(out.status.success(), "git {args:?}");
11397 };
11398 run(&["branch", "magi/x/A"]);
11399 run(&["checkout", "-q", "-b", "magi/x/B"]);
11400 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
11401 run(&["add", "-A"]);
11402 run(&["commit", "-q", "-m", "work"]);
11403 run(&["checkout", "-q", "main"]);
11404
11405 let probe = RunState::new(
11409 repo.clone(),
11410 "main".to_owned(),
11411 "deadbeef".to_owned(),
11412 "task".to_owned(),
11413 Config::default(),
11414 );
11415 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
11416 let bare = tmp.path().join("origin.git");
11417 let out = std::process::Command::new("git")
11418 .quiet()
11419 .args(["init", "-q", "--bare"])
11420 .arg(&bare)
11421 .output()
11422 .expect("spawn git");
11423 assert!(out.status.success(), "git init --bare");
11424 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
11425 run(&["push", "-q", "origin", "main"]);
11426
11427 let mut state = RunState::new(
11428 repo.clone(),
11429 "main".to_owned(),
11430 "deadbeef".to_owned(),
11431 "task".to_owned(),
11432 Config::default(),
11433 );
11434 state.seeds = vec![refs::Seed {
11435 token: "magi/27b2/A".to_owned(),
11436 kind: refs::SeedKind::Unresolved,
11437 sha: String::new(),
11438 branch: true,
11439 detail: "no branch or commit named magi/27b2/A".to_owned(),
11440 }];
11441
11442 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
11443 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
11444 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
11445 let detail = empty_candidate_detail(&state, "main");
11446 assert!(detail.starts_with("empty candidate"), "{detail}");
11447 assert!(detail.contains("magi/27b2/A"), "{detail}");
11448 }
11449
11450 #[tokio::test]
11459 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
11460 let tmp = tempfile::tempdir().expect("tempdir");
11461 let repo = tmp.path().join("repo");
11462 std::fs::create_dir_all(&repo).unwrap();
11463 init_repo(&repo);
11464
11465 let mut config = Config::default();
11466 config.merge.mode = MergeMode::Local;
11467
11468 let mut state = RunState::new(
11469 repo.clone(),
11470 "main".to_owned(),
11471 "deadbeef".to_owned(),
11472 "task".to_owned(),
11473 config,
11474 );
11475 state.candidates = vec![Candidate {
11476 index: 0,
11477 label: 'A',
11478 agent: "alpha".to_owned(),
11479 branch: "does-not-exist".to_owned(),
11480 worktree: repo.clone(),
11481 summary: String::new(),
11482 stat: String::new(),
11483 files: 0,
11484 commits: 0,
11485 empty: false,
11486 failed: None,
11487 verified_noop: None,
11488 duration_ms: 0,
11489 folded: false,
11490 }];
11491 state.tally = Some(Tally {
11492 first_choice: BTreeMap::from([('A', 1)]),
11493 borda: BTreeMap::new(),
11494 winner: 'A',
11495 rankings: 1,
11496 unanimous_initial: true,
11497 deliberated: false,
11498 changed_votes: 0,
11499 unanimous_final: true,
11500 tie_break: None,
11501 judges: 0,
11502 present: 0,
11503 quorum: 0,
11504 met_quorum: true,
11505 uncontested: Some("only candidate A produced a change".to_owned()),
11506 });
11507 state.reviews = vec![ReviewRound {
11508 round: 1,
11509 head: "deadbeef".to_owned(),
11510 verified_head: None,
11511 verified_at: None,
11512 reviews: Vec::new(),
11513 e2e: Vec::new(),
11514 fix: None,
11515 blocking: 0,
11516 answered: 0,
11517 expected: 0,
11518 clean: true,
11519 verify_retried: false,
11520 e2e_deferred: false,
11521 e2e_defer_reason: None,
11522 progressed: false,
11523 vote_split: false,
11524 reconsideration: Vec::new(),
11525 verdict: None,
11526 }];
11527 state.gate = vec![CommandOutcome {
11528 command: "test".to_owned(),
11529 code: Some(0),
11530 output_tail: String::new(),
11531 duration_ms: 0,
11532 resource_blocked: false,
11533 }];
11534 state.gate_ran = true;
11535 state.status = RunStatus::Ready;
11540 state.merge = Some(MergeOutcome {
11541 mode: MergeMode::Local,
11542 ok: false,
11543 detail: "already concluded".to_owned(),
11544 empty: false,
11545 });
11546
11547 let mut runner = Runner {
11548 state,
11549 roles: ResolvedRoles {
11550 implementers: Vec::new(),
11551 judges: Vec::new(),
11552 reviewers: Vec::new(),
11553 fixer: None,
11554 conductor: conductor(),
11555 implementer_roster: Vec::new(),
11556 judge_roster: Vec::new(),
11557 reviewer_roster: Vec::new(),
11558 },
11559 sem: Arc::new(Semaphore::new(1)),
11560 pause: Pause::new(),
11561 interrupt: Pause::new(),
11562 };
11563
11564 runner.merge().await.expect("merge");
11565
11566 assert_eq!(
11567 runner.state.status,
11568 RunStatus::Ready,
11569 "a concluded run's status must not change on reentry"
11570 );
11571 assert_eq!(
11572 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
11573 Some("already concluded"),
11574 "merge must not run again once the node already recorded an outcome"
11575 );
11576 }
11577
11578 #[tokio::test]
11587 async fn merge_refuses_a_gate_that_has_not_actually_run() {
11588 let tmp = tempfile::tempdir().expect("tempdir");
11589 let repo = tmp.path().join("repo");
11590 std::fs::create_dir_all(&repo).unwrap();
11591 init_repo(&repo);
11592
11593 let mut config = Config::default();
11594 config.merge.mode = MergeMode::Local;
11595
11596 let mut state = RunState::new(
11597 repo.clone(),
11598 "main".to_owned(),
11599 "deadbeef".to_owned(),
11600 "task".to_owned(),
11601 config,
11602 );
11603 state.candidates = vec![Candidate {
11604 index: 0,
11605 label: 'A',
11606 agent: "alpha".to_owned(),
11607 branch: "does-not-exist".to_owned(),
11608 worktree: repo.clone(),
11609 summary: String::new(),
11610 stat: String::new(),
11611 files: 0,
11612 commits: 0,
11613 empty: false,
11614 failed: None,
11615 verified_noop: None,
11616 duration_ms: 0,
11617 folded: false,
11618 }];
11619 state.tally = Some(Tally {
11620 first_choice: BTreeMap::from([('A', 1)]),
11621 borda: BTreeMap::new(),
11622 winner: 'A',
11623 rankings: 1,
11624 unanimous_initial: true,
11625 deliberated: false,
11626 changed_votes: 0,
11627 unanimous_final: true,
11628 tie_break: None,
11629 judges: 0,
11630 present: 0,
11631 quorum: 0,
11632 met_quorum: true,
11633 uncontested: Some("only candidate A produced a change".to_owned()),
11634 });
11635 state.reviews = vec![ReviewRound {
11636 round: 1,
11637 head: "deadbeef".to_owned(),
11638 verified_head: None,
11639 verified_at: None,
11640 reviews: Vec::new(),
11641 e2e: Vec::new(),
11642 fix: None,
11643 blocking: 0,
11644 answered: 0,
11645 expected: 0,
11646 clean: true,
11647 verify_retried: false,
11648 e2e_deferred: false,
11649 e2e_defer_reason: None,
11650 progressed: false,
11651 vote_split: false,
11652 reconsideration: Vec::new(),
11653 verdict: None,
11654 }];
11655 state.gate = Vec::new();
11657 state.gate_ran = false;
11658 state.status = RunStatus::Gating;
11659
11660 let mut runner = Runner {
11661 state,
11662 roles: ResolvedRoles {
11663 implementers: Vec::new(),
11664 judges: Vec::new(),
11665 reviewers: Vec::new(),
11666 fixer: None,
11667 conductor: conductor(),
11668 implementer_roster: Vec::new(),
11669 judge_roster: Vec::new(),
11670 reviewer_roster: Vec::new(),
11671 },
11672 sem: Arc::new(Semaphore::new(1)),
11673 pause: Pause::new(),
11674 interrupt: Pause::new(),
11675 };
11676
11677 runner.merge().await.expect("merge");
11678
11679 assert!(
11680 runner.state.merge.is_none(),
11681 "an empty gate must never be read as a passing one: {:?}",
11682 runner.state.merge
11683 );
11684 }
11685
11686 #[tokio::test]
11693 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
11694 let tmp = tempfile::tempdir().expect("tempdir");
11695 let repo = tmp.path().join("repo");
11696 std::fs::create_dir_all(&repo).unwrap();
11697 init_repo(&repo);
11698
11699 let config = Config::default();
11701
11702 let mut state = RunState::new(
11703 repo.clone(),
11704 "main".to_owned(),
11705 "deadbeef".to_owned(),
11706 "task".to_owned(),
11707 config,
11708 );
11709 state.candidates = vec![Candidate {
11710 index: 0,
11711 label: 'A',
11712 agent: "alpha".to_owned(),
11713 branch: "does-not-exist".to_owned(),
11714 worktree: repo.clone(),
11715 summary: String::new(),
11716 stat: String::new(),
11717 files: 0,
11718 commits: 0,
11719 empty: false,
11720 failed: None,
11721 verified_noop: None,
11722 duration_ms: 0,
11723 folded: false,
11724 }];
11725 state.tally = Some(Tally {
11726 first_choice: BTreeMap::from([('A', 1)]),
11727 borda: BTreeMap::new(),
11728 winner: 'A',
11729 rankings: 1,
11730 unanimous_initial: true,
11731 deliberated: false,
11732 changed_votes: 0,
11733 unanimous_final: true,
11734 tie_break: None,
11735 judges: 0,
11736 present: 0,
11737 quorum: 0,
11738 met_quorum: true,
11739 uncontested: Some("only candidate A produced a change".to_owned()),
11740 });
11741 state.reviews = vec![ReviewRound {
11742 round: 1,
11743 head: "deadbeef".to_owned(),
11744 verified_head: None,
11745 verified_at: None,
11746 reviews: Vec::new(),
11747 e2e: Vec::new(),
11748 fix: None,
11749 blocking: 0,
11750 answered: 0,
11751 expected: 0,
11752 clean: true,
11753 verify_retried: false,
11754 e2e_deferred: false,
11755 e2e_defer_reason: None,
11756 progressed: false,
11757 vote_split: false,
11758 reconsideration: Vec::new(),
11759 verdict: None,
11760 }];
11761
11762 let mut runner = Runner {
11763 state,
11764 roles: ResolvedRoles {
11765 implementers: Vec::new(),
11766 judges: Vec::new(),
11767 reviewers: Vec::new(),
11768 fixer: None,
11769 conductor: conductor(),
11770 implementer_roster: Vec::new(),
11771 judge_roster: Vec::new(),
11772 reviewer_roster: Vec::new(),
11773 },
11774 sem: Arc::new(Semaphore::new(1)),
11775 pause: Pause::new(),
11776 interrupt: Pause::new(),
11777 };
11778
11779 runner.gate().await.expect("gate");
11780 assert!(
11781 runner.state.gate_ran,
11782 "zero configured commands is still a real attempt, not an unrun gate"
11783 );
11784 assert!(runner.state.gate.is_empty());
11785 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
11786 assert_ne!(
11787 runner.state.status,
11788 RunStatus::Blocked,
11789 "a gate with nothing to check must not read as failed"
11790 );
11791
11792 runner.merge().await.expect("merge");
11793 assert_eq!(
11794 runner.state.status,
11795 RunStatus::Ready,
11796 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
11797 );
11798 }
11799
11800 #[tokio::test]
11811 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
11812 crate::run::pin_test_home();
11813 let home = crate::run::home();
11814
11815 let tmp = tempfile::tempdir().expect("tempdir");
11816 let repo = tmp.path().join("repo");
11817 std::fs::create_dir_all(&repo).unwrap();
11818 init_repo(&repo);
11819 let cache_dir = tmp.path().join("target");
11822
11823 let mut config = Config::default();
11824 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
11825 config.graph.timeout_verify = Some(2);
11828
11829 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
11830 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
11831 .expect("no io error acquiring directly")
11832 {
11833 crate::cache::AcquireOutcome::Acquired(g) => g,
11834 crate::cache::AcquireOutcome::Busy(b) => {
11835 panic!("expected the direct acquire to win the lease first: {b:?}")
11836 }
11837 };
11838
11839 let mut state = RunState::new(
11840 repo.clone(),
11841 "main".to_owned(),
11842 "deadbeef".to_owned(),
11843 "task".to_owned(),
11844 config,
11845 );
11846 state.candidates = vec![Candidate {
11847 index: 0,
11848 label: 'A',
11849 agent: "alpha".to_owned(),
11850 branch: "does-not-exist".to_owned(),
11851 worktree: repo.clone(),
11852 summary: String::new(),
11853 stat: String::new(),
11854 files: 0,
11855 commits: 0,
11856 empty: false,
11857 failed: None,
11858 verified_noop: None,
11859 duration_ms: 0,
11860 folded: false,
11861 }];
11862 state.tally = Some(Tally {
11863 first_choice: BTreeMap::from([('A', 1)]),
11864 borda: BTreeMap::new(),
11865 winner: 'A',
11866 rankings: 1,
11867 unanimous_initial: true,
11868 deliberated: false,
11869 changed_votes: 0,
11870 unanimous_final: true,
11871 tie_break: None,
11872 judges: 0,
11873 present: 0,
11874 quorum: 0,
11875 met_quorum: true,
11876 uncontested: Some("only candidate A produced a change".to_owned()),
11877 });
11878 state.reviews = vec![ReviewRound {
11879 round: 1,
11880 head: "deadbeef".to_owned(),
11881 verified_head: None,
11882 verified_at: None,
11883 reviews: Vec::new(),
11884 e2e: Vec::new(),
11885 fix: None,
11886 blocking: 0,
11887 answered: 0,
11888 expected: 0,
11889 clean: true,
11890 verify_retried: false,
11891 e2e_deferred: false,
11892 e2e_defer_reason: None,
11893 progressed: false,
11894 vote_split: false,
11895 reconsideration: Vec::new(),
11896 verdict: None,
11897 }];
11898
11899 let mut runner = Runner {
11900 state,
11901 roles: ResolvedRoles {
11902 implementers: Vec::new(),
11903 judges: Vec::new(),
11904 reviewers: Vec::new(),
11905 fixer: None,
11906 conductor: conductor(),
11907 implementer_roster: Vec::new(),
11908 judge_roster: Vec::new(),
11909 reviewer_roster: Vec::new(),
11910 },
11911 sem: Arc::new(Semaphore::new(1)),
11912 pause: Pause::new(),
11913 interrupt: Pause::new(),
11914 };
11915
11916 let started = std::time::Instant::now();
11917 runner.gate().await.expect("gate");
11918 assert!(
11919 started.elapsed() < Duration::from_secs(1),
11920 "a gate with nothing to run must never wait on a lease it never needed"
11921 );
11922 assert!(
11923 runner.state.gate_ran,
11924 "zero commands is still a real, immediate attempt"
11925 );
11926 assert!(runner.state.gate.is_empty());
11927 assert_ne!(
11928 runner.state.status,
11929 RunStatus::Blocked,
11930 "must not read as resource-blocked on a lease it never asked for"
11931 );
11932 }
11933
11934 #[tokio::test]
11944 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
11945 crate::run::pin_test_home();
11946
11947 let tmp = tempfile::tempdir().expect("tempdir");
11948 let repo = tmp.path().join("repo");
11949 std::fs::create_dir_all(&repo).unwrap();
11950 init_repo(&repo);
11951
11952 let mut config = Config::default();
11953 config.verify.gate = vec![
11954 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
11955 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
11956 .to_owned(),
11957 ];
11958
11959 let mut state = RunState::new(
11960 repo.clone(),
11961 "main".to_owned(),
11962 "deadbeef".to_owned(),
11963 "task".to_owned(),
11964 config,
11965 );
11966 let run_id = state.id.clone();
11967 state.candidates = vec![Candidate {
11968 index: 0,
11969 label: 'A',
11970 agent: "alpha".to_owned(),
11971 branch: "does-not-exist".to_owned(),
11972 worktree: repo.clone(),
11973 summary: String::new(),
11974 stat: String::new(),
11975 files: 0,
11976 commits: 0,
11977 empty: false,
11978 failed: None,
11979 verified_noop: None,
11980 duration_ms: 0,
11981 folded: false,
11982 }];
11983 state.tally = Some(Tally {
11984 first_choice: BTreeMap::from([('A', 1)]),
11985 borda: BTreeMap::new(),
11986 winner: 'A',
11987 rankings: 1,
11988 unanimous_initial: true,
11989 deliberated: false,
11990 changed_votes: 0,
11991 unanimous_final: true,
11992 tie_break: None,
11993 judges: 0,
11994 present: 0,
11995 quorum: 0,
11996 met_quorum: true,
11997 uncontested: Some("only candidate A produced a change".to_owned()),
11998 });
11999 state.reviews = vec![ReviewRound {
12000 round: 1,
12001 head: "deadbeef".to_owned(),
12002 verified_head: None,
12003 verified_at: None,
12004 reviews: Vec::new(),
12005 e2e: Vec::new(),
12006 fix: None,
12007 blocking: 0,
12008 answered: 0,
12009 expected: 0,
12010 clean: true,
12011 verify_retried: false,
12012 e2e_deferred: false,
12013 e2e_defer_reason: None,
12014 progressed: false,
12015 vote_split: false,
12016 reconsideration: Vec::new(),
12017 verdict: None,
12018 }];
12019
12020 let mut runner = Runner {
12021 state,
12022 roles: ResolvedRoles {
12023 implementers: Vec::new(),
12024 judges: Vec::new(),
12025 reviewers: Vec::new(),
12026 fixer: None,
12027 conductor: conductor(),
12028 implementer_roster: Vec::new(),
12029 judge_roster: Vec::new(),
12030 reviewer_roster: Vec::new(),
12031 },
12032 sem: Arc::new(Semaphore::new(1)),
12033 pause: Pause::new(),
12034 interrupt: Pause::new(),
12035 };
12036
12037 let started_marker = repo.join("started.marker");
12038 let release_marker = repo.join("release.marker");
12039 let poller = tokio::spawn(async move {
12040 for _ in 0..100 {
12045 if started_marker.exists()
12046 && let Ok(s) = crate::run::RunState::load(&run_id)
12047 && let Some(a) = s.active.get("gate")
12048 {
12049 std::fs::write(&release_marker, b"go").expect("release marker");
12050 return Some(a.clone());
12051 }
12052 tokio::time::sleep(Duration::from_millis(50)).await;
12053 }
12054 None
12055 });
12056
12057 runner.gate().await.expect("gate");
12058 let captured = poller.await.expect("poller task");
12059 let captured = captured.expect(
12060 "the poller never saw a `gate` task entry in run.json while the command was \
12061 still blocked on its own release marker",
12062 );
12063
12064 assert_eq!(captured.task.as_deref(), Some("gate"));
12065 assert_eq!(captured.node, "gate");
12066 assert_eq!(captured.index, Some(1));
12067 assert_eq!(captured.total, Some(1));
12068 assert!(
12069 captured
12070 .command
12071 .as_deref()
12072 .is_some_and(|c| c.contains("started.marker")),
12073 "{captured:?}"
12074 );
12075
12076 assert!(
12077 runner.state.active.is_empty(),
12078 "the entry must be cleared once the command actually finished: {:?}",
12079 runner.state.active
12080 );
12081 assert!(runner.state.gate_ran);
12082 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
12083 }
12084
12085 #[tokio::test]
12088 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
12089 use crate::verdict::{Finding, ReviewVote, Severity};
12090 crate::run::pin_test_home();
12091 let tmp = tempfile::tempdir().expect("tempdir");
12092 let repo = tmp.path().join("repo");
12093 std::fs::create_dir_all(&repo).unwrap();
12094 init_repo(&repo);
12095
12096 for (severity, vote, expect) in [
12097 (Severity::Major, ReviewVote::Reject, true),
12098 (Severity::Minor, ReviewVote::Reject, false),
12099 (Severity::Major, ReviewVote::Approve, false),
12100 ] {
12101 let mut round = review_round(false, 1, 1, 1, false, true);
12102 round.reviews = vec![ReviewRecord {
12103 reviewer: 1,
12104 agent: "alpha".to_owned(),
12105 summary: String::new(),
12106 findings: vec![Finding {
12107 id: "R1-1-1".to_owned(),
12108 severity,
12109 file: None,
12110 line: None,
12111 title: "t".to_owned(),
12112 detail: String::new(),
12113 }],
12114 vote: Some(vote),
12115 failed: None,
12116 duration_ms: 0,
12117 attempts: 0,
12118 }];
12119 let mut state = RunState::new(
12120 repo.clone(),
12121 "main".to_owned(),
12122 "deadbeef".to_owned(),
12123 "task".to_owned(),
12124 Config::default(),
12125 );
12126 state.reviews = vec![round];
12127 let mut runner = Runner {
12128 state,
12129 roles: ResolvedRoles {
12130 implementers: Vec::new(),
12131 judges: Vec::new(),
12132 reviewers: Vec::new(),
12133 fixer: None,
12134 conductor: conductor(),
12135 implementer_roster: Vec::new(),
12136 judge_roster: Vec::new(),
12137 reviewer_roster: Vec::new(),
12138 },
12139 sem: Arc::new(Semaphore::new(1)),
12140 pause: Pause::new(),
12141 interrupt: Pause::new(),
12142 };
12143 let shell = runner.state.config.shell();
12144 runner
12145 .stop_reviewing("round budget spent", &shell, &repo)
12146 .await
12147 .expect("stop_reviewing");
12148 assert_eq!(runner.state.status, RunStatus::Gating);
12149 assert_eq!(
12150 runner.state.contested_handoff.is_some(),
12151 expect,
12152 "{severity:?} + {vote:?}"
12153 );
12154 }
12155 }
12156
12157 #[tokio::test]
12170 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
12171 crate::run::pin_test_home();
12172 let home = crate::run::home();
12173
12174 let tmp = tempfile::tempdir().expect("tempdir");
12175 let repo = tmp.path().join("repo");
12176 std::fs::create_dir_all(&repo).unwrap();
12177 init_repo(&repo);
12178 let head = crate::git::rev_parse(&repo, "HEAD")
12179 .await
12180 .expect("rev-parse");
12181 let cache_dir = tmp.path().join("target");
12184
12185 let mut config = Config::default();
12186 config.verify.e2e = vec![format!(
12187 "CARGO_TARGET_DIR='{}' test -f README.md",
12188 cache_dir.display()
12189 )];
12190 config.graph.review_rounds = 1;
12191 config.graph.timeout_verify = Some(2);
12194
12195 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
12196 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
12197 .expect("no io error acquiring directly")
12198 {
12199 crate::cache::AcquireOutcome::Acquired(g) => g,
12200 crate::cache::AcquireOutcome::Busy(b) => {
12201 panic!("expected the direct acquire to win the lease first: {b:?}")
12202 }
12203 };
12204
12205 let mut state = RunState::new(
12206 repo.clone(),
12207 "main".to_owned(),
12208 head.clone(),
12209 "task".to_owned(),
12210 config,
12211 );
12212 state.candidates = vec![Candidate {
12213 index: 0,
12214 label: 'A',
12215 agent: "alpha".to_owned(),
12216 branch: "does-not-exist".to_owned(),
12217 worktree: repo.clone(),
12218 summary: String::new(),
12219 stat: String::new(),
12220 files: 0,
12221 commits: 0,
12222 empty: false,
12223 failed: None,
12224 verified_noop: None,
12225 duration_ms: 0,
12226 folded: false,
12227 }];
12228 state.tally = Some(Tally {
12229 first_choice: BTreeMap::from([('A', 1)]),
12230 borda: BTreeMap::new(),
12231 winner: 'A',
12232 rankings: 1,
12233 unanimous_initial: true,
12234 deliberated: false,
12235 changed_votes: 0,
12236 unanimous_final: true,
12237 tie_break: None,
12238 judges: 0,
12239 present: 0,
12240 quorum: 0,
12241 met_quorum: true,
12242 uncontested: Some("only candidate A produced a change".to_owned()),
12243 });
12244 state.reviews = vec![ReviewRound {
12248 round: 1,
12249 head: head.clone(),
12250 verified_head: None,
12251 verified_at: None,
12252 reviews: Vec::new(),
12253 e2e: Vec::new(),
12254 fix: None,
12255 blocking: 1,
12256 answered: 1,
12257 expected: 1,
12258 clean: false,
12259 verify_retried: false,
12260 e2e_deferred: true,
12261 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
12262 progressed: false,
12263 vote_split: false,
12264 reconsideration: Vec::new(),
12265 verdict: None,
12266 }];
12267
12268 let mut runner = Runner {
12269 state,
12270 roles: ResolvedRoles {
12271 implementers: Vec::new(),
12272 judges: Vec::new(),
12273 reviewers: Vec::new(),
12274 fixer: None,
12275 conductor: conductor(),
12276 implementer_roster: Vec::new(),
12277 judge_roster: Vec::new(),
12278 reviewer_roster: Vec::new(),
12279 },
12280 sem: Arc::new(Semaphore::new(1)),
12281 pause: Pause::new(),
12282 interrupt: Pause::new(),
12283 };
12284
12285 let shell = runner.state.config.shell();
12286 runner
12287 .stop_reviewing("round budget spent", &shell, &repo)
12288 .await
12289 .expect("stop_reviewing");
12290
12291 let last = runner.state.reviews.last().expect("round record");
12292 assert_eq!(
12293 last.e2e_status(),
12294 E2eStatus::ResourceBlocked,
12295 "the shared cache is still held; the attempt must read as blocked, not deferred or \
12296 failed: {last:?}"
12297 );
12298 assert_eq!(
12299 last.verified_head.as_deref(),
12300 Some(head.as_str()),
12301 "which commit this attempt targeted is known even though nothing finished checking \
12302 it"
12303 );
12304 let first_attempt_at = last
12305 .verified_at
12306 .expect("when this attempt ran is known too");
12307 assert_ne!(
12308 runner.state.status,
12309 RunStatus::Blocked,
12310 "contention is evidence about the machine, not the patch — it must not settle the \
12311 run as blocked: {:?}",
12312 runner.state.status
12313 );
12314 assert!(
12315 !runner
12316 .state
12317 .events
12318 .iter()
12319 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
12320 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
12321 runner.state.events
12322 );
12323
12324 runner
12329 .stop_reviewing("round budget spent", &shell, &repo)
12330 .await
12331 .expect("stop_reviewing retry");
12332 assert_eq!(
12333 runner.state.reviews.len(),
12334 1,
12335 "no new round was started: {:?}",
12336 runner.state.reviews
12337 );
12338 let last = runner.state.reviews.last().expect("round record");
12339 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
12340 assert!(
12341 last.verified_at.expect("still known") > first_attempt_at,
12342 "a second reentry must be a fresh attempt, not a stale copy of the first"
12343 );
12344 assert_ne!(runner.state.status, RunStatus::Blocked);
12345
12346 held.release();
12347 }
12348
12349 #[tokio::test]
12361 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
12362 crate::run::pin_test_home();
12363 let home = crate::run::home();
12364
12365 let tmp = tempfile::tempdir().expect("tempdir");
12366 let repo = tmp.path().join("repo");
12367 std::fs::create_dir_all(&repo).unwrap();
12368 init_repo(&repo);
12369 let head = crate::git::rev_parse(&repo, "HEAD")
12370 .await
12371 .expect("rev-parse");
12372 let cache_dir = tmp.path().join("target");
12373
12374 let mut config = Config::default();
12375 config.verify.e2e = vec![format!(
12376 "CARGO_TARGET_DIR='{}' test -f README.md",
12377 cache_dir.display()
12378 )];
12379 config.graph.review_rounds = 1;
12380 config.graph.timeout_verify = Some(2);
12381
12382 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
12383 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
12384 .expect("no io error acquiring directly")
12385 {
12386 crate::cache::AcquireOutcome::Acquired(g) => g,
12387 crate::cache::AcquireOutcome::Busy(b) => {
12388 panic!("expected the direct acquire to win the lease first: {b:?}")
12389 }
12390 };
12391
12392 let mut state = RunState::new(
12393 repo.clone(),
12394 "main".to_owned(),
12395 head.clone(),
12396 "task".to_owned(),
12397 config,
12398 );
12399 state.candidates = vec![Candidate {
12400 index: 0,
12401 label: 'A',
12402 agent: "alpha".to_owned(),
12403 branch: "does-not-exist".to_owned(),
12404 worktree: repo.clone(),
12405 summary: String::new(),
12406 stat: String::new(),
12407 files: 0,
12408 commits: 0,
12409 empty: false,
12410 failed: None,
12411 verified_noop: None,
12412 duration_ms: 0,
12413 folded: false,
12414 }];
12415 state.tally = Some(Tally {
12416 first_choice: BTreeMap::from([('A', 1)]),
12417 borda: BTreeMap::new(),
12418 winner: 'A',
12419 rankings: 1,
12420 unanimous_initial: true,
12421 deliberated: false,
12422 changed_votes: 0,
12423 unanimous_final: true,
12424 tie_break: None,
12425 judges: 0,
12426 present: 0,
12427 quorum: 0,
12428 met_quorum: true,
12429 uncontested: Some("only candidate A produced a change".to_owned()),
12430 });
12431 state.reviews = vec![ReviewRound {
12435 round: 1,
12436 head: head.clone(),
12437 verified_head: Some(head.clone()),
12438 verified_at: Some(jiff::Timestamp::now()),
12439 reviews: Vec::new(),
12440 e2e: vec![CommandOutcome {
12441 command: format!(
12442 "CARGO_TARGET_DIR='{}' test -f README.md",
12443 cache_dir.display()
12444 ),
12445 code: None,
12446 output_tail: "waiting for the shared build cache".to_owned(),
12447 duration_ms: 0,
12448 resource_blocked: true,
12449 }],
12450 fix: None,
12451 blocking: 1,
12452 answered: 1,
12453 expected: 1,
12454 clean: false,
12455 verify_retried: false,
12456 e2e_deferred: false,
12457 e2e_defer_reason: None,
12458 progressed: false,
12459 vote_split: false,
12460 reconsideration: Vec::new(),
12461 verdict: None,
12462 }];
12463
12464 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
12465 let mut runner = Runner {
12466 state,
12467 roles: ResolvedRoles {
12468 implementers: Vec::new(),
12469 judges: Vec::new(),
12470 reviewers: Vec::new(),
12471 fixer: None,
12472 conductor: conductor(),
12473 implementer_roster: Vec::new(),
12474 judge_roster: Vec::new(),
12475 reviewer_roster: Vec::new(),
12476 },
12477 sem: Arc::new(Semaphore::new(1)),
12478 pause: Pause::new(),
12479 interrupt: Pause::new(),
12480 };
12481
12482 runner.review_loop().await.expect("review_loop");
12487
12488 assert_eq!(
12489 runner.state.reviews.len(),
12490 1,
12491 "no new round was started on top of the unresolved one: {:?}",
12492 runner.state.reviews
12493 );
12494 let last = &runner.state.reviews[0];
12495 assert_eq!(
12496 last.e2e_status(),
12497 E2eStatus::ResourceBlocked,
12498 "still contended: {last:?}"
12499 );
12500 assert!(
12501 last.verified_at.expect("still known") > first_attempt_at,
12502 "review_loop must have actually retried the check, not left it exactly as found"
12503 );
12504 assert_ne!(
12505 runner.state.status,
12506 RunStatus::Blocked,
12507 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
12508 runner.state.status
12509 );
12510
12511 held.release();
12512 }
12513
12514 #[tokio::test]
12515 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
12516 crate::run::pin_test_home();
12517 let tmp = tempfile::tempdir().expect("tempdir");
12518 let repo = tmp.path().join("repo");
12519 std::fs::create_dir_all(&repo).unwrap();
12520 init_repo(&repo);
12521
12522 let mut config = Config::default();
12523 config.merge.mode = MergeMode::Pr;
12524 config.graph.land = true;
12525 config.graph.land_approval = false;
12526
12527 let mut state = RunState::new(
12528 repo.clone(),
12529 "main".to_owned(),
12530 "deadbeef".to_owned(),
12531 "task".to_owned(),
12532 config,
12533 );
12534 state.candidates = vec![Candidate {
12535 index: 0,
12536 label: 'A',
12537 agent: "alpha".to_owned(),
12538 branch: "does-not-exist".to_owned(),
12539 worktree: repo.clone(),
12540 summary: String::new(),
12541 stat: String::new(),
12542 files: 0,
12543 commits: 0,
12544 empty: false,
12545 failed: None,
12546 verified_noop: None,
12547 duration_ms: 0,
12548 folded: false,
12549 }];
12550 state.tally = Some(Tally {
12551 first_choice: BTreeMap::from([('A', 1)]),
12552 borda: BTreeMap::new(),
12553 winner: 'A',
12554 rankings: 1,
12555 unanimous_initial: true,
12556 deliberated: false,
12557 changed_votes: 0,
12558 unanimous_final: true,
12559 tie_break: None,
12560 judges: 0,
12561 present: 0,
12562 quorum: 0,
12563 met_quorum: true,
12564 uncontested: Some("only candidate A produced a change".to_owned()),
12565 });
12566 state.reviews = vec![ReviewRound {
12567 round: 1,
12568 head: "deadbeef".to_owned(),
12569 verified_head: None,
12570 verified_at: None,
12571 reviews: Vec::new(),
12572 e2e: Vec::new(),
12573 fix: None,
12574 blocking: 0,
12575 answered: 0,
12576 expected: 0,
12577 clean: true,
12578 verify_retried: false,
12579 e2e_deferred: false,
12580 e2e_defer_reason: None,
12581 progressed: false,
12582 vote_split: false,
12583 reconsideration: Vec::new(),
12584 verdict: None,
12585 }];
12586 state.gate = vec![CommandOutcome {
12587 command: "test".to_owned(),
12588 code: Some(0),
12589 output_tail: String::new(),
12590 duration_ms: 0,
12591 resource_blocked: false,
12592 }];
12593 state.gate_ran = true;
12594 state.status = RunStatus::Landing;
12598 state.merge = Some(MergeOutcome {
12599 mode: MergeMode::Pr,
12600 ok: true,
12601 detail: "https://example.invalid/x/y/pull/1".to_owned(),
12602 empty: false,
12603 });
12604
12605 ask_test_home();
12609 state.save().expect("save landing run");
12612 let store = ask::Questions::open();
12613 let q = ask_open_question(&store, &state.id);
12614
12615 let mut runner = Runner {
12616 state,
12617 roles: ResolvedRoles {
12618 implementers: Vec::new(),
12619 judges: Vec::new(),
12620 reviewers: Vec::new(),
12621 fixer: None,
12622 conductor: conductor(),
12623 implementer_roster: Vec::new(),
12624 judge_roster: Vec::new(),
12625 reviewer_roster: Vec::new(),
12626 },
12627 sem: Arc::new(Semaphore::new(1)),
12628 pause: Pause::new(),
12629 interrupt: Pause::new(),
12630 };
12631
12632 runner.execute().await.expect("execute");
12637
12638 assert_eq!(
12639 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
12640 Some("https://example.invalid/x/y/pull/1"),
12641 "reentry must not push again or open a second pull request over the \
12642 one `land` is already watching"
12643 );
12644 assert_ne!(
12645 runner.state.status,
12646 RunStatus::Landing,
12647 "land could not actually reach the fake pull request, so it must \
12648 have given up rather than left the run silently parked forever"
12649 );
12650 assert_eq!(runner.state.status, RunStatus::Blocked);
12654 assert!(
12655 store.get(&q.id).unwrap().status.open(),
12656 "Blocked is still alive; settle_questions must have been a no-op here"
12657 );
12658 }
12659
12660 fn state_with_round(round: ReviewRound) -> RunState {
12661 let mut s = RunState::new(
12662 PathBuf::from("/repo"),
12663 "main".to_owned(),
12664 "abc1234".to_owned(),
12665 "add retries".to_owned(),
12666 Config::default(),
12667 );
12668 s.reviews = vec![round];
12669 s
12670 }
12671
12672 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
12673 crate::verdict::Finding {
12674 id: id.to_owned(),
12675 severity,
12676 file: None,
12677 line: None,
12678 title: title.to_owned(),
12679 detail: String::new(),
12680 }
12681 }
12682
12683 #[test]
12684 fn pr_body_names_open_findings_and_declined_ones() {
12685 let round = ReviewRound {
12686 round: 2,
12687 head: "deadbee".to_owned(),
12688 verified_head: None,
12689 verified_at: None,
12690 reviews: vec![ReviewRecord {
12691 attempts: 0,
12692 reviewer: 1,
12693 agent: "alpha".to_owned(),
12694 summary: String::new(),
12695 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
12696 vote: None,
12697 failed: None,
12698 duration_ms: 0,
12699 }],
12700 e2e: vec![CommandOutcome {
12701 command: "cargo test".to_owned(),
12702 code: Some(0),
12703 output_tail: String::new(),
12704 duration_ms: 0,
12705 resource_blocked: false,
12706 }],
12707 verify_retried: false,
12708 e2e_deferred: false,
12709 e2e_defer_reason: None,
12710 fix: Some(FixRecord {
12711 agent: "alpha".to_owned(),
12712 addressed: Vec::new(),
12713 rejected: vec![crate::verdict::Rejection {
12714 id: "R1-1-1".to_owned(),
12715 why: "not reachable from any caller".to_owned(),
12716 }],
12717 notes: String::new(),
12718 committed: true,
12719 failed: None,
12720 duration_ms: 0,
12721 continuation: None,
12722 }),
12723 blocking: 0,
12724 answered: 1,
12725 expected: 1,
12726 clean: false,
12727 progressed: true,
12728 vote_split: false,
12729 reconsideration: Vec::new(),
12730 verdict: None,
12731 };
12732 let state = state_with_round(round);
12733 let body = pr_message(&state, 'A').body;
12734
12735 assert!(body.contains("add retries"), "the task must still be there");
12736 assert!(body.contains("R2-1-1"), "{body}");
12737 assert!(body.contains("unused import"), "{body}");
12738 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
12739 assert!(
12740 body.contains("not reachable from any caller"),
12741 "the reason it was declined: {body}"
12742 );
12743 }
12744
12745 #[test]
12746 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
12747 let round = ReviewRound {
12748 round: 1,
12749 head: "deadbee".to_owned(),
12750 verified_head: None,
12751 verified_at: None,
12752 reviews: vec![ReviewRecord {
12753 attempts: 0,
12754 reviewer: 1,
12755 agent: "alpha".to_owned(),
12756 summary: String::new(),
12757 findings: Vec::new(),
12758 vote: None,
12759 failed: None,
12760 duration_ms: 0,
12761 }],
12762 e2e: Vec::new(),
12763 verify_retried: false,
12764 e2e_deferred: false,
12765 e2e_defer_reason: None,
12766 fix: None,
12767 blocking: 0,
12768 answered: 1,
12769 expected: 1,
12770 clean: true,
12771 progressed: false,
12772 vote_split: false,
12773 reconsideration: Vec::new(),
12774 verdict: None,
12775 };
12776 let state = state_with_round(round);
12777 let body = pr_message(&state, 'A').body;
12778 assert!(!body.contains("Open review findings"), "{body}");
12779 assert!(!body.contains("Declined"), "{body}");
12780 }
12781
12782 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
12783 let mut state = RunState::new(
12784 PathBuf::from("/repo"),
12785 "main".to_owned(),
12786 "abc1234".to_owned(),
12787 instruction.to_owned(),
12788 Config::default(),
12789 );
12790 state.candidates.push(Candidate {
12791 index: 0,
12792 label: 'A',
12793 agent: "alpha".to_owned(),
12794 branch: "magi/x/A".to_owned(),
12795 worktree: PathBuf::from("/wt"),
12796 summary: summary.to_owned(),
12797 stat: String::new(),
12798 files: 1,
12799 commits: 1,
12800 empty: false,
12801 failed: None,
12802 verified_noop: None,
12803 folded: false,
12804 duration_ms: 0,
12805 });
12806 state
12807 }
12808
12809 fn review_state(subjects: &[&str]) -> RunState {
12810 let mut state = state_with_summary(
12811 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
12812 "",
12813 );
12814 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
12815 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
12816 state
12817 }
12818
12819 #[tokio::test]
12822 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
12823 ask_test_home();
12824 let tmp = tempfile::tempdir().unwrap();
12825 let repo = tmp.path().join("repo");
12826 std::fs::create_dir_all(&repo).unwrap();
12827 init_repo(&repo);
12828 let origin = tmp.path().join("origin.git");
12829 let g = |dir: &Path, args: &[&str]| {
12830 let out = std::process::Command::new("git")
12831 .args(args)
12832 .current_dir(dir)
12833 .quiet()
12834 .output()
12835 .expect("spawn git");
12836 assert!(
12837 out.status.success(),
12838 "git {args:?}: {}",
12839 String::from_utf8_lossy(&out.stderr)
12840 );
12841 };
12842 g(
12843 tmp.path(),
12844 &[
12845 "clone",
12846 "--bare",
12847 "-q",
12848 repo.to_str().unwrap(),
12849 origin.to_str().unwrap(),
12850 ],
12851 );
12852 g(
12853 &repo,
12854 &["remote", "add", "origin", origin.to_str().unwrap()],
12855 );
12856 let c1 = git::rev_parse(&repo, "main").await.unwrap();
12857
12858 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
12860 g(&repo, &["add", "-A"]);
12861 g(
12862 &repo,
12863 &["commit", "-q", "-m", "chore(deps): update a crate"],
12864 );
12865 g(&repo, &["push", "-q", "origin", "main"]);
12866 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
12867 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
12868 g(&repo, &["add", "-A"]);
12869 g(
12870 &repo,
12871 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
12872 );
12873 g(&repo, &["checkout", "-q", "main"]);
12874
12875 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
12876 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
12877 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
12879 assert_eq!(fallback, c1);
12880
12881 let mut state = review_state(&[
12882 "chore(deps): update a crate",
12883 "fix(daemon): apply a chosen action",
12884 ]);
12885 state.repo = repo.clone();
12886 state.base_branch = "main".to_owned();
12887 state.base_commit = c1;
12888 refresh_reviewed_commits(&mut state, "feat/own").await;
12889 assert_eq!(
12890 state.reviewed_commits,
12891 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
12892 );
12893 assert_eq!(
12894 review_title(&state).as_deref(),
12895 Some("fix(daemon): apply a chosen action")
12896 );
12897 assert_eq!(
12898 leaked_subjects(&state, "feat/own").await,
12899 Some(vec!["chore(deps): update a crate".to_owned()])
12900 );
12901 state.config.merge.remote = "nowhere".to_owned();
12904 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
12905 }
12906
12907 #[test]
12908 fn pr_message_review_single_commit_uses_its_subject() {
12909 let state = review_state(&["feat(nats): per-role user"]);
12910 let m = pr_message(&state, 'A');
12911 assert_eq!(m.title, "feat(nats): per-role user");
12912 assert!(!m.body.contains("Review the work already"), "{}", m.body);
12913 assert!(m.body.contains("## Commits under review"), "{}", m.body);
12914 }
12915
12916 #[test]
12917 fn pr_message_review_multi_commit_takes_the_oldest() {
12918 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
12919 let m = pr_message(&state, 'A');
12920 assert_eq!(m.title, "feat: the change");
12921 for s in ["feat: the change", "fix: typo", "fix: again"] {
12922 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
12923 }
12924 }
12925
12926 #[test]
12927 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
12928 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
12929 let mut state = review_state(&[first, "fix: later fixup"]);
12930 state.candidates[0].branch = "機能/ログイン".to_owned();
12931 let m = pr_message(&state, 'A');
12932 assert!(
12933 m.title.starts_with("chore: land candidate A of run"),
12934 "{}",
12935 m.title
12936 );
12937 }
12938 }
12939
12940 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
12941 BranchFacts {
12942 commits: commits
12943 .iter()
12944 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
12945 .collect(),
12946 stat: stat.to_owned(),
12947 }
12948 }
12949
12950 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
12951 let mut r = review_round(true, 0, 1, 1, true, true);
12952 r.round = round;
12953 r.fix = notes.map(|n| FixRecord {
12954 agent: "fixer".to_owned(),
12955 addressed: Vec::new(),
12956 rejected: Vec::new(),
12957 notes: n.to_owned(),
12958 committed: true,
12959 failed: None,
12960 duration_ms: 0,
12961 continuation: None,
12962 });
12963 r
12964 }
12965
12966 #[test]
12967 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
12968 let state = review_state(&["ignored"]);
12969 let f = facts(
12970 &[
12971 (
12972 "fix(login): resolve PATH on macOS",
12973 "Login shells skip rc files.",
12974 ),
12975 ("fix: address review", ""),
12976 ],
12977 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
12978 );
12979 let m = pr_message_with(&state, 'A', Some(&f));
12980 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
12981 assert!(
12982 m.body
12983 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
12984 "{}",
12985 m.body
12986 );
12987 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
12988 assert!(m.body.contains("## Diff stat"), "{}", m.body);
12989 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
12990 for banned in [
12991 "Review the work already",
12992 "no task statement",
12993 "Original task",
12994 ] {
12995 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
12996 }
12997 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
12998 }
12999
13000 #[test]
13001 fn pr_message_review_truncates_a_huge_first_commit_body() {
13002 let state = review_state(&["ignored"]);
13003 let f = facts(
13004 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
13005 "s",
13006 );
13007 let m = pr_message_with(&state, 'A', Some(&f));
13008 assert!(m.body.len() < 30_000, "{}", m.body.len());
13009 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
13010 assert!(
13011 m.body.contains("- ... 1 more commit(s)"),
13012 "{}",
13013 m.body.len()
13014 );
13015 assert!(m.body.ends_with("magi:candidate-a\n"));
13016 }
13017
13018 #[test]
13019 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
13020 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
13021 assert_eq!(m.title, "feat: x");
13022 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
13023 assert!(!m.body.contains("Diff stat"), "{}", m.body);
13024 assert!(!m.body.contains("no task statement"), "{}", m.body);
13025 }
13026
13027 #[test]
13028 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
13029 let mut state = review_state(&["日本語の件名"]);
13030 state.candidates[0].branch = "fix/macos-login-path".to_owned();
13031 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
13032 state.candidates[0].branch = "機能/ログイン".to_owned();
13033 assert!(
13034 pr_message(&state, 'A')
13035 .title
13036 .starts_with("chore: land candidate A")
13037 );
13038 }
13039
13040 #[test]
13041 fn pr_message_review_fixes_survive_a_clean_final_round() {
13042 let mut state = review_state(&["feat: x"]);
13043 state.reviews = vec![
13044 round_with_notes(1, Some("handled the PATH case")),
13045 round_with_notes(2, None),
13046 ];
13047 let body = pr_message(&state, 'A').body;
13048 assert!(
13049 body.contains("## Review fixes\n\nhandled the PATH case\n"),
13050 "{body}"
13051 );
13052 assert!(!body.contains("### Round"), "{body}");
13053
13054 state.reviews = vec![
13055 round_with_notes(1, Some("first fix")),
13056 round_with_notes(2, Some("")),
13057 round_with_notes(3, Some("second fix")),
13058 round_with_notes(4, None),
13059 ];
13060 let body = pr_message(&state, 'A').body;
13061 assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
13062 assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
13063 assert!(!body.contains("### Round 2"), "{body}");
13064 }
13065
13066 const FINAL: &str = "abcdef1234567890";
13067
13068 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
13069 let mut r = round_with_notes(round, None);
13070 r.verified_head = verified.map(str::to_owned);
13071 r
13072 }
13073
13074 fn ok_gate(state: &mut RunState, head: Option<&str>) {
13075 state.gate = vec![CommandOutcome {
13076 command: "gate".to_owned(),
13077 code: Some(0),
13078 output_tail: String::new(),
13079 duration_ms: 0,
13080 resource_blocked: false,
13081 }];
13082 state.gate_ran = true;
13083 state.gate_head = head.map(str::to_owned);
13084 }
13085
13086 fn body_at(state: &RunState) -> String {
13087 pr_message_raw(state, 'A', None, crate::pr_outcome::FinalHead::At(FINAL)).body
13088 }
13089
13090 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
13091
13092 #[test]
13093 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
13094 let mut state = review_state(&["feat: x"]);
13095 state.reviews = vec![
13096 round_with_notes(1, Some(SANDBOXED)),
13097 passing_round(2, Some(FINAL)),
13098 ];
13099 ok_gate(&mut state, Some(FINAL));
13100 let body = body_at(&state);
13101 assert!(body.contains("Handled the PATH case."), "{body}");
13102 assert!(!body.contains("could not commit"), "{body}");
13103 assert!(!body.contains("could not run"), "{body}");
13104 assert!(
13105 body.contains("The fix is committed on the branch"),
13106 "{body}"
13107 );
13108 assert!(body.contains("2 statement(s)"), "{body}");
13109 assert!(
13110 body.contains("Tests (`verify.e2e`): passed on the final commit"),
13111 "{body}"
13112 );
13113 assert!(
13114 body.contains("Gate (`verify.gate`): passed on the final commit"),
13115 "{body}"
13116 );
13117 assert!(
13119 state.reviews[0]
13120 .fix
13121 .as_ref()
13122 .unwrap()
13123 .notes
13124 .contains("could not commit")
13125 );
13126 }
13127
13128 #[test]
13129 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
13130 let mut state = review_state(&["feat: x"]);
13131 let mut r1 = round_with_notes(1, Some(SANDBOXED));
13132 r1.fix.as_mut().unwrap().committed = false;
13133 let mut r2 = passing_round(2, Some(FINAL));
13134 r2.e2e[0].code = Some(1);
13135 state.reviews = vec![r1, r2];
13136 let body = body_at(&state);
13137 assert!(body.contains("could not commit"), "{body}");
13138 assert!(body.contains("could not run"), "{body}");
13139 assert!(body.contains("This round produced no commit"), "{body}");
13140 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
13141 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
13142 assert!(!body.contains("passed"), "{body}");
13143 }
13144
13145 #[test]
13146 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
13147 let mut state = review_state(&["feat: x"]);
13148 state.reviews = vec![
13149 round_with_notes(1, Some(SANDBOXED)),
13150 passing_round(2, Some("0123456789abcdef")),
13151 ];
13152 ok_gate(&mut state, Some("0123456789abcdef"));
13153 let body = body_at(&state);
13154 assert!(body.contains("HTTP tests could not run"), "{body}");
13155 assert!(!body.contains("could not commit"), "{body}");
13157 assert!(
13158 body.contains("passed on an earlier commit (`0123456`)"),
13159 "{body}"
13160 );
13161 assert!(!body.contains("passed on the final commit"), "{body}");
13162
13163 ok_gate(&mut state, None);
13165 let body = body_at(&state);
13166 assert!(
13167 body.contains("passed on a commit that was not recorded"),
13168 "{body}"
13169 );
13170 }
13171
13172 #[test]
13173 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
13174 let state = state_with_summary(
13175 "add retries to the client",
13176 "TITLE: feat: retries\n\nDid it.",
13177 );
13178 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
13179 assert_eq!(m.title, "feat: retries");
13180 assert!(
13181 m.body.contains("<summary>Original task</summary>"),
13182 "{}",
13183 m.body
13184 );
13185 assert!(!m.body.contains("Commits under review"), "{}", m.body);
13186 assert!(
13187 m.body
13188 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
13189 "{}",
13190 m.body
13191 );
13192 }
13193
13194 #[test]
13195 fn pr_message_review_bounds_a_long_english_subject() {
13196 let long = format!("feat: {}", "word ".repeat(100));
13197 let m = pr_message(&review_state(&[&long]), 'A');
13198 assert!(m.title.starts_with("feat: word"), "{}", m.title);
13199 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
13200 }
13201
13202 #[test]
13203 fn pr_message_implementation_run_is_unchanged_by_review_support() {
13204 let state = state_with_summary("add retries\n\ndetails", "- did some things");
13205 let m = pr_message(&state, 'A');
13206 assert_eq!(m.title, "add retries");
13207 assert!(m.body.contains("<summary>Original task</summary>"));
13208 assert!(!m.body.contains("Commits under review"));
13209 assert_eq!(landing_subject_source(&state), state.instruction);
13210 }
13211
13212 #[test]
13213 fn review_run_squash_subject_is_the_change_not_the_prompt() {
13214 let state = review_state(&["feat: the change", "fix: typo"]);
13215 let source = landing_subject_source(&state);
13216 assert_eq!(land::merge_subject("", &source), "feat: the change");
13217 assert_eq!(
13218 land::merge_subject("magi: candidate A (uncommitted work)", &source),
13219 "feat: the change"
13220 );
13221 assert_eq!(
13223 land::merge_subject("feat: renamed by hand", &source),
13224 "feat: renamed by hand"
13225 );
13226 let blank = review_state(&["日本語"]);
13227 assert!(
13228 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
13229 );
13230 }
13231
13232 #[test]
13233 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
13234 let state = review_state(&["feat: the change"]);
13235 let source = landing_subject_source(&state);
13236 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
13237 assert_eq!(
13238 land::merge_subject(landing_title(&state, old), &source),
13239 "feat: the change"
13240 );
13241 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
13242 let task = state_with_summary("add retries", "");
13243 assert_eq!(landing_title(&task, old), old);
13244 }
13245
13246 #[test]
13247 fn pr_message_describes_the_change_not_the_task() {
13248 let state = state_with_summary(
13249 "今回やってほしいこと: results projector を直す",
13250 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
13251 );
13252 let m = pr_message(&state, 'A');
13253 assert_eq!(m.title, "fix(web): batch the runs list reads");
13254 assert!(
13255 m.body.starts_with("## Summary\n\n- reads run.json once"),
13256 "{}",
13257 m.body
13258 );
13259 assert!(!m.body.contains("TITLE:"), "{}", m.body);
13260 let task_at = m.body.find("今回やってほしいこと").unwrap();
13261 let details_at = m.body.find("<details>").unwrap();
13262 assert!(
13263 details_at < task_at,
13264 "the task lives inside <details>: {}",
13265 m.body
13266 );
13267 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
13268 assert!(m.body.contains("magi:candidate-a"));
13269 }
13270
13271 #[test]
13272 fn pr_message_falls_back_to_the_task_without_a_title_line() {
13273 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
13274 let m = pr_message(&state, 'A');
13275 assert_eq!(m.title, "add retries");
13276 assert!(
13277 m.body.contains("## Summary\n\n- did some things"),
13278 "{}",
13279 m.body
13280 );
13281
13282 let none = RunState::new(
13283 PathBuf::from("/repo"),
13284 "main".to_owned(),
13285 "abc1234".to_owned(),
13286 "add retries".to_owned(),
13287 Config::default(),
13288 );
13289 let m = pr_message(&none, 'A');
13290 assert_eq!(m.title, "add retries");
13291 assert!(!m.body.contains("## Summary"), "{}", m.body);
13292 }
13293
13294 #[test]
13295 fn pr_message_refuses_the_candidate_commit_subject() {
13296 for bad in [
13297 "TITLE: magi: candidate A (uncommitted work)",
13298 "TITLE: chore: stuff (uncommitted work)",
13299 "TITLE: ",
13300 ] {
13301 let state = state_with_summary("add retries", bad);
13302 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
13303 }
13304 }
13305
13306 #[test]
13307 fn pr_message_bounds_a_very_long_task_and_title() {
13308 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
13309 let state = state_with_summary(&long, "- nothing");
13310 let m = pr_message(&state, 'A');
13311 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
13312 assert!(!m.title.contains('\n'));
13313
13314 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
13315 let m = pr_message(&state, 'A');
13316 assert!(m.title.starts_with("feat: "));
13317 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
13318 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
13319 }
13320
13321 fn long_title_of(instruction: &str) -> String {
13322 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
13323 }
13324
13325 #[test]
13326 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
13327 let first = "Make the landing path keep a readable title for long tasks";
13328 let line = format!(
13329 "{first}. {}",
13330 "And then keep going with more words ".repeat(20)
13331 );
13332 let t = long_title_of(&line);
13333 assert_eq!(t, first);
13334 assert!(!t.starts_with("chore: land"));
13335 }
13336
13337 #[test]
13338 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
13339 let line = "word ".repeat(200);
13340 let t = long_title_of(&line);
13341 assert!(t.ends_with("word..."), "{t}");
13342 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
13343 }
13344
13345 #[test]
13346 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
13347 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
13348 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
13349 }
13350 }
13351
13352 #[test]
13353 fn pr_message_title_limit_is_exact() {
13354 let at = "a".repeat(PR_TITLE_MAX);
13355 assert_eq!(long_title_of(&at), at);
13356 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
13357 assert!(over.ends_with("..."), "{over}");
13358 assert_eq!(over.chars().count(), PR_TITLE_MAX);
13359 }
13360
13361 #[test]
13362 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
13363 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
13364 let t = long_title_of(&line);
13365 assert!(t.ends_with("..."), "{t}");
13366 assert!(t.is_ascii(), "{t}");
13367 }
13368
13369 #[test]
13370 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
13371 let line = format!(
13372 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
13373 "plus more filler words ".repeat(20)
13374 );
13375 let t = long_title_of(&line);
13376 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
13377 }
13378
13379 #[test]
13380 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
13381 let line = format!("\n\n# {}", "title words ".repeat(40));
13382 let state = state_with_summary(&line, "- nothing");
13383 let m = pr_message(&state, 'A');
13384 assert!(m.title.starts_with("title words"), "{}", m.title);
13385 assert_eq!(
13386 land::merge_subject(&m.title, &landing_subject_source(&state)),
13387 m.title
13388 );
13389 assert_eq!(
13391 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
13392 "feat: renamed by hand"
13393 );
13394 }
13395
13396 #[tokio::test]
13397 async fn github_text_rewrite_is_bounded_and_falls_back() {
13398 crate::run::pin_test_home();
13399 for (reply, accepted) in [
13400 (
13401 "TITLE: fix: retries\nAdd retries for failed requests.",
13402 true,
13403 ),
13404 (
13405 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
13406 false,
13407 ),
13408 (
13409 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
13410 true,
13411 ),
13412 (
13413 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
13414 true,
13415 ),
13416 (
13417 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
13418 true,
13419 ),
13420 (
13421 "TITLE: fix: retries\n日本語の説明です。 token=secret",
13422 false,
13423 ),
13424 ("TITLE: fix: retries\n", false),
13425 ] {
13426 let dir = tempfile::tempdir().unwrap();
13427 let mut runner = runner_at(RunStatus::Gating);
13428 runner.state = state_with_summary(
13429 "add retries",
13430 "TITLE: fix: retries\n日本語の説明を書きます。",
13431 );
13432 runner.state.repo = dir.path().to_owned();
13433 runner.state.candidates[0].worktree = dir.path().to_owned();
13434 let mut author = spec("alpha");
13435 author.command = vec![
13436 "sh".into(),
13437 "-c".into(),
13438 "printf '%s' \"$REWRITE_REPLY\"".into(),
13439 ];
13440 author.env.insert("REWRITE_REPLY".into(), reply.into());
13441 runner.state.config.agents = vec![author];
13442 let winner = runner.state.candidates[0].clone();
13443 let first = runner
13444 .guarded_pr_message(&winner, None, true)
13445 .await
13446 .unwrap();
13447 let message = if accepted {
13448 first.expect("a passing rewrite never asks")
13449 } else {
13450 assert!(first.is_none());
13452 assert!(runner.state.parked);
13453 let store = ask::Questions::open();
13454 let asked: Vec<_> = store
13455 .list()
13456 .into_iter()
13457 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
13458 .collect();
13459 assert_eq!(asked.len(), 1);
13460 assert!(!asked[0].detail.contains("secret"));
13461 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
13462 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
13463 assert_eq!(
13464 asked[0]
13465 .choices
13466 .contains(&crate::github_text::USE_REWRITE.to_owned()),
13467 rewrite.is_some()
13468 );
13469 store
13472 .update(&asked[0].id, |q| {
13473 q.abandon("test");
13474 Ok(())
13475 })
13476 .unwrap();
13477 let again = runner
13478 .guarded_pr_message(&winner, None, true)
13479 .await
13480 .unwrap()
13481 .expect("an abandoned question falls back");
13482 let n = store
13483 .list()
13484 .into_iter()
13485 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
13486 .count();
13487 assert_eq!(n, 1, "asked once per rejected text");
13488 again
13489 };
13490 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
13491 assert!(!message.body.contains("secret"));
13492 assert!(!message.body.contains("prefix"));
13493 assert!(!message.title.contains("secret"));
13494 assert_eq!(
13495 message.body.contains("Add retries for failed requests."),
13496 accepted
13497 );
13498 assert_eq!(
13499 runner.state.seats["github-text-rewrite"].turns, 1,
13500 "only one rewrite invocation"
13501 );
13502 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
13503 if accepted {
13504 assert!(runner.state.github_text.is_none());
13505 assert!(!runner.state.parked);
13506 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
13507 assert!(
13508 message
13509 .body
13510 .contains(&format!("magi:run/{}", runner.state.id))
13511 );
13512 }
13513 if !accepted {
13514 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
13515 assert!(
13516 message
13517 .body
13518 .contains(&format!("magi:run/{}", runner.state.id))
13519 );
13520 }
13521 }
13522 }
13523
13524 fn scripted(id: &str, reply: &str) -> AgentSpec {
13525 let mut a = spec(id);
13526 a.command = vec![
13527 "sh".into(),
13528 "-c".into(),
13529 "printf '%s' \"$REWRITE_REPLY\"".into(),
13530 ];
13531 a.env.insert("REWRITE_REPLY".into(), reply.into());
13532 a
13533 }
13534
13535 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
13536 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
13537
13538 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
13539 let mut runner = runner_at(RunStatus::Gating);
13540 runner.state = state_with_summary("add retries", JA_SUMMARY);
13541 if review_only {
13542 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
13543 }
13544 runner.state.repo = dir.to_owned();
13545 runner.state.candidates[0].worktree = dir.to_owned();
13546 runner.state.config.agents = agents;
13547 runner
13548 }
13549
13550 fn gate_questions(runner: &Runner) -> usize {
13551 ask::Questions::open()
13552 .list()
13553 .into_iter()
13554 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
13555 .count()
13556 }
13557
13558 #[tokio::test]
13559 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
13560 crate::run::pin_test_home();
13561 let dir = tempfile::tempdir().unwrap();
13562 let mut runner = rewrite_runner(
13563 dir.path(),
13564 true,
13565 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
13566 );
13567 runner.state.config.roles.language_judge =
13568 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
13569 let winner = runner.state.candidates[0].clone();
13570 let msg = runner
13571 .guarded_pr_message(&winner, None, true)
13572 .await
13573 .unwrap()
13574 .expect("an English rewrite posts without asking");
13575 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
13576 assert!(msg.body.contains("<details>kept</details>"));
13577 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
13578 assert_eq!(gate_questions(&runner), 0);
13579 assert!(runner.state.github_text.is_none());
13580 assert!(!runner.state.seats.contains_key("impl-A"));
13581 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
13582 }
13583
13584 #[tokio::test]
13585 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
13586 crate::run::pin_test_home();
13587 let dir = tempfile::tempdir().unwrap();
13588 let mut runner = rewrite_runner(
13589 dir.path(),
13590 false,
13591 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
13592 );
13593 runner.state.config.roles.language_judge =
13594 Some(crate::config::AgentChoice::One("beta".into()));
13595 runner.state.seats.insert(
13596 "impl-A".into(),
13597 SeatState::new("impl-A", "alpha", runner.state.seed),
13598 );
13599 let winner = runner.state.candidates[0].clone();
13600 let msg = runner
13601 .guarded_pr_message(&winner, None, true)
13602 .await
13603 .unwrap()
13604 .expect("rewritten");
13605 assert!(msg.body.contains("<details>kept</details>"));
13606 assert_eq!(runner.state.seats["impl-A"].turns, 0);
13607 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
13608 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
13609 }
13610
13611 #[tokio::test]
13612 async fn rewrite_reuses_a_resumable_implementer_session() {
13613 crate::run::pin_test_home();
13614 let dir = tempfile::tempdir().unwrap();
13615 let mut runner = rewrite_runner(
13616 dir.path(),
13617 false,
13618 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
13619 );
13620 runner.state.config.roles.language_judge =
13621 Some(crate::config::AgentChoice::One("beta".into()));
13622 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
13623 seat.turns = 1;
13624 runner.state.seats.insert("impl-A".into(), seat);
13625 let winner = runner.state.candidates[0].clone();
13626 runner
13627 .guarded_pr_message(&winner, None, true)
13628 .await
13629 .unwrap()
13630 .expect("rewritten");
13631 assert_eq!(runner.state.seats["impl-A"].turns, 2);
13632 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
13633 }
13634
13635 #[tokio::test]
13636 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
13637 crate::run::pin_test_home();
13638 let dir = tempfile::tempdir().unwrap();
13639 let mut broken = spec("bad");
13640 broken.command = vec!["false".into()];
13641 let mut runner = rewrite_runner(
13642 dir.path(),
13643 true,
13644 vec![
13645 broken,
13646 scripted("noformat", "Add retries."),
13647 scripted("good", EN_REWRITE),
13648 ],
13649 );
13650 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
13651 "bad".into(),
13652 "noformat".into(),
13653 "good".into(),
13654 ]));
13655 let winner = runner.state.candidates[0].clone();
13656 let msg = runner
13657 .guarded_pr_message(&winner, None, true)
13658 .await
13659 .unwrap()
13660 .expect("the third agent rewrites");
13661 assert!(msg.body.contains("`src/net.rs`"));
13662 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
13663 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
13664 }
13665
13666 #[tokio::test]
13667 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
13668 crate::run::pin_test_home();
13669 for (agents, role, saved) in [
13671 (
13672 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
13673 Some("beta"),
13674 true,
13675 ),
13676 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
13677 (Vec::new(), None, false),
13678 ] {
13679 let dir = tempfile::tempdir().unwrap();
13680 let mut runner = rewrite_runner(dir.path(), true, agents);
13681 runner.state.config.roles.language_judge =
13682 role.map(|r| crate::config::AgentChoice::One(r.into()));
13683 let winner = runner.state.candidates[0].clone();
13684 let first = runner
13685 .guarded_pr_message(&winner, None, true)
13686 .await
13687 .unwrap();
13688 assert!(first.is_none(), "withheld text parks the run");
13689 assert!(runner.state.parked);
13690 assert_eq!(gate_questions(&runner), 1);
13691 let g = runner.state.github_text.as_ref().unwrap();
13692 assert_eq!(g.rewrite.is_some(), saved);
13693 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
13694 }
13695 }
13696
13697 #[tokio::test]
13698 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
13699 crate::run::pin_test_home();
13700 let dir = tempfile::tempdir().unwrap();
13701 let mut runner = runner_at(RunStatus::Gating);
13702 runner.state =
13703 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
13704 runner.state.repo = dir.path().to_owned();
13705 runner.state.candidates[0].worktree = dir.path().to_owned();
13706 let mut author = spec("alpha");
13707 author.command = vec![
13708 "sh".into(),
13709 "-c".into(),
13710 "printf '%s' \"$REWRITE_REPLY\"".into(),
13711 ];
13712 author.env.insert(
13713 "REWRITE_REPLY".into(),
13714 format!(
13715 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
13716 "English sentence. ".repeat(600)
13717 ),
13718 );
13719 runner.state.config.agents = vec![author];
13720 let winner = runner.state.candidates[0].clone();
13721 assert!(
13722 runner
13723 .guarded_pr_message(&winner, None, true)
13724 .await
13725 .unwrap()
13726 .is_none()
13727 );
13728 let record = runner.state.github_text.clone().unwrap();
13729 let rewrite = record.rewrite.clone().unwrap();
13730 assert_eq!(rewrite.failed, ["body-language"]);
13731 assert!(
13732 rewrite
13733 .body
13734 .ends_with(&format!("magi:run/{}", runner.state.id))
13735 );
13736 let store = ask::Questions::open();
13737 let q = store.get(record.question.as_ref().unwrap()).unwrap();
13738 assert!(
13739 q.choices
13740 .contains(&crate::github_text::USE_REWRITE.to_owned())
13741 );
13742 for label in [
13743 "ORIGINAL",
13744 "FALLBACK",
13745 "REWRITE",
13746 "Rules still failed: body-language",
13747 "truncated",
13748 ] {
13749 assert!(q.detail.contains(label), "{label}");
13750 }
13751 let artifact = std::fs::read_to_string(crate::run::artifact_path(
13752 &runner.state,
13753 &crate::github_text::artifact_name(&record.fingerprint),
13754 ))
13755 .unwrap();
13756 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
13757 runner.state =
13759 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
13760 store
13761 .update(&q.id, |q| {
13762 q.say("まだ日本語")?;
13763 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
13764 })
13765 .unwrap();
13766 assert!(
13767 runner
13768 .guarded_pr_message(&winner, None, true)
13769 .await
13770 .unwrap()
13771 .is_none()
13772 );
13773 let retry = runner.state.github_text.as_ref().unwrap();
13774 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
13775 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
13776 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
13777 store
13778 .update(&q.id, |q| {
13779 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
13780 })
13781 .unwrap();
13782 let message = runner
13783 .guarded_pr_message(&winner, None, true)
13784 .await
13785 .unwrap()
13786 .unwrap();
13787 assert_eq!(message.title, rewrite.title);
13788 assert_eq!(message.body, rewrite.body);
13789 assert_eq!(
13790 runner.state.github_text.as_ref().unwrap().choice,
13791 Some(GithubTextChoice::Rewrite)
13792 );
13793 let again = runner
13794 .guarded_pr_message(&winner, None, true)
13795 .await
13796 .unwrap()
13797 .unwrap();
13798 assert_eq!(again.title, message.title);
13799 assert_eq!(again.body, message.body);
13800 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
13801 assert!(
13802 !runner
13803 .state
13804 .events
13805 .iter()
13806 .any(|e| e.message.contains(&rewrite.title)
13807 || e.message.contains("English sentence."))
13808 );
13809 }
13810
13811 #[tokio::test]
13812 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
13813 crate::run::pin_test_home();
13814 let dir = tempfile::tempdir().unwrap();
13815 let mut runner = runner_at(RunStatus::Gating);
13816 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
13817 runner.state.repo = dir.path().to_owned();
13818 runner.state.candidates[0].worktree = dir.path().to_owned();
13819 let winner = runner.state.candidates[0].clone();
13820 assert!(
13821 runner
13822 .guarded_pr_message(&winner, None, true)
13823 .await
13824 .unwrap()
13825 .is_none()
13826 );
13827 let record = runner.state.github_text.as_ref().unwrap();
13828 assert!(record.rewrite.is_none());
13829 let mut old = serde_json::to_value(record).unwrap();
13831 old.as_object_mut().unwrap().remove("rewrite");
13832 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
13833 assert!(old.rewrite.is_none());
13834 let id = record.question.clone().unwrap();
13835 ask::Questions::open()
13836 .update(&id, |q| {
13837 q.choices.push(crate::github_text::USE_REWRITE.into());
13838 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
13839 })
13840 .unwrap();
13841 let message = runner
13842 .guarded_pr_message(&winner, None, true)
13843 .await
13844 .unwrap()
13845 .unwrap();
13846 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
13847 assert_eq!(
13848 runner.state.github_text.as_ref().unwrap().choice,
13849 Some(GithubTextChoice::Fallback)
13850 );
13851 }
13852
13853 #[tokio::test]
13854 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
13855 crate::run::pin_test_home();
13856 let dir = tempfile::tempdir().unwrap();
13857 let mut runner = runner_at(RunStatus::Gating);
13858 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
13859 runner.state.repo = dir.path().to_owned();
13860 runner.state.candidates[0].worktree = dir.path().to_owned();
13861 let winner = runner.state.candidates[0].clone();
13862 assert!(
13863 runner
13864 .guarded_pr_message(&winner, None, true)
13865 .await
13866 .unwrap()
13867 .is_none()
13868 );
13869 let store = ask::Questions::open();
13870 let find = |run: &str| {
13871 store
13872 .list()
13873 .into_iter()
13874 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
13875 .unwrap()
13876 };
13877 let run = runner.state.id.clone();
13878 let q = find(&run);
13879 let fp = runner
13880 .state
13881 .github_text
13882 .as_ref()
13883 .unwrap()
13884 .fingerprint
13885 .clone();
13886 let name = crate::github_text::artifact_name(&fp);
13887 let path = crate::run::artifact_path(&runner.state, &name);
13888 assert!(q.detail.contains(&path.display().to_string()));
13889 assert!(q.detail.contains("title-language: title"));
13890 let saved = std::fs::read_to_string(&path).expect("artifact written");
13891 assert!(saved.contains("日本語のタイトル"));
13892 assert!(q.cwd.is_none());
13893 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
13894 let say = |id: &str, text: &str, choice: &str| {
13896 store
13897 .update(id, |q| {
13898 q.thread.push(ask::Turn {
13899 who: ask::Who::Operator,
13900 body: text.to_owned(),
13901 at: jiff::Timestamp::now(),
13902 note: None,
13903 });
13904 q.answer(ask::Answer::Choice(choice.to_owned()))
13905 })
13906 .unwrap();
13907 };
13908 say(&q.id, "まだ日本語", "use my text");
13909 assert!(
13910 runner
13911 .guarded_pr_message(&winner, None, true)
13912 .await
13913 .unwrap()
13914 .is_none()
13915 );
13916 let second = find(&run);
13917 assert_ne!(second.id, q.id);
13918 assert!(second.detail.contains("did not pass"));
13919 assert!(second.detail.contains("日本語のタイトル"));
13922 assert!(!second.detail.contains("まだ日本語"));
13923 assert!(second.detail.contains(&path.display().to_string()));
13924 say(&second.id, "fix: retry failed requests", "use my text");
13925 let message = runner
13926 .guarded_pr_message(&winner, None, true)
13927 .await
13928 .unwrap()
13929 .expect("a vetted replacement is posted");
13930 assert_eq!(message.title, "fix: retry failed requests");
13931 assert_eq!(
13932 runner
13933 .state
13934 .github_text
13935 .as_ref()
13936 .unwrap()
13937 .chosen_title
13938 .as_deref(),
13939 Some("fix: retry failed requests")
13940 );
13941 assert!(runner.state.github_text.as_ref().unwrap().resolved);
13942 let again = runner
13944 .guarded_pr_message(&winner, None, true)
13945 .await
13946 .unwrap()
13947 .unwrap();
13948 assert_eq!(again.title, message.title);
13949 assert_eq!(
13950 store
13951 .list()
13952 .into_iter()
13953 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
13954 .count(),
13955 2
13956 );
13957 }
13958
13959 #[test]
13960 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
13961 let mut state = state_with_summary(
13965 "add retries",
13966 "TITLE: fix(web): batch reads\n- reads run.json once",
13967 );
13968 state.config.graph.language = "ja".to_owned();
13969 let m = pr_message(&state, 'A');
13970 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
13971 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
13972
13973 let task = "今回やってほしいこと: results projector を直す";
13976 let mut state = state_with_summary(task, "- no title line");
13977 state.config.graph.language = "ja".to_owned();
13978 let m = pr_message(&state, 'A');
13979 assert_eq!(
13980 m.title,
13981 format!("chore: land candidate A of run {}", state.id)
13982 );
13983 assert!(
13984 m.body.contains(&format!(
13985 "<summary>Original task</summary>\n\n{task}\n\n</details>"
13986 )),
13987 "{}",
13988 m.body
13989 );
13990 }
13991
13992 #[test]
13993 fn pr_message_scrubs_home_paths_and_addresses() {
13994 let state = state_with_summary(
13995 "fix it in /Users/someone/src/x",
13996 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
13997 );
13998 let m = pr_message(&state, 'A');
13999 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
14000 assert!(!m.body.contains(leak), "{}", m.body);
14001 }
14002 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
14003 }
14004
14005 #[test]
14006 fn pr_message_survives_a_task_that_closes_details() {
14007 let state = state_with_summary("a </details> b", "TITLE: fix: x");
14008 let m = pr_message(&state, 'A');
14009 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
14010 }
14011
14012 #[test]
14013 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
14014 let cmd = manual_merge_command(
14015 MergeStyle::Squash,
14016 Path::new("/repo"),
14017 "b",
14018 "fix: \"quoted\" $(x) `y`\n\nbody",
14019 );
14020 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
14021 }
14022
14023 #[test]
14024 fn manual_merge_command_matches_the_configured_style() {
14025 let repo = Path::new("/repo");
14026 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
14027
14028 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
14029 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
14030
14031 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
14032 assert_eq!(
14033 squash,
14034 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
14035 \"Merge magi run 0832 (candidate A)\""
14036 );
14037
14038 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
14039 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
14040 }
14041
14042 #[test]
14043 fn a_nudge_gets_a_quarter_of_the_budget() {
14044 assert_eq!(retry_budget(secs(1200), true), secs(300));
14046 assert_eq!(retry_budget(secs(3600), true), secs(900));
14047 }
14048
14049 #[test]
14050 fn a_resent_prompt_keeps_the_whole_budget() {
14051 assert_eq!(retry_budget(secs(1200), false), secs(1200));
14054 assert_eq!(retry_budget(secs(60), false), secs(60));
14055 }
14056
14057 #[test]
14058 fn the_floor_never_exceeds_the_original_budget() {
14059 assert_eq!(retry_budget(secs(60), true), secs(60));
14063 assert_eq!(retry_budget(secs(480), true), secs(120));
14064 assert_eq!(retry_budget(secs(0), true), secs(0));
14065 }
14066
14067 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
14068 agent::CommandEvidence {
14069 id: "item1".to_owned(),
14070 description: "cargo test".to_owned(),
14071 exit_code,
14072 result_summary: String::new(),
14073 source: "codex".to_owned(),
14074 }
14075 }
14076
14077 #[test]
14078 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
14079 assert!(!has_unconfirmed_command(&[]));
14083 }
14084
14085 #[test]
14086 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
14087 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
14091 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
14092 assert!(!has_unconfirmed_command(&[
14093 evidence(Some(0)),
14094 evidence(Some(101))
14095 ]));
14096 }
14097
14098 #[test]
14099 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
14100 assert!(has_unconfirmed_command(&[
14101 evidence(Some(0)),
14102 evidence(None)
14103 ]));
14104 }
14105
14106 #[test]
14107 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
14108 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
14109 assert_eq!(
14110 verified_noop_claim(true, &[], text).as_deref(),
14111 Some("already fixed by b32cfc4, on main.")
14112 );
14113 }
14114
14115 #[test]
14116 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
14117 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
14120 assert!(verified_noop_claim(false, &[], text).is_none());
14121 }
14122
14123 #[test]
14124 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
14125 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
14126 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
14127 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
14129 }
14130
14131 #[test]
14132 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
14133 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
14134 }
14135
14136 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
14139 runner.state.candidates = shape
14140 .iter()
14141 .enumerate()
14142 .map(|(i, &(empty, verified))| Candidate {
14143 index: i,
14144 label: (b'A' + i as u8) as char,
14145 agent: "sonnet".to_owned(),
14146 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
14147 worktree: PathBuf::from(format!("/wt/{i}")),
14148 summary: String::new(),
14149 stat: String::new(),
14150 files: 0,
14151 commits: 0,
14152 empty,
14153 failed: None,
14154 verified_noop: verified.map(str::to_owned),
14155 duration_ms: 0,
14156 folded: false,
14157 })
14158 .collect();
14159 }
14160
14161 #[test]
14162 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
14163 ask_test_home();
14164 let mut runner = runner_at(RunStatus::Implementing);
14165 set_candidates(
14166 &mut runner,
14167 &[
14168 (true, Some("already on main at b32cfc4")),
14169 (true, Some("same fix, see the existing test")),
14170 ],
14171 );
14172
14173 runner
14174 .after_implement()
14175 .expect("a verified no-op is not an error");
14176
14177 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
14178 }
14179
14180 #[test]
14181 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
14182 ask_test_home();
14183 let mut runner = runner_at(RunStatus::Implementing);
14184 set_candidates(
14188 &mut runner,
14189 &[(true, Some("already on main at b32cfc4")), (true, None)],
14190 );
14191
14192 let err = runner
14193 .after_implement()
14194 .expect_err("an unverified empty candidate must still fail the run");
14195
14196 assert!(
14197 err.to_string().contains("no candidate produced a change"),
14198 "{err}"
14199 );
14200 assert_eq!(runner.state.status, RunStatus::Failed);
14201 }
14202
14203 #[test]
14204 fn after_implement_still_fails_an_ordinary_all_empty_run() {
14205 ask_test_home();
14206 let mut runner = runner_at(RunStatus::Implementing);
14207 set_candidates(&mut runner, &[(true, None), (true, None)]);
14208
14209 let err = runner
14210 .after_implement()
14211 .expect_err("no candidate declared anything; this is an ordinary failure");
14212
14213 assert!(
14214 err.to_string().contains("no candidate produced a change"),
14215 "{err}"
14216 );
14217 assert_eq!(runner.state.status, RunStatus::Failed);
14218 }
14219}
14220
14221#[derive(Debug)]
14224struct GraftRestoreFailed(String);
14225
14226impl std::fmt::Display for GraftRestoreFailed {
14227 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
14228 write!(
14229 f,
14230 "could not restore the winner to its pre-graft commit: {}",
14231 self.0
14232 )
14233 }
14234}
14235
14236impl std::error::Error for GraftRestoreFailed {}
14237
14238#[cfg(test)]
14242mod resource_wait_tests {
14243 use super::*;
14244 use std::collections::BTreeMap;
14245
14246 fn sh(id: &str, script: &str) -> AgentSpec {
14247 AgentSpec {
14248 id: id.to_owned(),
14249 kind: crate::config::AgentKind::Command,
14250 model: None,
14251 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
14252 extra_args: Vec::new(),
14253 env: BTreeMap::new(),
14254 prompt_delivery: None,
14255 }
14256 }
14257
14258 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
14259 SeatJob {
14260 seat: SeatState::new("impl-A", &spec.id, 1),
14261 spec,
14262 cwd: dir.to_path_buf(),
14263 prompt: "do it".to_owned(),
14264 timeout,
14265 allow_write: true,
14266 sessions: false,
14267 artifacts: dir.join("art"),
14268 stem: "impl-A".to_owned(),
14269 handover: None,
14270 }
14271 }
14272
14273 fn state(resource_wait_secs: u64) -> RunState {
14274 let mut config = Config::default();
14275 config.graph.timeout_resource_wait = resource_wait_secs;
14276 RunState::new(
14277 PathBuf::from("/repo"),
14278 "main".to_owned(),
14279 "abc1234".to_owned(),
14280 "task".to_owned(),
14281 config,
14282 )
14283 }
14284
14285 fn hold(cache: &Path) -> crate::cache::Guard {
14286 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
14287 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
14288 crate::cache::AcquireOutcome::Acquired(g) => g,
14289 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
14290 }
14291 }
14292
14293 async fn run(
14294 jobs: Vec<SeatJob>,
14295 sem: Arc<Semaphore>,
14296 cache: Option<&Path>,
14297 st: &mut RunState,
14298 ) -> Vec<(usize, SeatState, AgentOutcome)> {
14299 let prompts = Prompts::default();
14300 let ctx = WaveCtx {
14301 run: "r",
14302 node: "implement",
14303 prompts: &prompts,
14304 cache,
14305 round: None,
14306 carry_seats: false,
14307 pauses: None,
14308 };
14309 wave(jobs, sem, &ctx, st, 0).await
14310 }
14311
14312 #[tokio::test]
14313 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
14314 crate::run::pin_test_home();
14315 let tmp = tempfile::tempdir().unwrap();
14316 let cache = tmp.path().join("target-a");
14317 let guard = hold(&cache);
14318 tokio::spawn(async move {
14320 tokio::time::sleep(Duration::from_millis(1500)).await;
14321 guard.release();
14322 });
14323 let mut st = state(60);
14324 let results = run(
14325 vec![job(
14326 sh("alpha", "echo ok"),
14327 tmp.path(),
14328 Duration::from_secs(1),
14329 )],
14330 Arc::new(Semaphore::new(1)),
14331 Some(&cache),
14332 &mut st,
14333 )
14334 .await;
14335 assert!(
14336 matches!(results[0].2, AgentOutcome::Ok(_)),
14337 "the wait must not be charged to the agent"
14338 );
14339 assert!(st.resource_block.is_none());
14340 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
14341 assert!(st.resource_wait.is_none() && st.active.is_empty());
14342 }
14343
14344 #[tokio::test]
14345 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
14346 crate::run::pin_test_home();
14347 let tmp = tempfile::tempdir().unwrap();
14348 let cache = tmp.path().join("target-b");
14349 let _held = hold(&cache);
14350 let mut st = state(1);
14351 let results = run(
14352 vec![job(
14353 sh("alpha", "echo ok"),
14354 tmp.path(),
14355 Duration::from_secs(1),
14356 )],
14357 Arc::new(Semaphore::new(1)),
14358 Some(&cache),
14359 &mut st,
14360 )
14361 .await;
14362 let (_, seat, out) = &results[0];
14363 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14364 assert!(wave_blocked(&results));
14365 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
14366 assert_eq!(seat.turns, 0, "nothing was asked");
14367 assert!(
14368 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
14369 "no agent was spawned"
14370 );
14371 assert!(FailClass::of(out).is_none(), "not a failure class");
14372 let block = st.resource_block.as_ref().expect("recorded for the resume");
14373 assert_eq!(block.kind, "shared build cache");
14374 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
14375 assert!(st.active.is_empty());
14376 }
14377
14378 #[tokio::test]
14379 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
14380 crate::run::pin_test_home();
14381 let tmp = tempfile::tempdir().unwrap();
14382 let mut st = state(1);
14383 let results = run(
14384 vec![job(
14385 sh("alpha", "echo ok"),
14386 tmp.path(),
14387 Duration::from_secs(1),
14388 )],
14389 Arc::new(Semaphore::new(0)),
14390 None,
14391 &mut st,
14392 )
14393 .await;
14394 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
14395 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
14396 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
14397 }
14398
14399 #[tokio::test]
14400 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
14401 crate::run::pin_test_home();
14402 let tmp = tempfile::tempdir().unwrap();
14403 let mut st = state(60);
14404 let results = run(
14405 vec![job(
14406 sh("alpha", "sleep 30"),
14407 tmp.path(),
14408 Duration::from_secs(1),
14409 )],
14410 Arc::new(Semaphore::new(1)),
14411 None,
14412 &mut st,
14413 )
14414 .await;
14415 let out = &results[0].2;
14416 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
14417 let class = FailClass::of(out).expect("a failure");
14418 assert_eq!(class, FailClass::Timeout);
14419 assert!(should_hand_over(None, &class), "existing handover policy");
14420 assert!(st.resource_block.is_none());
14421 }
14422}